AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Foundational Cover Image Rework (#584)

Browse source 
* Updating wording on card item when total pages is 0, to be just "Cannot Read" since it could be a non-archive file

* Refactored cover images to be stored on disk. This first commit has the extraction to disk and the metadata service to handle updating when applicable.

* Refactored code to have the actual save to cover image directory done by ImageService.

* Implemented the ability to override cover images.

* Some cleanup on Image service

* Implemented the ability to cleanup old covers nightly

* Added a migration to migrate existing covers to new cover image format (files).

* Refactored all CoverImages to just be the filename, leaving the Join with Cover directory to higher level code.

* Ensure when getting user progress, we pick the first.

* Added cleanup cover images for deleted tags. Don't pull any cover images that are blank.

* After series update, clear out cover image. No change on UI, but just keeps things clear before metadata refresh hits

* Refactored image formats for covers to ImageService.

* Fixed an issue where after refactoring how images were stored, the cleanup service was deleting them after each scan.

* Changed how ShouldUpdateCoverImage works to check if file exists or not even if cover image is locked.

* Fixed unit tests

* Added caching back to cover images.

* Caching on series as well

* Code Cleanup items

* Ensure when checking if a file exists in MetadataService, that we join for cover image directory. After we scan library, do one last filter to delete any series that have 0 pages total.

* Catch exceptions so we don't run cover migration if this is first time run.

* After a scan, only clear out the cache directory and not do a deep clean.

* Implemented the ability to backup custom locked covers only.

* Fixed unit tests

* Trying to figure out why GA crashes when running MetadataServiceTests.cs

* Some debugging on GA tests not running

* Commented out tests that were causing issues in GA.

* Fixed an issue where series cover images wouldn't migrate

* Fixed the updating of links to actually do all series and not just locked
This commit is contained in:
Joseph Milazzo 2021-09-21 17:15:29 -07:00 • committed by GitHub
parent fd6925b126
commit 82b5b599e0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
50 changed files with 1928 additions and 234 deletions
Download patch file Download diff file Expand all files Collapse all files

160
API/Data/MigrateCoverImages.cs Normal file
View file

@ -0,0 +1,160 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Helpers;
using API.Services;
using Microsoft.EntityFrameworkCore;
namespace API.Data
{
/// <summary>
/// A data structure to migrate Cover Images from byte[] to files.
/// </summary>
internal class CoverMigration
{
public string Id { get; set; }
public byte[] CoverImage { get; set; }
public string ParentId { get; set; }
}
/// <summary>
/// In v0.4.6, Cover Images were migrated from byte[] in the DB to external files. This migration handles that work.
/// </summary>
public static class MigrateCoverImages
{
/// <summary>
/// Run first. Will extract byte[]s from DB and write them to the cover directory.
/// </summary>
public static void ExtractToImages(DbContext context)
{
Console.WriteLine("Migrating Cover Images to disk. Expect delay.");
DirectoryService.ExistOrCreate(DirectoryService.CoverImageDirectory);
Console.WriteLine("Extracting cover images for Series");
var lockedSeries = SqlHelper.RawSqlQuery(context, "Select Id, CoverImage From Series Where CoverImage IS NOT NULL", x =>
new CoverMigration()
{
Id = x[0] + string.Empty,
CoverImage = (byte[]) x[1],
ParentId = "0"
});
foreach (var series in lockedSeries)
{
if (series.CoverImage == null || !series.CoverImage.Any()) continue;
if (File.Exists(Path.Join(DirectoryService.CoverImageDirectory,
$"{ImageService.GetSeriesFormat(int.Parse(series.Id))}.png"))) continue;
try
{
var stream = new MemoryStream(series.CoverImage);
stream.Position = 0;
ImageService.WriteCoverThumbnail(stream, ImageService.GetSeriesFormat(int.Parse(series.Id)));
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
Console.WriteLine("Extracting cover images for Chapters");
var chapters = SqlHelper.RawSqlQuery(context, "Select Id, CoverImage, VolumeId From Chapter Where CoverImage IS NOT NULL;", x =>
new CoverMigration()
{
Id = x[0] + string.Empty,
CoverImage = (byte[]) x[1],
ParentId = x[2] + string.Empty
});
foreach (var chapter in chapters)
{
if (chapter.CoverImage == null || !chapter.CoverImage.Any()) continue;
if (File.Exists(Path.Join(DirectoryService.CoverImageDirectory,
$"{ImageService.GetChapterFormat(int.Parse(chapter.Id), int.Parse(chapter.ParentId))}.png"))) continue;
try
{
var stream = new MemoryStream(chapter.CoverImage);
stream.Position = 0;
ImageService.WriteCoverThumbnail(stream, $"{ImageService.GetChapterFormat(int.Parse(chapter.Id), int.Parse(chapter.ParentId))}");
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
Console.WriteLine("Extracting cover images for Collection Tags");
var tags = SqlHelper.RawSqlQuery(context, "Select Id, CoverImage From CollectionTag Where CoverImage IS NOT NULL;", x =>
new CoverMigration()
{
Id = x[0] + string.Empty,
CoverImage = (byte[]) x[1] ,
ParentId = "0"
});
foreach (var tag in tags)
{
if (tag.CoverImage == null || !tag.CoverImage.Any()) continue;
if (File.Exists(Path.Join(DirectoryService.CoverImageDirectory,
$"{ImageService.GetCollectionTagFormat(int.Parse(tag.Id))}.png"))) continue;
try
{
var stream = new MemoryStream(tag.CoverImage);
stream.Position = 0;
ImageService.WriteCoverThumbnail(stream, $"{ImageService.GetCollectionTagFormat(int.Parse(tag.Id))}");
}
catch (Exception e)
{
Console.WriteLine(e);
}
}
}
/// <summary>
/// Run after <see cref="ExtractToImages"/>. Will update the DB with names of files that were extracted.
/// </summary>
/// <param name="context"></param>
public static async Task UpdateDatabaseWithImages(DataContext context)
{
Console.WriteLine("Updating Series entities");
var seriesCovers = await context.Series.Where(s => !string.IsNullOrEmpty(s.CoverImage)).ToListAsync();
foreach (var series in seriesCovers)
{
if (!File.Exists(Path.Join(DirectoryService.CoverImageDirectory,
$"{ImageService.GetSeriesFormat(series.Id)}.png"))) continue;
series.CoverImage = $"{ImageService.GetSeriesFormat(series.Id)}.png";
}
await context.SaveChangesAsync();
Console.WriteLine("Updating Chapter entities");
var chapters = await context.Chapter.ToListAsync();
foreach (var chapter in chapters)
{
if (File.Exists(Path.Join(DirectoryService.CoverImageDirectory,
$"{ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId)}.png")))
{
chapter.CoverImage = $"{ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId)}.png";
}
}
await context.SaveChangesAsync();
Console.WriteLine("Updating Collection Tag entities");
var tags = await context.CollectionTag.ToListAsync();
foreach (var tag in tags)
{
if (File.Exists(Path.Join(DirectoryService.CoverImageDirectory,
$"{ImageService.GetCollectionTagFormat(tag.Id)}.png")))
{
tag.CoverImage = $"{ImageService.GetCollectionTagFormat(tag.Id)}.png";
}
}
await context.SaveChangesAsync();
Console.WriteLine("Cover Image Migration completed");
}
}
}

1042
API/Data/Migrations/20210916142418_EntityImageRefactor.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

97
API/Data/Migrations/20210916142418_EntityImageRefactor.cs Normal file
View file

@ -0,0 +1,97 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
namespace API.Data.Migrations
{
public partial class EntityImageRefactor : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "RowVersion",
table: "AppUserProgresses");
migrationBuilder.AlterColumn<string>(
name: "CoverImage",
table: "Volume",
type: "TEXT",
nullable: true,
oldClrType: typeof(byte[]),
oldType: "BLOB",
oldNullable: true);
migrationBuilder.AlterColumn<string>(
name: "CoverImage",
table: "Series",
type: "TEXT",
nullable: true,
oldClrType: typeof(byte[]),
oldType: "BLOB",
oldNullable: true);
migrationBuilder.AlterColumn<string>(
name: "CoverImage",
table: "CollectionTag",
type: "TEXT",
nullable: true,
oldClrType: typeof(byte[]),
oldType: "BLOB",
oldNullable: true);
migrationBuilder.AlterColumn<string>(
name: "CoverImage",
table: "Chapter",
type: "TEXT",
nullable: true,
oldClrType: typeof(byte[]),
oldType: "BLOB",
oldNullable: true);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.AlterColumn<byte[]>(
name: "CoverImage",
table: "Volume",
type: "BLOB",
nullable: true,
oldClrType: typeof(string),
oldType: "TEXT",
oldNullable: true);
migrationBuilder.AlterColumn<byte[]>(
name: "CoverImage",
table: "Series",
type: "BLOB",
nullable: true,
oldClrType: typeof(string),
oldType: "TEXT",
oldNullable: true);
migrationBuilder.AlterColumn<byte[]>(
name: "CoverImage",
table: "CollectionTag",
type: "BLOB",
nullable: true,
oldClrType: typeof(string),
oldType: "TEXT",
oldNullable: true);
migrationBuilder.AlterColumn<byte[]>(
name: "CoverImage",
table: "Chapter",
type: "BLOB",
nullable: true,
oldClrType: typeof(string),
oldType: "TEXT",
oldNullable: true);
migrationBuilder.AddColumn<uint>(
name: "RowVersion",
table: "AppUserProgresses",
type: "INTEGER",
nullable: false,
defaultValue: 0u);
}
}
}

20
API/Data/Migrations/DataContextModelSnapshot.cs
View file

@ -229,10 +229,6 @@ namespace API.Data.Migrations
b.Property<int>("PagesRead")
.HasColumnType("INTEGER");
b.Property<uint>("RowVersion")
.IsConcurrencyToken()
.HasColumnType("INTEGER");
b.Property<int>("SeriesId")
.HasColumnType("INTEGER");
@ -292,8 +288,8 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<byte[]>("CoverImage")
.HasColumnType("BLOB");
b.Property<string>("CoverImage")
.HasColumnType("TEXT");
b.Property<bool>("CoverImageLocked")
.HasColumnType("INTEGER");
@ -335,8 +331,8 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<byte[]>("CoverImage")
.HasColumnType("BLOB");
b.Property<string>("CoverImage")
.HasColumnType("TEXT");
b.Property<bool>("CoverImageLocked")
.HasColumnType("INTEGER");
@ -511,8 +507,8 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<byte[]>("CoverImage")
.HasColumnType("BLOB");
b.Property<string>("CoverImage")
.HasColumnType("TEXT");
b.Property<bool>("CoverImageLocked")
.HasColumnType("INTEGER");
@ -607,8 +603,8 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<byte[]>("CoverImage")
.HasColumnType("BLOB");
b.Property<string>("CoverImage")
.HasColumnType("TEXT");
b.Property<DateTime>("Created")
.HasColumnType("TEXT");

2
API/Data/Repositories/AppUserProgressRepository.cs
View file

@ -73,7 +73,7 @@ namespace API.Data.Repositories
{
return await _context.AppUserProgresses
.Where(p => p.ChapterId == chapterId && p.AppUserId == userId)
.SingleOrDefaultAsync();
.FirstOrDefaultAsync();
}
}
}

32
API/Data/Repositories/ChapterRepository.cs
View file

@ -1,5 +1,7 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using API.DTOs;
using API.DTOs.Reader;
@ -140,8 +142,9 @@ namespace API.Data.Repositories
/// </summary>
/// <param name="chapterId"></param>
/// <returns></returns>
public async Task<byte[]> GetChapterCoverImageAsync(int chapterId)
public async Task<string> GetChapterCoverImageAsync(int chapterId)
{
return await _context.Chapter
.Where(c => c.Id == chapterId)
.Select(c => c.CoverImage)
@ -149,10 +152,33 @@ namespace API.Data.Repositories
.SingleOrDefaultAsync();
}
public async Task<IList<string>> GetAllCoverImagesAsync()
{
return await _context.Chapter
.Select(c => c.CoverImage)
.Where(t => !string.IsNullOrEmpty(t))
.AsNoTracking()
.ToListAsync();
}
/// <summary>
/// Returns non-tracked files for a set of chapterIds
/// Returns cover images for locked chapters
/// </summary>
/// <param name="chapterIds"></param>
/// <returns></returns>
public async Task<IEnumerable<string>> GetCoverImagesForLockedChaptersAsync()
{
return await _context.Chapter
.Where(c => c.CoverImageLocked)
.Select(c => c.CoverImage)
.Where(t => !string.IsNullOrEmpty(t))
.AsNoTracking()
.ToListAsync();
}
/// <summary>
/// Returns non-tracked files for a set of <paramref name="chapterIds"/>
/// </summary>
/// <param name="chapterIds">List of chapter Ids</param>
/// <returns></returns>
public async Task<IList<MangaFile>> GetFilesForChaptersAsync(IReadOnlyList<int> chapterIds)
{

15
API/Data/Repositories/CollectionTagRepository.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.DTOs;
@ -48,11 +49,19 @@ namespace API.Data.Repositories
public async Task<IEnumerable<CollectionTag>> GetAllTagsAsync()
{
return await _context.CollectionTag
.Select(c => c)
.OrderBy(c => c.NormalizedTitle)
.ToListAsync();
}
public async Task<IList<string>> GetAllCoverImagesAsync()
{
return await _context.CollectionTag
.Select(t => t.CoverImage)
.Where(t => !string.IsNullOrEmpty(t))
.AsNoTracking()
.ToListAsync();
}
public async Task<IEnumerable<CollectionTagDto>> GetAllTagDtosAsync()
{
return await _context.CollectionTag
@ -100,9 +109,9 @@ namespace API.Data.Repositories
.ToListAsync();
}
public Task<byte[]> GetCoverImageAsync(int collectionTagId)
public async Task<string> GetCoverImageAsync(int collectionTagId)
{
return _context.CollectionTag
return await _context.CollectionTag
.Where(c => c.Id == collectionTagId)
.Select(c => c.CoverImage)
.AsNoTracking()

21
API/Data/Repositories/SeriesRepository.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Comparators;
@ -256,7 +257,7 @@ namespace API.Data.Repositories
}
}
public async Task<byte[]> GetSeriesCoverImageAsync(int seriesId)
public async Task<string> GetSeriesCoverImageAsync(int seriesId)
{
return await _context.Series
.Where(s => s.Id == seriesId)
@ -443,5 +444,23 @@ namespace API.Data.Repositories
.AsSplitQuery()
.ToListAsync();
}
public async Task<IList<string>> GetAllCoverImagesAsync()
{
return await _context.Series
.Select(s => s.CoverImage)
.Where(t => !string.IsNullOrEmpty(t))
.AsNoTracking()
.ToListAsync();
}
public async Task<IEnumerable<string>> GetLockedCoverImagesAsync()
{
return await _context.Series
.Where(s => s.CoverImageLocked && !string.IsNullOrEmpty(s.CoverImage))
.Select(s => s.CoverImage)
.AsNoTracking()
.ToListAsync();
}
}
}

5
API/Data/Repositories/VolumeRepository.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.DTOs;
@ -35,9 +36,9 @@ namespace API.Data.Repositories
.ToListAsync();
}
public async Task<byte[]> GetVolumeCoverImageAsync(int volumeId)
public async Task<string> GetVolumeCoverImageAsync(int volumeId)
{
return await _context.Volume
return await _context.Volume
.Where(v => v.Id == volumeId)
.Select(v => v.CoverImage)
.AsNoTracking()