Misc Enhancements (#1525)
* Moved the data connection for the Database out of appsettings.json and hardcoded it. This will allow for more customization and cleaner update process. * Removed unneeded code * Updated pdf viewer to 15.0.0 (pdf 2.6), which now supports east-asian fonts * Fixed up some regex parsing for volumes that have a float number. * Fixed a bug where the tooltip for Publication Status wouldn't show * Fixed some weird parsing rules where v1.1 would parse as volume 1 chapter 1 * Fixed a bug where bookmarking button was hidden for admins without bookmark role (due to migration) * Unified the star rating component in series detail to match metadata filter. * Fixed a bug in the bulk selection code when using shift selection, where the inverse of what was selected would be toggled. * Fixed some old code where if on all series page, only English as a language would return. We now return all languages of all libraries. * Updated api/metadata/languages documentation * Refactored some bookmark api names: get-bookmarks -> chapter-bookmarks, get-all-bookmarks -> all-bookmarks, get-series-bookmarks -> series-bookmarks, etc. * Refactored all cases of createSeriesFilter to filterUtiltityService. Added ability to search for a series on Bookmarks page. Fixed a bug where people filters wouldn't respect the disable flag froms ettings. * Cleaned up a bit of the circular downloader code. * Implemented Russian Parsing * Fixed an issue where some users that had a missing theme entry wouldn't be able to update their user preferences. * Refactored normalization to exclude !, thus allowing series with ! to be different from each other. * Fixed a migration exit case * Fixed broken unit test
This commit is contained in:
Joseph Milazzo
GitHub
parent
b7d88f08d8
commit
00f0ad5a3f
50 changed files with 508 additions and 419 deletions
|
|
@ -1,181 +0,0 @@
|
|||
using System;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using API.Comparators;
|
||||
using API.Helpers;
|
||||
using API.Services;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
|
||||
namespace API.Data;
|
||||
|
||||
/// <summary>
|
||||
/// A data structure to migrate Cover Images from byte[] to files.
|
||||
/// </summary>
|
||||
internal class CoverMigration
|
||||
{
|
||||
public string Id { get; set; }
|
||||
public byte[] CoverImage { get; set; }
|
||||
public string ParentId { get; set; }
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// In v0.4.6, Cover Images were migrated from byte[] in the DB to external files. This migration handles that work.
|
||||
/// </summary>
|
||||
public static class MigrateCoverImages
|
||||
{
|
||||
private static readonly ChapterSortComparerZeroFirst ChapterSortComparerForInChapterSorting = new ();
|
||||
|
||||
/// <summary>
|
||||
/// Run first. Will extract byte[]s from DB and write them to the cover directory.
|
||||
/// </summary>
|
||||
public static void ExtractToImages(DbContext context, IDirectoryService directoryService, IImageService imageService)
|
||||
{
|
||||
Console.WriteLine("Migrating Cover Images to disk. Expect delay.");
|
||||
directoryService.ExistOrCreate(directoryService.CoverImageDirectory);
|
||||
|
||||
Console.WriteLine("Extracting cover images for Series");
|
||||
var lockedSeries = SqlHelper.RawSqlQuery(context, "Select Id, CoverImage From Series Where CoverImage IS NOT NULL", x =>
|
||||
new CoverMigration()
|
||||
{
|
||||
Id = x[0] + string.Empty,
|
||||
CoverImage = (byte[]) x[1],
|
||||
ParentId = "0"
|
||||
});
|
||||
foreach (var series in lockedSeries)
|
||||
{
|
||||
if (series.CoverImage == null || !series.CoverImage.Any()) continue;
|
||||
if (File.Exists(directoryService.FileSystem.Path.Join(directoryService.CoverImageDirectory,
|
||||
$"{ImageService.GetSeriesFormat(int.Parse(series.Id))}.png"))) continue;
|
||||
|
||||
try
|
||||
{
|
||||
var stream = new MemoryStream(series.CoverImage);
|
||||
stream.Position = 0;
|
||||
imageService.WriteCoverThumbnail(stream, ImageService.GetSeriesFormat(int.Parse(series.Id)), directoryService.CoverImageDirectory);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e);
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Extracting cover images for Chapters");
|
||||
var chapters = SqlHelper.RawSqlQuery(context, "Select Id, CoverImage, VolumeId From Chapter Where CoverImage IS NOT NULL;", x =>
|
||||
new CoverMigration()
|
||||
{
|
||||
Id = x[0] + string.Empty,
|
||||
CoverImage = (byte[]) x[1],
|
||||
ParentId = x[2] + string.Empty
|
||||
});
|
||||
foreach (var chapter in chapters)
|
||||
{
|
||||
if (chapter.CoverImage == null || !chapter.CoverImage.Any()) continue;
|
||||
if (directoryService.FileSystem.File.Exists(directoryService.FileSystem.Path.Join(directoryService.CoverImageDirectory,
|
||||
$"{ImageService.GetChapterFormat(int.Parse(chapter.Id), int.Parse(chapter.ParentId))}.png"))) continue;
|
||||
|
||||
try
|
||||
{
|
||||
var stream = new MemoryStream(chapter.CoverImage);
|
||||
stream.Position = 0;
|
||||
imageService.WriteCoverThumbnail(stream, $"{ImageService.GetChapterFormat(int.Parse(chapter.Id), int.Parse(chapter.ParentId))}", directoryService.CoverImageDirectory);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e);
|
||||
}
|
||||
}
|
||||
|
||||
Console.WriteLine("Extracting cover images for Collection Tags");
|
||||
var tags = SqlHelper.RawSqlQuery(context, "Select Id, CoverImage From CollectionTag Where CoverImage IS NOT NULL;", x =>
|
||||
new CoverMigration()
|
||||
{
|
||||
Id = x[0] + string.Empty,
|
||||
CoverImage = (byte[]) x[1] ,
|
||||
ParentId = "0"
|
||||
});
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
if (tag.CoverImage == null || !tag.CoverImage.Any()) continue;
|
||||
if (directoryService.FileSystem.File.Exists(Path.Join(directoryService.CoverImageDirectory,
|
||||
$"{ImageService.GetCollectionTagFormat(int.Parse(tag.Id))}.png"))) continue;
|
||||
try
|
||||
{
|
||||
var stream = new MemoryStream(tag.CoverImage);
|
||||
stream.Position = 0;
|
||||
imageService.WriteCoverThumbnail(stream, $"{ImageService.GetCollectionTagFormat(int.Parse(tag.Id))}", directoryService.CoverImageDirectory);
|
||||
}
|
||||
catch (Exception e)
|
||||
{
|
||||
Console.WriteLine(e);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Run after <see cref="ExtractToImages"/>. Will update the DB with names of files that were extracted.
|
||||
/// </summary>
|
||||
/// <param name="context"></param>
|
||||
public static async Task UpdateDatabaseWithImages(DataContext context, IDirectoryService directoryService)
|
||||
{
|
||||
Console.WriteLine("Updating Series entities");
|
||||
var seriesCovers = await context.Series.Where(s => !string.IsNullOrEmpty(s.CoverImage)).ToListAsync();
|
||||
foreach (var series in seriesCovers)
|
||||
{
|
||||
if (!directoryService.FileSystem.File.Exists(directoryService.FileSystem.Path.Join(directoryService.CoverImageDirectory,
|
||||
$"{ImageService.GetSeriesFormat(series.Id)}.png"))) continue;
|
||||
series.CoverImage = $"{ImageService.GetSeriesFormat(series.Id)}.png";
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
Console.WriteLine("Updating Chapter entities");
|
||||
var chapters = await context.Chapter.ToListAsync();
|
||||
// ReSharper disable once ForeachCanBePartlyConvertedToQueryUsingAnotherGetEnumerator
|
||||
foreach (var chapter in chapters)
|
||||
{
|
||||
if (directoryService.FileSystem.File.Exists(directoryService.FileSystem.Path.Join(directoryService.CoverImageDirectory,
|
||||
$"{ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId)}.png")))
|
||||
{
|
||||
chapter.CoverImage = $"{ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId)}.png";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
Console.WriteLine("Updating Volume entities");
|
||||
var volumes = await context.Volume.Include(v => v.Chapters).ToListAsync();
|
||||
foreach (var volume in volumes)
|
||||
{
|
||||
var firstChapter = volume.Chapters.MinBy(x => double.Parse(x.Number), ChapterSortComparerForInChapterSorting);
|
||||
if (firstChapter == null) continue;
|
||||
if (directoryService.FileSystem.File.Exists(directoryService.FileSystem.Path.Join(directoryService.CoverImageDirectory,
|
||||
$"{ImageService.GetChapterFormat(firstChapter.Id, firstChapter.VolumeId)}.png")))
|
||||
{
|
||||
volume.CoverImage = $"{ImageService.GetChapterFormat(firstChapter.Id, firstChapter.VolumeId)}.png";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
Console.WriteLine("Updating Collection Tag entities");
|
||||
var tags = await context.CollectionTag.ToListAsync();
|
||||
// ReSharper disable once ForeachCanBePartlyConvertedToQueryUsingAnotherGetEnumerator
|
||||
foreach (var tag in tags)
|
||||
{
|
||||
if (directoryService.FileSystem.File.Exists(directoryService.FileSystem.Path.Join(directoryService.CoverImageDirectory,
|
||||
$"{ImageService.GetCollectionTagFormat(tag.Id)}.png")))
|
||||
{
|
||||
tag.CoverImage = $"{ImageService.GetCollectionTagFormat(tag.Id)}.png";
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
await context.SaveChangesAsync();
|
||||
|
||||
Console.WriteLine("Cover Image Migration completed");
|
||||
}
|
||||
|
||||
}
|
||||
120
API/Data/MigrateNormalizedEverything.cs
Normal file
120
API/Data/MigrateNormalizedEverything.cs
Normal file
|
|
@ -0,0 +1,120 @@
|
|||
using System;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using Kavita.Common.EnvironmentInfo;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace API.Data;
|
||||
|
||||
/// <summary>
|
||||
/// v0.6.0 introduced a change in how Normalization works and hence every normalized field needs to be re-calculated
|
||||
/// </summary>
|
||||
public static class MigrateNormalizedEverything
|
||||
{
|
||||
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
|
||||
{
|
||||
// if current version is > 0.5.6.3, then we can exit and not perform
|
||||
var settings = await unitOfWork.SettingsRepository.GetSettingsDtoAsync();
|
||||
if (Version.Parse(settings.InstallVersion) > new Version(0, 5, 6, 3))
|
||||
{
|
||||
return;
|
||||
}
|
||||
logger.LogCritical("Running MigrateNormalizedEverything migration. Please be patient, this may take some time depending on the size of your library. Do not abort, this can break your Database");
|
||||
|
||||
logger.LogInformation("Updating Normalization on Series...");
|
||||
foreach (var series in await dataContext.Series.ToListAsync())
|
||||
{
|
||||
series.NormalizedLocalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(series.LocalizedName ?? string.Empty);
|
||||
series.NormalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(series.Name ?? string.Empty);
|
||||
logger.LogInformation("Updated Series: {SeriesName}", series.Name);
|
||||
unitOfWork.SeriesRepository.Update(series);
|
||||
}
|
||||
|
||||
if (unitOfWork.HasChanges())
|
||||
{
|
||||
await unitOfWork.CommitAsync();
|
||||
}
|
||||
logger.LogInformation("Updating Normalization on Series...Done");
|
||||
|
||||
// Genres
|
||||
logger.LogInformation("Updating Normalization on Genres...");
|
||||
foreach (var genre in await dataContext.Genre.ToListAsync())
|
||||
{
|
||||
genre.NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(genre.Title ?? string.Empty);
|
||||
logger.LogInformation("Updated Genre: {Genre}", genre.Title);
|
||||
unitOfWork.GenreRepository.Attach(genre);
|
||||
}
|
||||
|
||||
if (unitOfWork.HasChanges())
|
||||
{
|
||||
await unitOfWork.CommitAsync();
|
||||
}
|
||||
logger.LogInformation("Updating Normalization on Genres...Done");
|
||||
|
||||
// Tags
|
||||
logger.LogInformation("Updating Normalization on Tags...");
|
||||
foreach (var tag in await dataContext.Tag.ToListAsync())
|
||||
{
|
||||
tag.NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(tag.Title ?? string.Empty);
|
||||
logger.LogInformation("Updated Tag: {Tag}", tag.Title);
|
||||
unitOfWork.TagRepository.Attach(tag);
|
||||
}
|
||||
|
||||
if (unitOfWork.HasChanges())
|
||||
{
|
||||
await unitOfWork.CommitAsync();
|
||||
}
|
||||
logger.LogInformation("Updating Normalization on Tags...Done");
|
||||
|
||||
// People
|
||||
logger.LogInformation("Updating Normalization on People...");
|
||||
foreach (var person in await dataContext.Person.ToListAsync())
|
||||
{
|
||||
person.NormalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(person.Name ?? string.Empty);
|
||||
logger.LogInformation("Updated Person: {Person}", person.Name);
|
||||
unitOfWork.PersonRepository.Attach(person);
|
||||
}
|
||||
|
||||
if (unitOfWork.HasChanges())
|
||||
{
|
||||
await unitOfWork.CommitAsync();
|
||||
}
|
||||
logger.LogInformation("Updating Normalization on People...Done");
|
||||
|
||||
// Collections
|
||||
logger.LogInformation("Updating Normalization on Collections...");
|
||||
foreach (var collection in await dataContext.CollectionTag.ToListAsync())
|
||||
{
|
||||
collection.NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(collection.Title ?? string.Empty);
|
||||
logger.LogInformation("Updated Collection: {Collection}", collection.Title);
|
||||
unitOfWork.CollectionTagRepository.Update(collection);
|
||||
}
|
||||
|
||||
if (unitOfWork.HasChanges())
|
||||
{
|
||||
await unitOfWork.CommitAsync();
|
||||
}
|
||||
logger.LogInformation("Updating Normalization on Collections...Done");
|
||||
|
||||
// Reading Lists
|
||||
logger.LogInformation("Updating Normalization on Reading Lists...");
|
||||
foreach (var readingList in await dataContext.ReadingList.ToListAsync())
|
||||
{
|
||||
readingList.NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(readingList.Title ?? string.Empty);
|
||||
logger.LogInformation("Updated Reading List: {ReadingList}", readingList.Title);
|
||||
unitOfWork.ReadingListRepository.Update(readingList);
|
||||
}
|
||||
|
||||
if (unitOfWork.HasChanges())
|
||||
{
|
||||
await unitOfWork.CommitAsync();
|
||||
}
|
||||
logger.LogInformation("Updating Normalization on Reading Lists...Done");
|
||||
|
||||
|
||||
logger.LogInformation("MigrateNormalizedEverything migration finished");
|
||||
|
||||
}
|
||||
|
||||
}
|
||||
|
|
@ -44,6 +44,7 @@ public interface ILibraryRepository
|
|||
IEnumerable<JumpKeyDto> GetJumpBarAsync(int libraryId);
|
||||
Task<IList<AgeRatingDto>> GetAllAgeRatingsDtosForLibrariesAsync(List<int> libraryIds);
|
||||
Task<IList<LanguageDto>> GetAllLanguagesForLibrariesAsync(List<int> libraryIds);
|
||||
Task<IList<LanguageDto>> GetAllLanguagesForLibrariesAsync();
|
||||
IEnumerable<PublicationStatusDto> GetAllPublicationStatusesDtosForLibrariesAsync(List<int> libraryIds);
|
||||
Task<bool> DoAnySeriesFoldersMatch(IEnumerable<string> folders);
|
||||
Library GetLibraryByFolder(string folder);
|
||||
|
|
@ -311,6 +312,26 @@ public class LibraryRepository : ILibraryRepository
|
|||
.ToList();
|
||||
}
|
||||
|
||||
public async Task<IList<LanguageDto>> GetAllLanguagesForLibrariesAsync()
|
||||
{
|
||||
var ret = await _context.Series
|
||||
.Select(s => s.Metadata.Language)
|
||||
.AsSplitQuery()
|
||||
.AsNoTracking()
|
||||
.Distinct()
|
||||
.ToListAsync();
|
||||
|
||||
return ret
|
||||
.Where(s => !string.IsNullOrEmpty(s))
|
||||
.Select(s => new LanguageDto()
|
||||
{
|
||||
Title = CultureInfo.GetCultureInfo(s).DisplayName,
|
||||
IsoCode = s
|
||||
})
|
||||
.OrderBy(s => s.Title)
|
||||
.ToList();
|
||||
}
|
||||
|
||||
public IEnumerable<PublicationStatusDto> GetAllPublicationStatusesDtosForLibrariesAsync(List<int> libraryIds)
|
||||
{
|
||||
return _context.Series
|
||||
|
|
|
|||
|
|
@ -5,12 +5,14 @@ using System.Linq;
|
|||
using System.Threading.Tasks;
|
||||
using API.Constants;
|
||||
using API.DTOs;
|
||||
using API.DTOs.Filtering;
|
||||
using API.DTOs.Reader;
|
||||
using API.Entities;
|
||||
using AutoMapper;
|
||||
using AutoMapper.QueryableExtensions;
|
||||
using Microsoft.AspNetCore.Identity;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using SixLabors.ImageSharp.PixelFormats;
|
||||
|
||||
namespace API.Data.Repositories;
|
||||
|
||||
|
|
@ -44,7 +46,7 @@ public interface IUserRepository
|
|||
Task<IEnumerable<BookmarkDto>> GetBookmarkDtosForSeries(int userId, int seriesId);
|
||||
Task<IEnumerable<BookmarkDto>> GetBookmarkDtosForVolume(int userId, int volumeId);
|
||||
Task<IEnumerable<BookmarkDto>> GetBookmarkDtosForChapter(int userId, int chapterId);
|
||||
Task<IEnumerable<BookmarkDto>> GetAllBookmarkDtos(int userId);
|
||||
Task<IEnumerable<BookmarkDto>> GetAllBookmarkDtos(int userId, FilterDto filter);
|
||||
Task<IEnumerable<AppUserBookmark>> GetAllBookmarksAsync();
|
||||
Task<AppUserBookmark> GetBookmarkForPage(int page, int chapterId, int userId);
|
||||
Task<AppUserBookmark> GetBookmarkAsync(int bookmarkId);
|
||||
|
|
@ -309,12 +311,63 @@ public class UserRepository : IUserRepository
|
|||
.ToListAsync();
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<BookmarkDto>> GetAllBookmarkDtos(int userId)
|
||||
/// <summary>
|
||||
/// Get all bookmarks for the user
|
||||
/// </summary>
|
||||
/// <param name="userId"></param>
|
||||
/// <param name="filter">Only supports SeriesNameQuery</param>
|
||||
/// <returns></returns>
|
||||
public async Task<IEnumerable<BookmarkDto>> GetAllBookmarkDtos(int userId, FilterDto filter)
|
||||
{
|
||||
return await _context.AppUserBookmark
|
||||
var query = _context.AppUserBookmark
|
||||
.Where(x => x.AppUserId == userId)
|
||||
.OrderBy(x => x.Page)
|
||||
.AsNoTracking()
|
||||
.AsNoTracking();
|
||||
|
||||
if (!string.IsNullOrEmpty(filter.SeriesNameQuery))
|
||||
{
|
||||
var seriesNameQueryNormalized = Services.Tasks.Scanner.Parser.Parser.Normalize(filter.SeriesNameQuery);
|
||||
var filterSeriesQuery = query.Join(_context.Series, b => b.SeriesId, s => s.Id, (bookmark, series) => new
|
||||
{
|
||||
bookmark,
|
||||
series
|
||||
})
|
||||
.Where(o => EF.Functions.Like(o.series.Name, $"%{filter.SeriesNameQuery}%")
|
||||
|| EF.Functions.Like(o.series.OriginalName, $"%{filter.SeriesNameQuery}%")
|
||||
|| EF.Functions.Like(o.series.LocalizedName, $"%{filter.SeriesNameQuery}%")
|
||||
|| EF.Functions.Like(o.series.NormalizedName, $"%{seriesNameQueryNormalized}%")
|
||||
);
|
||||
|
||||
// This doesn't work on bookmarks themselves, only the series. For now, I don't think there is much value add
|
||||
// if (filter.SortOptions != null)
|
||||
// {
|
||||
// if (filter.SortOptions.IsAscending)
|
||||
// {
|
||||
// filterSeriesQuery = filter.SortOptions.SortField switch
|
||||
// {
|
||||
// SortField.SortName => filterSeriesQuery.OrderBy(s => s.series.SortName),
|
||||
// SortField.CreatedDate => filterSeriesQuery.OrderBy(s => s.bookmark.Created),
|
||||
// SortField.LastModifiedDate => filterSeriesQuery.OrderBy(s => s.bookmark.LastModified),
|
||||
// _ => filterSeriesQuery
|
||||
// };
|
||||
// }
|
||||
// else
|
||||
// {
|
||||
// filterSeriesQuery = filter.SortOptions.SortField switch
|
||||
// {
|
||||
// SortField.SortName => filterSeriesQuery.OrderByDescending(s => s.series.SortName),
|
||||
// SortField.CreatedDate => filterSeriesQuery.OrderByDescending(s => s.bookmark.Created),
|
||||
// SortField.LastModifiedDate => filterSeriesQuery.OrderByDescending(s => s.bookmark.LastModified),
|
||||
// _ => filterSeriesQuery
|
||||
// };
|
||||
// }
|
||||
// }
|
||||
|
||||
query = filterSeriesQuery.Select(o => o.bookmark);
|
||||
}
|
||||
|
||||
|
||||
return await query
|
||||
.ProjectTo<BookmarkDto>(_mapper.ConfigurationProvider)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue