AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

AVIF Support & Much More! (#1992)

Browse source 
* Expand the list of potential favicon icons to grab.

* Added a url mapping functionality to use alternative urls for fetching icons

* Initial commit to streamline media encoding. No DB migration yet, No UI changes, no Task changes.

* Started refactoring code so that webp queries use encoding format instead.

* More refactoring to remove hardcoded webp references.

* Moved manual migrations to their own folder to keep things organized. Manually drop the obsolete webp keys.

* Removed old apis for converting media and now have one. Reworked where the conversion code was located and streamlined events and whatnot.

* Make favicon encode setting aware

* Cleaned up favicon conversion

* Updated format counter to now just use Extension from MangaFile now that it's been out a while.

* Tweaked jumpbar code to reduce a lookup to hashmap.

* Added AVIF (8-bit only) support.

* In UpdatePeopleList, use FirstOrDefault as Single adds extra checks that may not be needed.

* You can now remove weblinks from edit series page and you can leave empty cells, they will just be removed on backend.

* Forgot a file

* Don't prompt to write a review, just show the pencil. It's the same amount of clicks if you do, less if you dont.

* Fixed Refresh token using wrong Claim to look up the user.

* Refactored how we refresh authentication to perform it every 10 m ins to ensure we always stay authenticated.

* Changed Version update code to run more throughout the day. Updated some hangfire to newer method signatures.
This commit is contained in:
Joe Milazzo 2023-05-12 15:31:23 -05:00 • committed by GitHub
parent c1989e2819
commit 70690b747e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
73 changed files with 778 additions and 566 deletions
Download patch file Download diff file Expand all files Collapse all files

223
API/Services/BookmarkService.cs
View file

@ -7,7 +7,6 @@ using API.Data;
using API.DTOs.Reader;
using API.Entities;
using API.Entities.Enums;
using API.SignalR;
using Hangfire;
using Microsoft.Extensions.Logging;
@ -19,9 +18,6 @@ public interface IBookmarkService
Task<bool> BookmarkPage(AppUser userWithBookmarks, BookmarkDto bookmarkDto, string imageToBookmark);
Task<bool> RemoveBookmarkPage(AppUser userWithBookmarks, BookmarkDto bookmarkDto);
Task<IEnumerable<string>> GetBookmarkFilesById(IEnumerable<int> bookmarkIds);
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
Task ConvertAllBookmarkToWebP();
Task ConvertAllCoverToWebP();
}
public class BookmarkService : IBookmarkService
@ -30,17 +26,15 @@ public class BookmarkService : IBookmarkService
private readonly ILogger<BookmarkService> _logger;
private readonly IUnitOfWork _unitOfWork;
private readonly IDirectoryService _directoryService;
private readonly IImageService _imageService;
private readonly IEventHub _eventHub;
private readonly IMediaConversionService _mediaConversionService;
public BookmarkService(ILogger<BookmarkService> logger, IUnitOfWork unitOfWork,
IDirectoryService directoryService, IImageService imageService, IEventHub eventHub)
IDirectoryService directoryService, IMediaConversionService mediaConversionService)
{
_logger = logger;
_unitOfWork = unitOfWork;
_directoryService = directoryService;
_imageService = imageService;
_eventHub = eventHub;
_mediaConversionService = mediaConversionService;
}
/// <summary>
@ -77,21 +71,25 @@ public class BookmarkService : IBookmarkService
/// This is a job that runs after a bookmark is saved
/// </summary>
/// <remarks>This must be public</remarks>
public async Task ConvertBookmarkToWebP(int bookmarkId)
public async Task ConvertBookmarkToEncoding(int bookmarkId)
{
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
var convertBookmarkToWebP =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertBookmarkToWebP;
var encodeFormat =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
if (!convertBookmarkToWebP) return;
if (encodeFormat == EncodeFormat.PNG)
{
_logger.LogError("Cannot convert media to PNG");
return;
}
// Validate the bookmark still exists
var bookmark = await _unitOfWork.UserRepository.GetBookmarkAsync(bookmarkId);
if (bookmark == null) return;
bookmark.FileName = await SaveAsWebP(bookmarkDirectory, bookmark.FileName,
BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId));
bookmark.FileName = await _mediaConversionService.SaveAsEncodingFormat(bookmarkDirectory, bookmark.FileName,
BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId), encodeFormat);
_unitOfWork.UserRepository.Update(bookmark);
await _unitOfWork.CommitAsync();
@ -137,10 +135,10 @@ public class BookmarkService : IBookmarkService
_unitOfWork.UserRepository.Add(bookmark);
await _unitOfWork.CommitAsync();
if (settings.ConvertBookmarkToWebP)
if (settings.EncodeMediaAs == EncodeFormat.WEBP)
{
// Enqueue a task to convert the bookmark to webP
BackgroundJob.Enqueue(() => ConvertBookmarkToWebP(bookmark.Id));
BackgroundJob.Enqueue(() => ConvertBookmarkToEncoding(bookmark.Id));
}
}
catch (Exception ex)
@ -192,198 +190,9 @@ public class BookmarkService : IBookmarkService
b.FileName)));
}
/// <summary>
/// This is a long-running job that will convert all bookmarks into WebP. Do not invoke anyway except via Hangfire.
/// </summary>
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
public async Task ConvertAllBookmarkToWebP()
{
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(0F, ProgressEventType.Started));
var bookmarks = (await _unitOfWork.UserRepository.GetAllBookmarksAsync())
.Where(b => !b.FileName.EndsWith(".webp")).ToList();
var count = 1F;
foreach (var bookmark in bookmarks)
{
bookmark.FileName = await SaveAsWebP(bookmarkDirectory, bookmark.FileName,
BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId));
_unitOfWork.UserRepository.Update(bookmark);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(count / bookmarks.Count, ProgressEventType.Started));
count++;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(1F, ProgressEventType.Ended));
_logger.LogInformation("[BookmarkService] Converted bookmarks to WebP");
}
/// <summary>
/// This is a long-running job that will convert all covers into WebP. Do not invoke anyway except via Hangfire.
/// </summary>
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
public async Task ConvertAllCoverToWebP()
{
_logger.LogInformation("[BookmarkService] Starting conversion of all covers to webp");
var coverDirectory = _directoryService.CoverImageDirectory;
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(0F, ProgressEventType.Started));
var chapterCovers = await _unitOfWork.ChapterRepository.GetAllChaptersWithNonWebPCovers();
var seriesCovers = await _unitOfWork.SeriesRepository.GetAllWithNonWebPCovers();
var readingListCovers = await _unitOfWork.ReadingListRepository.GetAllWithNonWebPCovers();
var libraryCovers = await _unitOfWork.LibraryRepository.GetAllWithNonWebPCovers();
var collectionCovers = await _unitOfWork.CollectionTagRepository.GetAllWithNonWebPCovers();
var totalCount = chapterCovers.Count + seriesCovers.Count + readingListCovers.Count +
libraryCovers.Count + collectionCovers.Count;
var count = 1F;
_logger.LogInformation("[BookmarkService] Starting conversion of chapters");
foreach (var chapter in chapterCovers)
{
if (string.IsNullOrEmpty(chapter.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, chapter.CoverImage, coverDirectory);
chapter.CoverImage = Path.GetFileName(newFile);
_unitOfWork.ChapterRepository.Update(chapter);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
_logger.LogInformation("[BookmarkService] Starting conversion of series");
foreach (var series in seriesCovers)
{
if (string.IsNullOrEmpty(series.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, series.CoverImage, coverDirectory);
series.CoverImage = Path.GetFileName(newFile);
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
_logger.LogInformation("[BookmarkService] Starting conversion of libraries");
foreach (var library in libraryCovers)
{
if (string.IsNullOrEmpty(library.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, library.CoverImage, coverDirectory);
library.CoverImage = Path.GetFileName(newFile);
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
_logger.LogInformation("[BookmarkService] Starting conversion of reading lists");
foreach (var readingList in readingListCovers)
{
if (string.IsNullOrEmpty(readingList.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, readingList.CoverImage, coverDirectory);
readingList.CoverImage = Path.GetFileName(newFile);
_unitOfWork.ReadingListRepository.Update(readingList);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
_logger.LogInformation("[BookmarkService] Starting conversion of collections");
foreach (var collection in collectionCovers)
{
if (string.IsNullOrEmpty(collection.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, collection.CoverImage, coverDirectory);
collection.CoverImage = Path.GetFileName(newFile);
_unitOfWork.CollectionTagRepository.Update(collection);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
// Now null out all series and volumes that aren't webp or custom
var nonCustomOrConvertedVolumeCovers = await _unitOfWork.VolumeRepository.GetAllWithNonWebPCovers();
foreach (var volume in nonCustomOrConvertedVolumeCovers)
{
if (string.IsNullOrEmpty(volume.CoverImage)) continue;
volume.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter
_unitOfWork.VolumeRepository.Update(volume);
await _unitOfWork.CommitAsync();
}
var nonCustomOrConvertedSeriesCovers = await _unitOfWork.SeriesRepository.GetAllWithNonWebPCovers(false);
foreach (var series in nonCustomOrConvertedSeriesCovers)
{
if (string.IsNullOrEmpty(series.CoverImage)) continue;
series.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(1F, ProgressEventType.Ended));
_logger.LogInformation("[BookmarkService] Converted covers to WebP");
}
/// <summary>
/// Converts an image file, deletes original and returns the new path back
/// </summary>
/// <param name="imageDirectory">Full Path to where files are stored</param>
/// <param name="filename">The file to convert</param>
/// <param name="targetFolder">Full path to where files should be stored or any stem</param>
/// <returns></returns>
public async Task<string> SaveAsWebP(string imageDirectory, string filename, string targetFolder)
{
// This must be Public as it's used in via Hangfire as a background task
var fullSourcePath = _directoryService.FileSystem.Path.Join(imageDirectory, filename);
var fullTargetDirectory = fullSourcePath.Replace(new FileInfo(filename).Name, string.Empty);
var newFilename = string.Empty;
_logger.LogDebug("Converting {Source} image into WebP at {Target}", fullSourcePath, fullTargetDirectory);
try
{
// Convert target file to webp then delete original target file and update bookmark
try
{
var targetFile = await _imageService.ConvertToWebP(fullSourcePath, fullTargetDirectory);
var targetName = new FileInfo(targetFile).Name;
newFilename = Path.Join(targetFolder, targetName);
_directoryService.DeleteFiles(new[] {fullSourcePath});
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not convert image {FilePath}", filename);
newFilename = filename;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not convert image to WebP");
}
return newFilename;
}
private static string BookmarkStem(int userId, int seriesId, int chapterId)
public static string BookmarkStem(int userId, int seriesId, int chapterId)
{
return Path.Join($"{userId}", $"{seriesId}", $"{chapterId}");
}