AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

UX Changes, Tasks, WebP, and More! (#1280)

Browse source 
* When account updates occur for a user, send an event to them to tell them to refresh their account information (if they are on the site at the time). This way if we revoke permissions, the site will reactively adapt.

* Some cleanup on the user preferences to remove some calls we don't need anymore.

* Removed old bulk cleanup bookmark code as it's no longer needed.

* Tweaked the messaging for stat collection to reflect what we collect now versus when this was initially implemented.

* Implemented the ability for users to configure their servers to save bookmarks as webP. Reorganized the tabs for Admin dashboard to account for upcoming features.

* Implemented the ability to bulk convert bookmarks (as many times as the user wants).

Added a display of Reoccurring Jobs to the Tasks admin tab. Currently it's just placeholder, but will be enhanced further later in the release.

* Tweaked the wording around the convert switch.

* Moved System actions to the task tab

* Added a controller just for Tachiyomi so we can have dedicated APIs for that client. Deprecated an existing API on the Reader route.

* Fixed the unit tests
This commit is contained in:
Joseph Milazzo 2022-05-23 18:19:52 -05:00 • committed by GitHub
parent dd83b6a9a1
commit e0a2fc615f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 971 additions and 271 deletions
Download patch file Download diff file Expand all files Collapse all files

113
API/Services/BookmarkService.cs
View file

@ -8,6 +8,7 @@ using API.DTOs.Reader;
using API.Entities;
using API.Entities.Enums;
using API.SignalR;
using Hangfire;
using Microsoft.Extensions.Logging;
namespace API.Services;
@ -18,6 +19,7 @@ public interface IBookmarkService
Task<bool> BookmarkPage(AppUser userWithBookmarks, BookmarkDto bookmarkDto, string imageToBookmark);
Task<bool> RemoveBookmarkPage(AppUser userWithBookmarks, BookmarkDto bookmarkDto);
Task<IEnumerable<string>> GetBookmarkFilesById(IEnumerable<int> bookmarkIds);
Task ConvertAllBookmarkToWebP();
}
@ -26,12 +28,17 @@ public class BookmarkService : IBookmarkService
private readonly ILogger<BookmarkService> _logger;
private readonly IUnitOfWork _unitOfWork;
private readonly IDirectoryService _directoryService;
private readonly IImageService _imageService;
private readonly IEventHub _eventHub;
public BookmarkService(ILogger<BookmarkService> logger, IUnitOfWork unitOfWork, IDirectoryService directoryService)
public BookmarkService(ILogger<BookmarkService> logger, IUnitOfWork unitOfWork,
IDirectoryService directoryService, IImageService imageService, IEventHub eventHub)
{
_logger = logger;
_unitOfWork = unitOfWork;
_directoryService = directoryService;
_imageService = imageService;
_eventHub = eventHub;
}
/// <summary>
@ -87,18 +94,28 @@ public class BookmarkService : IBookmarkService
var targetFolderStem = BookmarkStem(userWithBookmarks.Id, bookmarkDto.SeriesId, bookmarkDto.ChapterId);
var targetFilepath = Path.Join(bookmarkDirectory, targetFolderStem);
userWithBookmarks.Bookmarks ??= new List<AppUserBookmark>();
userWithBookmarks.Bookmarks.Add(new AppUserBookmark()
var bookmark = new AppUserBookmark()
{
Page = bookmarkDto.Page,
VolumeId = bookmarkDto.VolumeId,
SeriesId = bookmarkDto.SeriesId,
ChapterId = bookmarkDto.ChapterId,
FileName = Path.Join(targetFolderStem, fileInfo.Name)
});
};
_directoryService.CopyFileToDirectory(imageToBookmark, targetFilepath);
userWithBookmarks.Bookmarks ??= new List<AppUserBookmark>();
userWithBookmarks.Bookmarks.Add(bookmark);
_unitOfWork.UserRepository.Update(userWithBookmarks);
await _unitOfWork.CommitAsync();
var convertToWebP = bool.Parse((await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.ConvertBookmarkToWebP)).Value);
if (convertToWebP)
{
// Enqueue a task to convert the bookmark to webP
BackgroundJob.Enqueue(() => ConvertBookmarkToWebP(bookmark.Id));
}
}
catch (Exception ex)
{
@ -153,6 +170,94 @@ public class BookmarkService : IBookmarkService
b.FileName)));
}
/// <summary>
/// This is a long-running job that will convert all bookmarks into WebP. Do not invoke anyway except via Hangfire.
/// </summary>
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
public async Task ConvertAllBookmarkToWebP()
{
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(0F, ProgressEventType.Started));
var bookmarks = (await _unitOfWork.UserRepository.GetAllBookmarksAsync())
.Where(b => !b.FileName.EndsWith(".webp")).ToList();
var count = 1F;
foreach (var bookmark in bookmarks)
{
await SaveBookmarkAsWebP(bookmarkDirectory, bookmark);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(count / bookmarks.Count, ProgressEventType.Started));
count++;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(1F, ProgressEventType.Ended));
_logger.LogInformation("[BookmarkService] Converted bookmarks to WebP");
}
/// <summary>
/// This is a job that runs after a bookmark is saved
/// </summary>
public async Task ConvertBookmarkToWebP(int bookmarkId)
{
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
var convertBookmarkToWebP =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertBookmarkToWebP;
if (!convertBookmarkToWebP) return;
// Validate the bookmark still exists
var bookmark = await _unitOfWork.UserRepository.GetBookmarkAsync(bookmarkId);
if (bookmark == null) return;
await SaveBookmarkAsWebP(bookmarkDirectory, bookmark);
await _unitOfWork.CommitAsync();
}
/// <summary>
/// Converts bookmark file, deletes original, marks bookmark as dirty. Does not commit.
/// </summary>
/// <param name="bookmarkDirectory"></param>
/// <param name="bookmark"></param>
private async Task SaveBookmarkAsWebP(string bookmarkDirectory, AppUserBookmark bookmark)
{
var fullSourcePath = _directoryService.FileSystem.Path.Join(bookmarkDirectory, bookmark.FileName);
var fullTargetDirectory = fullSourcePath.Replace(new FileInfo(bookmark.FileName).Name, string.Empty);
var targetFolderStem = BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId);
_logger.LogDebug("Converting {Source} bookmark into WebP at {Target}", fullSourcePath, fullTargetDirectory);
try
{
// Convert target file to webp then delete original target file and update bookmark
var originalFile = bookmark.FileName;
try
{
var targetFile = await _imageService.ConvertToWebP(fullSourcePath, fullTargetDirectory);
var targetName = new FileInfo(targetFile).Name;
bookmark.FileName = Path.Join(targetFolderStem, targetName);
_directoryService.DeleteFiles(new[] {fullSourcePath});
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not convert file {FilePath}", bookmark.FileName);
bookmark.FileName = originalFile;
}
_unitOfWork.UserRepository.Update(bookmark);
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not convert bookmark to WebP");
}
}
private static string BookmarkStem(int userId, int seriesId, int chapterId)
{
return Path.Join($"{userId}", $"{seriesId}", $"{chapterId}");

22
API/Services/ImageService.cs
View file

@ -1,7 +1,9 @@
using System;
using System.IO;
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
using NetVips;
using SixLabors.ImageSharp;
using Image = NetVips.Image;
namespace API.Services;
@ -19,6 +21,12 @@ public interface IImageService
string CreateThumbnailFromBase64(string encodedImage, string fileName);
string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory);
/// <summary>
/// Converts the passed image to webP and outputs it in the same directory
/// </summary>
/// <param name="filePath">Full path to the image to convert</param>
/// <returns>File of written webp image</returns>
Task<string> ConvertToWebP(string filePath, string outputPath);
}
public class ImageService : IImageService
@ -95,6 +103,18 @@ public class ImageService : IImageService
return filename;
}
public async Task<string> ConvertToWebP(string filePath, string outputPath)
{
var file = _directoryService.FileSystem.FileInfo.FromFileName(filePath);
var fileName = file.Name.Replace(file.Extension, string.Empty);
var outputFile = Path.Join(outputPath, fileName + ".webp");
using var sourceImage = await SixLabors.ImageSharp.Image.LoadAsync(filePath);
await sourceImage.SaveAsWebpAsync(outputFile);
return outputFile;
}
/// <inheritdoc />
public string CreateThumbnailFromBase64(string encodedImage, string fileName)

4
API/Services/TaskScheduler.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Collections.Generic;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
@ -6,6 +7,7 @@ using API.Entities.Enums;
using API.Helpers.Converters;
using API.Services.Tasks;
using Hangfire;
using Hangfire.Storage;
using Microsoft.Extensions.Logging;
namespace API.Services;
@ -23,6 +25,8 @@ public interface ITaskScheduler
void CancelStatsTasks();
Task RunStatCollection();
void ScanSiteThemes();
}
public class TaskScheduler : ITaskScheduler
{

39
API/Services/Tasks/CleanupService.cs
View file

@ -1,4 +1,5 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
@ -19,7 +20,6 @@ namespace API.Services.Tasks
Task DeleteChapterCoverImages();
Task DeleteTagCoverImages();
Task CleanupBackups();
Task CleanupBookmarks();
}
/// <summary>
/// Cleans up after operations on reoccurring basis
@ -65,7 +65,6 @@ namespace API.Services.Tasks
await SendProgress(0.7F, "Cleaning deleted cover images");
await DeleteTagCoverImages();
await DeleteReadingListCoverImages();
await SendProgress(0.8F, "Cleaning deleted cover images");
await SendProgress(1F, "Cleanup finished");
_logger.LogInformation("Cleanup finished");
}
@ -152,7 +151,7 @@ namespace API.Services.Tasks
/// </summary>
public async Task CleanupBackups()
{
const int dayThreshold = 30;
const int dayThreshold = 30; // TODO: We can make this a config option
_logger.LogInformation("Beginning cleanup of Database backups at {Time}", DateTime.Now);
var backupDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BackupDirectory)).Value;
@ -176,39 +175,5 @@ namespace API.Services.Tasks
}
_logger.LogInformation("Finished cleanup of Database backups at {Time}", DateTime.Now);
}
/// <summary>
/// Removes all files in the BookmarkDirectory that don't currently have bookmarks in the Database
/// </summary>
public Task CleanupBookmarks()
{
// TODO: This is disabled for now while we test and validate a new method of deleting bookmarks
return Task.CompletedTask;
// Search all files in bookmarks/ except bookmark files and delete those
// var bookmarkDirectory =
// (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
// var allBookmarkFiles = _directoryService.GetFiles(bookmarkDirectory, searchOption: SearchOption.AllDirectories).Select(Parser.Parser.NormalizePath);
// var bookmarks = (await _unitOfWork.UserRepository.GetAllBookmarksAsync())
// .Select(b => Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(bookmarkDirectory,
// b.FileName)));
//
//
// var filesToDelete = allBookmarkFiles.AsEnumerable().Except(bookmarks).ToList();
// _logger.LogDebug("[Bookmarks] Bookmark cleanup wants to delete {Count} files", filesToDelete.Count);
//
// if (filesToDelete.Count == 0) return;
//
// _directoryService.DeleteFiles(filesToDelete);
//
// // Clear all empty directories
// foreach (var directory in _directoryService.FileSystem.Directory.GetDirectories(bookmarkDirectory, "", SearchOption.AllDirectories))
// {
// if (_directoryService.FileSystem.Directory.GetFiles(directory, "", SearchOption.AllDirectories).Length == 0 &&
// _directoryService.FileSystem.Directory.GetDirectories(directory).Length == 0)
// {
// _directoryService.FileSystem.Directory.Delete(directory, false);
// }
// }
}
}
}

3
API/Services/Tasks/ScannerService.cs
View file

@ -789,7 +789,7 @@ public class ScannerService : IScannerService
// Update all the metadata on the Chapters
foreach (var chapter in volume.Chapters)
{
var firstFile = chapter.Files.OrderBy(x => x.Chapter).FirstOrDefault();
var firstFile = chapter.Files.MinBy(x => x.Chapter);
if (firstFile == null || _cacheHelper.HasFileNotChangedSinceCreationOrLastScan(chapter, false, firstFile)) continue;
try
{
@ -923,6 +923,7 @@ public class ScannerService : IScannerService
}
if (comicInfo == null) return;
_logger.LogDebug("[ScannerService] Read ComicInfo for {File}", firstFile.FilePath);
chapter.AgeRating = ComicInfo.ConvertAgeRatingToEnum(comicInfo.AgeRating);