Bookmark Refactor (#893)

* Fixed a bug which didn't take sort direction when not changing sort field

* Added foundation for Bookmark refactor

* Code broken, need to take a break. Issue is Getting bookmark image needs authentication but UI doesn't send.

* Implemented the ability to send bookmarked files to the web. Implemented ability to clear bookmarks on disk on a re-occuring basis.

* Updated the bookmark design to have it's own card that is self contained. View bookmarks modal has been updated to better lay out the cards.

* Refactored download bookmark codes to select files from bookmark directory directly rather than open underlying files.

* Wrote the basic logic to kick start the bookmark migration.

Added Installed Version into the DB to allow us to know more accurately when to run migrations

* Implemented the ability to change the bookmarks directory

* Updated all references to BookmarkDirectory to use setting from the DB.

Updated Server Settings page to use 2 col for some rows.

* Refactored some code to DirectoryService (hasWriteAccess) and fixed up some unit tests from a previous PR.

* Treat folders that start with ._ as blacklisted.

* Implemented Reset User preferences. Some extra code to prep for the migration.

* Implemented a migration for existing bookmarks to using new filesystem based bookmarks

API/Data/MigrateBookmarks.cs

@@ -0,0 +1,102 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using API.Comparators;
+using API.Entities.Enums;
+using API.Services;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data;
+
+/// <summary>
+/// Responsible to migrate existing bookmarks to files
+/// </summary>
+public static class MigrateBookmarks
+{
+    private static readonly Version VersionBookmarksChanged = new Version(0, 4, 9, 27);
+    /// <summary>
+    /// This will migrate existing bookmarks to bookmark folder based
+    /// </summary>
+    /// <remarks>Bookmark directory is configurable. This will always use the default bookmark directory.</remarks>
+    /// <param name="directoryService"></param>
+    /// <returns></returns>
+    public static async Task Migrate(IDirectoryService directoryService, IUnitOfWork unitOfWork,
+        ILogger<Program> logger, ICacheService cacheService)
+    {
+        var bookmarkDirectory = (await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory))
+            .Value;
+        if (string.IsNullOrEmpty(bookmarkDirectory))
+        {
+            bookmarkDirectory = directoryService.BookmarkDirectory;
+        }
+
+        if (directoryService.Exists(bookmarkDirectory)) return;
+
+        logger.LogInformation("Bookmark migration is needed....This may take some time");
+
+        var allBookmarks = (await unitOfWork.UserRepository.GetAllBookmarksAsync()).ToList();
+
+        var uniqueChapterIds = allBookmarks.Select(b => b.ChapterId).Distinct().ToList();
+        var uniqueUserIds = allBookmarks.Select(b => b.AppUserId).Distinct().ToList();
+        foreach (var userId in uniqueUserIds)
+        {
+            foreach (var chapterId in uniqueChapterIds)
+            {
+                var chapterBookmarks = allBookmarks.Where(b => b.ChapterId == chapterId).ToList();
+                var chapterPages = chapterBookmarks
+                    .Select(b => b.Page).ToList();
+                var seriesId = chapterBookmarks
+                    .Select(b => b.SeriesId).First();
+                var mangaFiles = await unitOfWork.ChapterRepository.GetFilesForChapterAsync(chapterId);
+                var chapterExtractPath = directoryService.FileSystem.Path.Join(directoryService.TempDirectory, $"bookmark_c{chapterId}_u{userId}_s{seriesId}");
+
+                var numericComparer = new NumericComparer();
+                if (!mangaFiles.Any()) continue;
+
+                switch (mangaFiles.First().Format)
+                {
+                    case MangaFormat.Image:
+                        directoryService.ExistOrCreate(chapterExtractPath);
+                        directoryService.CopyFilesToDirectory(mangaFiles.Select(f => f.FilePath), chapterExtractPath);
+                        break;
+                    case MangaFormat.Archive:
+                    case MangaFormat.Pdf:
+                        cacheService.ExtractChapterFiles(chapterExtractPath, mangaFiles.ToList());
+                        break;
+                    case MangaFormat.Epub:
+                        continue;
+                    default:
+                        continue;
+                }
+
+                var files = directoryService.GetFilesWithExtension(chapterExtractPath, Parser.Parser.ImageFileExtensions);
+                // Filter out images that aren't in bookmarks
+                Array.Sort(files, numericComparer);
+                foreach (var chapterPage in chapterPages)
+                {
+                    var file = files.ElementAt(chapterPage);
+                    var bookmark = allBookmarks.FirstOrDefault(b =>
+                        b.ChapterId == chapterId && b.SeriesId == seriesId && b.AppUserId == userId &&
+                        b.Page == chapterPage);
+                    if (bookmark == null) continue;
+
+                    var filename = directoryService.FileSystem.Path.GetFileName(file);
+                    var newLocation = directoryService.FileSystem.Path.Join(
+                        ReaderService.FormatBookmarkFolderPath(String.Empty, userId, seriesId, chapterId),
+                        filename);
+                    bookmark.FileName = newLocation;
+                    directoryService.CopyFileToDirectory(file,
+                        ReaderService.FormatBookmarkFolderPath(bookmarkDirectory, userId, seriesId, chapterId));
+                    unitOfWork.UserRepository.Update(bookmark);
+                }
+            }
+            // Clear temp after each user to avoid too much space being eaten
+            directoryService.ClearDirectory(directoryService.TempDirectory);
+        }
+
+        await unitOfWork.CommitAsync();
+        // Run CleanupService as we cache a ton of files
+        directoryService.ClearDirectory(directoryService.TempDirectory);
+
+    }
+}

API/Data/Migrations/20211217013734_BookmarkRefactor.cs

@@ -0,0 +1,25 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace API.Data.Migrations
+{
+    public partial class BookmarkRefactor : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<string>(
+                name: "FileName",
+                table: "AppUserBookmark",
+                type: "TEXT",
+                nullable: true);
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "FileName",
+                table: "AppUserBookmark");
+        }
+    }
+}

API/Data/Migrations/DataContextModelSnapshot.cs

@@ -134,6 +134,9 @@ namespace API.Data.Migrations
                     b.Property<int>("ChapterId")
                         .HasColumnType("INTEGER");
 
+                    b.Property<string>("FileName")
+                        .HasColumnType("TEXT");
+
                     b.Property<int>("Page")
                         .HasColumnType("INTEGER");
 

API/Data/Repositories/UserRepository.cs

@@ -39,12 +39,15 @@ public interface IUserRepository
     Task<IEnumerable<BookmarkDto>> GetBookmarkDtosForVolume(int userId, int volumeId);
     Task<IEnumerable<BookmarkDto>> GetBookmarkDtosForChapter(int userId, int chapterId);
     Task<IEnumerable<BookmarkDto>> GetAllBookmarkDtos(int userId);
+    Task<IEnumerable<AppUserBookmark>> GetAllBookmarksAsync();
     Task<AppUserBookmark> GetBookmarkForPage(int page, int chapterId, int userId);
+    Task<AppUserBookmark> GetBookmarkAsync(int bookmarkId);
     Task<int> GetUserIdByApiKeyAsync(string apiKey);
     Task<AppUser> GetUserByUsernameAsync(string username, AppUserIncludes includeFlags = AppUserIncludes.None);
     Task<AppUser> GetUserByIdAsync(int userId, AppUserIncludes includeFlags = AppUserIncludes.None);
     Task<int> GetUserIdByUsernameAsync(string username);
     Task<AppUser> GetUserWithReadingListsByUsernameAsync(string username);
+    Task<IList<AppUserBookmark>> GetAllBookmarksByIds(IList<int> bookmarkIds);
 }
 
 public class UserRepository : IUserRepository
@@ -112,6 +115,11 @@ public class UserRepository : IUserRepository
         return await query.SingleOrDefaultAsync();
     }
 
+    public async Task<IEnumerable<AppUserBookmark>> GetAllBookmarksAsync()
+    {
+        return await _context.AppUserBookmark.ToListAsync();
+    }
+
     public async Task<AppUserBookmark> GetBookmarkForPage(int page, int chapterId, int userId)
     {
         return await _context.AppUserBookmark
@@ -119,6 +127,13 @@ public class UserRepository : IUserRepository
             .SingleOrDefaultAsync();
     }
 
+    public async Task<AppUserBookmark> GetBookmarkAsync(int bookmarkId)
+    {
+        return await _context.AppUserBookmark
+            .Where(b => b.Id == bookmarkId)
+            .SingleOrDefaultAsync();
+    }
+
     private static IQueryable<AppUser> AddIncludesToQuery(IQueryable<AppUser> query, AppUserIncludes includeFlags)
     {
         if (includeFlags.HasFlag(AppUserIncludes.Bookmarks))
@@ -171,6 +186,18 @@ public class UserRepository : IUserRepository
             .SingleOrDefaultAsync(x => x.UserName == username);
     }
 
+    /// <summary>
+    /// Returns all Bookmarks for a given set of Ids
+    /// </summary>
+    /// <param name="bookmarkIds"></param>
+    /// <returns></returns>
+    public async Task<IList<AppUserBookmark>> GetAllBookmarksByIds(IList<int> bookmarkIds)
+    {
+        return await _context.AppUserBookmark
+            .Where(b => bookmarkIds.Contains(b.Id))
+            .ToListAsync();
+    }
+
     public async Task<IEnumerable<AppUser>> GetAdminUsersAsync()
     {
         return await _userManager.GetUsersInRoleAsync(PolicyConstants.AdminRole);

API/Data/Seed.cs

@@ -8,6 +8,7 @@ using API.Entities;
 using API.Entities.Enums;
 using API.Services;
 using Kavita.Common;
+using Kavita.Common.EnvironmentInfo;
 using Microsoft.AspNetCore.Identity;
 using Microsoft.EntityFrameworkCore;
 
@@ -15,6 +16,11 @@ namespace API.Data
 {
     public static class Seed
     {
+        /// <summary>
+        /// Generated on Startup. Seed.SeedSettings must run before
+        /// </summary>
+        public static IList<ServerSetting> DefaultSettings;
+
         public static async Task SeedRoles(RoleManager<AppRole> roleManager)
         {
             var roles = typeof(PolicyConstants)
@@ -39,7 +45,7 @@ namespace API.Data
         {
             await context.Database.EnsureCreatedAsync();
 
-            IList<ServerSetting> defaultSettings = new List<ServerSetting>()
+            DefaultSettings = new List<ServerSetting>()
             {
                 new () {Key = ServerSettingKey.CacheDirectory, Value = directoryService.CacheDirectory},
                 new () {Key = ServerSettingKey.TaskScan, Value = "daily"},
@@ -52,9 +58,11 @@ namespace API.Data
                 new () {Key = ServerSettingKey.EnableAuthentication, Value = "true"},
                 new () {Key = ServerSettingKey.BaseUrl, Value = "/"},
                 new () {Key = ServerSettingKey.InstallId, Value = HashUtil.AnonymousToken()},
+                new () {Key = ServerSettingKey.InstallVersion, Value = BuildInfo.Version.ToString()},
+                new () {Key = ServerSettingKey.BookmarkDirectory, Value = directoryService.BookmarkDirectory},
             };
 
-            foreach (var defaultSetting in defaultSettings)
+            foreach (var defaultSetting in DefaultSettings)
             {
                 var existing = context.ServerSetting.FirstOrDefault(s => s.Key == defaultSetting.Key);
                 if (existing == null)