CBL Import Rework (#1862)

* Fixed a typo in a log

* Invalid XML files now "validate" correctly by sending back a failure.

* Cleaned up messaging on backend and frontend to provide some linking on series name when collision, handle corrupt xml files, etc.

* When reading list conflict occurs, show the reading list name that's conflicting. Started refactoring the code to allow multiple files to be imported at once.

* Started adding new CBL elements for some enhancements I have planned with maintainers.

* Default to empty string for IpAddress to allow to fallback into existing experience

* Tweaked the layout of reading list page (not complete), moved some not used much controls to page extras and reordered the buttons for reading list

* Edit Reading Lists now allows selection of cover image from existing items

* Fixed a bug where cover chooser base64 to image would fail to write webp files.

* Refactored the validate step to now handle multiple files in one go.

* Clean up code

* Don't show CBL name if there were xml errors that prevented showing it

* Don't allow user to go prev step after they perform the import.

* Cleaned up the heading code for accordions

* Fixed a bug with import keeping failed items

* Sort the failures to the bottom of result windows

* CBL import is pretty solid. Need one pass from Robbie on Reading List Page

API/Controllers/CBLController.cs

@@ -1,4 +1,6 @@
-using System.IO;
+using System;
+using System.Collections.Generic;
+using System.IO;
 using System.Threading.Tasks;
 using API.DTOs.ReadingLists.CBL;
 using API.Extensions;
@@ -32,10 +34,43 @@ public class CblController : BaseApiController
     public async Task<ActionResult<CblImportSummaryDto>> ValidateCbl([FromForm(Name = "cbl")] IFormFile file)
     {
         var userId = User.GetUserId();
-        var cbl = await SaveAndLoadCblFile(userId, file);
-
-        var importSummary = await _readingListService.ValidateCblFile(userId, cbl);
-        return Ok(importSummary);
+        try
+        {
+            var cbl = await SaveAndLoadCblFile(file);
+            var importSummary = await _readingListService.ValidateCblFile(userId, cbl);
+            importSummary.FileName = file.FileName;
+            return Ok(importSummary);
+        }
+        catch (ArgumentNullException)
+        {
+            return Ok(new CblImportSummaryDto()
+            {
+                FileName = file.FileName,
+                Success = CblImportResult.Fail,
+                Results = new List<CblBookResult>()
+                {
+                    new CblBookResult()
+                    {
+                        Reason = CblImportReason.InvalidFile
+                    }
+                }
+            });
+        }
+        catch (InvalidOperationException)
+        {
+            return Ok(new CblImportSummaryDto()
+            {
+                FileName = file.FileName,
+                Success = CblImportResult.Fail,
+                Results = new List<CblBookResult>()
+                {
+                    new CblBookResult()
+                    {
+                        Reason = CblImportReason.InvalidFile
+                    }
+                }
+            });
+        }
     }
 
 
@@ -48,13 +83,47 @@ public class CblController : BaseApiController
     [HttpPost("import")]
     public async Task<ActionResult<CblImportSummaryDto>> ImportCbl([FromForm(Name = "cbl")] IFormFile file, [FromForm(Name = "dryRun")] bool dryRun = false)
     {
-        var userId = User.GetUserId();
-        var cbl = await SaveAndLoadCblFile(userId, file);
+        try
+        {
+            var userId = User.GetUserId();
+            var cbl = await SaveAndLoadCblFile(file);
+            var importSummary = await _readingListService.CreateReadingListFromCbl(userId, cbl, dryRun);
+            importSummary.FileName = file.FileName;
+            return Ok(importSummary);
+        } catch (ArgumentNullException)
+        {
+            return Ok(new CblImportSummaryDto()
+            {
+                FileName = file.FileName,
+                Success = CblImportResult.Fail,
+                Results = new List<CblBookResult>()
+                {
+                    new CblBookResult()
+                    {
+                        Reason = CblImportReason.InvalidFile
+                    }
+                }
+            });
+        }
+        catch (InvalidOperationException)
+        {
+            return Ok(new CblImportSummaryDto()
+            {
+                FileName = file.FileName,
+                Success = CblImportResult.Fail,
+                Results = new List<CblBookResult>()
+                {
+                    new CblBookResult()
+                    {
+                        Reason = CblImportReason.InvalidFile
+                    }
+                }
+            });
+        }
 
-        return Ok(await _readingListService.CreateReadingListFromCbl(userId, cbl, dryRun));
     }
 
-    private async Task<CblReadingList> SaveAndLoadCblFile(int userId, IFormFile file)
+    private async Task<CblReadingList> SaveAndLoadCblFile(IFormFile file)
     {
         var filename = Path.GetRandomFileName();
         var outputFile = Path.Join(_directoryService.TempDirectory, filename);

API/Controllers/ReaderController.cs

@@ -506,45 +506,6 @@ public class ReaderController : BaseApiController
         return Ok(await _unitOfWork.AppUserProgressRepository.HasAnyProgressOnSeriesAsync(seriesId, userId));
     }
 
-    /// <summary>
-    /// Marks every chapter that is sorted below the passed number as Read. This will not mark any specials as read.
-    /// </summary>
-    /// <remarks>This is built for Tachiyomi and is not expected to be called by any other place</remarks>
-    /// <returns></returns>
-    [Obsolete("Deprecated. Use 'Tachiyomi/mark-chapter-until-as-read'")]
-    [HttpPost("mark-chapter-until-as-read")]
-    public async Task<ActionResult<bool>> MarkChaptersUntilAsRead(int seriesId, float chapterNumber)
-    {
-        var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername(), AppUserIncludes.Progress);
-        if (user == null) return Unauthorized();
-        user.Progresses ??= new List<AppUserProgress>();
-
-        // Tachiyomi sends chapter 0.0f when there's no chapters read.
-        // Due to the encoding for volumes this marks all chapters in volume 0 (loose chapters) as read so we ignore it
-        if (chapterNumber == 0.0f) return true;
-
-        if (chapterNumber < 1.0f)
-        {
-            // This is a hack to track volume number. We need to map it back by x100
-            var volumeNumber = int.Parse($"{chapterNumber * 100f}");
-            await _readerService.MarkVolumesUntilAsRead(user, seriesId, volumeNumber);
-        }
-        else
-        {
-            await _readerService.MarkChaptersUntilAsRead(user, seriesId, chapterNumber);
-        }
-
-
-        _unitOfWork.UserRepository.Update(user);
-
-        if (!_unitOfWork.HasChanges()) return Ok(true);
-        if (await _unitOfWork.CommitAsync()) return Ok(true);
-
-        await _unitOfWork.RollbackAsync();
-        return Ok(false);
-    }
-
-
     /// <summary>
     /// Returns a list of bookmarked pages for a given Chapter
     /// </summary>

API/Controllers/ReadingListController.cs

@@ -1,8 +1,10 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using API.Constants;
 using API.Data;
 using API.Data.Repositories;
+using API.DTOs;
 using API.DTOs.ReadingLists;
 using API.Extensions;
 using API.Helpers;
@@ -421,6 +423,18 @@ public class ReadingListController : BaseApiController
         return Ok("Nothing to do");
     }
 
+    /// <summary>
+    /// Returns a list of characters associated with the reading list
+    /// </summary>
+    /// <param name="readingListId"></param>
+    /// <returns></returns>
+    [HttpGet("characters")]
+    [ResponseCache(CacheProfileName = ResponseCacheProfiles.TenMinute)]
+    public ActionResult<IEnumerable<PersonDto>> GetCharactersForList(int readingListId)
+    {
+        return Ok(_unitOfWork.ReadingListRepository.GetReadingListCharactersAsync(readingListId));
+    }
+
 
 
     /// <summary>

API/Controllers/SettingsController.cs

@@ -80,7 +80,7 @@ public class SettingsController : BaseApiController
     {
         _logger.LogInformation("{UserName} is resetting IP Addresses Setting", User.GetUsername());
         var ipAddresses = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.IpAddresses);
-        ipAddresses.Value = Configuration.DefaultIPAddresses;
+        ipAddresses.Value = Configuration.DefaultIpAddresses;
         _unitOfWork.SettingsRepository.Update(ipAddresses);
 
         if (!await _unitOfWork.CommitAsync())

API/DTOs/ReadingLists/CBL/CblImportSummary.cs

@@ -68,6 +68,11 @@ public enum CblImportReason
     /// </summary>
     [Description("Success")]
     Success = 8,
+    /// <summary>
+    /// The file does not match the XML spec
+    /// </summary>
+    [Description("Invalid File")]
+    InvalidFile = 9,
 }
 
 public class CblBookResult
@@ -79,6 +84,18 @@ public class CblBookResult
     public string Series { get; set; }
     public string Volume { get; set; }
     public string Number { get; set; }
+    /// <summary>
+    /// Used on Series conflict
+    /// </summary>
+    public int LibraryId { get; set; }
+    /// <summary>
+    /// Used on Series conflict
+    /// </summary>
+    public int SeriesId { get; set; }
+    /// <summary>
+    /// The name of the reading list
+    /// </summary>
+    public string ReadingListName { get; set; }
     public CblImportReason Reason { get; set; }
 
     public CblBookResult(CblBook book)
@@ -100,6 +117,10 @@ public class CblBookResult
 public class CblImportSummaryDto
 {
     public string CblName { get; set; }
+    /// <summary>
+    /// Used only for Kavita's UI, the filename of the cbl
+    /// </summary>
+    public string FileName { get; set; }
     public ICollection<CblBookResult> Results { get; set; }
     public CblImportResult Success { get; set; }
     public ICollection<CblBookResult> SuccessfulInserts { get; set; }

API/DTOs/ReadingLists/CBL/CblReadingList.cs

@@ -21,6 +21,44 @@ public class CblReadingList
     [XmlElement(ElementName="Name")]
     public string Name { get; set; }
 
+    /// <summary>
+    /// Summary of the Reading List
+    /// </summary>
+    /// <remarks>This is not a standard, adding based on discussion with CBL Maintainers</remarks>
+    [XmlElement(ElementName="Summary")]
+    public string Summary { get; set; }
+
+    /// <summary>
+    /// Start Year of the Reading List. Overrides calculation
+    /// </summary>
+    /// <remarks>This is not a standard, adding based on discussion with CBL Maintainers</remarks>
+    [XmlElement(ElementName="StartYear")]
+    public int StartYear { get; set; }
+
+    /// <summary>
+    /// Start Year of the Reading List. Overrides calculation
+    /// </summary>
+    /// <remarks>This is not a standard, adding based on discussion with CBL Maintainers</remarks>
+    [XmlElement(ElementName="StartMonth")]
+    public int StartMonth { get; set; }
+
+    /// <summary>
+    /// End Year of the Reading List. Overrides calculation
+    /// </summary>
+    /// <remarks>This is not a standard, adding based on discussion with CBL Maintainers</remarks>
+    [XmlElement(ElementName="EndYear")]
+    public int EndYear { get; set; }
+
+    /// <summary>
+    /// End Year of the Reading List. Overrides calculation
+    /// </summary>
+    /// <remarks>This is not a standard, adding based on discussion with CBL Maintainers</remarks>
+    [XmlElement(ElementName="EndMonth")]
+    public int EndMonth { get; set; }
+
+    /// <summary>
+    /// Issues of the Reading List
+    /// </summary>
     [XmlElement(ElementName="Books")]
     public CblBooks Books { get; set; }
 }

API/DTOs/ReadingLists/ReadingListDto.cs

@@ -14,4 +14,5 @@ public class ReadingListDto
     /// This is used to tell the UI if it should request a Cover Image or not. If null or empty, it has not been set.
     /// </summary>
     public string CoverImage { get; set; } = string.Empty;
+
 }

API/Data/Repositories/ReadingListRepository.cs

@@ -1,13 +1,16 @@
 using System.Collections.Generic;
 using System.Linq;
 using System.Threading.Tasks;
+using API.DTOs;
 using API.DTOs.ReadingLists;
 using API.Entities;
+using API.Entities.Enums;
 using API.Extensions;
 using API.Helpers;
 using API.Services;
 using AutoMapper;
 using AutoMapper.QueryableExtensions;
+using Microsoft.AspNetCore.Identity;
 using Microsoft.EntityFrameworkCore;
 
 namespace API.Data.Repositories;
@@ -32,6 +35,7 @@ public interface IReadingListRepository
     Task<IList<string>> GetAllCoverImagesAsync();
     Task<bool> ReadingListExists(string name);
     Task<List<ReadingList>> GetAllReadingListsAsync();
+    IEnumerable<PersonDto> GetReadingListCharactersAsync(int readingListId);
 }
 
 public class ReadingListRepository : IReadingListRepository
@@ -92,6 +96,16 @@ public class ReadingListRepository : IReadingListRepository
             .ToListAsync();
     }
 
+    public IEnumerable<PersonDto> GetReadingListCharactersAsync(int readingListId)
+    {
+        return _context.ReadingListItem
+            .Where(item => item.ReadingListId == readingListId)
+            .SelectMany(item => item.Chapter.People.Where(p => p.Role == PersonRole.Character))
+            .OrderBy(p => p.NormalizedName)
+            .ProjectTo<PersonDto>(_mapper.ConfigurationProvider)
+            .AsEnumerable();
+    }
+
     public void Remove(ReadingListItem item)
     {
         _context.ReadingListItem.Remove(item);

API/Entities/ReadingList.cs

@@ -15,7 +15,7 @@ public class ReadingList : IEntityDate
     /// <summary>
     /// A normalized string used to check if the reading list already exists in the DB
     /// </summary>
-    public string? NormalizedTitle { get; set; }
+    public required string NormalizedTitle { get; set; }
     public string? Summary { get; set; }
     /// <summary>
     /// Reading lists that are promoted are only done by admins
@@ -39,6 +39,14 @@ public class ReadingList : IEntityDate
     public DateTime LastModified { get; set; }
     public DateTime CreatedUtc { get; set; }
     public DateTime LastModifiedUtc { get; set; }
+    // /// <summary>
+    // /// Minimum Year and Month the Reading List starts
+    // /// </summary>
+    // public DateOnly StartingYear { get; set; }
+    // /// <summary>
+    // ///  Maximum Year and Month the Reading List starts
+    // /// </summary>
+    // public DateOnly EndingYear { get; set; }
 
     // Relationships
     public int AppUserId { get; set; }

API/Program.cs

@@ -173,7 +173,7 @@ public class Program
                 webBuilder.UseKestrel((opts) =>
                 {
                     var ipAddresses = Configuration.IpAddresses;
-                    if (new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker || string.IsNullOrEmpty(ipAddresses))
+                    if (new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker || string.IsNullOrEmpty(ipAddresses) || ipAddresses.Equals(Configuration.DefaultIpAddresses))
                     {
                         opts.ListenAnyIP(HttpPort, options => { options.Protocols = HttpProtocols.Http1AndHttp2; });
                     }

API/Services/ImageService.cs

@@ -149,9 +149,9 @@ public class ImageService : IImageService
         try
         {
             using var thumbnail = Image.ThumbnailBuffer(Convert.FromBase64String(encodedImage), thumbnailWidth);
-            var filename = fileName + (saveAsWebP ? ".webp" : ".png");
-            thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, fileName + ".png"));
-            return filename;
+            fileName += (saveAsWebP ? ".webp" : ".png");
+            thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, fileName));
+            return fileName;
         }
         catch (Exception e)
         {

API/Services/ReadingListService.cs

@@ -367,9 +367,11 @@ public class ReadingListService : IReadingListService
         // Is there another reading list with the same name?
         if (await _unitOfWork.ReadingListRepository.ReadingListExists(cblReading.Name))
         {
+            importSummary.Success = CblImportResult.Fail;
             importSummary.Results.Add(new CblBookResult()
             {
-                Reason = CblImportReason.NameConflict
+                Reason = CblImportReason.NameConflict,
+                ReadingListName = cblReading.Name
             });
         }
 
@@ -391,24 +393,16 @@ public class ReadingListService : IReadingListService
         if (!conflicts.Any()) return importSummary;
 
         importSummary.Success = CblImportResult.Fail;
-        if (conflicts.Count == cblReading.Books.Book.Count)
+        foreach (var conflict in conflicts)
         {
             importSummary.Results.Add(new CblBookResult()
             {
-                Reason = CblImportReason.AllChapterMissing,
+                Reason = CblImportReason.SeriesCollision,
+                Series = conflict.Name,
+                LibraryId = conflict.LibraryId,
+                SeriesId = conflict.Id,
             });
         }
-        else
-        {
-            foreach (var conflict in conflicts)
-            {
-                importSummary.Results.Add(new CblBookResult()
-                {
-                    Reason = CblImportReason.SeriesCollision,
-                    Series = conflict.Name
-                });
-            }
-        }
 
         return importSummary;
     }
@@ -484,6 +478,7 @@ public class ReadingListService : IReadingListService
                 importSummary.Results.Add(new CblBookResult(book)
                 {
                     Reason = CblImportReason.VolumeMissing,
+                    LibraryId = bookSeries.LibraryId,
                     Order = i
                 });
                 continue;
@@ -499,6 +494,7 @@ public class ReadingListService : IReadingListService
                 importSummary.Results.Add(new CblBookResult(book)
                 {
                     Reason = CblImportReason.ChapterMissing,
+                    LibraryId = bookSeries.LibraryId,
                     Order = i
                 });
                 continue;
@@ -523,11 +519,16 @@ public class ReadingListService : IReadingListService
             importSummary.Success = CblImportResult.Fail;
         }
 
-        await CalculateReadingListAgeRating(readingList);
-
         if (dryRun) return importSummary;
 
-        if (!_unitOfWork.HasChanges()) return importSummary;
+        await CalculateReadingListAgeRating(readingList);
+        if (!string.IsNullOrEmpty(readingList.Summary?.Trim()))
+        {
+            readingList.Summary = readingList.Summary?.Trim();
+        }
+
+        // If there are no items, don't create a blank list
+        if (!_unitOfWork.HasChanges() || !readingList.Items.Any()) return importSummary;
         await _unitOfWork.CommitAsync();
 
 

API/config/appsettings.Development.json

@@ -1,5 +1,5 @@
 {
   "TokenKey": "super secret unguessable key",
   "Port": 5000,
-  "IpAddresses": "0.0.0.0,::"
+  "IpAddresses": ""
 }