MAL Interest Stacks (#2932)

API/API.csproj

@@ -53,7 +53,7 @@
   </PropertyGroup>
 
   <ItemGroup>
-      <PackageReference Include="CsvHelper" Version="31.0.3" />
+      <PackageReference Include="CsvHelper" Version="32.0.1" />
       <PackageReference Include="MailKit" Version="4.5.0" />
       <PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.4">
           <PrivateAssets>all</PrivateAssets>
@@ -66,10 +66,10 @@
     <PackageReference Include="Flurl" Version="3.0.7" />
     <PackageReference Include="Flurl.Http" Version="3.2.4" />
     <PackageReference Include="Hangfire" Version="1.8.12" />
-    <PackageReference Include="Hangfire.InMemory" Version="0.8.1" />
+    <PackageReference Include="Hangfire.InMemory" Version="0.9.0" />
     <PackageReference Include="Hangfire.MaximumConcurrentExecutions" Version="1.1.0" />
     <PackageReference Include="Hangfire.Storage.SQLite" Version="0.4.2" />
-    <PackageReference Include="HtmlAgilityPack" Version="1.11.60" />
+    <PackageReference Include="HtmlAgilityPack" Version="1.11.61" />
     <PackageReference Include="MarkdownDeep.NET.Core" Version="1.5.0.4" />
     <PackageReference Include="Hangfire.AspNetCore" Version="1.8.12" />
     <PackageReference Include="Microsoft.AspNetCore.SignalR" Version="1.1.0" />
@@ -93,14 +93,14 @@
     <PackageReference Include="Serilog.Sinks.Console" Version="5.0.1" />
     <PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
     <PackageReference Include="Serilog.Sinks.SignalR.Core" Version="0.1.2" />
-    <PackageReference Include="SharpCompress" Version="0.36.0" />
+    <PackageReference Include="SharpCompress" Version="0.37.2" />
     <PackageReference Include="SixLabors.ImageSharp" Version="3.1.4" />
-    <PackageReference Include="SonarAnalyzer.CSharp" Version="9.23.2.88755">
+    <PackageReference Include="SonarAnalyzer.CSharp" Version="9.24.0.89429">
       <PrivateAssets>all</PrivateAssets>
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
-    <PackageReference Include="Swashbuckle.AspNetCore.Filters" Version="8.0.1" />
+    <PackageReference Include="Swashbuckle.AspNetCore.Filters" Version="8.0.2" />
     <PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="7.5.1" />
     <PackageReference Include="System.IO.Abstractions" Version="21.0.2" />
     <PackageReference Include="System.Drawing.Common" Version="8.0.4" />
@@ -194,6 +194,7 @@
     <Content Include="EmailTemplates\**">
       <CopyToOutputDirectory>Always</CopyToOutputDirectory>
     </Content>
+    <Folder Include="Extensions\KavitaPlus\" />
     <None Include="I18N\**" />
   </ItemGroup>
 

API/Controllers/AccountController.cs

@@ -898,8 +898,6 @@ public class AccountController : BaseApiController
     {
 
         var settings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
-        if (!settings.IsEmailSetup()) return Ok(await _localizationService.Get("en", "email-not-enabled"));
-
         var user = await _unitOfWork.UserRepository.GetUserByEmailAsync(email);
         if (user == null)
         {
@@ -914,11 +912,7 @@ public class AccountController : BaseApiController
         if (string.IsNullOrEmpty(user.Email) || !user.EmailConfirmed)
             return BadRequest(await _localizationService.Translate(user.Id, "confirm-email"));
 
-        if (!_emailService.IsValidEmail(user.Email))
-        {
-            _logger.LogCritical("[Forgot Password]: User is trying to do a forgot password flow, but their email ({Email}) isn't valid. No email will be send. Admin must change it in UI", user.Email);
-            return Ok(await _localizationService.Translate(user.Id, "invalid-email"));
-        }
+
 
         var token = await _userManager.GeneratePasswordResetTokenAsync(user);
         var emailLink = await _emailService.GenerateEmailLink(Request, token, "confirm-reset-password", user.Email);
@@ -927,6 +921,13 @@ public class AccountController : BaseApiController
         await _unitOfWork.CommitAsync();
         _logger.LogCritical("[Forgot Password]: Email Link for {UserName}: {Link}", user.UserName, emailLink);
 
+        if (!settings.IsEmailSetup()) return Ok(await _localizationService.Get("en", "email-not-enabled"));
+        if (!_emailService.IsValidEmail(user.Email))
+        {
+            _logger.LogCritical("[Forgot Password]: User is trying to do a forgot password flow, but their email ({Email}) isn't valid. No email will be send. Admin must change it in UI or from url above", user.Email);
+            return Ok(await _localizationService.Translate(user.Id, "invalid-email"));
+        }
+
         var installId = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallId)).Value;
         BackgroundJob.Enqueue(() => _emailService.SendForgotPasswordEmail(new PasswordResetEmailDto()
         {

API/Controllers/CollectionController.cs

@@ -12,8 +12,11 @@ using API.Extensions;
 using API.Helpers.Builders;
 using API.Services;
 using API.Services.Plus;
+using API.SignalR;
+using Hangfire;
 using Kavita.Common;
 using Microsoft.AspNetCore.Mvc;
+using Microsoft.Extensions.Logging;
 
 namespace API.Controllers;
 
@@ -28,15 +31,23 @@ public class CollectionController : BaseApiController
     private readonly ICollectionTagService _collectionService;
     private readonly ILocalizationService _localizationService;
     private readonly IExternalMetadataService _externalMetadataService;
+    private readonly ISmartCollectionSyncService _collectionSyncService;
+    private readonly ILogger<CollectionController> _logger;
+    private readonly IEventHub _eventHub;
 
     /// <inheritdoc />
     public CollectionController(IUnitOfWork unitOfWork, ICollectionTagService collectionService,
-        ILocalizationService localizationService, IExternalMetadataService externalMetadataService)
+        ILocalizationService localizationService, IExternalMetadataService externalMetadataService,
+        ISmartCollectionSyncService collectionSyncService, ILogger<CollectionController> logger,
+        IEventHub eventHub)
     {
         _unitOfWork = unitOfWork;
         _collectionService = collectionService;
         _localizationService = localizationService;
         _externalMetadataService = externalMetadataService;
+        _collectionSyncService = collectionSyncService;
+        _logger = logger;
+        _eventHub = eventHub;
     }
 
     /// <summary>
@@ -49,6 +60,18 @@ public class CollectionController : BaseApiController
         return Ok(await _unitOfWork.CollectionTagRepository.GetCollectionDtosAsync(User.GetUserId(), !ownedOnly));
     }
 
+    /// <summary>
+    /// Returns a single Collection tag by Id for a given user
+    /// </summary>
+    /// <param name="collectionId"></param>
+    /// <returns></returns>
+    [HttpGet("single")]
+    public async Task<ActionResult<IEnumerable<AppUserCollectionDto>>> GetTag(int collectionId)
+    {
+        var collections = await _unitOfWork.CollectionTagRepository.GetCollectionDtosAsync(User.GetUserId(), false);
+        return Ok(collections.FirstOrDefault(c => c.Id == collectionId));
+    }
+
     /// <summary>
     /// Returns all collections that contain the Series for the user with the option to allow for promoted collections (non-user owned)
     /// </summary>
@@ -86,6 +109,8 @@ public class CollectionController : BaseApiController
         {
             if (await _collectionService.UpdateTag(updatedTag, User.GetUserId()))
             {
+                await _eventHub.SendMessageAsync(MessageFactory.CollectionUpdated,
+                    MessageFactory.CollectionUpdatedEvent(updatedTag.Id), false);
                 return Ok(await _localizationService.Translate(User.GetUserId(), "collection-updated-successfully"));
             }
         }
@@ -129,12 +154,12 @@ public class CollectionController : BaseApiController
 
 
     /// <summary>
-    /// Promote/UnPromote multiple collections in one go
+    /// Delete multiple collections in one go
     /// </summary>
     /// <param name="dto"></param>
     /// <returns></returns>
     [HttpPost("delete-multiple")]
-    public async Task<ActionResult> DeleteMultipleCollections(PromoteCollectionsDto dto)
+    public async Task<ActionResult> DeleteMultipleCollections(DeleteCollectionsDto dto)
     {
         // This needs to take into account owner as I can select other users cards
         var user = await _unitOfWork.UserRepository.GetUserByIdAsync(User.GetUserId(), AppUserIncludes.Collections);
@@ -178,7 +203,7 @@ public class CollectionController : BaseApiController
             return BadRequest(_localizationService.Translate(User.GetUserId(), "collection-doesnt-exists"));
         }
 
-        var series = await _unitOfWork.SeriesRepository.GetSeriesByIdsAsync(dto.SeriesIds.ToList());
+        var series = await _unitOfWork.SeriesRepository.GetSeriesByIdsAsync(dto.SeriesIds.ToList(), false);
         foreach (var s in series)
         {
             if (tag.Items.Contains(s)) continue;
@@ -253,4 +278,45 @@ public class CollectionController : BaseApiController
     {
         return Ok(await _externalMetadataService.GetStacksForUser(User.GetUserId()));
     }
+
+    /// <summary>
+    /// Imports a MAL Stack into Kavita
+    /// </summary>
+    /// <param name="dto"></param>
+    /// <returns></returns>
+    [HttpPost("import-stack")]
+    public async Task<ActionResult> ImportMalStack(MalStackDto dto)
+    {
+        var user = await _unitOfWork.UserRepository.GetUserByIdAsync(User.GetUserId(), AppUserIncludes.Collections);
+        if (user == null) return Unauthorized();
+
+        // Validation check to ensure stack doesn't exist already
+        if (await _unitOfWork.CollectionTagRepository.CollectionExists(dto.Title, user.Id))
+        {
+            return BadRequest(_localizationService.Translate(user.Id, "collection-already-exists"));
+        }
+
+        try
+        {
+            // Create new collection
+            var newCollection = new AppUserCollectionBuilder(dto.Title)
+                .WithSource(ScrobbleProvider.Mal)
+                .WithSourceUrl(dto.Url)
+                .Build();
+            user.Collections.Add(newCollection);
+
+            _unitOfWork.UserRepository.Update(user);
+            await _unitOfWork.CommitAsync();
+
+            // Trigger Stack Refresh for just one stack (not all)
+            BackgroundJob.Enqueue(() => _collectionSyncService.Sync(newCollection.Id));
+            return Ok();
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "There was an issue importing MAL Stack");
+        }
+
+        return BadRequest(_localizationService.Translate(user.Id, "error-import-stack"));
+    }
 }

API/DTOs/Collection/AppUserCollectionDto.cs

@@ -36,4 +36,12 @@ public class AppUserCollectionDto
     /// For Non-Kavita sourced collections, the url to sync from
     /// </summary>
     public string? SourceUrl { get; set; }
+    /// <summary>
+    /// Total number of items as of the last sync. Not applicable for Kavita managed collections.
+    /// </summary>
+    public int TotalSourceCount { get; set; }
+    /// <summary>
+    /// A <br/> separated string of all missing series
+    /// </summary>
+    public string? MissingSeriesFromSource { get; set; }
 }

API/DTOs/Collection/DeleteCollectionsDto.cs

@@ -1,8 +1,10 @@
 using System.Collections.Generic;
+using System.ComponentModel.DataAnnotations;
 
 namespace API.DTOs.Collection;
 
 public class DeleteCollectionsDto
 {
+    [Required]
     public IList<int> CollectionIds { get; set; }
 }

API/DTOs/CollectionTags/CollectionTagDto.cs

@@ -1,5 +1,8 @@
-namespace API.DTOs.CollectionTags;
+using System;
 
+namespace API.DTOs.CollectionTags;
+
+[Obsolete("Use AppUserCollectionDto")]
 public class CollectionTagDto
 {
     public int Id { get; set; }

API/DTOs/CollectionTags/UpdateSeriesForTagDto.cs

@@ -1,9 +1,11 @@
-using System.Collections.Generic;
+using System;
+using System.Collections.Generic;
+using API.DTOs.Collection;
 
 namespace API.DTOs.CollectionTags;
 
 public class UpdateSeriesForTagDto
 {
-    public CollectionTagDto Tag { get; init; } = default!;
+    public AppUserCollectionDto Tag { get; init; } = default!;
     public IEnumerable<int> SeriesIdsToRemove { get; init; } = default!;
 }

API/DTOs/UpdateLibraryDto.cs

@@ -36,5 +36,6 @@ public class UpdateLibraryDto
     /// <summary>
     /// A set of Glob patterns that the scanner will exclude processing
     /// </summary>
+    [Required]
     public ICollection<string> ExcludePatterns { get; init; }
 }

API/Data/Migrations/20240503120147_SmartCollectionFields.cs

@@ -0,0 +1,39 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace API.Data.Migrations
+{
+    /// <inheritdoc />
+    public partial class SmartCollectionFields : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<string>(
+                name: "MissingSeriesFromSource",
+                table: "AppUserCollection",
+                type: "TEXT",
+                nullable: true);
+
+            migrationBuilder.AddColumn<int>(
+                name: "TotalSourceCount",
+                table: "AppUserCollection",
+                type: "INTEGER",
+                nullable: false,
+                defaultValue: 0);
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "MissingSeriesFromSource",
+                table: "AppUserCollection");
+
+            migrationBuilder.DropColumn(
+                name: "TotalSourceCount",
+                table: "AppUserCollection");
+        }
+    }
+}

API/Data/Migrations/DataContextModelSnapshot.cs

@@ -224,6 +224,9 @@ namespace API.Data.Migrations
                     b.Property<DateTime>("LastSyncUtc")
                         .HasColumnType("TEXT");
 
+                    b.Property<string>("MissingSeriesFromSource")
+                        .HasColumnType("TEXT");
+
                     b.Property<string>("NormalizedTitle")
                         .HasColumnType("TEXT");
 
@@ -242,6 +245,9 @@ namespace API.Data.Migrations
                     b.Property<string>("Title")
                         .HasColumnType("TEXT");
 
+                    b.Property<int>("TotalSourceCount")
+                        .HasColumnType("INTEGER");
+
                     b.HasKey("Id");
 
                     b.HasIndex("AppUserId");

API/Data/Repositories/CollectionTagRepository.cs

@@ -9,6 +9,7 @@ using API.Entities.Enums;
 using API.Extensions;
 using API.Extensions.QueryExtensions;
 using API.Extensions.QueryExtensions.Filtering;
+using API.Services.Plus;
 using AutoMapper;
 using AutoMapper.QueryableExtensions;
 using Microsoft.EntityFrameworkCore;
@@ -57,6 +58,7 @@ public interface ICollectionTagRepository
     Task<IList<AppUserCollection>> GetCollectionsForUserAsync(int userId, CollectionIncludes includes = CollectionIncludes.None);
     Task UpdateCollectionAgeRating(AppUserCollection tag);
     Task<IEnumerable<AppUserCollection>> GetCollectionsByIds(IEnumerable<int> tags, CollectionIncludes includes = CollectionIncludes.None);
+    Task<IList<AppUserCollection>> GetAllCollectionsForSyncing(DateTime expirationTime);
 }
 public class CollectionTagRepository : ICollectionTagRepository
 {
@@ -207,6 +209,16 @@ public class CollectionTagRepository : ICollectionTagRepository
             .ToListAsync();
     }
 
+    public async Task<IList<AppUserCollection>> GetAllCollectionsForSyncing(DateTime expirationTime)
+    {
+        return await _context.AppUserCollection
+            .Where(c => c.Source == ScrobbleProvider.Mal)
+            .Where(c => c.LastSyncUtc <= expirationTime)
+            .Include(c => c.Items)
+            .AsSplitQuery()
+            .ToListAsync();
+    }
+
 
     public async Task<AppUserCollection?> GetCollectionAsync(int tagId, CollectionIncludes includes = CollectionIncludes.None)
     {

API/Data/Repositories/SeriesRepository.cs

@@ -73,6 +73,7 @@ public interface ISeriesRepository
     void Update(Series series);
     void Remove(Series series);
     void Remove(IEnumerable<Series> series);
+    void Detach(Series series);
     Task<bool> DoesSeriesNameExistInLibrary(string name, int libraryId, MangaFormat format);
     /// <summary>
     /// Adds user information like progress, ratings, etc
@@ -96,7 +97,7 @@ public interface ISeriesRepository
     Task<SeriesDto?> GetSeriesDtoByIdAsync(int seriesId, int userId);
     Task<Series?> GetSeriesByIdAsync(int seriesId, SeriesIncludes includes = SeriesIncludes.Volumes | SeriesIncludes.Metadata);
     Task<IList<SeriesDto>> GetSeriesDtoByIdsAsync(IEnumerable<int> seriesIds, AppUser user);
-    Task<IList<Series>> GetSeriesByIdsAsync(IList<int> seriesIds);
+    Task<IList<Series>> GetSeriesByIdsAsync(IList<int> seriesIds, bool fullSeries = true);
     Task<int[]> GetChapterIdsForSeriesAsync(IList<int> seriesIds);
     Task<IDictionary<int, IList<int>>> GetChapterIdWithSeriesIdForSeriesAsync(int[] seriesIds);
     /// <summary>
@@ -138,6 +139,7 @@ public interface ISeriesRepository
     Task<IEnumerable<Series>> GetAllSeriesByNameAsync(IList<string> normalizedNames,
         int userId, SeriesIncludes includes = SeriesIncludes.None);
     Task<Series?> GetFullSeriesByAnyName(string seriesName, string localizedName, int libraryId, MangaFormat format, bool withFullIncludes = true);
+    Task<Series?> GetSeriesByAnyName(string seriesName, string localizedName, IList<MangaFormat> formats, int userId);
     public Task<IList<Series>> GetAllSeriesByAnyName(string seriesName, string localizedName, int libraryId,
         MangaFormat format);
     Task<IList<Series>> RemoveSeriesNotInList(IList<ParsedSeries> seenSeries, int libraryId);
@@ -204,6 +206,11 @@ public class SeriesRepository : ISeriesRepository
         _context.Series.RemoveRange(series);
     }
 
+    public void Detach(Series series)
+    {
+        _context.Entry(series).State = EntityState.Detached;
+    }
+
     /// <summary>
     /// Returns if a series name and format exists already in a library
     /// </summary>
@@ -531,15 +538,19 @@ public class SeriesRepository : ISeriesRepository
     /// Returns Full Series including all external links
     /// </summary>
     /// <param name="seriesIds"></param>
+    /// <param name="fullSeries">Include all the includes or just the Series</param>
     /// <returns></returns>
-    public async Task<IList<Series>> GetSeriesByIdsAsync(IList<int> seriesIds)
+    public async Task<IList<Series>> GetSeriesByIdsAsync(IList<int> seriesIds, bool fullSeries = true)
     {
-        return await _context.Series
-            .Include(s => s.Volumes)
+        var query = _context.Series
+            .Where(s => seriesIds.Contains(s.Id))
+            .AsSplitQuery();
+
+        if (!fullSeries) return await query.ToListAsync();
+
+        return await query.Include(s => s.Volumes)
             .Include(s => s.Relations)
             .Include(s => s.Metadata)
-            .ThenInclude(m => m.CollectionTags)
-
 
             .Include(s => s.ExternalSeriesMetadata)
 
@@ -549,9 +560,6 @@ public class SeriesRepository : ISeriesRepository
             .ThenInclude(e => e.ExternalReviews)
             .Include(s => s.ExternalSeriesMetadata)
             .ThenInclude(e => e.ExternalRecommendations)
-
-            .Where(s => seriesIds.Contains(s.Id))
-            .AsSplitQuery()
             .ToListAsync();
     }
 
@@ -1670,6 +1678,26 @@ public class SeriesRepository : ISeriesRepository
     #nullable enable
     }
 
+    public async Task<Series?> GetSeriesByAnyName(string seriesName, string localizedName, IList<MangaFormat> formats, int userId)
+    {
+        var libraryIds = GetLibraryIdsForUser(userId);
+        var normalizedSeries = seriesName.ToNormalized();
+        var normalizedLocalized = localizedName.ToNormalized();
+        return await _context.Series
+            .Where(s => libraryIds.Contains(s.LibraryId))
+            .Where(s => formats.Contains(s.Format))
+            .Where(s =>
+                s.NormalizedName.Equals(normalizedSeries)
+                || s.NormalizedName.Equals(normalizedLocalized)
+
+                || s.NormalizedLocalizedName.Equals(normalizedSeries)
+                || (!string.IsNullOrEmpty(normalizedLocalized) && s.NormalizedLocalizedName.Equals(normalizedLocalized))
+
+                || (s.OriginalName != null && s.OriginalName.Equals(seriesName))
+            )
+            .FirstOrDefaultAsync();
+    }
+
     public async Task<IList<Series>> GetAllSeriesByAnyName(string seriesName, string localizedName, int libraryId,
         MangaFormat format)
     {

API/Entities/AppUserCollection.cs

@@ -52,7 +52,14 @@ public class AppUserCollection : IEntityDate
     /// For Non-Kavita sourced collections, the url to sync from
     /// </summary>
     public string? SourceUrl { get; set; }
-
+    /// <summary>
+    /// Total number of items as of the last sync. Not applicable for Kavita managed collections.
+    /// </summary>
+    public int TotalSourceCount { get; set; }
+    /// <summary>
+    /// A <br/> separated string of all missing series
+    /// </summary>
+    public string? MissingSeriesFromSource { get; set; }
 
     // Relationship
     public AppUser AppUser { get; set; } = null!;

API/Extensions/ApplicationServiceExtensions.cs

@@ -74,6 +74,7 @@ public static class ApplicationServiceExtensions
         services.AddScoped<IScrobblingService, ScrobblingService>();
         services.AddScoped<ILicenseService, LicenseService>();
         services.AddScoped<IExternalMetadataService, ExternalMetadataService>();
+        services.AddScoped<ISmartCollectionSyncService, SmartCollectionSyncService>();
 
         services.AddSqLite();
         services.AddSignalR(opt => opt.EnableDetailedErrors = true);

API/Helpers/Builders/AppUserCollectionBuilder.cs

@@ -69,4 +69,10 @@ public class AppUserCollectionBuilder : IEntityBuilder<AppUserCollection>
         _collection.CoverImage = cover;
         return this;
     }
+
+    public AppUserCollectionBuilder WithSourceUrl(string url)
+    {
+        _collection.SourceUrl = url;
+        return this;
+    }
 }

API/Helpers/LibraryTypeHelper.cs

@@ -9,11 +9,16 @@ public static class LibraryTypeHelper
 {
     public static MediaFormat GetFormat(LibraryType libraryType)
     {
+        // TODO: Refactor this to an extension on LibraryType
         return libraryType switch
         {
             LibraryType.Manga => MediaFormat.Manga,
             LibraryType.Comic => MediaFormat.Comic,
             LibraryType.LightNovel => MediaFormat.LightNovel,
+            LibraryType.Book => MediaFormat.LightNovel,
+            LibraryType.Image => MediaFormat.Manga,
+            LibraryType.ComicVine => MediaFormat.Comic,
+            _ => throw new ArgumentOutOfRangeException(nameof(libraryType), libraryType, null)
         };
     }
 }

API/I18N/en.json

@@ -49,6 +49,8 @@
     "collection-deleted": "Collection deleted",
     "generic-error": "Something went wrong, please try again",
     "collection-doesnt-exist": "Collection does not exist",
+    "collection-already-exists":"Collection already exists",
+    "error-import-stack": "There was an issue importing MAL stack",
 
     "device-doesnt-exist": "Device does not exist",
     "generic-device-create": "There was an error when creating the device",

API/Services/Plus/ExternalMetadataService.cs

@@ -82,7 +82,7 @@ public class ExternalMetadataService : IExternalMetadataService
         Reviews = ArraySegment<UserReviewDto>.Empty
     };
     // Allow 50 requests per 24 hours
-    private static readonly RateLimiter RateLimiter = new RateLimiter(50, TimeSpan.FromHours(12), false);
+    private static readonly RateLimiter RateLimiter = new RateLimiter(50, TimeSpan.FromHours(24), false);
 
     public ExternalMetadataService(IUnitOfWork unitOfWork, ILogger<ExternalMetadataService> logger, IMapper mapper, ILicenseService licenseService)
     {

API/Services/Plus/SmartCollectionSyncService.cs

@@ -0,0 +1,266 @@
+using System;
+using System.Collections.Generic;
+using System.Globalization;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+using API.Data;
+using API.Data.Repositories;
+using API.DTOs.Scrobbling;
+using API.Entities;
+using API.Entities.Enums;
+using API.Extensions;
+using API.Helpers;
+using API.SignalR;
+using Flurl.Http;
+using Kavita.Common;
+using Kavita.Common.EnvironmentInfo;
+using Microsoft.Extensions.Logging;
+
+namespace API.Services.Plus;
+#nullable enable
+
+sealed class SeriesCollection
+{
+    public required IList<ExternalMetadataIdsDto> Series { get; set; }
+    public required string Summary { get; set; }
+    public required string Title { get; set; }
+    /// <summary>
+    /// Total items in the source, not what was matched
+    /// </summary>
+    public int TotalItems { get; set; }
+}
+
+/// <summary>
+/// Responsible to synchronize Collection series from non-Kavita sources
+/// </summary>
+public interface ISmartCollectionSyncService
+{
+    /// <summary>
+    /// Synchronize all collections
+    /// </summary>
+    /// <returns></returns>
+    Task Sync();
+    /// <summary>
+    /// Synchronize a collection
+    /// </summary>
+    /// <param name="collectionId"></param>
+    /// <returns></returns>
+    Task Sync(int collectionId);
+}
+
+public class SmartCollectionSyncService : ISmartCollectionSyncService
+{
+    private readonly IUnitOfWork _unitOfWork;
+    private readonly ILogger<SmartCollectionSyncService> _logger;
+    private readonly IEventHub _eventHub;
+    private readonly ILicenseService _licenseService;
+
+    private const int SyncDelta = -2;
+    // Allow 50 requests per 24 hours
+    private static readonly RateLimiter RateLimiter = new RateLimiter(50, TimeSpan.FromHours(24), false);
+
+
+    public SmartCollectionSyncService(IUnitOfWork unitOfWork, ILogger<SmartCollectionSyncService> logger,
+        IEventHub eventHub, ILicenseService licenseService)
+    {
+        _unitOfWork = unitOfWork;
+        _logger = logger;
+        _eventHub = eventHub;
+        _licenseService = licenseService;
+    }
+
+    /// <summary>
+    /// For every Sync-eligible collection, synchronize with upstream
+    /// </summary>
+    /// <returns></returns>
+    public async Task Sync()
+    {
+        if (!await _licenseService.HasActiveLicense()) return;
+        var expirationTime = DateTime.UtcNow.AddDays(SyncDelta).Truncate(TimeSpan.TicksPerHour);
+        var collections = (await _unitOfWork.CollectionTagRepository.GetAllCollectionsForSyncing(expirationTime))
+            .Where(CanSync)
+            .ToList();
+
+        _logger.LogInformation("Found {Count} collections to synchronize", collections.Count);
+        foreach (var collection in collections)
+        {
+            try
+            {
+                await SyncCollection(collection);
+            }
+            catch (RateLimitException)
+            {
+                break;
+            }
+        }
+
+        _logger.LogInformation("Synchronization complete");
+    }
+
+    public async Task Sync(int collectionId)
+    {
+        if (!await _licenseService.HasActiveLicense()) return;
+        var collection = await _unitOfWork.CollectionTagRepository.GetCollectionAsync(collectionId, CollectionIncludes.Series);
+        if (!CanSync(collection))
+        {
+            _logger.LogInformation("Requested to sync {CollectionName} but not applicable to sync", collection!.Title);
+            return;
+        }
+
+        try
+        {
+            await SyncCollection(collection!);
+        } catch (RateLimitException) {/* Swallow */}
+    }
+
+    private static bool CanSync(AppUserCollection? collection)
+    {
+        if (collection is not {Source: ScrobbleProvider.Mal}) return false;
+        if (string.IsNullOrEmpty(collection.SourceUrl)) return false;
+        if (collection.LastSyncUtc.Truncate(TimeSpan.TicksPerHour) >= DateTime.UtcNow.AddDays(SyncDelta).Truncate(TimeSpan.TicksPerHour)) return false;
+        return true;
+    }
+
+    private async Task SyncCollection(AppUserCollection collection)
+    {
+        if (!RateLimiter.TryAcquire(string.Empty))
+        {
+            // Request not allowed due to rate limit
+            _logger.LogDebug("Rate Limit hit for Smart Collection Sync");
+            throw new RateLimitException();
+        }
+
+        var info = await GetStackInfo(GetStackId(collection.SourceUrl!));
+        if (info == null)
+        {
+            _logger.LogInformation("Unable to find collection through Kavita+");
+            return;
+        }
+
+        // Check each series in the collection against what's in the target
+        // For everything that's not there, link it up for this user.
+        _logger.LogInformation("Starting Sync on {CollectionName} with {SeriesCount} Series", info.Title, info.TotalItems);
+
+        var missingCount = 0;
+        var missingSeries = new StringBuilder();
+        foreach (var seriesInfo in info.Series.OrderBy(s => s.SeriesName))
+        {
+            try
+            {
+                // Normalize series name and localized name
+                var normalizedSeriesName = seriesInfo.SeriesName?.ToNormalized();
+                var normalizedLocalizedSeriesName = seriesInfo.LocalizedSeriesName?.ToNormalized();
+
+                // Search for existing series in the collection
+                var formats = GetMangaFormats(seriesInfo.PlusMediaFormat);
+                var existingSeries = collection.Items.FirstOrDefault(s =>
+                    (s.Name.ToNormalized() == normalizedSeriesName ||
+                     s.NormalizedName == normalizedSeriesName ||
+                     s.LocalizedName.ToNormalized() == normalizedLocalizedSeriesName ||
+                     s.NormalizedLocalizedName == normalizedLocalizedSeriesName ||
+
+                     s.NormalizedName == normalizedLocalizedSeriesName ||
+                     s.NormalizedLocalizedName == normalizedSeriesName)
+                    && formats.Contains(s.Format));
+
+                if (existingSeries != null) continue;
+
+                // Series not found in the collection, try to find it in the server
+                var newSeries = await _unitOfWork.SeriesRepository.GetSeriesByAnyName(seriesInfo.SeriesName,
+                    seriesInfo.LocalizedSeriesName,
+                    formats, collection.AppUserId);
+
+                collection.Items ??= new List<Series>();
+                if (newSeries != null)
+                {
+                    // Add the new series to the collection
+                    collection.Items.Add(newSeries);
+
+                }
+                else
+                {
+                    _logger.LogDebug("{Series} not found in the server", seriesInfo.SeriesName);
+                    missingCount++;
+                    missingSeries.Append(
+                        $"<a href='{ScrobblingService.MalWeblinkWebsite}{seriesInfo.MalId}' target='_blank' rel='noopener noreferrer'>{seriesInfo.SeriesName}</a>");
+                    missingSeries.Append("<br/>");
+                }
+            }
+            catch (Exception ex)
+            {
+                _logger.LogError(ex, "An exception occured when linking up a series to the collection. Skipping");
+                missingCount++;
+                missingSeries.Append(
+                    $"<a href='{ScrobblingService.MalWeblinkWebsite}{seriesInfo.MalId}' target='_blank' rel='noopener noreferrer'>{seriesInfo.SeriesName}</a>");
+                missingSeries.Append("<br/>");
+            }
+
+        }
+
+        // At this point, all series in the info have been checked and added if necessary
+        // You may want to commit changes to the database if needed
+        collection.LastSyncUtc = DateTime.UtcNow.Truncate(TimeSpan.TicksPerHour);
+        collection.TotalSourceCount = info.TotalItems;
+        collection.Summary = info.Summary;
+        collection.MissingSeriesFromSource = missingSeries.ToString();
+
+        _unitOfWork.CollectionTagRepository.Update(collection);
+
+        try
+        {
+            await _unitOfWork.CommitAsync();
+
+            await _unitOfWork.CollectionTagRepository.UpdateCollectionAgeRating(collection);
+
+            await _eventHub.SendMessageAsync(MessageFactory.CollectionUpdated,
+                MessageFactory.CollectionUpdatedEvent(collection.Id), false);
+
+            _logger.LogInformation("Finished Syncing Collection {CollectionName} - Missing {MissingCount} series",
+                collection.Title, missingCount);
+        }
+        catch (Exception ex)
+        {
+            _logger.LogError(ex, "There was an error during saving the collection");
+        }
+    }
+
+    private static IList<MangaFormat> GetMangaFormats(MediaFormat? mediaFormat)
+    {
+        if (mediaFormat == null) return [MangaFormat.Archive];
+        return mediaFormat switch
+        {
+            MediaFormat.Manga => [MangaFormat.Archive, MangaFormat.Image],
+            MediaFormat.Comic => [MangaFormat.Archive],
+            MediaFormat.LightNovel => [MangaFormat.Epub, MangaFormat.Pdf],
+            MediaFormat.Book => [MangaFormat.Epub, MangaFormat.Pdf],
+            MediaFormat.Unknown => [MangaFormat.Archive],
+            _ => [MangaFormat.Archive]
+        };
+    }
+
+    private static long GetStackId(string url)
+    {
+        var tokens = url.Split("/");
+        return long.Parse(tokens[^1], CultureInfo.InvariantCulture);
+    }
+
+    private async Task<SeriesCollection?> GetStackInfo(long stackId)
+    {
+        _logger.LogDebug("Fetching Kavita+ for MAL Stack");
+
+        var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
+
+        var seriesForStack = await ($"{Configuration.KavitaPlusApiUrl}/api/metadata/v2/stack?stackId=" + stackId)
+            .WithHeader("Accept", "application/json")
+            .WithHeader("User-Agent", "Kavita")
+            .WithHeader("x-license-key", license)
+            .WithHeader("x-installId", HashUtil.ServerToken())
+            .WithHeader("x-kavita-version", BuildInfo.Version)
+            .WithHeader("Content-Type", "application/json")
+            .WithTimeout(TimeSpan.FromSeconds(Configuration.DefaultTimeOutSecs))
+            .GetJsonAsync<SeriesCollection>();
+
+        return seriesForStack;
+    }
+}

API/Services/TaskScheduler.cs

@@ -57,6 +57,7 @@ public class TaskScheduler : ITaskScheduler
     private readonly IScrobblingService _scrobblingService;
     private readonly ILicenseService _licenseService;
     private readonly IExternalMetadataService _externalMetadataService;
+    private readonly ISmartCollectionSyncService _smartCollectionSyncService;
 
     public static BackgroundJobServer Client => new ();
     public const string ScanQueue = "scan";
@@ -74,6 +75,7 @@ public class TaskScheduler : ITaskScheduler
     public const string ProcessProcessedScrobblingEventsId = "process-processed-scrobbling-events";
     public const string LicenseCheckId = "license-check";
     public const string KavitaPlusDataRefreshId = "kavita+-data-refresh";
+    public const string KavitaPlusStackSyncId = "kavita+-stack-sync";
 
     private static readonly ImmutableArray<string> ScanTasks =
         ["ScannerService", "ScanLibrary", "ScanLibraries", "ScanFolder", "ScanSeries"];
@@ -91,7 +93,7 @@ public class TaskScheduler : ITaskScheduler
         ICleanupService cleanupService, IStatsService statsService, IVersionUpdaterService versionUpdaterService,
         IThemeService themeService, IWordCountAnalyzerService wordCountAnalyzerService, IStatisticService statisticService,
         IMediaConversionService mediaConversionService, IScrobblingService scrobblingService, ILicenseService licenseService,
-        IExternalMetadataService externalMetadataService)
+        IExternalMetadataService externalMetadataService, ISmartCollectionSyncService smartCollectionSyncService)
     {
         _cacheService = cacheService;
         _logger = logger;
@@ -109,6 +111,7 @@ public class TaskScheduler : ITaskScheduler
         _scrobblingService = scrobblingService;
         _licenseService = licenseService;
         _externalMetadataService = externalMetadataService;
+        _smartCollectionSyncService = smartCollectionSyncService;
     }
 
     public async Task ScheduleTasks()
@@ -186,6 +189,10 @@ public class TaskScheduler : ITaskScheduler
         RecurringJob.AddOrUpdate(KavitaPlusDataRefreshId,
             () => _externalMetadataService.FetchExternalDataTask(), Cron.Daily(Rnd.Next(1, 4)),
             RecurringJobOptions);
+
+        RecurringJob.AddOrUpdate(KavitaPlusStackSyncId,
+            () => _smartCollectionSyncService.Sync(), Cron.Daily(Rnd.Next(1, 4)),
+            RecurringJobOptions);
     }
 
     #region StatsTasks

API/Services/Tasks/Scanner/ProcessSeries.cs

@@ -32,7 +32,7 @@ public interface IProcessSeries
     Task Prime();
 
     void Reset();
-    Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false);
+    Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, int totalToProcess, bool forceUpdate = false);
 }
 
 /// <summary>
@@ -99,7 +99,7 @@ public class ProcessSeries : IProcessSeries
         _tagManagerService.Reset();
     }
 
-    public async Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false)
+    public async Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, int totalToProcess, bool forceUpdate = false)
     {
         if (!parsedInfos.Any()) return;
 
@@ -107,7 +107,7 @@ public class ProcessSeries : IProcessSeries
         var scanWatch = Stopwatch.StartNew();
         var seriesName = parsedInfos[0].Series;
         await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
-            MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Updated, seriesName));
+            MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Updated, seriesName, totalToProcess));
         _logger.LogInformation("[ScannerService] Beginning series update on {SeriesName}, Forced: {ForceUpdate}", seriesName, forceUpdate);
 
         // Check if there is a Series

API/Services/Tasks/ScannerService.cs

@@ -147,7 +147,7 @@ public class ScannerService : IScannerService
         {
             if (ex.Message.Equals("Sequence contains more than one element."))
             {
-                _logger.LogCritical("[ScannerService] Multiple series map to this folder. Library scan will be used for ScanFolder");
+                _logger.LogCritical(ex, "[ScannerService] Multiple series map to this folder. Library scan will be used for ScanFolder");
             }
         }
 
@@ -245,7 +245,7 @@ public class ScannerService : IScannerService
         var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
 
         await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
-            MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
+            MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name, 1));
 
         _logger.LogInformation("Beginning file scan on {SeriesName}", series.Name);
         var (scanElapsedTime, processedSeries) = await ScanFiles(library, new []{ folderPath },
@@ -309,15 +309,18 @@ public class ScannerService : IScannerService
             await _processSeries.Prime();
         }
 
+        var seriesLeftToProcess = toProcess.Count;
         foreach (var pSeries in toProcess)
         {
             // Process Series
-            await _processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, bypassFolderOptimizationChecks);
+            await _processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, seriesLeftToProcess, bypassFolderOptimizationChecks);
+            seriesLeftToProcess--;
         }
 
         _processSeries.Reset();
 
-        await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
+        await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
+            MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name, 0));
         // Tell UI that this series is done
         await _eventHub.SendMessageAsync(MessageFactory.ScanSeries,
             MessageFactory.ScanSeriesEvent(library.Id, seriesId, series.Name));
@@ -543,7 +546,8 @@ public class ScannerService : IScannerService
                 "[ScannerService] There was a critical error that resulted in a failed scan. Please check logs and rescan");
         }
 
-        await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, string.Empty));
+        await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
+            MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, string.Empty));
         await _metadataService.RemoveAbandonedMetadataKeys();
 
         BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.CacheDirectory));
@@ -589,12 +593,14 @@ public class ScannerService : IScannerService
 
         var totalFiles = 0;
         //var tasks = new List<Task>();
+        var seriesLeftToProcess = toProcess.Count;
         foreach (var pSeries in toProcess)
         {
             totalFiles += parsedSeries[pSeries].Count;
             //tasks.Add(_processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, forceUpdate));
             // We can't do Task.WhenAll because of concurrency issues.
-            await _processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, forceUpdate);
+            await _processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, seriesLeftToProcess, forceUpdate);
+            seriesLeftToProcess--;
         }
 
         //await Task.WhenAll(tasks);

API/SignalR/MessageFactory.cs

@@ -41,9 +41,9 @@ public static class MessageFactory
     /// </summary>
     public const string OnlineUsers = "OnlineUsers";
     /// <summary>
-    /// When a series is added to a collection
+    /// When a Collection has been updated
     /// </summary>
-    public const string SeriesAddedToCollection = "SeriesAddedToCollection";
+    public const string CollectionUpdated = "CollectionUpdated";
     /// <summary>
     /// Event sent out during backing up the database
     /// </summary>
@@ -310,17 +310,17 @@ public static class MessageFactory
         };
     }
 
-    public static SignalRMessage SeriesAddedToCollectionEvent(int tagId, int seriesId)
+
+    public static SignalRMessage CollectionUpdatedEvent(int collectionId)
     {
         return new SignalRMessage
         {
-            Name = SeriesAddedToCollection,
+            Name = CollectionUpdated,
             Progress = ProgressType.None,
             EventType = ProgressEventType.Single,
             Body = new
             {
-                TagId = tagId,
-                SeriesId = seriesId
+                TagId = collectionId,
             }
         };
     }
@@ -428,7 +428,7 @@ public static class MessageFactory
     /// <param name="eventType"></param>
     /// <param name="seriesName"></param>
     /// <returns></returns>
-    public static SignalRMessage LibraryScanProgressEvent(string libraryName, string eventType, string seriesName = "")
+    public static SignalRMessage LibraryScanProgressEvent(string libraryName, string eventType, string seriesName = "", int? totalToProcess = null)
     {
         return new SignalRMessage()
         {
@@ -437,7 +437,12 @@ public static class MessageFactory
             SubTitle = seriesName,
             EventType = eventType,
             Progress = ProgressType.Indeterminate,
-            Body = null
+            Body = new
+            {
+                SeriesName = seriesName,
+                LibraryName = libraryName,
+                LeftToProcess = totalToProcess
+            }
         };
     }
 

API/Startup.cs

@@ -139,7 +139,7 @@ public class Startup
             {
                 Version = BuildInfo.Version.ToString(),
                 Title = "Kavita",
-                Description = "Kavita provides a set of APIs that are authenticated by JWT. JWT token can be copied from local storage.",
+                Description = "Kavita provides a set of APIs that are authenticated by JWT. JWT token can be copied from local storage. Assume all fields of a payload are required.",
                 License = new OpenApiLicense
                 {
                     Name = "GPL-3.0",