Next Estimated Chapter (#2342)

API/Controllers/FilterController.cs

@@ -46,8 +46,6 @@ public class FilterController : BaseApiController
             return BadRequest("You cannot use the name of a system provided stream");
         }
 
-        // I might just want to use DashboardStream instead of a separate entity. It will drastically simplify implementation
-
         var existingFilter =
             user.SmartFilters.FirstOrDefault(f => f.Name.Equals(dto.Name, StringComparison.InvariantCultureIgnoreCase));
         if (existingFilter != null)

API/Controllers/SeriesController.cs

@@ -610,4 +610,18 @@ public class SeriesController : BaseApiController
         }
     }
 
+    /// <summary>
+    /// Based on the delta times between when chapters are added, for series that are not Completed/Cancelled/Hiatus, forecast the next
+    /// date when it will be available.
+    /// </summary>
+    /// <param name="seriesId"></param>
+    /// <returns></returns>
+    [HttpGet("next-expected")]
+    public async Task<ActionResult<NextExpectedChapterDto>> GetNextExpectedChapter(int seriesId)
+    {
+        var userId = User.GetUserId();
+
+        return Ok(await _seriesService.GetEstimatedChapterCreationDate(seriesId, userId));
+    }
+
 }

API/DTOs/Filtering/v2/FilterField.cs

@@ -44,6 +44,6 @@ public enum FilterField
     /// <summary>
     /// Last time User Read
     /// </summary>
-    ReadingDate = 27
+    ReadingDate = 27,
 
 }

API/DTOs/SeriesDetail/NextExpectedChapterDto.cs

@@ -0,0 +1,17 @@
+using System;
+
+namespace API.DTOs.SeriesDetail;
+
+public class NextExpectedChapterDto
+{
+    public float ChapterNumber { get; set; }
+    public int VolumeNumber { get; set; }
+    /// <summary>
+    /// Null if not applicable
+    /// </summary>
+    public DateTime? ExpectedDate { get; set; }
+    /// <summary>
+    /// The localized title to render on the card
+    /// </summary>
+    public string Title { get; set; }
+}

API/DTOs/SeriesDetail/SeriesDetailDto.cs

@@ -1,6 +1,7 @@
 using System.Collections.Generic;
 
 namespace API.DTOs.SeriesDetail;
+#nullable enable
 
 /// <summary>
 /// This is a special DTO for a UI page in Kavita. This performs sorting and grouping and returns exactly what UI requires for layout.
@@ -32,5 +33,4 @@ public class SeriesDetailDto
     /// How many chapters are there
     /// </summary>
     public int TotalCount { get; set; }
-
 }

API/Data/Metadata/ComicInfo.cs

@@ -1,5 +1,7 @@
 using System;
+using System.Globalization;
 using System.Linq;
+using System.Threading;
 using API.Entities;
 using API.Entities.Enums;
 using API.Services;
@@ -168,6 +170,11 @@ public class ComicInfo
                 info.Isbn = info.GTIN;
             }
         }
+
+        if (!string.IsNullOrEmpty(info.Number))
+        {
+            info.Number = info.Number.Replace(",", "."); // Corrective measure for non English OSes
+        }
     }
 
     /// <summary>
@@ -176,13 +183,21 @@ public class ComicInfo
     /// <returns></returns>
     public int CalculatedCount()
     {
-        if (!string.IsNullOrEmpty(Number) && float.Parse(Number) > 0)
+        try
         {
-            return (int) Math.Floor(float.Parse(Number));
+            if (float.TryParse(Number, out var chpCount) && chpCount > 0)
+            {
+                return (int) Math.Floor(chpCount);
+            }
+
+            if (float.TryParse(Volume, out var volCount) && volCount > 0)
+            {
+                return (int) Math.Floor(volCount);
+            }
         }
-        if (!string.IsNullOrEmpty(Volume) && float.Parse(Volume) > 0)
+        catch (Exception)
         {
-            return Math.Max(Count, (int) Math.Floor(float.Parse(Volume)));
+            return 0;
         }
 
         return 0;

API/Data/Repositories/AppUserProgressRepository.cs

@@ -205,5 +205,4 @@ public class AppUserProgressRepository : IAppUserProgressRepository
             .Where(p => p.ChapterId == chapterId && p.AppUserId == userId)
             .FirstOrDefaultAsync();
     }
-    #nullable disable
 }

API/Data/Repositories/ChapterRepository.cs

@@ -40,6 +40,7 @@ public interface IChapterRepository
     Task<IList<Chapter>> GetAllChaptersWithCoversInDifferentEncoding(EncodeFormat format);
     Task<IEnumerable<string>> GetCoverImagesForLockedChaptersAsync();
     Task<ChapterDto> AddChapterModifiers(int userId, ChapterDto chapter);
+    IEnumerable<Chapter> GetChaptersForSeries(int seriesId);
 }
 public class ChapterRepository : IChapterRepository
 {
@@ -264,4 +265,12 @@ public class ChapterRepository : IChapterRepository
 
         return chapter;
     }
+
+    public IEnumerable<Chapter> GetChaptersForSeries(int seriesId)
+    {
+        return _context.Chapter
+            .Where(c => c.Volume.SeriesId == seriesId)
+            .Include(c => c.Volume)
+            .AsEnumerable();
+    }
 }

API/Data/Repositories/SeriesRepository.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.Linq;
 using System.Text.RegularExpressions;
 using System.Threading.Tasks;
@@ -347,9 +348,9 @@ public class SeriesRepository : ISeriesRepository
             .Where(l => libraryIds.Contains(l.Id))
             .Where(l => EF.Functions.Like(l.Name, $"%{searchQuery}%"))
             .IsRestricted(QueryContext.Search)
-            .OrderBy(l => l.Name.ToLower())
             .AsSplitQuery()
             .Take(maxRecords)
+            .OrderBy(l => l.Name.ToLower())
             .ProjectTo<LibraryDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -366,10 +367,10 @@ public class SeriesRepository : ISeriesRepository
                          || (hasYearInQuery && s.Metadata.ReleaseYear == yearComparison))
             .RestrictAgainstAgeRestriction(userRating)
             .Include(s => s.Library)
-            .OrderBy(s => s.SortName!.ToLower())
             .AsNoTracking()
             .AsSplitQuery()
             .Take(maxRecords)
+            .OrderBy(s => s.SortName!.ToLower())
             .ProjectTo<SearchResultDto>(_mapper.ConfigurationProvider)
             .AsEnumerable();
 
@@ -379,6 +380,7 @@ public class SeriesRepository : ISeriesRepository
             .RestrictAgainstAgeRestriction(userRating)
             .AsSplitQuery()
             .Take(maxRecords)
+            .OrderBy(r => r.NormalizedTitle)
             .ProjectTo<ReadingListDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -399,8 +401,9 @@ public class SeriesRepository : ISeriesRepository
             .Where(sm => seriesIds.Contains(sm.SeriesId))
             .SelectMany(sm => sm.People.Where(t => t.Name != null && EF.Functions.Like(t.Name, $"%{searchQuery}%")))
             .AsSplitQuery()
-            .Take(maxRecords)
             .Distinct()
+            .Take(maxRecords)
+            .OrderBy(p => p.NormalizedName)
             .ProjectTo<PersonDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -408,9 +411,9 @@ public class SeriesRepository : ISeriesRepository
             .Where(sm => seriesIds.Contains(sm.SeriesId))
             .SelectMany(sm => sm.Genres.Where(t => EF.Functions.Like(t.Title, $"%{searchQuery}%")))
             .AsSplitQuery()
-            .OrderBy(t => t.NormalizedTitle)
             .Distinct()
             .Take(maxRecords)
+            .OrderBy(t => t.NormalizedTitle)
             .ProjectTo<GenreTagDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -418,9 +421,9 @@ public class SeriesRepository : ISeriesRepository
             .Where(sm => seriesIds.Contains(sm.SeriesId))
             .SelectMany(sm => sm.Tags.Where(t => EF.Functions.Like(t.Title, $"%{searchQuery}%")))
             .AsSplitQuery()
-            .OrderBy(t => t.NormalizedTitle)
             .Distinct()
             .Take(maxRecords)
+            .OrderBy(t => t.NormalizedTitle)
             .ProjectTo<TagDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -435,6 +438,7 @@ public class SeriesRepository : ISeriesRepository
             .Where(m => EF.Functions.Like(m.FilePath, $"%{searchQuery}%") && fileIds.Contains(m.Id))
             .AsSplitQuery()
             .Take(maxRecords)
+            .OrderBy(f => f.FilePath)
             .ProjectTo<MangaFileDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -447,12 +451,19 @@ public class SeriesRepository : ISeriesRepository
             .Where(c => c.Files.All(f => fileIds.Contains(f.Id)))
             .AsSplitQuery()
             .Take(maxRecords)
+            .OrderBy(c => c.TitleName)
             .ProjectTo<ChapterDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
         return result;
     }
 
+    /// <summary>
+    /// Includes Progress for the user
+    /// </summary>
+    /// <param name="seriesId"></param>
+    /// <param name="userId"></param>
+    /// <returns></returns>
     public async Task<SeriesDto?> GetSeriesDtoByIdAsync(int seriesId, int userId)
     {
         var series = await _context.Series.Where(x => x.Id == seriesId)
@@ -955,7 +966,7 @@ public class SeriesRepository : ISeriesRepository
 
 
         return ApplyLimit(query
-            .Sort(filter.SortOptions)
+            .Sort(userId, filter.SortOptions)
             .AsSplitQuery(), filter.LimitTo);
     }
 
@@ -1108,7 +1119,7 @@ public class SeriesRepository : ISeriesRepository
                                                || EF.Functions.Like(s.LocalizedName!, $"%{filter.SeriesNameQuery}%"))
             .Where(s => userLibraries.Contains(s.LibraryId)
                         && formats.Contains(s.Format))
-            .Sort(filter.SortOptions)
+            .Sort(userId, filter.SortOptions)
             .AsNoTracking();
 
         return query.AsSplitQuery();
@@ -1971,4 +1982,5 @@ public class SeriesRepository : ISeriesRepository
             .IsRestricted(queryContext)
             .Select(lib => lib.Id);
     }
+
 }

API/Data/UnitOfWork.cs

@@ -2,6 +2,7 @@
 using System.Threading.Tasks;
 using API.Data.Repositories;
 using API.Entities;
+using API.Services;
 using AutoMapper;
 using Microsoft.AspNetCore.Identity;
 
@@ -40,6 +41,7 @@ public class UnitOfWork : IUnitOfWork
     private readonly DataContext _context;
     private readonly IMapper _mapper;
     private readonly UserManager<AppUser> _userManager;
+    private readonly ILocalizationService _localizationService;
 
     public UnitOfWork(DataContext context, IMapper mapper, UserManager<AppUser> userManager)
     {
@@ -99,6 +101,16 @@ public class UnitOfWork : IUnitOfWork
         return _context.ChangeTracker.HasChanges();
     }
 
+    public async Task BeginTransactionAsync()
+    {
+        await _context.Database.BeginTransactionAsync();
+    }
+
+    public async Task CommitTransactionAsync()
+    {
+        await _context.Database.CommitTransactionAsync();
+    }
+
     /// <summary>
     /// Rollback transaction
     /// </summary>

API/Entities/Chapter.cs

@@ -67,13 +67,13 @@ public class Chapter : IEntityDate, IHasReadTimeEstimate
     /// </summary>
     public string? Language { get; set; }
     /// <summary>
-    /// Total number of issues or volumes in the series
+    /// Total number of issues or volumes in the series. This is straight from ComicInfo
     /// </summary>
-    /// <remarks>Users may use Volume count or issue count. Kavita performs some light logic to help Count match up with TotalCount</remarks>
     public int TotalCount { get; set; } = 0;
     /// <summary>
     /// Number of the Total Count (progress the Series is complete)
     /// </summary>
+    /// <remarks>This is either the highest of ComicInfo Count field and (nonparsed volume/chapter number)</remarks>
     public int Count { get; set; } = 0;
     /// <summary>
     /// SeriesGroup tag in ComicInfo

API/Entities/Metadata/SeriesMetadata.cs

@@ -36,7 +36,7 @@ public class SeriesMetadata : IHasConcurrencyToken
     /// </summary>
     public string Language { get; set; } = string.Empty;
     /// <summary>
-    /// Total number of issues/volumes in the series
+    /// Total expected number of issues/volumes in the series from ComicInfo.xml
     /// </summary>
     public int TotalCount { get; set; } = 0;
     /// <summary>

API/Extensions/QueryExtensions/Filtering/SeriesSort.cs

@@ -11,7 +11,7 @@ public static class SeriesSort
     /// <param name="query"></param>
     /// <param name="sortOptions"></param>
     /// <returns></returns>
-    public static IQueryable<Series> Sort(this IQueryable<Series> query, SortOptions? sortOptions)
+    public static IQueryable<Series> Sort(this IQueryable<Series> query, int userId, SortOptions? sortOptions)
     {
         // If no sort options, default to using SortName
         sortOptions ??= new SortOptions()
@@ -28,7 +28,9 @@ public static class SeriesSort
             SortField.LastChapterAdded => query.DoOrderBy(s => s.LastChapterAdded, sortOptions),
             SortField.TimeToRead => query.DoOrderBy(s => s.AvgHoursToRead, sortOptions),
             SortField.ReleaseYear => query.DoOrderBy(s => s.Metadata.ReleaseYear, sortOptions),
-            SortField.ReadProgress => query.DoOrderBy(s => s.Progress.Where(p => p.SeriesId == s.Id).Select(p => p.LastModified).Max(), sortOptions),
+            SortField.ReadProgress => query.DoOrderBy(s => s.Progress.Where(p => p.SeriesId == s.Id && p.AppUserId == userId)
+                .Select(p => p.LastModified)
+                .Max(), sortOptions),
             _ => query
         };
 

API/Helpers/SmartFilterHelper.cs

@@ -80,7 +80,7 @@ public static class SmartFilterHelper
         if (statements == null || statements.Count == 0)
             return string.Empty;
 
-        var encodedStatements = StatementsKey + HttpUtility.UrlEncode(string.Join(",", statements.Select(EncodeFilterStatementDto)));
+        var encodedStatements = StatementsKey + Uri.EscapeDataString(string.Join(",", statements.Select(EncodeFilterStatementDto)));
         return encodedStatements;
     }
 
@@ -88,7 +88,7 @@ public static class SmartFilterHelper
     {
         var encodedComparison = $"comparison={(int) statement.Comparison}";
         var encodedField = $"field={(int) statement.Field}";
-        var encodedValue = $"value={HttpUtility.UrlEncode(statement.Value)}";
+        var encodedValue = $"value={Uri.EscapeDataString(statement.Value)}";
 
         return $"{encodedComparison}&{encodedField}&{encodedValue}";
     }

API/Services/BookService.cs

@@ -1026,16 +1026,26 @@ public class BookService : IBookService
 
         if (chaptersList.Count != 0) return chaptersList;
         // Generate from TOC from links (any point past this, Kavita is generating as a TOC doesn't exist)
-        var tocPage = book.Content.Html.Local.Select(s => s.Key).FirstOrDefault(k => k.Equals("TOC.XHTML", StringComparison.InvariantCultureIgnoreCase) ||
+        var tocPage = book.Content.Html.Local.Select(s => s.Key)
+            .FirstOrDefault(k => k.Equals("TOC.XHTML", StringComparison.InvariantCultureIgnoreCase) ||
             k.Equals("NAVIGATION.XHTML", StringComparison.InvariantCultureIgnoreCase));
         if (string.IsNullOrEmpty(tocPage)) return chaptersList;
 
+
         // Find all anchor tags, for each anchor we get inner text, to lower then title case on UI. Get href and generate page content
         if (!book.Content.Html.TryGetLocalFileRefByKey(tocPage, out var file)) return chaptersList;
         var content = await file.ReadContentAsync();
 
         var doc = new HtmlDocument();
         doc.LoadHtml(content);
+
+        // TODO: We may want to check if there is a toc.ncs file to better handle nested toc
+        // We could do a fallback first with ol/lis
+        //var sections = doc.DocumentNode.SelectNodes("//ol");
+        //if (sections == null)
+
+
+
         var anchors = doc.DocumentNode.SelectNodes("//a");
         if (anchors == null) return chaptersList;
 

API/Services/Plus/ScrobblingService.cs

@@ -206,8 +206,7 @@ public class ScrobblingService : IScrobblingService
             ScrobbleEventType.Review);
         if (existingEvt is {IsProcessed: false})
         {
-            _logger.LogDebug("Overriding scrobble event for {Series} from Review {Tagline}/{Body} -> {UpdatedTagline}{UpdatedBody}",
-                existingEvt.Series.Name, existingEvt.ReviewTitle, existingEvt.ReviewBody, reviewTitle, reviewBody);
+            _logger.LogDebug("Overriding Review scrobble event for {Series}", existingEvt.Series.Name);
             existingEvt.ReviewBody = reviewBody;
             existingEvt.ReviewTitle = reviewTitle;
             _unitOfWork.ScrobbleRepository.Update(existingEvt);

API/Services/SeriesService.cs

@@ -1,5 +1,6 @@
 using System;
 using System.Collections.Generic;
+using System.Globalization;
 using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
@@ -17,6 +18,7 @@ using API.Helpers.Builders;
 using API.Services.Plus;
 using API.SignalR;
 using Hangfire;
+using Kavita.Common;
 using Microsoft.Extensions.Logging;
 
 namespace API.Services;
@@ -36,6 +38,7 @@ public interface ISeriesService
     Task<string> FormatChapterTitle(int userId, bool isSpecial, LibraryType libraryType, string? chapterTitle,
         bool withHash);
     Task<string> FormatChapterName(int userId, LibraryType libraryType, bool withHash = false);
+    Task<NextExpectedChapterDto> GetEstimatedChapterCreationDate(int seriesId, int userId);
 }
 
 public class SeriesService : ISeriesService
@@ -399,6 +402,7 @@ public class SeriesService : ISeriesService
     public async Task<SeriesDetailDto> GetSeriesDetail(int seriesId, int userId)
     {
         var series = await _unitOfWork.SeriesRepository.GetSeriesDtoByIdAsync(seriesId, userId);
+        if (series == null) throw new KavitaException(await _localizationService.Translate(userId, "series-doesnt-exist"));
         var libraryIds = _unitOfWork.LibraryRepository.GetLibraryIdsForUserIdAsync(userId);
         if (!libraryIds.Contains(series.LibraryId))
             throw new UnauthorizedAccessException("user-no-access-library-from-series");
@@ -488,7 +492,7 @@ public class SeriesService : ISeriesService
             Volumes = processedVolumes,
             StorylineChapters = storylineChapters,
             TotalCount = chapters.Count,
-            UnreadCount = chapters.Count(c => c.Pages > 0 && c.PagesRead < c.Pages)
+            UnreadCount = chapters.Count(c => c.Pages > 0 && c.PagesRead < c.Pages),
         };
     }
 
@@ -642,4 +646,88 @@ public class SeriesService : ISeriesService
             _unitOfWork.SeriesRepository.Update(series);
         }
     }
+
+    public async Task<NextExpectedChapterDto> GetEstimatedChapterCreationDate(int seriesId, int userId)
+    {
+        var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId, SeriesIncludes.Metadata | SeriesIncludes.Library);
+        if (series == null) throw new KavitaException(await _localizationService.Translate(userId, "series-doesnt-exist"));
+        var libraryIds = _unitOfWork.LibraryRepository.GetLibraryIdsForUserIdAsync(userId);
+        if (!libraryIds.Contains(series.LibraryId)) //// TODO: Rewrite this to use a new method which checks permissions all in the DB to be streamlined and less memory
+            throw new UnauthorizedAccessException("user-no-access-library-from-series");
+        if (series?.Metadata.PublicationStatus is not (PublicationStatus.OnGoing or PublicationStatus.Ended) || series.Library.Type == LibraryType.Book)
+        {
+            return new NextExpectedChapterDto()
+            {
+                ExpectedDate = null,
+                ChapterNumber = 0,
+                VolumeNumber = 0
+            };
+        }
+
+        var chapters = _unitOfWork.ChapterRepository.GetChaptersForSeries(seriesId)
+            .Where(c => !c.IsSpecial)
+            .OrderBy(c => c.CreatedUtc)
+            .ToList();
+
+        // Calculate the time differences between consecutive chapters
+        var timeDifferences = chapters
+            .Select((chapter, index) => new
+            {
+                ChapterNumber = chapter.Number,
+                VolumeNumber = chapter.Volume.Number,
+                TimeDifference = index == 0 ? TimeSpan.Zero : (chapter.CreatedUtc - chapters.ElementAt(index - 1).CreatedUtc)
+            })
+            .ToList();
+
+        // Calculate the average time difference between chapters
+        var averageTimeDifference = timeDifferences
+            .Average(td => td.TimeDifference.TotalDays);
+
+        // Calculate the forecast for when the next chapter is expected
+        var nextChapterExpected = chapters.Any()
+            ? chapters.Max(c => c.CreatedUtc) + TimeSpan.FromDays(averageTimeDifference)
+            : (DateTime?) null;
+
+        if (nextChapterExpected != null && nextChapterExpected < DateTime.UtcNow)
+        {
+            nextChapterExpected = DateTime.UtcNow + TimeSpan.FromDays(averageTimeDifference);
+        }
+
+        var lastChapter = timeDifferences.Last();
+        float.TryParse(lastChapter.ChapterNumber, NumberStyles.Number, CultureInfo.InvariantCulture,
+            out var lastChapterNumber);
+
+        var result = new NextExpectedChapterDto()
+        {
+            ChapterNumber = 0,
+            VolumeNumber = 0,
+            ExpectedDate = nextChapterExpected,
+            Title = string.Empty
+        };
+
+        if (lastChapterNumber > 0)
+        {
+            result.ChapterNumber = lastChapterNumber + 1;
+            result.VolumeNumber = lastChapter.VolumeNumber;
+            result.Title = series.Library.Type switch
+            {
+                LibraryType.Manga => await _localizationService.Translate(userId, "chapter-num",
+                    new object[] {result.ChapterNumber}),
+                LibraryType.Comic => await _localizationService.Translate(userId, "issue-num",
+                    new object[] {"#", result.ChapterNumber}),
+                LibraryType.Book => await _localizationService.Translate(userId, "book-num",
+                    new object[] {result.ChapterNumber}),
+                _ => "Chapter " + result.ChapterNumber
+            };
+        }
+        else
+        {
+            result.VolumeNumber = lastChapter.VolumeNumber + 1;
+            result.Title = await _localizationService.Translate(userId, "vol-num",
+                new object[] {result.VolumeNumber});
+        }
+
+
+        return result;
+    }
 }

API/Services/Tasks/Scanner/ProcessSeries.cs

@@ -288,14 +288,12 @@ public class ProcessSeries : IProcessSeries
         series.Metadata.TotalCount = chapters.Max(chapter => chapter.TotalCount);
         // The actual number of count's defined across all chapter's metadata
         series.Metadata.MaxCount = chapters.Max(chapter => chapter.Count);
-        // To not have to rely completely on ComicInfo, try to parse out if the series is complete by checking parsed filenames as well.
-        if (series.Metadata.MaxCount != series.Metadata.TotalCount)
-        {
-            var maxVolume = series.Volumes.Max(v => (int) Parser.Parser.MaxNumberFromRange(v.Name));
-            var maxChapter = chapters.Max(c => (int) Parser.Parser.MaxNumberFromRange(c.Range));
-            if (maxVolume == series.Metadata.TotalCount) series.Metadata.MaxCount = maxVolume;
-            else if (maxChapter == series.Metadata.TotalCount) series.Metadata.MaxCount = maxChapter;
-        }
+
+        var maxVolume = series.Volumes.Max(v => (int) Parser.Parser.MaxNumberFromRange(v.Name));
+        var maxChapter = chapters.Max(c => (int) Parser.Parser.MaxNumberFromRange(c.Range));
+        var maxActual = Math.Max(maxVolume, maxChapter);
+
+        series.Metadata.MaxCount = maxActual;
 
 
         if (!series.Metadata.PublicationStatusLocked)