Random Changes and Enhancements (#1819)

* When skipping over folders in a scan, inform the ui

* Try out new backout condition for library watcher.

* Tweaked the code for folder watching to be more intense on killing if stuck in inotify loop.

* Streamlined my implementation of enhanced LibraryWatcher

* Added new extension method to make complex where statements cleaner.

* Added an implementation to flatten series and not show them if they have relationships defined. Only the parent would show. Currently disabled until i figure out how to apply it.

* Added the ability to collapse series that are not the primary entry point to reading. Configurable in library settings, only applies when all libraries in a filter have the property to true.

* Exclude from parsing .@_thumb directories, a QNAP system folder.

Show number of items a JumpKey has

* Refactored some time reading to display in days, months, years or minutes.

API/Controllers/LibraryController.cs

@@ -335,6 +335,7 @@ public class LibraryController : BaseApiController
         library.IncludeInRecommended = dto.IncludeInRecommended;
         library.IncludeInSearch = dto.IncludeInSearch;
         library.ManageCollections = dto.ManageCollections;
+        library.CollapseSeriesRelationships = dto.CollapseSeriesRelationships;
 
         _unitOfWork.LibraryRepository.Update(library);
 

API/DTOs/LibraryDto.cs

@@ -37,5 +37,9 @@ public class LibraryDto
     /// Include library series in Search
     /// </summary>
     public bool IncludeInSearch { get; set; } = true;
+    /// <summary>
+    /// When showing series, only parent series or series with no relationships will be returned
+    /// </summary>
+    public bool CollapseSeriesRelationships { get; set; } = false;
     public ICollection<string> Folders { get; init; }
 }

API/DTOs/UpdateLibraryDto.cs

@@ -24,5 +24,7 @@ public class UpdateLibraryDto
     public bool IncludeInSearch { get; init; }
     [Required]
     public bool ManageCollections { get; init; }
+    [Required]
+    public bool CollapseSeriesRelationships { get; init; }
 
 }

API/Data/Migrations/20230220203128_CollapseSeriesRelationships.cs

@@ -0,0 +1,26 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace API.Data.Migrations
+{
+    public partial class CollapseSeriesRelationships : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.AddColumn<bool>(
+                name: "CollapseSeriesRelationships",
+                table: "Library",
+                type: "INTEGER",
+                nullable: false,
+                defaultValue: false);
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropColumn(
+                name: "CollapseSeriesRelationships",
+                table: "Library");
+        }
+    }
+}

API/Data/Migrations/DataContextModelSnapshot.cs

@@ -598,6 +598,9 @@ namespace API.Data.Migrations
                         .ValueGeneratedOnAdd()
                         .HasColumnType("INTEGER");
 
+                    b.Property<bool>("CollapseSeriesRelationships")
+                        .HasColumnType("INTEGER");
+
                     b.Property<string>("CoverImage")
                         .HasColumnType("TEXT");
 

API/Data/Repositories/SeriesRepository.cs

@@ -743,8 +743,12 @@ public class SeriesRepository : ISeriesRepository
 
     private async Task<IQueryable<Series>> CreateFilteredSearchQueryable(int userId, int libraryId, FilterDto filter, QueryContext queryContext)
     {
+        // NOTE: Why do we even have libraryId when the filter has the actual libraryIds?
         var userLibraries = await GetUserLibrariesForFilteredQuery(libraryId, userId, queryContext);
         var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
+        var onlyParentSeries = await _context.Library.AsNoTracking()
+            .Where(l => filter.Libraries.Contains(l.Id))
+            .AllAsync(l => l.CollapseSeriesRelationships);
 
         var formats = ExtractFilters(libraryId, userId, filter, ref userLibraries,
             out var allPeopleIds, out var hasPeopleFilter, out var hasGenresFilter,
@@ -753,30 +757,33 @@ public class SeriesRepository : ISeriesRepository
             out var hasPublicationFilter, out var hasSeriesNameFilter, out var hasReleaseYearMinFilter, out var hasReleaseYearMaxFilter);
 
         var query = _context.Series
-            .Where(s => userLibraries.Contains(s.LibraryId)
-                        && formats.Contains(s.Format)
-                        && (!hasGenresFilter || s.Metadata.Genres.Any(g => filter.Genres.Contains(g.Id)))
-                        && (!hasPeopleFilter || s.Metadata.People.Any(p => allPeopleIds.Contains(p.Id)))
-                        && (!hasCollectionTagFilter ||
-                            s.Metadata.CollectionTags.Any(t => filter.CollectionTags.Contains(t.Id)))
-                        && (!hasRatingFilter || s.Ratings.Any(r => r.Rating >= filter.Rating && r.AppUserId == userId))
-                        && (!hasProgressFilter || seriesIds.Contains(s.Id))
-                        && (!hasAgeRating || filter.AgeRating.Contains(s.Metadata.AgeRating))
-                        && (!hasTagsFilter || s.Metadata.Tags.Any(t => filter.Tags.Contains(t.Id)))
-                        && (!hasLanguageFilter || filter.Languages.Contains(s.Metadata.Language))
-                        && (!hasReleaseYearMinFilter || s.Metadata.ReleaseYear >= filter.ReleaseYearRange.Min)
-                        && (!hasReleaseYearMaxFilter || s.Metadata.ReleaseYear <= filter.ReleaseYearRange.Max)
-                        && (!hasPublicationFilter || filter.PublicationStatus.Contains(s.Metadata.PublicationStatus)))
-            .Where(s => !hasSeriesNameFilter ||
-                        EF.Functions.Like(s.Name, $"%{filter.SeriesNameQuery}%")
-                                             || EF.Functions.Like(s.OriginalName, $"%{filter.SeriesNameQuery}%")
-                                             || EF.Functions.Like(s.LocalizedName, $"%{filter.SeriesNameQuery}%"));
+            .AsNoTracking()
+            .WhereIf(hasGenresFilter, s => s.Metadata.Genres.Any(g => filter.Genres.Contains(g.Id)))
+            .WhereIf(hasPeopleFilter, s => s.Metadata.People.Any(p => allPeopleIds.Contains(p.Id)))
+            .WhereIf(hasCollectionTagFilter,
+                s => s.Metadata.CollectionTags.Any(t => filter.CollectionTags.Contains(t.Id)))
+            .WhereIf(hasRatingFilter, s => s.Ratings.Any(r => r.Rating >= filter.Rating && r.AppUserId == userId))
+            .WhereIf(hasProgressFilter, s => seriesIds.Contains(s.Id))
+            .WhereIf(hasAgeRating, s => filter.AgeRating.Contains(s.Metadata.AgeRating))
+            .WhereIf(hasTagsFilter, s => s.Metadata.Tags.Any(t => filter.Tags.Contains(t.Id)))
+            .WhereIf(hasLanguageFilter, s => filter.Languages.Contains(s.Metadata.Language))
+            .WhereIf(hasReleaseYearMinFilter, s => s.Metadata.ReleaseYear >= filter.ReleaseYearRange.Min)
+            .WhereIf(hasReleaseYearMaxFilter, s => s.Metadata.ReleaseYear <= filter.ReleaseYearRange.Max)
+            .WhereIf(hasPublicationFilter, s => filter.PublicationStatus.Contains(s.Metadata.PublicationStatus))
+            .WhereIf(hasSeriesNameFilter, s => EF.Functions.Like(s.Name, $"%{filter.SeriesNameQuery}%")
+                                               || EF.Functions.Like(s.OriginalName, $"%{filter.SeriesNameQuery}%")
+                                               || EF.Functions.Like(s.LocalizedName, $"%{filter.SeriesNameQuery}%"))
+
+            .WhereIf(onlyParentSeries,
+                s => s.RelationOf.Count == 0 || s.RelationOf.All(p => p.RelationKind == RelationKind.Prequel))
+            .Where(s => userLibraries.Contains(s.LibraryId))
+            .Where(s => formats.Contains(s.Format));
+
         if (userRating.AgeRating != AgeRating.NotApplicable)
         {
             query = query.RestrictAgainstAgeRestriction(userRating);
         }
 
-        query = query.AsNoTracking();
 
         // If no sort options, default to using SortName
         filter.SortOptions ??= new SortOptions()
@@ -825,24 +832,23 @@ public class SeriesRepository : ISeriesRepository
             out var hasPublicationFilter, out var hasSeriesNameFilter, out var hasReleaseYearMinFilter, out var hasReleaseYearMaxFilter);
 
         var query = sQuery
+            .WhereIf(hasGenresFilter, s => s.Metadata.Genres.Any(g => filter.Genres.Contains(g.Id)))
+            .WhereIf(hasPeopleFilter, s => s.Metadata.People.Any(p => allPeopleIds.Contains(p.Id)))
+            .WhereIf(hasCollectionTagFilter,
+                s => s.Metadata.CollectionTags.Any(t => filter.CollectionTags.Contains(t.Id)))
+            .WhereIf(hasRatingFilter, s => s.Ratings.Any(r => r.Rating >= filter.Rating && r.AppUserId == userId))
+            .WhereIf(hasProgressFilter, s => seriesIds.Contains(s.Id))
+            .WhereIf(hasAgeRating, s => filter.AgeRating.Contains(s.Metadata.AgeRating))
+            .WhereIf(hasTagsFilter, s => s.Metadata.Tags.Any(t => filter.Tags.Contains(t.Id)))
+            .WhereIf(hasLanguageFilter, s => filter.Languages.Contains(s.Metadata.Language))
+            .WhereIf(hasReleaseYearMinFilter, s => s.Metadata.ReleaseYear >= filter.ReleaseYearRange.Min)
+            .WhereIf(hasReleaseYearMaxFilter, s => s.Metadata.ReleaseYear <= filter.ReleaseYearRange.Max)
+            .WhereIf(hasPublicationFilter, s => filter.PublicationStatus.Contains(s.Metadata.PublicationStatus))
+            .WhereIf(hasSeriesNameFilter, s => EF.Functions.Like(s.Name, $"%{filter.SeriesNameQuery}%")
+                                               || EF.Functions.Like(s.OriginalName, $"%{filter.SeriesNameQuery}%")
+                                               || EF.Functions.Like(s.LocalizedName, $"%{filter.SeriesNameQuery}%"))
             .Where(s => userLibraries.Contains(s.LibraryId)
-                        && formats.Contains(s.Format)
-                        && (!hasGenresFilter || s.Metadata.Genres.Any(g => filter.Genres.Contains(g.Id)))
-                        && (!hasPeopleFilter || s.Metadata.People.Any(p => allPeopleIds.Contains(p.Id)))
-                        && (!hasCollectionTagFilter ||
-                            s.Metadata.CollectionTags.Any(t => filter.CollectionTags.Contains(t.Id)))
-                        && (!hasRatingFilter || s.Ratings.Any(r => r.Rating >= filter.Rating && r.AppUserId == userId))
-                        && (!hasProgressFilter || seriesIds.Contains(s.Id))
-                        && (!hasAgeRating || filter.AgeRating.Contains(s.Metadata.AgeRating))
-                        && (!hasTagsFilter || s.Metadata.Tags.Any(t => filter.Tags.Contains(t.Id)))
-                        && (!hasLanguageFilter || filter.Languages.Contains(s.Metadata.Language))
-                        && (!hasReleaseYearMinFilter || s.Metadata.ReleaseYear >= filter.ReleaseYearRange.Min)
-                        && (!hasReleaseYearMaxFilter || s.Metadata.ReleaseYear <= filter.ReleaseYearRange.Max)
-                        && (!hasPublicationFilter || filter.PublicationStatus.Contains(s.Metadata.PublicationStatus)))
-            .Where(s => !hasSeriesNameFilter ||
-                        EF.Functions.Like(s.Name, $"%{filter.SeriesNameQuery}%")
-                                             || EF.Functions.Like(s.OriginalName, $"%{filter.SeriesNameQuery}%")
-                                             || EF.Functions.Like(s.LocalizedName, $"%{filter.SeriesNameQuery}%"))
+                        && formats.Contains(s.Format))
             .AsNoTracking();
 
         // If no sort options, default to using SortName

API/Entities/Library.cs

@@ -31,6 +31,10 @@ public class Library : IEntityDate
     /// Should this library create and manage collections from Metadata
     /// </summary>
     public bool ManageCollections { get; set; } = true;
+    /// <summary>
+    /// When showing series, only parent series or series with no relationships will be returned
+    /// </summary>
+    public bool CollapseSeriesRelationships { get; set; } = false;
     public DateTime Created { get; set; }
     public DateTime LastModified { get; set; }
     public DateTime CreatedUtc { get; set; }

API/Extensions/QueryableExtensions.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Linq;
+using System.Linq.Expressions;
 using System.Threading.Tasks;
 using API.Data.Misc;
 using API.Data.Repositories;
@@ -234,4 +235,10 @@ public static class QueryableExtensions
 
     public static IEnumerable<DateTime> Range(this DateTime startDate, int numberOfDays) =>
         Enumerable.Range(0, numberOfDays).Select(e => startDate.AddDays(e));
+
+    public static IQueryable<T> WhereIf<T>(this IQueryable<T> queryable, bool condition,
+        Expression<Func<T, bool>> predicate)
+    {
+        return condition ? queryable.Where(predicate) : queryable;
+    }
 }

API/Services/DirectoryService.cs

@@ -89,7 +89,7 @@ public class DirectoryService : IDirectoryService
     private readonly ILogger<DirectoryService> _logger;
 
     private static readonly Regex ExcludeDirectories = new Regex(
-        @"@eaDir|\.DS_Store|\.qpkg|__MACOSX|@Recently-Snapshot|@recycle",
+        @"@eaDir|\.DS_Store|\.qpkg|__MACOSX|@Recently-Snapshot|@recycle|\.@__thumb",
         RegexOptions.Compiled | RegexOptions.IgnoreCase,
         Tasks.Scanner.Parser.Parser.RegexTimeout);
     private static readonly Regex FileCopyAppend = new Regex(@"\(\d+\)",

API/Services/Tasks/Scanner/LibraryWatcher.cs

@@ -5,6 +5,7 @@ using System.IO;
 using System.Linq;
 using System.Threading.Tasks;
 using API.Data;
+using API.Entities.Enums;
 using Hangfire;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
@@ -55,6 +56,8 @@ public class LibraryWatcher : ILibraryWatcher
     /// Counts within a time frame how many times the buffer became full. Is used to reschedule LibraryWatcher to start monitoring much later rather than instantly
     /// </summary>
     private int _bufferFullCounter;
+    private int _restartCounter;
+    private DateTime _lastErrorTime = DateTime.MinValue;
     /// <summary>
     /// Used to lock buffer Full Counter
     /// </summary>
@@ -180,12 +183,21 @@ public class LibraryWatcher : ILibraryWatcher
         lock (Lock)
         {
             _bufferFullCounter += 1;
-            condition = _bufferFullCounter >= 3;
+            _lastErrorTime = DateTime.Now;
+            condition = _bufferFullCounter >= 3 && (DateTime.Now - _lastErrorTime).TotalMinutes <= 10;
+        }
+
+        if (_restartCounter >= 3)
+        {
+            _logger.LogInformation("[LibraryWatcher] Too many restarts occured, you either have limited inotify or an OS constraint. Kavita will turn off folder watching to prevent high utilization of resources");
+            Task.Run(TurnOffWatching);
+            return;
         }
 
         if (condition)
         {
-            _logger.LogInformation("[LibraryWatcher] Internal buffer has been overflown multiple times in past 10 minutes. Suspending file watching for an hour");
+            _logger.LogInformation("[LibraryWatcher] Internal buffer has been overflown multiple times in past 10 minutes. Suspending file watching for an hour. Restart count: {RestartCount}", _restartCounter);
+            _restartCounter++;
             StopWatching();
             BackgroundJob.Schedule(() => RestartWatching(), TimeSpan.FromHours(1));
             return;
@@ -194,6 +206,16 @@ public class LibraryWatcher : ILibraryWatcher
         BackgroundJob.Schedule(() => UpdateLastBufferOverflow(), TimeSpan.FromMinutes(10));
     }
 
+    private async Task TurnOffWatching()
+    {
+        var setting = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.EnableFolderWatching);
+        setting.Value = "false";
+        _unitOfWork.SettingsRepository.Update(setting);
+        await _unitOfWork.CommitAsync();
+        StopWatching();
+        _logger.LogInformation("[LibraryWatcher] Folder watching has been disabled");
+    }
+
 
     /// <summary>
     /// Processes the file or folder change. If the change is a file change and not from a supported extension, it will be ignored.

API/Services/Tasks/Scanner/ParseScannedFiles.cs

@@ -289,6 +289,8 @@ public class ParseScannedFiles
                 }).ToList();
                 await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
                 _logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
+                await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
+                    MessageFactory.FileScanProgressEvent("Skipped " + normalizedFolder, libraryName, ProgressEventType.Updated));
                 return;
             }