Scan Loop Fortification (#1573)

* Cleanup some messaging in the scan loop to be more context bearing

* Added Response Caching to Series Detail for 1 min, due to the heavy nature of the call.

* Refactored code to make it so that processing of series runs sync correctly.

Added a log to inform the user of corrupted volume from buggy code in v0.5.6.

* Moved folder watching out of experimental

* Fixed an issue where empty folders could break the scan loop

* Another fix for when dates aren't valid, the scanner wouldn't get the proper min and would throw an exception (develop)

* Implemented the ability to edit release year from the UI for a series.

* Added a unit test for some new logic

* Code smells

* Rewrote the handler for suspending watching to be more resilient and ensure no two threads have a race condition.

* More error handling for when a ScanFolder is invoked but multiple series belong to that folder, log it to the user and default to a library scan.

* ScanSeries now will check for kavitaignores higher than it's own folder and respect library level.

* Fixed an issue where image series with a folder name containing the word "folder" could get ignored as it thought the image was a cover image.

When a series folder is moved or deleted, skip parent ignore finding.

* Removed some old files, added in scanFolder a check if the series found for a folder is in a book library and if so to always do a library scan (as books are often nested into one folder with  multiple series). Added some unit tests

* Refactored some scan loop logic into ComicInfo, wrote tests and updated some documentation to make the fields more clear.

* Added a test for GetLastWriteTime based on recent bug

* Cleaned up some redundant code

* Fixed a bad merge

* Code smells

* Removed a package that's no longer used.

* Ensure we check against ScanQueue on ScanFolder enqueuing

* Documentation and more bullet proofing to ensure Hangfire checks work more as expected

API/Services/TaskScheduler.cs

@@ -2,6 +2,7 @@
 using System.Collections.Generic;
 using System.Collections.Immutable;
 using System.Linq;
+using System.Threading;
 using System.Threading.Tasks;
 using API.Data;
 using API.Entities.Enums;
@@ -18,6 +19,7 @@ public interface ITaskScheduler
     Task ScheduleTasks();
     Task ScheduleStatsTasks();
     void ScheduleUpdaterTasks();
+    void ScanFolder(string folderPath, TimeSpan delay);
     void ScanFolder(string folderPath);
     void ScanLibrary(int libraryId, bool force = false);
     void CleanupChapters(int[] chapterIds);
@@ -179,9 +181,32 @@ public class TaskScheduler : ITaskScheduler
         RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local);
     }
 
+    public void ScanFolder(string folderPath, TimeSpan delay)
+    {
+        var normalizedFolder = Tasks.Scanner.Parser.Parser.NormalizePath(folderPath);
+        if (HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] { normalizedFolder }))
+        {
+            _logger.LogInformation("Skipped scheduling ScanFolder for {Folder} as a job already queued",
+                normalizedFolder);
+            return;
+        }
+
+        _logger.LogInformation("Scheduling ScanFolder for {Folder}", normalizedFolder);
+        BackgroundJob.Schedule(() => _scannerService.ScanFolder(normalizedFolder), delay);
+    }
+
     public void ScanFolder(string folderPath)
     {
-        _scannerService.ScanFolder(Tasks.Scanner.Parser.Parser.NormalizePath(folderPath));
+        var normalizedFolder = Tasks.Scanner.Parser.Parser.NormalizePath(folderPath);
+        if (HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {normalizedFolder}))
+        {
+            _logger.LogInformation("Skipped scheduling ScanFolder for {Folder} as a job already queued",
+                normalizedFolder);
+            return;
+        }
+
+        _logger.LogInformation("Scheduling ScanFolder for {Folder}", normalizedFolder);
+        _scannerService.ScanFolder(normalizedFolder);
     }
 
     #endregion
@@ -298,15 +323,32 @@ public class TaskScheduler : ITaskScheduler
         await _versionUpdaterService.PushUpdate(update);
     }
 
+    /// <summary>
+    /// If there is an enqueued or scheduled tak for <see cref="ScannerService.ScanLibrary"/> method
+    /// </summary>
+    /// <param name="libraryId"></param>
+    /// <returns></returns>
     public static bool HasScanTaskRunningForLibrary(int libraryId)
     {
         return
-            HasAlreadyEnqueuedTask("ScannerService", "ScanLibrary", new object[] {libraryId, true}, ScanQueue) ||
-            HasAlreadyEnqueuedTask("ScannerService", "ScanLibrary", new object[] {libraryId, false}, ScanQueue);
+            HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, true}, ScanQueue) ||
+            HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, false}, ScanQueue);
     }
 
     /// <summary>
-    /// Checks if this same invocation is already enqueued
+    /// If there is an enqueued or scheduled tak for <see cref="ScannerService.ScanSeries"/> method
+    /// </summary>
+    /// <param name="seriesId"></param>
+    /// <returns></returns>
+    public static bool HasScanTaskRunningForSeries(int seriesId)
+    {
+        return
+            HasAlreadyEnqueuedTask(ScannerService.Name, "ScanSeries", new object[] {seriesId, true}, ScanQueue) ||
+            HasAlreadyEnqueuedTask(ScannerService.Name, "ScanSeries", new object[] {seriesId, false}, ScanQueue);
+    }
+
+    /// <summary>
+    /// Checks if this same invocation is already enqueued or scheduled
     /// </summary>
     /// <param name="methodName">Method name that was enqueued</param>
     /// <param name="className">Class name the method resides on</param>
@@ -316,16 +358,33 @@ public class TaskScheduler : ITaskScheduler
     public static bool HasAlreadyEnqueuedTask(string className, string methodName, object[] args, string queue = DefaultQueue)
     {
         var enqueuedJobs =  JobStorage.Current.GetMonitoringApi().EnqueuedJobs(queue, 0, int.MaxValue);
-        return enqueuedJobs.Any(j => j.Value.InEnqueuedState &&
+        var ret = enqueuedJobs.Any(j => j.Value.InEnqueuedState &&
                                      j.Value.Job.Method.DeclaringType != null && j.Value.Job.Args.SequenceEqual(args) &&
                                      j.Value.Job.Method.Name.Equals(methodName) &&
                                      j.Value.Job.Method.DeclaringType.Name.Equals(className));
+        if (ret) return true;
+
+        var scheduledJobs = JobStorage.Current.GetMonitoringApi().ScheduledJobs(0, int.MaxValue);
+        return scheduledJobs.Any(j =>
+            j.Value.Job.Method.DeclaringType != null && j.Value.Job.Args.SequenceEqual(args) &&
+            j.Value.Job.Method.Name.Equals(methodName) &&
+            j.Value.Job.Method.DeclaringType.Name.Equals(className));
     }
 
+    /// <summary>
+    /// Checks against any jobs that are running or about to run
+    /// </summary>
+    /// <param name="classNames"></param>
+    /// <param name="queue"></param>
+    /// <returns></returns>
     public static bool RunningAnyTasksByMethod(IEnumerable<string> classNames, string queue = DefaultQueue)
     {
         var enqueuedJobs =  JobStorage.Current.GetMonitoringApi().EnqueuedJobs(queue, 0, int.MaxValue);
-        return enqueuedJobs.Any(j => !j.Value.InEnqueuedState &&
+        var ret = enqueuedJobs.Any(j => !j.Value.InEnqueuedState &&
                                      classNames.Contains(j.Value.Job.Method.DeclaringType?.Name));
+        if (ret) return true;
+
+        var runningJobs = JobStorage.Current.GetMonitoringApi().ProcessingJobs(0, int.MaxValue);
+        return runningJobs.Any(j => classNames.Contains(j.Value.Job.Method.DeclaringType?.Name));
     }
 }