AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Misc Polish (#1569)

Browse source 
* Introduced a lock for DB work during the scan to hopefully reduce the concurrency issues

* Don't allow multiple theme scans to occur

* Fixed bulk actions not having all actions due to nested actionable menu changes

* Refactored the Scan loop to be synchronous to avoid any issues. After first loop, no real performance issues.

* Updated the LibraryWatcher when under many internal buffer full issues, to suspend watching for a full hour, to allow whatever downloading to complete.

* Removed Semaphore as it's not needed anymore

* Updated the output for logger to explicitly say from Kavita (if you're pushing to Seq)

* Fixed a broken test

* Fixed ReleaseYear not populating due to a change from a contributor around how to populate ReleaseYear.

* Ensure when scan folder runs, that we don't double enqueue the same tasks.

* Fixed user settings not loading the correct tab

* Changed the Release Year -> Release

* Added more refresh hooks in reader to hopefully ensure faster refreshes

* Reset images between chapter loads to help flush image faster. Don't show broken image icon when an image is still loading.

* Fixed the prefetcher not properly loading the correct images and hence, allowing a bit of lag between chapter loads.

* Code smells
This commit is contained in:
Joe Milazzo 2022-10-04 19:40:34 -05:00 • committed by GitHub
parent 097ec32842
commit 58bbba29cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 208 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

34
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -51,6 +51,14 @@ public class LibraryWatcher : ILibraryWatcher
/// <remarks>The Job will be enqueued instantly</remarks>
private readonly TimeSpan _queueWaitTime;
/// <summary>
/// Counts within a time frame how many times the buffer became full. Is used to reschedule LibraryWatcher to start monitoring much later rather than instantly
/// </summary>
private int _bufferFullCounter = 0;
private DateTime _lastBufferOverflow = DateTime.MinValue;
public LibraryWatcher(IDirectoryService directoryService, IUnitOfWork unitOfWork, ILogger<LibraryWatcher> logger, IScannerService scannerService, IHostEnvironment environment)
{
@ -118,6 +126,9 @@ public class LibraryWatcher : ILibraryWatcher
public async Task RestartWatching()
{
_logger.LogDebug("[LibraryWatcher] Restarting watcher");
UpdateBufferOverflow();
StopWatching();
await StartWatching();
}
@ -151,6 +162,15 @@ public class LibraryWatcher : ILibraryWatcher
private void OnError(object sender, ErrorEventArgs e)
{
_logger.LogError(e.GetException(), "[LibraryWatcher] An error occured, likely too many changes occured at once or the folder being watched was deleted. Restarting Watchers");
_bufferFullCounter += 1;
_lastBufferOverflow = DateTime.Now;
if (_bufferFullCounter >= 3)
{
_logger.LogInformation("[LibraryWatcher] Internal buffer has been overflown multiple times in past 10 minutes. Suspending file watching for an hour");
BackgroundJob.Schedule(() => RestartWatching(), TimeSpan.FromHours(1));
return;
}
Task.Run(RestartWatching);
}
@ -162,8 +182,11 @@ public class LibraryWatcher : ILibraryWatcher
/// <remarks>This is public only because Hangfire will invoke it. Do not call external to this class.</remarks>
/// <param name="filePath">File or folder that changed</param>
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
// ReSharper disable once MemberCanBePrivate.Global
public async Task ProcessChange(string filePath, bool isDirectoryChange = false)
{
UpdateBufferOverflow();
var sw = Stopwatch.StartNew();
_logger.LogDebug("[LibraryWatcher] Processing change of {FilePath}", filePath);
try
@ -232,4 +255,15 @@ public class LibraryWatcher : ILibraryWatcher
// Select the first folder and join with library folder, this should give us the folder to scan.
return Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()));
}
private void UpdateBufferOverflow()
{
if (_bufferFullCounter == 0) return;
// If the last buffer overflow is over 5 mins back, we can remove a buffer count
if (_lastBufferOverflow < DateTime.Now.Subtract(TimeSpan.FromMinutes(5)))
{
_bufferFullCounter = Math.Min(0, _bufferFullCounter - 1);
_lastBufferOverflow = DateTime.Now;
}
}
}

6
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -223,7 +223,7 @@ public class ParseScannedFiles
/// <returns></returns>
public async Task ScanLibrariesForSeries(LibraryType libraryType,
IEnumerable<string> folders, string libraryName, bool isLibraryScan,
IDictionary<string, IList<SeriesModified>> seriesPaths, Action<Tuple<bool, IList<ParserInfo>>> processSeriesInfos, bool forceCheck = false)
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<Tuple<bool, IList<ParserInfo>>, Task> processSeriesInfos, bool forceCheck = false)
{
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", libraryName, ProgressEventType.Started));
@ -242,7 +242,7 @@ public class ParseScannedFiles
Series = fp.SeriesName,
Format = fp.Format,
}).ToList();
processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
_logger.LogDebug("Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
return;
}
@ -280,7 +280,7 @@ public class ParseScannedFiles
{
if (scannedSeries[series].Count > 0 && processSeriesInfos != null)
{
processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
}
}
}, forceCheck);

10
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.Data.Metadata;
@ -48,8 +49,6 @@ public class ProcessSeries : IProcessSeries
private volatile IList<Person> _people;
private volatile IList<Tag> _tags;
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
IFileService fileService, IMetadataService metadataService, IWordCountAnalyzerService wordCountAnalyzerService)
@ -167,7 +166,9 @@ public class ProcessSeries : IProcessSeries
catch (Exception ex)
{
await _unitOfWork.RollbackAsync();
_logger.LogCritical(ex, "[ScannerService] There was an issue writing to the database for series {@SeriesName}", series.Name);
_logger.LogCritical(ex,
"[ScannerService] There was an issue writing to the database for series {@SeriesName}",
series.Name);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"There was an issue writing to the DB for Series {series}",
@ -234,7 +235,7 @@ public class ProcessSeries : IProcessSeries
var chapters = series.Volumes.SelectMany(volume => volume.Chapters).ToList();
// Update Metadata based on Chapter metadata
series.Metadata.ReleaseYear = chapters.Min(c => c.ReleaseDate.Year);
series.Metadata.ReleaseYear = chapters.Select(v => v.ReleaseDate.Year).Where(y => y >= 1000).Min();
if (series.Metadata.ReleaseYear < 1000)
{
@ -439,6 +440,7 @@ public class ProcessSeries : IProcessSeries
_logger.LogDebug("[ScannerService] Updating {DistinctVolumes} volumes on {SeriesName}", distinctVolumes.Count, series.Name);
foreach (var volumeNumber in distinctVolumes)
{
_logger.LogDebug("[ScannerService] Looking up volume for {volumeNumber}", volumeNumber);
var volume = series.Volumes.SingleOrDefault(s => s.Name == volumeNumber);
if (volume == null)
{