AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Misc Polish (#1569)

Browse source 
* Introduced a lock for DB work during the scan to hopefully reduce the concurrency issues

* Don't allow multiple theme scans to occur

* Fixed bulk actions not having all actions due to nested actionable menu changes

* Refactored the Scan loop to be synchronous to avoid any issues. After first loop, no real performance issues.

* Updated the LibraryWatcher when under many internal buffer full issues, to suspend watching for a full hour, to allow whatever downloading to complete.

* Removed Semaphore as it's not needed anymore

* Updated the output for logger to explicitly say from Kavita (if you're pushing to Seq)

* Fixed a broken test

* Fixed ReleaseYear not populating due to a change from a contributor around how to populate ReleaseYear.

* Ensure when scan folder runs, that we don't double enqueue the same tasks.

* Fixed user settings not loading the correct tab

* Changed the Release Year -> Release

* Added more refresh hooks in reader to hopefully ensure faster refreshes

* Reset images between chapter loads to help flush image faster. Don't show broken image icon when an image is still loading.

* Fixed the prefetcher not properly loading the correct images and hence, allowing a bit of lag between chapter loads.

* Code smells
This commit is contained in:
Joe Milazzo 2022-10-04 19:40:34 -05:00 • committed by GitHub
parent 097ec32842
commit 58bbba29cc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 208 additions and 45 deletions
Download patch file Download diff file Expand all files Collapse all files

2
API/Controllers/LibraryController.cs
View file

@ -213,9 +213,11 @@ public class LibraryController : BaseApiController
{
var userId = await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(dto.ApiKey);
var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId);
// Validate user has Admin privileges
var isAdmin = await _unitOfWork.UserRepository.IsUserAdminAsync(user);
if (!isAdmin) return BadRequest("API key must belong to an admin");
if (dto.FolderPath.Contains("..")) return BadRequest("Invalid Path");
dto.FolderPath = Services.Tasks.Scanner.Parser.Parser.NormalizePath(dto.FolderPath);

2
API/Entities/AppUserBookmark.cs
View file

@ -11,8 +11,8 @@ public class AppUserBookmark : IEntityDate
{
public int Id { get; set; }
public int Page { get; set; }
public int VolumeId { get; set; }
public int SeriesId { get; set; }
public int VolumeId { get; set; }
public int ChapterId { get; set; }
/// <summary>

2
API/Logging/LogLevelOptions.cs
View file

@ -40,7 +40,7 @@ public static class LogLevelOptions
public static LoggerConfiguration CreateConfig(LoggerConfiguration configuration)
{
const string outputTemplate = "[{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {CorrelationId} {ThreadId}] [{Level}] {SourceContext} {Message:lj}{NewLine}{Exception}";
const string outputTemplate = "[Kavita] [{Timestamp:yyyy-MM-dd HH:mm:ss.fff zzz} {CorrelationId} {ThreadId}] [{Level}] {SourceContext} {Message:lj}{NewLine}{Exception}";
return configuration
.MinimumLevel
.ControlledBy(LogLevelSwitch)

8
API/Services/TaskScheduler.cs
View file

@ -158,7 +158,13 @@ public class TaskScheduler : ITaskScheduler
public void ScanSiteThemes()
{
_logger.LogInformation("Starting Site Theme scan");
if (HasAlreadyEnqueuedTask("ThemeService", "Scan", Array.Empty<object>(), ScanQueue))
{
_logger.LogInformation("A Theme Scan is already running");
return;
}
_logger.LogInformation("Enqueueing Site Theme scan");
BackgroundJob.Enqueue(() => _themeService.Scan());
}

34
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -51,6 +51,14 @@ public class LibraryWatcher : ILibraryWatcher
/// <remarks>The Job will be enqueued instantly</remarks>
private readonly TimeSpan _queueWaitTime;
/// <summary>
/// Counts within a time frame how many times the buffer became full. Is used to reschedule LibraryWatcher to start monitoring much later rather than instantly
/// </summary>
private int _bufferFullCounter = 0;
private DateTime _lastBufferOverflow = DateTime.MinValue;
public LibraryWatcher(IDirectoryService directoryService, IUnitOfWork unitOfWork, ILogger<LibraryWatcher> logger, IScannerService scannerService, IHostEnvironment environment)
{
@ -118,6 +126,9 @@ public class LibraryWatcher : ILibraryWatcher
public async Task RestartWatching()
{
_logger.LogDebug("[LibraryWatcher] Restarting watcher");
UpdateBufferOverflow();
StopWatching();
await StartWatching();
}
@ -151,6 +162,15 @@ public class LibraryWatcher : ILibraryWatcher
private void OnError(object sender, ErrorEventArgs e)
{
_logger.LogError(e.GetException(), "[LibraryWatcher] An error occured, likely too many changes occured at once or the folder being watched was deleted. Restarting Watchers");
_bufferFullCounter += 1;
_lastBufferOverflow = DateTime.Now;
if (_bufferFullCounter >= 3)
{
_logger.LogInformation("[LibraryWatcher] Internal buffer has been overflown multiple times in past 10 minutes. Suspending file watching for an hour");
BackgroundJob.Schedule(() => RestartWatching(), TimeSpan.FromHours(1));
return;
}
Task.Run(RestartWatching);
}
@ -162,8 +182,11 @@ public class LibraryWatcher : ILibraryWatcher
/// <remarks>This is public only because Hangfire will invoke it. Do not call external to this class.</remarks>
/// <param name="filePath">File or folder that changed</param>
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
// ReSharper disable once MemberCanBePrivate.Global
public async Task ProcessChange(string filePath, bool isDirectoryChange = false)
{
UpdateBufferOverflow();
var sw = Stopwatch.StartNew();
_logger.LogDebug("[LibraryWatcher] Processing change of {FilePath}", filePath);
try
@ -232,4 +255,15 @@ public class LibraryWatcher : ILibraryWatcher
// Select the first folder and join with library folder, this should give us the folder to scan.
return Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()));
}
private void UpdateBufferOverflow()
{
if (_bufferFullCounter == 0) return;
// If the last buffer overflow is over 5 mins back, we can remove a buffer count
if (_lastBufferOverflow < DateTime.Now.Subtract(TimeSpan.FromMinutes(5)))
{
_bufferFullCounter = Math.Min(0, _bufferFullCounter - 1);
_lastBufferOverflow = DateTime.Now;
}
}
}

6
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -223,7 +223,7 @@ public class ParseScannedFiles
/// <returns></returns>
public async Task ScanLibrariesForSeries(LibraryType libraryType,
IEnumerable<string> folders, string libraryName, bool isLibraryScan,
IDictionary<string, IList<SeriesModified>> seriesPaths, Action<Tuple<bool, IList<ParserInfo>>> processSeriesInfos, bool forceCheck = false)
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<Tuple<bool, IList<ParserInfo>>, Task> processSeriesInfos, bool forceCheck = false)
{
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", libraryName, ProgressEventType.Started));
@ -242,7 +242,7 @@ public class ParseScannedFiles
Series = fp.SeriesName,
Format = fp.Format,
}).ToList();
processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
_logger.LogDebug("Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
return;
}
@ -280,7 +280,7 @@ public class ParseScannedFiles
{
if (scannedSeries[series].Count > 0 && processSeriesInfos != null)
{
processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
}
}
}, forceCheck);

10
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.Data.Metadata;
@ -48,8 +49,6 @@ public class ProcessSeries : IProcessSeries
private volatile IList<Person> _people;
private volatile IList<Tag> _tags;
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
IFileService fileService, IMetadataService metadataService, IWordCountAnalyzerService wordCountAnalyzerService)
@ -167,7 +166,9 @@ public class ProcessSeries : IProcessSeries
catch (Exception ex)
{
await _unitOfWork.RollbackAsync();
_logger.LogCritical(ex, "[ScannerService] There was an issue writing to the database for series {@SeriesName}", series.Name);
_logger.LogCritical(ex,
"[ScannerService] There was an issue writing to the database for series {@SeriesName}",
series.Name);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"There was an issue writing to the DB for Series {series}",
@ -234,7 +235,7 @@ public class ProcessSeries : IProcessSeries
var chapters = series.Volumes.SelectMany(volume => volume.Chapters).ToList();
// Update Metadata based on Chapter metadata
series.Metadata.ReleaseYear = chapters.Min(c => c.ReleaseDate.Year);
series.Metadata.ReleaseYear = chapters.Select(v => v.ReleaseDate.Year).Where(y => y >= 1000).Min();
if (series.Metadata.ReleaseYear < 1000)
{
@ -439,6 +440,7 @@ public class ProcessSeries : IProcessSeries
_logger.LogDebug("[ScannerService] Updating {DistinctVolumes} volumes on {SeriesName}", distinctVolumes.Count, series.Name);
foreach (var volumeNumber in distinctVolumes)
{
_logger.LogDebug("[ScannerService] Looking up volume for {volumeNumber}", volumeNumber);
var volume = series.Volumes.SingleOrDefault(s => s.Name == volumeNumber);
if (volume == null)
{

24
API/Services/Tasks/ScannerService.cs
View file

@ -102,6 +102,12 @@ public class ScannerService : IScannerService
var seriesId = await _unitOfWork.SeriesRepository.GetSeriesIdByFolder(folder);
if (seriesId > 0)
{
if (TaskScheduler.HasAlreadyEnqueuedTask(Name, "ScanSeries",
new object[] {seriesId, true}))
{
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder);
return;
}
BackgroundJob.Enqueue(() => ScanSeries(seriesId, true));
return;
}
@ -119,6 +125,12 @@ public class ScannerService : IScannerService
var library = libraries.FirstOrDefault(l => l.Folders.Select(Scanner.Parser.Parser.NormalizePath).Contains(libraryFolder));
if (library != null)
{
if (TaskScheduler.HasAlreadyEnqueuedTask(Name, "ScanLibrary",
new object[] {library.Id, false}))
{
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
return;
}
BackgroundJob.Enqueue(() => ScanLibrary(library.Id, false));
}
}
@ -175,13 +187,11 @@ public class ScannerService : IScannerService
}
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
var processTasks = new List<Task>();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
await _processSeries.Prime();
void TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return;
@ -198,7 +208,7 @@ public class ScannerService : IScannerService
return;
}
processTasks.Add(_processSeries.ProcessSeriesAsync(parsedFiles, library));
await _processSeries.ProcessSeriesAsync(parsedFiles, library);
parsedSeries.Add(foundParsedSeries, parsedFiles);
}
@ -424,7 +434,7 @@ public class ScannerService : IScannerService
await _processSeries.Prime();
var processTasks = new List<Task>();
void TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
@ -452,7 +462,7 @@ public class ScannerService : IScannerService
seenSeries.Add(foundParsedSeries);
processTasks.Add(_processSeries.ProcessSeriesAsync(parsedFiles, library));
await _processSeries.ProcessSeriesAsync(parsedFiles, library);
}
@ -512,7 +522,7 @@ public class ScannerService : IScannerService
}
private async Task<long> ScanFiles(Library library, IEnumerable<string> dirs,
bool isLibraryScan, Action<Tuple<bool, IList<ParserInfo>>> processSeriesInfos = null, bool forceChecks = false)
bool isLibraryScan, Func<Tuple<bool, IList<ParserInfo>>, Task> processSeriesInfos = null, bool forceChecks = false)
{
var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService, _eventHub);
var scanWatch = Stopwatch.StartNew();