AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Event Widget Updates + Format Downloads + Scanner Work (#3024)

Browse source
This commit is contained in:
Joe Milazzo 2024-06-27 16:35:50 -05:00 • committed by GitHub
parent 30a8a2555f
commit a427d02ed1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 971 additions and 694 deletions
Download patch file Download diff file Expand all files Collapse all files

51
API/Services/Tasks/ScannerService.cs
View file

@ -76,7 +76,7 @@ public enum ScanCancelReason
public class ScannerService : IScannerService
{
public const string Name = "ScannerService";
public const int Timeout = 60 * 60 * 60;
private const int Timeout = 60 * 60 * 60; // 2.5 days
private readonly IUnitOfWork _unitOfWork;
private readonly ILogger<ScannerService> _logger;
private readonly IMetadataService _metadataService;
@ -157,11 +157,11 @@ public class ScannerService : IScannerService
}
// TODO: Figure out why we have the library type restriction here
if (series != null && series.Library.Type is not (LibraryType.Book or LibraryType.LightNovel))
if (series != null)// && series.Library.Type is not (LibraryType.Book or LibraryType.LightNovel)
{
if (TaskScheduler.HasScanTaskRunningForSeries(series.Id))
{
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder);
_logger.LogDebug("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder);
return;
}
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder}, Series matched to folder and ScanSeries enqueued for 1 minute", folder);
@ -185,7 +185,7 @@ public class ScannerService : IScannerService
{
if (TaskScheduler.HasScanTaskRunningForLibrary(library.Id))
{
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
_logger.LogDebug("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
return;
}
BackgroundJob.Schedule(() => ScanLibrary(library.Id, false, true), TimeSpan.FromMinutes(1));
@ -198,20 +198,21 @@ public class ScannerService : IScannerService
/// <param name="seriesId"></param>
/// <param name="bypassFolderOptimizationChecks">Not Used. Scan series will always force</param>
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(Timeout)]
[AutomaticRetry(Attempts = 200, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true)
{
if (TaskScheduler.HasAlreadyEnqueuedTask(Name, "ScanSeries", [seriesId, bypassFolderOptimizationChecks], TaskScheduler.ScanQueue))
{
_logger.LogInformation("[ScannerService] Scan series invoked but a task is already running/enqueued. Dropping request");
return;
}
var sw = Stopwatch.StartNew();
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update
// if (TaskScheduler.HasScanTaskRunningForSeries(seriesId))
// {
// _logger.LogInformation("[ScannerService] Scan series invoked but a task is already running/enqueued. Rescheduling request for 1 mins");
// BackgroundJob.Schedule(() => ScanSeries(seriesId, bypassFolderOptimizationChecks), TimeSpan.FromMinutes(1));
// return;
// }
var existingChapterIdsToClean = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
@ -444,7 +445,7 @@ public class ScannerService : IScannerService
// Check if any of the folder roots are not available (ie disconnected from network, etc) and fail if any of them are
if (folders.Any(f => !_directoryService.IsDriveMounted(f)))
{
_logger.LogCritical("Some of the root folders for library ({LibraryName} are not accessible. Please check that drives are connected and rescan. Scan will be aborted", libraryName);
_logger.LogCritical("[ScannerService] Some of the root folders for library ({LibraryName} are not accessible. Please check that drives are connected and rescan. Scan will be aborted", libraryName);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted",
@ -458,7 +459,7 @@ public class ScannerService : IScannerService
if (folders.Any(f => _directoryService.IsDirectoryEmpty(f)))
{
// That way logging and UI informing is all in one place with full context
_logger.LogError("Some of the root folders for the library are empty. " +
_logger.LogError("[ScannerService] Some of the root folders for the library are empty. " +
"Either your mount has been disconnected or you are trying to delete all series in the library. " +
"Scan has be aborted. " +
"Check that your mount is connected or change the library's root folder and rescan");
@ -479,22 +480,23 @@ public class ScannerService : IScannerService
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibraries(bool forceUpdate = false)
{
_logger.LogInformation("Starting Scan of All Libraries, Forced: {Forced}", forceUpdate);
_logger.LogInformation("[ScannerService] Starting Scan of All Libraries, Forced: {Forced}", forceUpdate);
foreach (var lib in await _unitOfWork.LibraryRepository.GetLibrariesAsync())
{
if (TaskScheduler.RunningAnyTasksByMethod(TaskScheduler.ScanTasks, TaskScheduler.ScanQueue))
// BUG: This will trigger the first N libraries to scan over and over if there is always an interruption later in the chain
if (TaskScheduler.HasScanTaskRunningForLibrary(lib.Id))
{
_logger.LogInformation("[ScannerService] Scan library invoked via nightly scan job but a task is already running. Rescheduling for 4 hours");
await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.InfoEvent($"Scan libraries task delayed",
$"A scan was ongoing during processing of the scan libraries task. Task has been rescheduled for {DateTime.UtcNow.AddHours(4)} UTC"));
BackgroundJob.Schedule(() => ScanLibraries(forceUpdate), TimeSpan.FromHours(4));
return;
// We don't need to send SignalR event as this is a background job that user doesn't need insight into
_logger.LogInformation("[ScannerService] Scan library invoked via nightly scan job but a task is already running for {LibraryName}. Rescheduling for 4 hours", lib.Name);
await Task.Delay(TimeSpan.FromHours(4));
//BackgroundJob.Schedule(() => ScanLibraries(forceUpdate), TimeSpan.FromHours(4));
//return;
}
await ScanLibrary(lib.Id, forceUpdate, true);
}
_processSeries.Reset();
_logger.LogInformation("Scan of All Libraries Finished");
_logger.LogInformation("[ScannerService] Scan of All Libraries Finished");
}
@ -526,23 +528,27 @@ public class ScannerService : IScannerService
var shouldUseLibraryScan = !(await _unitOfWork.LibraryRepository.DoAnySeriesFoldersMatch(libraryFolderPaths));
if (!shouldUseLibraryScan)
{
_logger.LogError("Library {LibraryName} consists of one or more Series folders, using series scan", library.Name);
_logger.LogError("[ScannerService] Library {LibraryName} consists of one or more Series folders, using series scan", library.Name);
}
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 1: Scan Files", library.Name);
var (scanElapsedTime, processedSeries) = await ScanFiles(library, libraryFolderPaths,
shouldUseLibraryScan, forceUpdate);
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 2: Track Found Series", library.Name);
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
TrackFoundSeriesAndFiles(parsedSeries, processedSeries);
// We need to remove any keys where there is no actual parser info
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 3: Process Parsed Series", library.Name);
var totalFiles = await ProcessParsedSeries(forceUpdate, parsedSeries, library, scanElapsedTime);
UpdateLastScanned(library);
_unitOfWork.LibraryRepository.Update(library);
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 4: Save Library", library.Name);
if (await _unitOfWork.CommitAsync())
{
if (isSingleScan)
@ -563,6 +569,7 @@ public class ScannerService : IScannerService
totalFiles, parsedSeries.Count, sw.ElapsedMilliseconds, library.Name);
}
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 5: Remove Deleted Series", library.Name);
await RemoveSeriesNotFound(parsedSeries, library);
}
else