AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scan Loop Fixes (#1459)

Browse source 
* Added Last Folder Scanned time to series info modal.

Tweaked the info event detail modal to have a primary and thus be auto-dismissable

* Added an error event when multiple series are found in processing a series.

* Fixed a bug where a series could get stuck with other series due to a bad select query.

Started adding the force flag hook for the UI and designing the confirm.

Confirm service now also has ability to hide the close button.

Updated error events and logging in the loop, to be more informative

* Fixed a bug where confirm service wasn't showing the proper body content.

* Hooked up force scan series

* refresh metadata now has force update

* Fixed up the messaging with the prompt on scan, hooked it up properly in the scan library to avoid the check if the whole library needs to even be scanned. Fixed a bug where NormalizedLocalizedName wasn't being calculated on new entities.

Started adding unit tests for this problematic repo method.

* Fixed a bug where we updated NormalizedLocalizedName before we set it.

* Send an info to the UI when series are spread between multiple library level folders.

* Added some logger output when there are no files found in a folder. Return early if there are no files found, so we can avoid some small loops of code.

* Fixed an issue where multiple series in a folder with localized series would cause unintended grouping. This is not supported and hence we will warn them and allow the bad grouping.

* Added a case where scan series fails due to the folder being removed. We will now log an error

* Normalize paths when finding the highest directory till root.

* Fixed an issue with Scan Series where changing a series' folder to a different path but the original series folder existed with another series in it, would cause the series to not be deleted.

* Fixed some bugs around specials causing a series merge issue on scan series.

* Removed a bug marker

* Cleaned up some of the scan loop and removed a test I don't need.

* Remove any prompts for force flow, it doesn't work well. Leave the API as is though.

* Fixed up a check for duplicate ScanLibrary calls
This commit is contained in:
Joseph Milazzo 2022-08-22 12:14:31 -05:00 • committed by GitHub
parent 354be09c4c
commit 1c9544fc47
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
27 changed files with 367 additions and 222 deletions
Download patch file Download diff file Expand all files Collapse all files

94
API/Services/Tasks/ScannerService.cs
View file

@ -29,7 +29,7 @@ public interface IScannerService
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task ScanLibrary(int libraryId);
Task ScanLibrary(int libraryId, bool forceUpdate = false);
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
@ -62,6 +62,10 @@ public enum ScanCancelReason
/// There has been no change to the filesystem since last scan
/// </summary>
NoChange = 2,
/// <summary>
/// The underlying folder is missing
/// </summary>
FolderMissing = 3
}
/**
@ -117,10 +121,15 @@ public class ScannerService : IScannerService
var library = libraries.FirstOrDefault(l => l.Folders.Select(Parser.Parser.NormalizePath).Contains(libraryFolder));
if (library != null)
{
BackgroundJob.Enqueue(() => ScanLibrary(library.Id));
BackgroundJob.Enqueue(() => ScanLibrary(library.Id, false));
}
}
/// <summary>
///
/// </summary>
/// <param name="seriesId"></param>
/// <param name="bypassFolderOptimizationChecks">Not Used. Scan series will always force</param>
[Queue(TaskScheduler.ScanQueue)]
public async Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true)
{
@ -130,12 +139,7 @@ public class ScannerService : IScannerService
var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders);
var libraryPaths = library.Folders.Select(f => f.Path).ToList();
if (await ShouldScanSeries(seriesId, library, libraryPaths, series, bypassFolderOptimizationChecks) != ScanCancelReason.NoCancel) return;
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
var seenSeries = new List<ParsedSeries>();
var processTasks = new List<Task>();
if (await ShouldScanSeries(seriesId, library, libraryPaths, series, true) != ScanCancelReason.NoCancel) return;
var folderPath = series.FolderPath;
if (string.IsNullOrEmpty(folderPath) || !_directoryService.Exists(folderPath))
@ -150,22 +154,32 @@ public class ScannerService : IScannerService
}
folderPath = seriesDirs.Keys.FirstOrDefault();
// We should check if folderPath is a library folder path and if so, return early and tell user to correct their setup.
if (libraryPaths.Contains(folderPath))
{
_logger.LogCritical("[ScannerSeries] {SeriesName} scan aborted. Files for series are not in a nested folder under library path. Correct this and rescan", series.Name);
await _eventHub.SendMessageAsync(MessageFactory.Error, MessageFactory.ErrorEvent($"{series.Name} scan aborted", "Files for series are not in a nested folder under library path. Correct this and rescan."));
return;
}
}
if (string.IsNullOrEmpty(folderPath))
{
_logger.LogCritical("Scan Series could not find a single, valid folder root for files");
_logger.LogCritical("[ScannerSeries] Scan Series could not find a single, valid folder root for files");
await _eventHub.SendMessageAsync(MessageFactory.Error, MessageFactory.ErrorEvent($"{series.Name} scan aborted", "Scan Series could not find a single, valid folder root for files"));
return;
}
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
var processTasks = new List<Task>();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
await _processSeries.Prime();
void TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return;
@ -176,44 +190,21 @@ public class ScannerService : IScannerService
Format = parsedFiles.First().Format
};
if (skippedScan)
if (!foundParsedSeries.NormalizedName.Equals(series.NormalizedName))
{
seenSeries.AddRange(parsedFiles.Select(pf => new ParsedSeries()
{
Name = pf.Series,
NormalizedName = Parser.Parser.Normalize(pf.Series),
Format = pf.Format
}));
return;
}
seenSeries.Add(foundParsedSeries);
processTasks.Add(_processSeries.ProcessSeriesAsync(parsedFiles, library));
parsedSeries.Add(foundParsedSeries, parsedFiles);
}
_logger.LogInformation("Beginning file scan on {SeriesName}", series.Name);
var scanElapsedTime = await ScanFiles(library, new []{folderPath}, false, TrackFiles, bypassFolderOptimizationChecks);
var scanElapsedTime = await ScanFiles(library, new []{folderPath}, false, TrackFiles, true);
_logger.LogInformation("ScanFiles for {Series} took {Time}", series.Name, scanElapsedTime);
await Task.WhenAll(processTasks);
// At this point, we've already inserted the series into the DB OR we haven't and seenSeries has our series
// We now need to do any leftover work, like removing
// We need to handle if parsedSeries is empty but seenSeries has our series
if (seenSeries.Any(s => s.NormalizedName.Equals(series.NormalizedName)) && parsedSeries.Keys.Count == 0)
{
// Nothing has changed
_logger.LogInformation("[ScannerService] {SeriesName} scan has no work to do. All folders have not been changed since last scan", series.Name);
await _eventHub.SendMessageAsync(MessageFactory.Info,
MessageFactory.InfoEvent($"{series.Name} scan has no work to do",
"All folders have not been changed since last scan. Scan will be aborted."));
_processSeries.EnqueuePostSeriesProcessTasks(series.LibraryId, seriesId, false);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
return;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
// Remove any parsedSeries keys that don't belong to our series. This can occur when users store 2 series in the same folder
@ -222,8 +213,8 @@ public class ScannerService : IScannerService
// If nothing was found, first validate any of the files still exist. If they don't then we have a deletion and can skip the rest of the logic flow
if (parsedSeries.Count == 0)
{
var anyFilesExist =
(await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id)).Any(m => File.Exists(m.FilePath));
var seriesFiles = (await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id));
var anyFilesExist = seriesFiles.Where(f => f.FilePath.Contains(series.FolderPath)).Any(m => File.Exists(m.FilePath));
if (!anyFilesExist)
{
@ -287,21 +278,34 @@ public class ScannerService : IScannerService
}
// If all series Folder paths haven't been modified since last scan, abort
// NOTE: On windows, the parent folder will not update LastWriteTime if a subfolder was updated with files. Need to do a bit of light I/O.
if (!bypassFolderChecks)
{
var allFolders = seriesFolderPaths.SelectMany(path => _directoryService.GetDirectories(path)).ToList();
allFolders.AddRange(seriesFolderPaths);
if (allFolders.All(folder => _directoryService.GetLastWriteTime(folder) <= series.LastFolderScanned))
try
{
_logger.LogInformation(
"[ScannerService] {SeriesName} scan has no work to do. All folders have not been changed since last scan",
if (allFolders.All(folder => _directoryService.GetLastWriteTime(folder) <= series.LastFolderScanned))
{
_logger.LogInformation(
"[ScannerService] {SeriesName} scan has no work to do. All folders have not been changed since last scan",
series.Name);
await _eventHub.SendMessageAsync(MessageFactory.Info,
MessageFactory.InfoEvent($"{series.Name} scan has no work to do",
"All folders have not been changed since last scan. Scan will be aborted."));
return ScanCancelReason.NoChange;
}
}
catch (IOException ex)
{
// If there is an exception it means that the folder doesn't exist. So we should delete the series
_logger.LogError(ex, "[ScannerService] Scan series for {SeriesName} found the folder path no longer exists",
series.Name);
await _eventHub.SendMessageAsync(MessageFactory.Info,
MessageFactory.InfoEvent($"{series.Name} scan has no work to do", "All folders have not been changed since last scan. Scan will be aborted."));
return ScanCancelReason.NoChange;
MessageFactory.ErrorEvent($"{series.Name} scan has no work to do",
"The folder the series is in is missing. Delete series manually or perform a library scan."));
return ScanCancelReason.NoCancel;
}
}
@ -393,7 +397,7 @@ public class ScannerService : IScannerService
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibrary(int libraryId)
public async Task ScanLibrary(int libraryId, bool forceUpdate = false)
{
var sw = Stopwatch.StartNew();
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.Folders);
@ -405,7 +409,7 @@ public class ScannerService : IScannerService
var wasLibraryUpdatedSinceLastScan = (library.LastModified.Truncate(TimeSpan.TicksPerMinute) >
library.LastScanned.Truncate(TimeSpan.TicksPerMinute))
&& library.LastScanned != DateTime.MinValue;
if (!wasLibraryUpdatedSinceLastScan)
if (!forceUpdate && !wasLibraryUpdatedSinceLastScan)
{
var haveFoldersChangedSinceLastScan = library.Folders
.All(f => _directoryService.GetLastWriteTime(f.Path).Truncate(TimeSpan.TicksPerMinute) > f.LastScanned.Truncate(TimeSpan.TicksPerMinute));