AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

More Scan Loop Bugfixes (#1471)

Browse source 
* Updated scan time for watcher to 30 seconds for non-dev. Moved ScanFolder off the Scan queue as it doesn't need to be there. Updated loggers

* Fixed jumpbar missing

* Tweaked the messaging for CoverGen

* When we return early due to nothing being done on library and series scan, make sure we kick off other tasks that need to occur.

* Fixed a foreign constraint issue on Volumes when we were adding to a new series.

* Fixed a case where when picking normalized series, capitalization differences wouldn't stack when they should.

* Reduced the logging output on dev and prod settings.

* Fixed a bug in the code that finds the highest directory from a file, where we were not checking against a normalized path.

* Cleaned up some code

* Fixed broken unit tests
This commit is contained in:
Joseph Milazzo 2022-08-24 11:27:32 -05:00 • committed by GitHub
parent fc0121e7a8
commit 1e535a8184
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 77 additions and 62 deletions
Download patch file Download diff file Expand all files Collapse all files

32
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -78,7 +78,7 @@ public class LibraryWatcher : ILibraryWatcher
_logger = logger;
_scannerService = scannerService;
_queueWaitTime = environment.IsDevelopment() ? TimeSpan.FromSeconds(10) : TimeSpan.FromMinutes(5);
_queueWaitTime = environment.IsDevelopment() ? TimeSpan.FromSeconds(10) : TimeSpan.FromSeconds(30);
}
@ -142,18 +142,18 @@ public class LibraryWatcher : ILibraryWatcher
private void OnChanged(object sender, FileSystemEventArgs e)
{
if (e.ChangeType != WatcherChangeTypes.Changed) return;
Console.WriteLine($"Changed: {e.FullPath}, {e.Name}");
_logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}", e.FullPath, e.Name);
ProcessChange(e.FullPath);
}
private void OnCreated(object sender, FileSystemEventArgs e)
{
Console.WriteLine($"Created: {e.FullPath}, {e.Name}");
_logger.LogDebug("[LibraryWatcher] Created: {FullPath}, {Name}", e.FullPath, e.Name);
ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name));
}
private void OnDeleted(object sender, FileSystemEventArgs e) {
Console.WriteLine($"Deleted: {e.FullPath}, {e.Name}");
_logger.LogDebug("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name);
// On deletion, we need another type of check. We need to check if e.Name has an extension or not
// NOTE: File deletion will trigger a folder change event, so this might not be needed
@ -164,9 +164,9 @@ public class LibraryWatcher : ILibraryWatcher
private void OnRenamed(object sender, RenamedEventArgs e)
{
Console.WriteLine($"Renamed:");
Console.WriteLine($" Old: {e.OldFullPath}");
Console.WriteLine($" New: {e.FullPath}");
_logger.LogDebug($"[LibraryWatcher] Renamed:");
_logger.LogDebug(" Old: {OldFullPath}", e.OldFullPath);
_logger.LogDebug(" New: {FullPath}", e.FullPath);
ProcessChange(e.FullPath, _directoryService.FileSystem.Directory.Exists(e.FullPath));
}
@ -179,14 +179,6 @@ public class LibraryWatcher : ILibraryWatcher
{
// We need to check if directory or not
if (!isDirectoryChange && !new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return;
// Don't do anything if a Library or ScanSeries in progress
// if (TaskScheduler.RunningAnyTasksByMethod(new[] {"MetadataService", "ScannerService"}))
// {
// // NOTE: I'm not sure we need this to be honest. Now with the speed of the new loop and the queue, we should just put in queue for processing
// _logger.LogDebug("Suppressing Change due to scan being inprogress");
// return;
// }
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
if (string.IsNullOrEmpty(parentDirectory)) return;
@ -206,14 +198,12 @@ public class LibraryWatcher : ILibraryWatcher
FolderPath = fullPath,
QueueTime = DateTime.Now
};
if (_scanQueue.Contains(queueItem, _folderScanQueueableComparer))
if (!_scanQueue.Contains(queueItem, _folderScanQueueableComparer))
{
ProcessQueue();
return;
_logger.LogDebug("[LibraryWatcher] Queuing job for {Folder}", fullPath);
_scanQueue.Enqueue(queueItem);
}
_scanQueue.Enqueue(queueItem);
ProcessQueue();
}
@ -228,7 +218,7 @@ public class LibraryWatcher : ILibraryWatcher
var item = _scanQueue.Peek();
if (item.QueueTime < DateTime.Now.Subtract(_queueWaitTime))
{
_logger.LogDebug("Scheduling ScanSeriesFolder for {Folder}", item.FolderPath);
_logger.LogDebug("[LibraryWatcher] Scheduling ScanSeriesFolder for {Folder}", item.FolderPath);
BackgroundJob.Enqueue(() => _scannerService.ScanFolder(item.FolderPath));
_scanQueue.Dequeue();
i++;

7
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -320,7 +320,10 @@ namespace API.Services.Tasks.Scanner
// NOTE: If we have multiple series in a folder with a localized title, then this will fail. It will group into one series. User needs to fix this themselves.
string nonLocalizedSeries;
var nonLocalizedSeriesFound = infos.Where(i => !i.IsSpecial).Select(i => i.Series).Distinct().ToList();
// Normalize this as many of the cases is a capitalization difference
var nonLocalizedSeriesFound = infos
.Where(i => !i.IsSpecial)
.Select(i => i.Series).DistinctBy(Parser.Parser.Normalize).ToList();
if (nonLocalizedSeriesFound.Count == 1)
{
nonLocalizedSeries = nonLocalizedSeriesFound.First();
@ -330,7 +333,7 @@ namespace API.Services.Tasks.Scanner
// There can be a case where there are multiple series in a folder that causes merging.
if (nonLocalizedSeriesFound.Count > 2)
{
_logger.LogError("[ScannerService] There are multiple series within one folder that contain localized series. This will cause them to group incorrectly. Please separate series into their own dedicated folder: {LocalizedSeries}", string.Join(", ", nonLocalizedSeriesFound));
_logger.LogError("[ScannerService] There are multiple series within one folder that contain localized series. This will cause them to group incorrectly. Please separate series into their own dedicated folder or ensure there is only 2 potential series (localized and series): {LocalizedSeries}", string.Join(", ", nonLocalizedSeriesFound));
}
nonLocalizedSeries = nonLocalizedSeriesFound.FirstOrDefault(s => !s.Equals(localizedSeries));
}

3
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -183,7 +183,7 @@ public class ProcessSeries : IProcessSeries
}
_logger.LogInformation("[ScannerService] Finished series update on {SeriesName} in {Milliseconds} ms", seriesName, scanWatch.ElapsedMilliseconds);
EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id, false);
EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id);
}
private async Task UpdateSeriesFolderPath(IEnumerable<ParserInfo> parsedInfos, Library library, Series series)
@ -431,7 +431,6 @@ public class ProcessSeries : IProcessSeries
volume = DbFactory.Volume(volumeNumber);
volume.SeriesId = series.Id;
series.Volumes.Add(volume);
_unitOfWork.VolumeRepository.Add(volume);
}
volume.Name = volumeNumber;