AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Folder Watching Polish + Epub Fix (#1550)

Browse source 
* Fixed entrypoint writing bad json (from develop)

* Fixed a bug where log file could write out a crap ton of information (serializing Series object) when a db error occurs.

* Fixed an issue with scan loop where concurrency issues could occur on new series being added.

* Tweaked the logger to suppress some noisy logs when using Debug log level.

* Fixed a regression with epub parsing from v3.2 of Vers-One's release

* Fixed up folder watching to work more reliable. Validated in production.

* Code cleanup
This commit is contained in:
Joseph Milazzo 2022-09-20 18:14:40 -05:00 • committed by GitHub
parent 356d2c8015
commit 4ecb44d3d0
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 67 additions and 44 deletions
Download patch file Download diff file Expand all files Collapse all files

53
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -40,12 +40,11 @@ public class LibraryWatcher : ILibraryWatcher
private readonly ILogger<LibraryWatcher> _logger;
private readonly IScannerService _scannerService;
private readonly Dictionary<string, IList<FileSystemWatcher>> _watcherDictionary = new ();
private static readonly Dictionary<string, IList<FileSystemWatcher>> WatcherDictionary = new ();
/// <summary>
/// This is just here to prevent GC from Disposing our watchers
/// </summary>
private readonly IList<FileSystemWatcher> _fileWatchers = new List<FileSystemWatcher>();
private IList<string> _libraryFolders = new List<string>();
private static readonly IList<FileSystemWatcher> FileWatchers = new List<FileSystemWatcher>();
/// <summary>
/// The amount of time until the Schedule ScanFolder task should be executed
/// </summary>
@ -68,13 +67,14 @@ public class LibraryWatcher : ILibraryWatcher
{
_logger.LogInformation("[LibraryWatcher] Starting file watchers");
_libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
var libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
.SelectMany(l => l.Folders)
.Distinct()
.Select(Parser.Parser.NormalizePath)
.Where(_directoryService.Exists)
.ToList();
foreach (var libraryFolder in _libraryFolders)
foreach (var libraryFolder in libraryFolders)
{
_logger.LogDebug("[LibraryWatcher] Watching {FolderPath}", libraryFolder);
var watcher = new FileSystemWatcher(libraryFolder);
@ -87,21 +87,21 @@ public class LibraryWatcher : ILibraryWatcher
watcher.Filter = "*.*";
watcher.IncludeSubdirectories = true;
watcher.EnableRaisingEvents = true;
_fileWatchers.Add(watcher);
if (!_watcherDictionary.ContainsKey(libraryFolder))
FileWatchers.Add(watcher);
if (!WatcherDictionary.ContainsKey(libraryFolder))
{
_watcherDictionary.Add(libraryFolder, new List<FileSystemWatcher>());
WatcherDictionary.Add(libraryFolder, new List<FileSystemWatcher>());
}
_watcherDictionary[libraryFolder].Add(watcher);
WatcherDictionary[libraryFolder].Add(watcher);
}
_logger.LogInformation("[LibraryWatcher] Watching {Count} folders", _fileWatchers.Count);
_logger.LogInformation("[LibraryWatcher] Watching {Count} folders", FileWatchers.Count);
}
public void StopWatching()
{
_logger.LogInformation("[LibraryWatcher] Stopping watching folders");
foreach (var fileSystemWatcher in _watcherDictionary.Values.SelectMany(watcher => watcher))
foreach (var fileSystemWatcher in WatcherDictionary.Values.SelectMany(watcher => watcher))
{
fileSystemWatcher.EnableRaisingEvents = false;
fileSystemWatcher.Changed -= OnChanged;
@ -110,12 +110,13 @@ public class LibraryWatcher : ILibraryWatcher
fileSystemWatcher.Error -= OnError;
fileSystemWatcher.Dispose();
}
_fileWatchers.Clear();
_watcherDictionary.Clear();
FileWatchers.Clear();
WatcherDictionary.Clear();
}
public async Task RestartWatching()
{
_logger.LogDebug("[LibraryWatcher] Restarting watcher");
StopWatching();
await StartWatching();
}
@ -160,7 +161,7 @@ public class LibraryWatcher : ILibraryWatcher
/// <remarks>This is public only because Hangfire will invoke it. Do not call external to this class.</remarks>
/// <param name="filePath">File or folder that changed</param>
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
public void ProcessChange(string filePath, bool isDirectoryChange = false)
public async Task ProcessChange(string filePath, bool isDirectoryChange = false)
{
var sw = Stopwatch.StartNew();
_logger.LogDebug("[LibraryWatcher] Processing change of {FilePath}", filePath);
@ -174,26 +175,33 @@ public class LibraryWatcher : ILibraryWatcher
return;
}
var fullPath = GetFolder(filePath, _libraryFolders);
var libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
.SelectMany(l => l.Folders)
.Distinct()
.Select(Parser.Parser.NormalizePath)
.Where(_directoryService.Exists)
.ToList();
var fullPath = GetFolder(filePath, libraryFolders);
_logger.LogDebug("Folder path: {FolderPath}", fullPath);
if (string.IsNullOrEmpty(fullPath))
{
_logger.LogDebug("[LibraryWatcher] Change from {FilePath} could not find root level folder, ignoring change", filePath);
return;
}
// Check if this task has already enqueued or is being processed, before enquing
// Check if this task has already enqueued or is being processed, before enqueing
var alreadyScheduled =
TaskScheduler.HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {fullPath});
_logger.LogDebug("[LibraryWatcher] {FullPath} already enqueued: {Value}", fullPath, alreadyScheduled);
if (!alreadyScheduled)
{
_logger.LogDebug("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath);
_logger.LogInformation("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath);
BackgroundJob.Schedule(() => _scannerService.ScanFolder(fullPath), _queueWaitTime);
}
else
{
_logger.LogDebug("[LibraryWatcher] Skipped scheduling ScanFolder for {Folder} as a job already queued",
_logger.LogInformation("[LibraryWatcher] Skipped scheduling ScanFolder for {Folder} as a job already queued",
fullPath);
}
}
@ -207,18 +215,17 @@ public class LibraryWatcher : ILibraryWatcher
private string GetFolder(string filePath, IList<string> libraryFolders)
{
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
if (string.IsNullOrEmpty(parentDirectory))
{
return string.Empty;
}
_logger.LogDebug("[LibraryWatcher] Parent Directory: {ParentDirectory}", parentDirectory);
if (string.IsNullOrEmpty(parentDirectory)) return string.Empty;
// We need to find the library this creation belongs to
// Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault
var libraryFolder = libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f));
_logger.LogDebug("[LibraryWatcher] Library Folder: {LibraryFolder}", libraryFolder);
if (string.IsNullOrEmpty(libraryFolder)) return string.Empty;
var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList();
_logger.LogDebug("[LibraryWatcher] Root Folders: {RootFolders}", rootFolder);
if (!rootFolder.Any()) return string.Empty;
// Select the first folder and join with library folder, this should give us the folder to scan.

10
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -44,9 +44,9 @@ public class ProcessSeries : IProcessSeries
private readonly IMetadataService _metadataService;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private IList<Genre> _genres;
private IList<Person> _people;
private IList<Tag> _tags;
private volatile IList<Genre> _genres;
private volatile IList<Person> _people;
private volatile IList<Tag> _tags;
@ -108,6 +108,7 @@ public class ProcessSeries : IProcessSeries
{
seriesAdded = true;
series = DbFactory.Series(firstInfo.Series, firstInfo.LocalizedSeries);
_unitOfWork.SeriesRepository.Add(series);
}
if (series.LibraryId == 0) series.LibraryId = library.Id;
@ -156,7 +157,6 @@ public class ProcessSeries : IProcessSeries
await UpdateSeriesFolderPath(parsedInfos, library, series);
series.LastFolderScanned = DateTime.Now;
_unitOfWork.SeriesRepository.Attach(series);
if (_unitOfWork.HasChanges())
{
@ -167,7 +167,7 @@ public class ProcessSeries : IProcessSeries
catch (Exception ex)
{
await _unitOfWork.RollbackAsync();
_logger.LogCritical(ex, "[ScannerService] There was an issue writing to the for series {@SeriesName}", series);
_logger.LogCritical(ex, "[ScannerService] There was an issue writing to the database for series {@SeriesName}", series.Name);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"There was an issue writing to the DB for Series {series}",