AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Misc Polish and Fixes (#1542)

Browse source 
* Moved LibraryWatcher to utilize a queue for calculating the change event to ensure the Watcher doesn't get overwhelmed on large moves.

* Fixed a security vulnerability (https://huntr.dev/bounties/8a3e652f-d6bf-436e-877e-0eaf5c69ef95/). This will be disclosed in Stable release changelog.

* Tweaked the log message template

* Removed some dead code from Configuration json patcher

* Fixed a bug with the ComicInfo finding to properly handle root level.

Fixed a bug where sometimes scanner wouldn't choose the first file with ComicInfo for filling out information.

* Added new setting for managing how many logs files are allowed, just like how backups work.

* Added unit tests for new CleanupLogs code

* Fixed a bug where manga reader background color wasn't actually sending from the UI

* Added new stats for tracking to help understand usage in the app and what features are used or not.

* Fixed Stats url

* Fixed a bug where volumes that had larger than 1 difference wouldn't properly return next/prev chapter (for continuous reader)

* Remove a redundant test step in build pipeline, since it's already done at PR stage.

* Updated dockerfile to use the new Heath check endpoint

* Allow force to pass through to scan loop

* Removed some old config stuff from a safety check on config in entrypoint.sh

* Fixed broken unit tests due to new RBS check and how we setup mock data.
This commit is contained in:
Joseph Milazzo 2022-09-18 12:24:30 -05:00 • committed by GitHub
parent c58c7deaf9
commit e89a06865c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 702 additions and 308 deletions
Download patch file Download diff file Expand all files Collapse all files

80
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -91,8 +91,11 @@ public class LibraryWatcher : ILibraryWatcher
/// This is just here to prevent GC from Disposing our watchers
/// </summary>
private readonly IList<FileSystemWatcher> _fileWatchers = new List<FileSystemWatcher>();
private IList<string> _libraryFolders = new List<string>();
private static IList<string> _libraryFolders = new List<string>();
/// <summary>
/// The amount of time until the Schedule ScanFolder task should be executed
/// </summary>
/// <remarks>The Job will be enqueued instantly</remarks>
private readonly TimeSpan _queueWaitTime;
@ -109,7 +112,7 @@ public class LibraryWatcher : ILibraryWatcher
public async Task StartWatching()
{
_logger.LogInformation("Starting file watchers");
_logger.LogInformation("[LibraryWatcher] Starting file watchers");
_libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
.SelectMany(l => l.Folders)
@ -119,7 +122,7 @@ public class LibraryWatcher : ILibraryWatcher
.ToList();
foreach (var libraryFolder in _libraryFolders)
{
_logger.LogDebug("Watching {FolderPath}", libraryFolder);
_logger.LogDebug("[LibraryWatcher] Watching {FolderPath}", libraryFolder);
var watcher = new FileSystemWatcher(libraryFolder);
watcher.Changed += OnChanged;
@ -138,17 +141,19 @@ public class LibraryWatcher : ILibraryWatcher
_watcherDictionary[libraryFolder].Add(watcher);
}
_logger.LogInformation("[LibraryWatcher] Watching {Count} folders", _fileWatchers.Count);
}
public void StopWatching()
{
_logger.LogInformation("Stopping watching folders");
_logger.LogInformation("[LibraryWatcher] Stopping watching folders");
foreach (var fileSystemWatcher in _watcherDictionary.Values.SelectMany(watcher => watcher))
{
fileSystemWatcher.EnableRaisingEvents = false;
fileSystemWatcher.Changed -= OnChanged;
fileSystemWatcher.Created -= OnCreated;
fileSystemWatcher.Deleted -= OnDeleted;
fileSystemWatcher.Error -= OnError;
fileSystemWatcher.Dispose();
}
_fileWatchers.Clear();
@ -165,13 +170,13 @@ public class LibraryWatcher : ILibraryWatcher
{
if (e.ChangeType != WatcherChangeTypes.Changed) return;
_logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}", e.FullPath, e.Name);
ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name)));
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name))));
}
private void OnCreated(object sender, FileSystemEventArgs e)
{
_logger.LogDebug("[LibraryWatcher] Created: {FullPath}, {Name}", e.FullPath, e.Name);
ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name));
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name)));
}
/// <summary>
@ -183,7 +188,7 @@ public class LibraryWatcher : ILibraryWatcher
var isDirectory = string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name));
if (!isDirectory) return;
_logger.LogDebug("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name);
ProcessChange(e.FullPath, true);
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, true));
}
@ -198,35 +203,42 @@ public class LibraryWatcher : ILibraryWatcher
/// Processes the file or folder change. If the change is a file change and not from a supported extension, it will be ignored.
/// </summary>
/// <remarks>This will ignore image files that are added to the system. However, they may still trigger scans due to folder changes.</remarks>
/// <remarks>This is public only because Hangfire will invoke it. Do not call external to this class.</remarks>
/// <param name="filePath">File or folder that changed</param>
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
private void ProcessChange(string filePath, bool isDirectoryChange = false)
public void ProcessChange(string filePath, bool isDirectoryChange = false)
{
var sw = Stopwatch.StartNew();
_logger.LogDebug("[LibraryWatcher] Processing change of {FilePath}", filePath);
try
{
// We need to check if directory or not
// If not a directory change AND file is not an archive or book, ignore
if (!isDirectoryChange &&
!(Parser.Parser.IsArchive(filePath) || Parser.Parser.IsBook(filePath))) return;
!(Parser.Parser.IsArchive(filePath) || Parser.Parser.IsBook(filePath)))
{
_logger.LogDebug("[LibraryWatcher] Change from {FilePath} is not an archive or book, ignoring change", filePath);
return;
}
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
if (string.IsNullOrEmpty(parentDirectory)) return;
// var libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
// .SelectMany(l => l.Folders)
// .Distinct()
// .Select(Parser.Parser.NormalizePath)
// .Where(_directoryService.Exists)
// .ToList();
// We need to find the library this creation belongs to
// Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault
var libraryFolder = _libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f));
if (string.IsNullOrEmpty(libraryFolder)) return;
var fullPath = GetFolder(filePath, _libraryFolders);
if (string.IsNullOrEmpty(fullPath))
{
_logger.LogDebug("[LibraryWatcher] Change from {FilePath} could not find root level folder, ignoring change", filePath);
return;
}
var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList();
if (!rootFolder.Any()) return;
// Select the first folder and join with library folder, this should give us the folder to scan.
var fullPath =
Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()));
// Check if this task has already enqueued or is being processed, before enquing
var alreadyScheduled =
TaskScheduler.HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {fullPath});
_logger.LogDebug("{FullPath} already enqueued: {Value}", fullPath, alreadyScheduled);
_logger.LogDebug("[LibraryWatcher] {FullPath} already enqueued: {Value}", fullPath, alreadyScheduled);
if (!alreadyScheduled)
{
_logger.LogDebug("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath);
@ -242,9 +254,27 @@ public class LibraryWatcher : ILibraryWatcher
{
_logger.LogError(ex, "[LibraryWatcher] An error occured when processing a watch event");
}
_logger.LogDebug("ProcessChange occured in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds);
_logger.LogDebug("[LibraryWatcher] ProcessChange ran in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds);
}
private string GetFolder(string filePath, IList<string> libraryFolders)
{
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
if (string.IsNullOrEmpty(parentDirectory))
{
return string.Empty;
}
if (string.IsNullOrEmpty(parentDirectory)) return string.Empty;
// We need to find the library this creation belongs to
// Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault
var libraryFolder = libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f));
if (string.IsNullOrEmpty(libraryFolder)) return string.Empty;
var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList();
if (!rootFolder.Any()) return string.Empty;
// Select the first folder and join with library folder, this should give us the folder to scan.
return Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()));
}
}