AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Folder Watching (#1467)

Browse source 
* Hooked in a server setting to enable/disable folder watching

* Validated the file rename change event

* Validated delete file works

* Tweaked some logic to determine if a change occurs on a folder or a file.

* Added a note for an upcoming branch

* Some minor changes in the loop that just shift where code runs.

* Implemented ScanFolder api

* Ensure we restart watchers when we modify a library folder.

* Fixed a unit test
This commit is contained in:
Joseph Milazzo 2022-08-23 11:45:11 -05:00 • committed by GitHub
parent 87ad5844f0
commit 037a1a5a8c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 269 additions and 102 deletions
Download patch file Download diff file Expand all files Collapse all files

138
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -1,5 +1,4 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@ -14,7 +13,20 @@ namespace API.Services.Tasks.Scanner;
public interface ILibraryWatcher
{
Task StartWatchingLibraries();
/// <summary>
/// Start watching all library folders
/// </summary>
/// <returns></returns>
Task StartWatching();
/// <summary>
/// Stop watching all folders
/// </summary>
void StopWatching();
/// <summary>
/// Essentially stops then starts watching. Useful if there is a change in folders or libraries
/// </summary>
/// <returns></returns>
Task RestartWatching();
}
internal class FolderScanQueueable
@ -51,17 +63,12 @@ public class LibraryWatcher : ILibraryWatcher
private readonly ILogger<LibraryWatcher> _logger;
private readonly IScannerService _scannerService;
private readonly IList<FileSystemWatcher> _watchers = new List<FileSystemWatcher>();
private readonly Dictionary<string, IList<FileSystemWatcher>> _watcherDictionary = new ();
private IList<string> _libraryFolders = new List<string>();
// TODO: This needs to be blocking so we can consume from another thread
private readonly Queue<FolderScanQueueable> _scanQueue = new Queue<FolderScanQueueable>();
//public readonly BlockingCollection<FolderScanQueueable> ScanQueue = new BlockingCollection<FolderScanQueueable>();
private readonly TimeSpan _queueWaitTime;
private readonly FolderScanQueueableComparer _folderScanQueueableComparer = new FolderScanQueueableComparer();
public LibraryWatcher(IDirectoryService directoryService, IUnitOfWork unitOfWork, ILogger<LibraryWatcher> logger, IScannerService scannerService, IHostEnvironment environment)
@ -75,43 +82,63 @@ public class LibraryWatcher : ILibraryWatcher
}
public async Task StartWatchingLibraries()
public async Task StartWatching()
{
_logger.LogInformation("Starting file watchers");
_libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).SelectMany(l => l.Folders).ToList();
foreach (var library in await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
_libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
.SelectMany(l => l.Folders)
.Distinct()
.Select(Parser.Parser.NormalizePath)
.ToList();
foreach (var libraryFolder in _libraryFolders)
{
foreach (var libraryFolder in library.Folders)
_logger.LogInformation("Watching {FolderPath}", libraryFolder);
var watcher = new FileSystemWatcher(libraryFolder);
watcher.NotifyFilter = NotifyFilters.CreationTime
| NotifyFilters.DirectoryName
| NotifyFilters.FileName
| NotifyFilters.LastWrite
| NotifyFilters.Size;
watcher.Changed += OnChanged;
watcher.Created += OnCreated;
watcher.Deleted += OnDeleted;
watcher.Renamed += OnRenamed;
watcher.Filter = "*.*";
watcher.IncludeSubdirectories = true;
watcher.EnableRaisingEvents = true;
if (!_watcherDictionary.ContainsKey(libraryFolder))
{
_logger.LogInformation("Watching {FolderPath}", libraryFolder);
var watcher = new FileSystemWatcher(libraryFolder);
watcher.NotifyFilter = NotifyFilters.CreationTime
| NotifyFilters.DirectoryName
| NotifyFilters.FileName
| NotifyFilters.LastWrite
| NotifyFilters.Size;
watcher.Changed += OnChanged;
watcher.Created += OnCreated;
watcher.Deleted += OnDeleted;
watcher.Renamed += OnRenamed;
watcher.Filter = "*.*"; // TODO: Configure with Parser files
watcher.IncludeSubdirectories = true;
watcher.EnableRaisingEvents = true;
_logger.LogInformation("Watching {Folder}", libraryFolder);
_watchers.Add(watcher);
if (!_watcherDictionary.ContainsKey(libraryFolder))
{
_watcherDictionary.Add(libraryFolder, new List<FileSystemWatcher>());
}
_watcherDictionary[libraryFolder].Add(watcher);
_watcherDictionary.Add(libraryFolder, new List<FileSystemWatcher>());
}
_watcherDictionary[libraryFolder].Add(watcher);
}
}
public void StopWatching()
{
_logger.LogInformation("Stopping watching folders");
foreach (var fileSystemWatcher in _watcherDictionary.Values.SelectMany(watcher => watcher))
{
fileSystemWatcher.EnableRaisingEvents = false;
fileSystemWatcher.Changed -= OnChanged;
fileSystemWatcher.Created -= OnCreated;
fileSystemWatcher.Deleted -= OnDeleted;
fileSystemWatcher.Renamed -= OnRenamed;
fileSystemWatcher.Dispose();
}
_watcherDictionary.Clear();
}
public async Task RestartWatching()
{
StopWatching();
await StartWatching();
}
private void OnChanged(object sender, FileSystemEventArgs e)
{
if (e.ChangeType != WatcherChangeTypes.Changed) return;
@ -122,12 +149,15 @@ public class LibraryWatcher : ILibraryWatcher
private void OnCreated(object sender, FileSystemEventArgs e)
{
Console.WriteLine($"Created: {e.FullPath}, {e.Name}");
ProcessChange(e.FullPath);
ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name));
}
private void OnDeleted(object sender, FileSystemEventArgs e) {
Console.WriteLine($"Deleted: {e.FullPath}, {e.Name}");
ProcessChange(e.FullPath);
// On deletion, we need another type of check. We need to check if e.Name has an extension or not
// NOTE: File deletion will trigger a folder change event, so this might not be needed
ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name)));
}
@ -137,18 +167,25 @@ public class LibraryWatcher : ILibraryWatcher
Console.WriteLine($"Renamed:");
Console.WriteLine($" Old: {e.OldFullPath}");
Console.WriteLine($" New: {e.FullPath}");
ProcessChange(e.FullPath);
ProcessChange(e.FullPath, _directoryService.FileSystem.Directory.Exists(e.FullPath));
}
private void ProcessChange(string filePath)
/// <summary>
/// Processes the file or folder change.
/// </summary>
/// <param name="filePath">File or folder that changed</param>
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
private void ProcessChange(string filePath, bool isDirectoryChange = false)
{
if (!new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return;
// We need to check if directory or not
if (!isDirectoryChange && !new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return;
// Don't do anything if a Library or ScanSeries in progress
if (TaskScheduler.RunningAnyTasksByMethod(new[] {"MetadataService", "ScannerService"}))
{
_logger.LogDebug("Suppressing Change due to scan being inprogress");
return;
}
// if (TaskScheduler.RunningAnyTasksByMethod(new[] {"MetadataService", "ScannerService"}))
// {
// // NOTE: I'm not sure we need this to be honest. Now with the speed of the new loop and the queue, we should just put in queue for processing
// _logger.LogDebug("Suppressing Change due to scan being inprogress");
// return;
// }
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
@ -156,21 +193,20 @@ public class LibraryWatcher : ILibraryWatcher
// We need to find the library this creation belongs to
// Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault
var libraryFolder = _libraryFolders.Select(Parser.Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory));
var libraryFolder = _libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f));
if (string.IsNullOrEmpty(libraryFolder)) return;
var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList();
if (!rootFolder.Any()) return;
// Select the first folder and join with library folder, this should give us the folder to scan.
var fullPath = _directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First());
var fullPath = Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()));
var queueItem = new FolderScanQueueable()
{
FolderPath = fullPath,
QueueTime = DateTime.Now
};
if (_scanQueue.Contains(queueItem, new FolderScanQueueableComparer()))
if (_scanQueue.Contains(queueItem, _folderScanQueueableComparer))
{
ProcessQueue();
return;
@ -205,7 +241,7 @@ public class LibraryWatcher : ILibraryWatcher
if (_scanQueue.Count > 0)
{
Task.Delay(TimeSpan.FromSeconds(10)).ContinueWith(t=> ProcessQueue());
Task.Delay(_queueWaitTime).ContinueWith(t=> ProcessQueue());
}
}