Folder Watching (#1467)
* Hooked in a server setting to enable/disable folder watching * Validated the file rename change event * Validated delete file works * Tweaked some logic to determine if a change occurs on a folder or a file. * Added a note for an upcoming branch * Some minor changes in the loop that just shift where code runs. * Implemented ScanFolder api * Ensure we restart watchers when we modify a library folder. * Fixed a unit test
This commit is contained in:
Joseph Milazzo
GitHub
parent
87ad5844f0
commit
037a1a5a8c
19 changed files with 269 additions and 102 deletions
|
|
@ -1,4 +1,4 @@
|
|||
using System;
|
||||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.Collections.Immutable;
|
||||
using System.IO;
|
||||
|
|
@ -511,7 +511,7 @@ namespace API.Services
|
|||
var fullPath = Path.Join(folder, parts.Last());
|
||||
if (!dirs.ContainsKey(fullPath))
|
||||
{
|
||||
dirs.Add(fullPath, string.Empty);
|
||||
dirs.Add(Parser.Parser.NormalizePath(fullPath), string.Empty);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -560,7 +560,7 @@ namespace API.Services
|
|||
{
|
||||
try
|
||||
{
|
||||
return Parser.Parser.NormalizePath(Directory.GetParent(fileOrFolder).FullName);
|
||||
return Parser.Parser.NormalizePath(Directory.GetParent(fileOrFolder)?.FullName);
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -1,6 +1,7 @@
|
|||
using System;
|
||||
using System.Threading;
|
||||
using System.Threading.Tasks;
|
||||
using API.Data;
|
||||
using API.Services.Tasks.Scanner;
|
||||
using Microsoft.Extensions.DependencyInjection;
|
||||
using Microsoft.Extensions.Hosting;
|
||||
|
|
@ -38,8 +39,20 @@ namespace API.Services.HostedServices
|
|||
//If stats startup fail the user can keep using the app
|
||||
}
|
||||
|
||||
var libraryWatcher = scope.ServiceProvider.GetRequiredService<ILibraryWatcher>();
|
||||
//await libraryWatcher.StartWatchingLibraries(); // TODO: Enable this in the next PR
|
||||
try
|
||||
{
|
||||
var unitOfWork = scope.ServiceProvider.GetRequiredService<IUnitOfWork>();
|
||||
if ((await unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EnableFolderWatching)
|
||||
{
|
||||
var libraryWatcher = scope.ServiceProvider.GetRequiredService<ILibraryWatcher>();
|
||||
await libraryWatcher.StartWatching();
|
||||
}
|
||||
}
|
||||
catch (Exception)
|
||||
{
|
||||
// Fail silently
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
public Task StopAsync(CancellationToken cancellationToken) => Task.CompletedTask;
|
||||
|
|
|
|||
|
|
@ -19,6 +19,7 @@ public interface ITaskScheduler
|
|||
Task ScheduleTasks();
|
||||
Task ScheduleStatsTasks();
|
||||
void ScheduleUpdaterTasks();
|
||||
void ScanFolder(string folderPath);
|
||||
void ScanLibrary(int libraryId, bool force = false);
|
||||
void CleanupChapters(int[] chapterIds);
|
||||
void RefreshMetadata(int libraryId, bool forceUpdate = true);
|
||||
|
|
@ -161,6 +162,12 @@ public class TaskScheduler : ITaskScheduler
|
|||
// Schedule update check between noon and 6pm local time
|
||||
RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local);
|
||||
}
|
||||
|
||||
public void ScanFolder(string folderPath)
|
||||
{
|
||||
_scannerService.ScanFolder(Parser.Parser.NormalizePath(folderPath));
|
||||
}
|
||||
|
||||
#endregion
|
||||
|
||||
public void ScanLibraries()
|
||||
|
|
|
|||
|
|
@ -1,5 +1,4 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
|
|
@ -14,7 +13,20 @@ namespace API.Services.Tasks.Scanner;
|
|||
|
||||
public interface ILibraryWatcher
|
||||
{
|
||||
Task StartWatchingLibraries();
|
||||
/// <summary>
|
||||
/// Start watching all library folders
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
Task StartWatching();
|
||||
/// <summary>
|
||||
/// Stop watching all folders
|
||||
/// </summary>
|
||||
void StopWatching();
|
||||
/// <summary>
|
||||
/// Essentially stops then starts watching. Useful if there is a change in folders or libraries
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
Task RestartWatching();
|
||||
}
|
||||
|
||||
internal class FolderScanQueueable
|
||||
|
|
@ -51,17 +63,12 @@ public class LibraryWatcher : ILibraryWatcher
|
|||
private readonly ILogger<LibraryWatcher> _logger;
|
||||
private readonly IScannerService _scannerService;
|
||||
|
||||
private readonly IList<FileSystemWatcher> _watchers = new List<FileSystemWatcher>();
|
||||
|
||||
private readonly Dictionary<string, IList<FileSystemWatcher>> _watcherDictionary = new ();
|
||||
|
||||
private IList<string> _libraryFolders = new List<string>();
|
||||
|
||||
// TODO: This needs to be blocking so we can consume from another thread
|
||||
private readonly Queue<FolderScanQueueable> _scanQueue = new Queue<FolderScanQueueable>();
|
||||
//public readonly BlockingCollection<FolderScanQueueable> ScanQueue = new BlockingCollection<FolderScanQueueable>();
|
||||
private readonly TimeSpan _queueWaitTime;
|
||||
|
||||
private readonly FolderScanQueueableComparer _folderScanQueueableComparer = new FolderScanQueueableComparer();
|
||||
|
||||
|
||||
public LibraryWatcher(IDirectoryService directoryService, IUnitOfWork unitOfWork, ILogger<LibraryWatcher> logger, IScannerService scannerService, IHostEnvironment environment)
|
||||
|
|
@ -75,43 +82,63 @@ public class LibraryWatcher : ILibraryWatcher
|
|||
|
||||
}
|
||||
|
||||
public async Task StartWatchingLibraries()
|
||||
public async Task StartWatching()
|
||||
{
|
||||
_logger.LogInformation("Starting file watchers");
|
||||
_libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).SelectMany(l => l.Folders).ToList();
|
||||
|
||||
foreach (var library in await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
|
||||
_libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
|
||||
.SelectMany(l => l.Folders)
|
||||
.Distinct()
|
||||
.Select(Parser.Parser.NormalizePath)
|
||||
.ToList();
|
||||
foreach (var libraryFolder in _libraryFolders)
|
||||
{
|
||||
foreach (var libraryFolder in library.Folders)
|
||||
_logger.LogInformation("Watching {FolderPath}", libraryFolder);
|
||||
var watcher = new FileSystemWatcher(libraryFolder);
|
||||
watcher.NotifyFilter = NotifyFilters.CreationTime
|
||||
| NotifyFilters.DirectoryName
|
||||
| NotifyFilters.FileName
|
||||
| NotifyFilters.LastWrite
|
||||
| NotifyFilters.Size;
|
||||
|
||||
watcher.Changed += OnChanged;
|
||||
watcher.Created += OnCreated;
|
||||
watcher.Deleted += OnDeleted;
|
||||
watcher.Renamed += OnRenamed;
|
||||
|
||||
watcher.Filter = "*.*";
|
||||
watcher.IncludeSubdirectories = true;
|
||||
watcher.EnableRaisingEvents = true;
|
||||
if (!_watcherDictionary.ContainsKey(libraryFolder))
|
||||
{
|
||||
_logger.LogInformation("Watching {FolderPath}", libraryFolder);
|
||||
var watcher = new FileSystemWatcher(libraryFolder);
|
||||
watcher.NotifyFilter = NotifyFilters.CreationTime
|
||||
| NotifyFilters.DirectoryName
|
||||
| NotifyFilters.FileName
|
||||
| NotifyFilters.LastWrite
|
||||
| NotifyFilters.Size;
|
||||
|
||||
watcher.Changed += OnChanged;
|
||||
watcher.Created += OnCreated;
|
||||
watcher.Deleted += OnDeleted;
|
||||
watcher.Renamed += OnRenamed;
|
||||
|
||||
watcher.Filter = "*.*"; // TODO: Configure with Parser files
|
||||
watcher.IncludeSubdirectories = true;
|
||||
watcher.EnableRaisingEvents = true;
|
||||
_logger.LogInformation("Watching {Folder}", libraryFolder);
|
||||
_watchers.Add(watcher);
|
||||
if (!_watcherDictionary.ContainsKey(libraryFolder))
|
||||
{
|
||||
_watcherDictionary.Add(libraryFolder, new List<FileSystemWatcher>());
|
||||
}
|
||||
|
||||
_watcherDictionary[libraryFolder].Add(watcher);
|
||||
_watcherDictionary.Add(libraryFolder, new List<FileSystemWatcher>());
|
||||
}
|
||||
|
||||
_watcherDictionary[libraryFolder].Add(watcher);
|
||||
}
|
||||
}
|
||||
|
||||
public void StopWatching()
|
||||
{
|
||||
_logger.LogInformation("Stopping watching folders");
|
||||
foreach (var fileSystemWatcher in _watcherDictionary.Values.SelectMany(watcher => watcher))
|
||||
{
|
||||
fileSystemWatcher.EnableRaisingEvents = false;
|
||||
fileSystemWatcher.Changed -= OnChanged;
|
||||
fileSystemWatcher.Created -= OnCreated;
|
||||
fileSystemWatcher.Deleted -= OnDeleted;
|
||||
fileSystemWatcher.Renamed -= OnRenamed;
|
||||
fileSystemWatcher.Dispose();
|
||||
}
|
||||
_watcherDictionary.Clear();
|
||||
}
|
||||
|
||||
public async Task RestartWatching()
|
||||
{
|
||||
StopWatching();
|
||||
await StartWatching();
|
||||
}
|
||||
|
||||
private void OnChanged(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
if (e.ChangeType != WatcherChangeTypes.Changed) return;
|
||||
|
|
@ -122,12 +149,15 @@ public class LibraryWatcher : ILibraryWatcher
|
|||
private void OnCreated(object sender, FileSystemEventArgs e)
|
||||
{
|
||||
Console.WriteLine($"Created: {e.FullPath}, {e.Name}");
|
||||
ProcessChange(e.FullPath);
|
||||
ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name));
|
||||
}
|
||||
|
||||
private void OnDeleted(object sender, FileSystemEventArgs e) {
|
||||
Console.WriteLine($"Deleted: {e.FullPath}, {e.Name}");
|
||||
ProcessChange(e.FullPath);
|
||||
|
||||
// On deletion, we need another type of check. We need to check if e.Name has an extension or not
|
||||
// NOTE: File deletion will trigger a folder change event, so this might not be needed
|
||||
ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name)));
|
||||
}
|
||||
|
||||
|
||||
|
|
@ -137,18 +167,25 @@ public class LibraryWatcher : ILibraryWatcher
|
|||
Console.WriteLine($"Renamed:");
|
||||
Console.WriteLine($" Old: {e.OldFullPath}");
|
||||
Console.WriteLine($" New: {e.FullPath}");
|
||||
ProcessChange(e.FullPath);
|
||||
ProcessChange(e.FullPath, _directoryService.FileSystem.Directory.Exists(e.FullPath));
|
||||
}
|
||||
|
||||
private void ProcessChange(string filePath)
|
||||
/// <summary>
|
||||
/// Processes the file or folder change.
|
||||
/// </summary>
|
||||
/// <param name="filePath">File or folder that changed</param>
|
||||
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
|
||||
private void ProcessChange(string filePath, bool isDirectoryChange = false)
|
||||
{
|
||||
if (!new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return;
|
||||
// We need to check if directory or not
|
||||
if (!isDirectoryChange && !new Regex(Parser.Parser.SupportedExtensions).IsMatch(new FileInfo(filePath).Extension)) return;
|
||||
// Don't do anything if a Library or ScanSeries in progress
|
||||
if (TaskScheduler.RunningAnyTasksByMethod(new[] {"MetadataService", "ScannerService"}))
|
||||
{
|
||||
_logger.LogDebug("Suppressing Change due to scan being inprogress");
|
||||
return;
|
||||
}
|
||||
// if (TaskScheduler.RunningAnyTasksByMethod(new[] {"MetadataService", "ScannerService"}))
|
||||
// {
|
||||
// // NOTE: I'm not sure we need this to be honest. Now with the speed of the new loop and the queue, we should just put in queue for processing
|
||||
// _logger.LogDebug("Suppressing Change due to scan being inprogress");
|
||||
// return;
|
||||
// }
|
||||
|
||||
|
||||
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
|
||||
|
|
@ -156,21 +193,20 @@ public class LibraryWatcher : ILibraryWatcher
|
|||
|
||||
// We need to find the library this creation belongs to
|
||||
// Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault
|
||||
var libraryFolder = _libraryFolders.Select(Parser.Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory));
|
||||
|
||||
var libraryFolder = _libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f));
|
||||
if (string.IsNullOrEmpty(libraryFolder)) return;
|
||||
|
||||
var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList();
|
||||
if (!rootFolder.Any()) return;
|
||||
|
||||
// Select the first folder and join with library folder, this should give us the folder to scan.
|
||||
var fullPath = _directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First());
|
||||
var fullPath = Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()));
|
||||
var queueItem = new FolderScanQueueable()
|
||||
{
|
||||
FolderPath = fullPath,
|
||||
QueueTime = DateTime.Now
|
||||
};
|
||||
if (_scanQueue.Contains(queueItem, new FolderScanQueueableComparer()))
|
||||
if (_scanQueue.Contains(queueItem, _folderScanQueueableComparer))
|
||||
{
|
||||
ProcessQueue();
|
||||
return;
|
||||
|
|
@ -205,7 +241,7 @@ public class LibraryWatcher : ILibraryWatcher
|
|||
|
||||
if (_scanQueue.Count > 0)
|
||||
{
|
||||
Task.Delay(TimeSpan.FromSeconds(10)).ContinueWith(t=> ProcessQueue());
|
||||
Task.Delay(_queueWaitTime).ContinueWith(t=> ProcessQueue());
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -116,14 +116,16 @@ public class ProcessSeries : IProcessSeries
|
|||
{
|
||||
_logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName);
|
||||
|
||||
var firstParsedInfo = parsedInfos[0];
|
||||
|
||||
UpdateVolumes(series, parsedInfos);
|
||||
series.Pages = series.Volumes.Sum(v => v.Pages);
|
||||
|
||||
series.NormalizedName = Parser.Parser.Normalize(series.Name);
|
||||
series.OriginalName ??= parsedInfos[0].Series;
|
||||
series.OriginalName ??= firstParsedInfo.Series;
|
||||
if (series.Format == MangaFormat.Unknown)
|
||||
{
|
||||
series.Format = parsedInfos[0].Format;
|
||||
series.Format = firstParsedInfo.Format;
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(series.SortName))
|
||||
|
|
@ -133,9 +135,9 @@ public class ProcessSeries : IProcessSeries
|
|||
if (!series.SortNameLocked)
|
||||
{
|
||||
series.SortName = series.Name;
|
||||
if (!string.IsNullOrEmpty(parsedInfos[0].SeriesSort))
|
||||
if (!string.IsNullOrEmpty(firstParsedInfo.SeriesSort))
|
||||
{
|
||||
series.SortName = parsedInfos[0].SeriesSort;
|
||||
series.SortName = firstParsedInfo.SeriesSort;
|
||||
}
|
||||
}
|
||||
|
||||
|
|
@ -147,27 +149,11 @@ public class ProcessSeries : IProcessSeries
|
|||
series.NormalizedLocalizedName = Parser.Parser.Normalize(series.LocalizedName);
|
||||
}
|
||||
|
||||
// Update series FolderPath here (TODO: Move this into it's own private method)
|
||||
var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(library.Folders.Select(l => l.Path), parsedInfos.Select(f => f.FullFilePath).ToList());
|
||||
if (seriesDirs.Keys.Count == 0)
|
||||
{
|
||||
_logger.LogCritical("Scan Series has files spread outside a main series folder. This has negative performance effects. Please ensure all series are under a single folder from library");
|
||||
await _eventHub.SendMessageAsync(MessageFactory.Info,
|
||||
MessageFactory.InfoEvent($"{series.Name} has files spread outside a single series folder",
|
||||
"This has negative performance effects. Please ensure all series are under a single folder from library"));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Don't save FolderPath if it's a library Folder
|
||||
if (!library.Folders.Select(f => f.Path).Contains(seriesDirs.Keys.First()))
|
||||
{
|
||||
series.FolderPath = Parser.Parser.NormalizePath(seriesDirs.Keys.First());
|
||||
}
|
||||
}
|
||||
|
||||
series.Metadata ??= DbFactory.SeriesMetadata(new List<CollectionTag>());
|
||||
UpdateSeriesMetadata(series, library.Type);
|
||||
|
||||
// Update series FolderPath here
|
||||
await UpdateSeriesFolderPath(parsedInfos, library, series);
|
||||
|
||||
series.LastFolderScanned = DateTime.Now;
|
||||
_unitOfWork.SeriesRepository.Attach(series);
|
||||
|
||||
|
|
@ -200,6 +186,28 @@ public class ProcessSeries : IProcessSeries
|
|||
EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id, false);
|
||||
}
|
||||
|
||||
private async Task UpdateSeriesFolderPath(IEnumerable<ParserInfo> parsedInfos, Library library, Series series)
|
||||
{
|
||||
var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(library.Folders.Select(l => l.Path),
|
||||
parsedInfos.Select(f => f.FullFilePath).ToList());
|
||||
if (seriesDirs.Keys.Count == 0)
|
||||
{
|
||||
_logger.LogCritical(
|
||||
"Scan Series has files spread outside a main series folder. This has negative performance effects. Please ensure all series are under a single folder from library");
|
||||
await _eventHub.SendMessageAsync(MessageFactory.Info,
|
||||
MessageFactory.InfoEvent($"{series.Name} has files spread outside a single series folder",
|
||||
"This has negative performance effects. Please ensure all series are under a single folder from library"));
|
||||
}
|
||||
else
|
||||
{
|
||||
// Don't save FolderPath if it's a library Folder
|
||||
if (!library.Folders.Select(f => f.Path).Contains(seriesDirs.Keys.First()))
|
||||
{
|
||||
series.FolderPath = Parser.Parser.NormalizePath(seriesDirs.Keys.First());
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public void EnqueuePostSeriesProcessTasks(int libraryId, int seriesId, bool forceUpdate = false)
|
||||
{
|
||||
BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(libraryId, seriesId, forceUpdate));
|
||||
|
|
@ -208,6 +216,7 @@ public class ProcessSeries : IProcessSeries
|
|||
|
||||
private static void UpdateSeriesMetadata(Series series, LibraryType libraryType)
|
||||
{
|
||||
series.Metadata ??= DbFactory.SeriesMetadata(new List<CollectionTag>());
|
||||
var isBook = libraryType == LibraryType.Book;
|
||||
var firstChapter = SeriesService.GetFirstChapterForMetadata(series, isBook);
|
||||
|
||||
|
|
|
|||
|
|
@ -100,8 +100,6 @@ public class ScannerService : IScannerService
|
|||
[Queue(TaskScheduler.ScanQueue)]
|
||||
public async Task ScanFolder(string folder)
|
||||
{
|
||||
// NOTE: I might want to move a lot of this code to the LibraryWatcher or something and just pack libraryId and seriesId
|
||||
// Validate if we are scanning a new series (that belongs to a library) or an existing series
|
||||
var seriesId = await _unitOfWork.SeriesRepository.GetSeriesIdByFolder(folder);
|
||||
if (seriesId > 0)
|
||||
{
|
||||
|
|
@ -109,6 +107,7 @@ public class ScannerService : IScannerService
|
|||
return;
|
||||
}
|
||||
|
||||
// This is basically rework of what's already done in Library Watcher but is needed if invoked via API
|
||||
var parentDirectory = _directoryService.GetParentDirectoryName(folder);
|
||||
if (string.IsNullOrEmpty(parentDirectory)) return; // This should never happen as it's calculated before enqueing
|
||||
|
||||
|
|
@ -456,6 +455,8 @@ public class ScannerService : IScannerService
|
|||
Format = parsedFiles.First().Format
|
||||
};
|
||||
|
||||
// NOTE: Could we check if there are multiple found series (different series) and process each one?
|
||||
|
||||
if (skippedScan)
|
||||
{
|
||||
seenSeries.AddRange(parsedFiles.Select(pf => new ParsedSeries()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue