AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Simplify Folder Watcher (#1484)

Browse source 
* Refactored Library Watcher to use Hangfire under the hood.

* Support .kavitaignore at root level.

* Refactored a lot of the library watching code to process faster and handle when FileSystemWatcher runs out of internal buffer space. It's still not perfect, but good enough for basic use.

* Make folder watching as experimental and default it to off by default.

* Revert #1479

* Tweaked the messaging for OPDS to remove a note about download role.

Moved some code closer to where it's used.

* Cleaned up how the events widget reports

* Fixed a null issue when deleting series in the UI

* Cleaned up some debug code

* Added more information for when we skip a scan

* Cleaned up some logging messages in CoverGen tasks

* More log message tweaks

* Added some debug to help identify a rare issue

* Fixed a bug where save bookmarks as webp could get reset to false when saving other server settings

* Updated some documentation on library watcher.

* Make LibraryWatcher fire every 5 mins
This commit is contained in:
Joseph Milazzo 2022-08-28 15:20:46 -05:00 • committed by GitHub
parent b64ed6df8d
commit b07aaf1eb5
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 187 additions and 259 deletions
Download patch file Download diff file Expand all files Collapse all files

41
API/Services/DirectoryService.cs
View file

@ -10,6 +10,7 @@ using API.DTOs.System;
using API.Entities.Enums;
using API.Extensions;
using Kavita.Common.Helpers;
using Microsoft.Extensions.FileSystemGlobbing;
using Microsoft.Extensions.Logging;
namespace API.Services
@ -64,14 +65,17 @@ namespace API.Services
SearchOption searchOption = SearchOption.TopDirectoryOnly);
IEnumerable<string> GetDirectories(string folderPath);
IEnumerable<string> GetDirectories(string folderPath, GlobMatcher matcher);
string GetParentDirectoryName(string fileOrFolder);
#nullable enable
IList<string> ScanFiles(string folderPath, GlobMatcher? matcher = null);
DateTime GetLastWriteTime(string folderPath);
GlobMatcher CreateMatcherFromFile(string filePath);
#nullable disable
}
public class DirectoryService : IDirectoryService
{
public const string KavitaIgnoreFile = ".kavitaignore";
public IFileSystem FileSystem { get; }
public string CacheDirectory { get; }
public string CoverImageDirectory { get; }
@ -531,6 +535,21 @@ namespace API.Services
.Where(path => ExcludeDirectories.Matches(path).Count == 0);
}
/// <summary>
/// Gets a set of directories from the folder path. Automatically excludes directories that shouldn't be in scope.
/// </summary>
/// <param name="folderPath"></param>
/// <param name="matcher">A set of glob rules that will filter directories out</param>
/// <returns>List of directory paths, empty if path doesn't exist</returns>
public IEnumerable<string> GetDirectories(string folderPath, GlobMatcher matcher)
{
if (matcher == null) return GetDirectories(folderPath);
return GetDirectories(folderPath)
.Where(folder => !matcher.ExcludeMatches(
$"{FileSystem.DirectoryInfo.FromDirectoryName(folder).Name}{FileSystem.Path.AltDirectorySeparatorChar}"));
}
/// <summary>
/// Returns all directories, including subdirectories. Automatically excludes directories that shouldn't be in scope.
/// </summary>
@ -580,7 +599,7 @@ namespace API.Services
var files = new List<string>();
if (!Exists(folderPath)) return files;
var potentialIgnoreFile = FileSystem.Path.Join(folderPath, ".kavitaignore");
var potentialIgnoreFile = FileSystem.Path.Join(folderPath, KavitaIgnoreFile);
if (matcher == null)
{
matcher = CreateMatcherFromFile(potentialIgnoreFile);
@ -591,17 +610,7 @@ namespace API.Services
}
IEnumerable<string> directories;
if (matcher == null)
{
directories = GetDirectories(folderPath);
}
else
{
directories = GetDirectories(folderPath)
.Where(folder => matcher != null &&
!matcher.ExcludeMatches($"{FileSystem.DirectoryInfo.FromDirectoryName(folder).Name}{FileSystem.Path.AltDirectorySeparatorChar}"));
}
var directories = GetDirectories(folderPath, matcher);
foreach (var directory in directories)
{
@ -640,8 +649,12 @@ namespace API.Services
return directories.Max(d => FileSystem.Directory.GetLastWriteTime(d));
}
private GlobMatcher CreateMatcherFromFile(string filePath)
/// <summary>
/// Generates a GlobMatcher from a .kavitaignore file found at path. Returns null otherwise.
/// </summary>
/// <param name="filePath"></param>
/// <returns></returns>
public GlobMatcher CreateMatcherFromFile(string filePath)
{
if (!FileSystem.File.Exists(filePath))
{