More Bugfixes (#2989)
This commit is contained in:
Joe Milazzo
GitHub
parent
1ae723b405
commit
a3e020fe17
49 changed files with 579 additions and 272 deletions
|
|
@ -56,9 +56,9 @@ public class LibraryWatcher : ILibraryWatcher
|
|||
/// <summary>
|
||||
/// Counts within a time frame how many times the buffer became full. Is used to reschedule LibraryWatcher to start monitoring much later rather than instantly
|
||||
/// </summary>
|
||||
private int _bufferFullCounter;
|
||||
private int _restartCounter;
|
||||
private DateTime _lastErrorTime = DateTime.MinValue;
|
||||
private static int _bufferFullCounter;
|
||||
private static int _restartCounter;
|
||||
private static DateTime _lastErrorTime = DateTime.MinValue;
|
||||
/// <summary>
|
||||
/// Used to lock buffer Full Counter
|
||||
/// </summary>
|
||||
|
|
@ -262,17 +262,19 @@ public class LibraryWatcher : ILibraryWatcher
|
|||
return;
|
||||
}
|
||||
|
||||
_taskScheduler.ScanFolder(fullPath, _queueWaitTime);
|
||||
_taskScheduler.ScanFolder(fullPath, filePath, _queueWaitTime);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "[LibraryWatcher] An error occured when processing a watch event");
|
||||
}
|
||||
_logger.LogDebug("[LibraryWatcher] ProcessChange completed in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds);
|
||||
_logger.LogTrace("[LibraryWatcher] ProcessChange completed in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds);
|
||||
}
|
||||
|
||||
private string GetFolder(string filePath, IEnumerable<string> libraryFolders)
|
||||
{
|
||||
// TODO: I can optimize this to avoid a library scan and instead do a Series Scan by finding the series that has a lowestFolderPath higher or equal to the filePath
|
||||
|
||||
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
|
||||
_logger.LogTrace("[LibraryWatcher] Parent Directory: {ParentDirectory}", parentDirectory);
|
||||
if (string.IsNullOrEmpty(parentDirectory)) return string.Empty;
|
||||
|
|
|
|||
|
|
@ -114,7 +114,6 @@ public class ParseScannedFiles
|
|||
_eventHub = eventHub;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// This will Scan all files in a folder path. For each folder within the folderPath, FolderAction will be invoked for all files contained
|
||||
/// </summary>
|
||||
|
|
@ -122,48 +121,53 @@ public class ParseScannedFiles
|
|||
/// <param name="seriesPaths">A dictionary mapping a normalized path to a list of <see cref="SeriesModified"/> to help scanner skip I/O</param>
|
||||
/// <param name="folderPath">A library folder or series folder</param>
|
||||
/// <param name="forceCheck">If we should bypass any folder last write time checks on the scan and force I/O</param>
|
||||
public IList<ScanResult> ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
|
||||
public async Task<IList<ScanResult>> ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
|
||||
IDictionary<string, IList<SeriesModified>> seriesPaths, Library library, bool forceCheck = false)
|
||||
{
|
||||
string normalizedPath;
|
||||
var result = new List<ScanResult>();
|
||||
var fileExtensions = string.Join("|", library.LibraryFileTypes.Select(l => l.FileTypeGroup.GetRegex()));
|
||||
var matcher = BuildMatcher(library);
|
||||
|
||||
var result = new List<ScanResult>();
|
||||
|
||||
if (scanDirectoryByDirectory)
|
||||
{
|
||||
// This is used in library scan, so we should check first for a ignore file and use that here as well
|
||||
var matcher = new GlobMatcher();
|
||||
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
|
||||
{
|
||||
matcher.AddExclude(pattern.Pattern);
|
||||
}
|
||||
|
||||
var directories = _directoryService.GetDirectories(folderPath, matcher).ToList();
|
||||
var directories = _directoryService.GetDirectories(folderPath, matcher).Select(Parser.Parser.NormalizePath);
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
// Since this is a loop, we need a list return
|
||||
normalizedPath = Parser.Parser.NormalizePath(directory);
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
|
||||
MessageFactory.FileScanProgressEvent(directory, library.Name, ProgressEventType.Updated));
|
||||
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, directory, forceCheck))
|
||||
{
|
||||
result.Add(new ScanResult()
|
||||
if (result.Exists(r => r.Folder == directory))
|
||||
{
|
||||
Files = ArraySegment<string>.Empty,
|
||||
Folder = directory,
|
||||
LibraryRoot = folderPath,
|
||||
HasChanged = false
|
||||
});
|
||||
continue;
|
||||
}
|
||||
result.Add(CreateScanResult(directory, folderPath, false, ArraySegment<string>.Empty));
|
||||
}
|
||||
else if (seriesPaths.TryGetValue(normalizedPath, out var series) && series.All(s => !string.IsNullOrEmpty(s.LowestFolderPath)))
|
||||
else if (seriesPaths.TryGetValue(directory, out var series) && series.All(s => !string.IsNullOrEmpty(s.LowestFolderPath)))
|
||||
{
|
||||
// If there are multiple series inside this path, let's check each of them to see which was modified and only scan those
|
||||
// This is very helpful for ComicVine libraries by Publisher
|
||||
_logger.LogDebug("[ProcessFiles] {Directory} is dirty and has multiple series folders, checking if we can avoid a full scan", directory);
|
||||
foreach (var seriesModified in series)
|
||||
{
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesModified, seriesModified.LowestFolderPath!))
|
||||
var hasFolderChangedSinceLastScan = library.LastScanned.Truncate(TimeSpan.TicksPerSecond) <
|
||||
_directoryService
|
||||
.GetLastWriteTime(seriesModified.LowestFolderPath!)
|
||||
.Truncate(TimeSpan.TicksPerSecond);
|
||||
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
|
||||
MessageFactory.FileScanProgressEvent(seriesModified.LowestFolderPath!, library.Name, ProgressEventType.Updated));
|
||||
|
||||
if (!hasFolderChangedSinceLastScan)
|
||||
{
|
||||
result.Add(CreateScanResult(directory, folderPath, false, ArraySegment<string>.Empty));
|
||||
_logger.LogDebug("[ProcessFiles] {Directory} subfolder {Folder} did not change since last scan, adding entry to skip", directory, seriesModified.LowestFolderPath);
|
||||
result.Add(CreateScanResult(seriesModified.LowestFolderPath!, folderPath, false, ArraySegment<string>.Empty));
|
||||
}
|
||||
else
|
||||
{
|
||||
_logger.LogDebug("[ProcessFiles] {Directory} subfolder {Folder} changed, adding folders", directory, seriesModified.LowestFolderPath);
|
||||
result.Add(CreateScanResult(directory, folderPath, true,
|
||||
_directoryService.ScanFiles(seriesModified.LowestFolderPath!, fileExtensions, matcher)));
|
||||
}
|
||||
|
|
@ -173,19 +177,22 @@ public class ParseScannedFiles
|
|||
{
|
||||
// For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication
|
||||
result.Add(CreateScanResult(directory, folderPath, true,
|
||||
_directoryService.ScanFiles(directory, fileExtensions)));
|
||||
_directoryService.ScanFiles(directory, fileExtensions, matcher)));
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
normalizedPath = Parser.Parser.NormalizePath(folderPath);
|
||||
var normalizedPath = Parser.Parser.NormalizePath(folderPath);
|
||||
var libraryRoot =
|
||||
library.Folders.FirstOrDefault(f =>
|
||||
Parser.Parser.NormalizePath(folderPath).Contains(Parser.Parser.NormalizePath(f.Path)))?.Path ??
|
||||
normalizedPath.Contains(Parser.Parser.NormalizePath(f.Path)))?.Path ??
|
||||
folderPath;
|
||||
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
|
||||
MessageFactory.FileScanProgressEvent(normalizedPath, library.Name, ProgressEventType.Updated));
|
||||
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
|
||||
{
|
||||
result.Add(CreateScanResult(folderPath, libraryRoot, false, ArraySegment<string>.Empty));
|
||||
|
|
@ -193,13 +200,24 @@ public class ParseScannedFiles
|
|||
else
|
||||
{
|
||||
result.Add(CreateScanResult(folderPath, libraryRoot, true,
|
||||
_directoryService.ScanFiles(folderPath, fileExtensions)));
|
||||
_directoryService.ScanFiles(folderPath, fileExtensions, matcher)));
|
||||
}
|
||||
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static GlobMatcher BuildMatcher(Library library)
|
||||
{
|
||||
var matcher = new GlobMatcher();
|
||||
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
|
||||
{
|
||||
matcher.AddExclude(pattern.Pattern);
|
||||
}
|
||||
|
||||
return matcher;
|
||||
}
|
||||
|
||||
private static ScanResult CreateScanResult(string folderPath, string libraryRoot, bool hasChanged,
|
||||
IList<string> files)
|
||||
{
|
||||
|
|
@ -243,7 +261,7 @@ public class ParseScannedFiles
|
|||
NormalizedName = normalizedSeries
|
||||
};
|
||||
|
||||
scannedSeries.AddOrUpdate(existingKey, new List<ParserInfo>() {info}, (_, oldValue) =>
|
||||
scannedSeries.AddOrUpdate(existingKey, [info], (_, oldValue) =>
|
||||
{
|
||||
oldValue ??= new List<ParserInfo>();
|
||||
if (!oldValue.Contains(info))
|
||||
|
|
@ -338,7 +356,7 @@ public class ParseScannedFiles
|
|||
{
|
||||
try
|
||||
{
|
||||
var scanResults = ProcessFiles(folderPath, isLibraryScan, seriesPaths, library, forceCheck);
|
||||
var scanResults = await ProcessFiles(folderPath, isLibraryScan, seriesPaths, library, forceCheck);
|
||||
|
||||
foreach (var scanResult in scanResults)
|
||||
{
|
||||
|
|
@ -414,15 +432,19 @@ public class ParseScannedFiles
|
|||
/// <param name="library"></param>
|
||||
private async Task ProcessScanResult(ScanResult result, IDictionary<string, IList<SeriesModified>> seriesPaths, Library library)
|
||||
{
|
||||
// TODO: This should return the result as we are modifying it as a side effect
|
||||
|
||||
// If the folder hasn't changed, generate fake ParserInfos for the Series that were in that folder.
|
||||
var normalizedFolder = Parser.Parser.NormalizePath(result.Folder);
|
||||
if (!result.HasChanged)
|
||||
{
|
||||
var normalizedFolder = Parser.Parser.NormalizePath(result.Folder);
|
||||
result.ParserInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
|
||||
{
|
||||
Series = fp.SeriesName,
|
||||
Format = fp.Format,
|
||||
}).ToList();
|
||||
result.ParserInfos = seriesPaths[normalizedFolder]
|
||||
.Select(fp => new ParserInfo()
|
||||
{
|
||||
Series = fp.SeriesName,
|
||||
Format = fp.Format,
|
||||
})
|
||||
.ToList();
|
||||
|
||||
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", normalizedFolder);
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
|
||||
|
|
@ -431,25 +453,24 @@ public class ParseScannedFiles
|
|||
}
|
||||
|
||||
var files = result.Files;
|
||||
var folder = result.Folder;
|
||||
var libraryRoot = result.LibraryRoot;
|
||||
|
||||
// When processing files for a folder and we do enter, we need to parse the information and combine parser infos
|
||||
// NOTE: We might want to move the merge step later in the process, like return and combine.
|
||||
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, folder);
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
|
||||
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
|
||||
|
||||
if (files.Count == 0)
|
||||
{
|
||||
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", folder);
|
||||
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", normalizedFolder);
|
||||
result.ParserInfos = ArraySegment<ParserInfo>.Empty;
|
||||
return;
|
||||
}
|
||||
|
||||
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, normalizedFolder);
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
|
||||
MessageFactory.FileScanProgressEvent($"{files.Count} files in {normalizedFolder}", library.Name, ProgressEventType.Updated));
|
||||
|
||||
// Multiple Series can exist within a folder. We should instead put these infos on the result and perform merging above
|
||||
IList<ParserInfo> infos = files
|
||||
.Select(file => _readingItemService.ParseFile(file, folder, libraryRoot, library.Type))
|
||||
.Select(file => _readingItemService.ParseFile(file, normalizedFolder, result.LibraryRoot, library.Type))
|
||||
.Where(info => info != null)
|
||||
.ToList()!;
|
||||
|
||||
|
|
|
|||
|
|
@ -26,7 +26,7 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag
|
|||
{
|
||||
Filename = Path.GetFileName(filePath),
|
||||
Format = Parser.ParseFormat(filePath),
|
||||
Title = Parser.RemoveExtensionIfSupported(fileName),
|
||||
Title = Parser.RemoveExtensionIfSupported(fileName)!,
|
||||
FullFilePath = Parser.NormalizePath(filePath),
|
||||
Series = string.Empty,
|
||||
ComicInfo = comicInfo
|
||||
|
|
@ -76,6 +76,9 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag
|
|||
ret.Chapters = Parser.DefaultChapter;
|
||||
ret.Volumes = Parser.SpecialVolume;
|
||||
|
||||
// NOTE: This uses rootPath. LibraryRoot works better for manga, but it's not always that way.
|
||||
// It might be worth writing some logic if the file is a special, to take the folder above the Specials/
|
||||
// if present
|
||||
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -101,7 +101,7 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
|
|||
}
|
||||
}
|
||||
|
||||
protected void UpdateFromComicInfo(ParserInfo info)
|
||||
protected static void UpdateFromComicInfo(ParserInfo info)
|
||||
{
|
||||
if (info.ComicInfo == null) return;
|
||||
|
||||
|
|
@ -109,6 +109,10 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
|
|||
{
|
||||
info.Volumes = info.ComicInfo.Volume;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
|
||||
{
|
||||
info.Chapters = info.ComicInfo.Number;
|
||||
}
|
||||
if (!string.IsNullOrEmpty(info.ComicInfo.Series))
|
||||
{
|
||||
info.Series = info.ComicInfo.Series.Trim();
|
||||
|
|
@ -125,16 +129,6 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
|
|||
info.Volumes = Parser.SpecialVolume;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
|
||||
{
|
||||
info.Chapters = info.ComicInfo.Number;
|
||||
if (info.IsSpecial && Parser.DefaultChapter != info.Chapters)
|
||||
{
|
||||
info.IsSpecial = false;
|
||||
info.Volumes = Parser.SpecialVolume;
|
||||
}
|
||||
}
|
||||
|
||||
// Patch is SeriesSort from ComicInfo
|
||||
if (!string.IsNullOrEmpty(info.ComicInfo.TitleSort))
|
||||
{
|
||||
|
|
|
|||
|
|
@ -103,7 +103,11 @@ public static class Parser
|
|||
private static readonly Regex CoverImageRegex = new Regex(@"(?<![[a-z]\d])(?:!?)(?<!back)(?<!back_)(?<!back-)(cover|folder)(?![\w\d])",
|
||||
MatchOptions, RegexTimeout);
|
||||
|
||||
private static readonly Regex NormalizeRegex = new Regex(@"[^\p{L}0-9\+!\*]",
|
||||
/// <summary>
|
||||
/// Normalize everything within Kavita. Some characters don't fall under Unicode, like full-width characters and need to be
|
||||
/// added on a case-by-case basis.
|
||||
/// </summary>
|
||||
private static readonly Regex NormalizeRegex = new Regex(@"[^\p{L}0-9\+!＊！＋]",
|
||||
MatchOptions, RegexTimeout);
|
||||
|
||||
/// <summary>
|
||||
|
|
|
|||
|
|
@ -45,7 +45,7 @@ public interface IScannerService
|
|||
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
|
||||
Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true);
|
||||
|
||||
Task ScanFolder(string folder);
|
||||
Task ScanFolder(string folder, string originalPath);
|
||||
Task AnalyzeFiles();
|
||||
|
||||
}
|
||||
|
|
@ -135,30 +135,35 @@ public class ScannerService : IScannerService
|
|||
/// Given a generic folder path, will invoke a Series scan or Library scan.
|
||||
/// </summary>
|
||||
/// <remarks>This will Schedule the job to run 1 minute in the future to allow for any close-by duplicate requests to be dropped</remarks>
|
||||
/// <param name="folder"></param>
|
||||
public async Task ScanFolder(string folder)
|
||||
/// <param name="folder">Normalized folder</param>
|
||||
/// <param name="originalPath">If invoked from LibraryWatcher, this maybe a nested folder and can allow for optimization</param>
|
||||
public async Task ScanFolder(string folder, string originalPath)
|
||||
{
|
||||
Series? series = null;
|
||||
try
|
||||
{
|
||||
series = await _unitOfWork.SeriesRepository.GetSeriesByFolderPath(folder, SeriesIncludes.Library);
|
||||
series = await _unitOfWork.SeriesRepository.GetSeriesThatContainsLowestFolderPath(originalPath,
|
||||
SeriesIncludes.Library) ??
|
||||
await _unitOfWork.SeriesRepository.GetSeriesByFolderPath(originalPath, SeriesIncludes.Library) ??
|
||||
await _unitOfWork.SeriesRepository.GetSeriesByFolderPath(folder, SeriesIncludes.Library);
|
||||
}
|
||||
catch (InvalidOperationException ex)
|
||||
{
|
||||
if (ex.Message.Equals("Sequence contains more than one element."))
|
||||
{
|
||||
_logger.LogCritical(ex, "[ScannerService] Multiple series map to this folder. Library scan will be used for ScanFolder");
|
||||
_logger.LogCritical(ex, "[ScannerService] Multiple series map to this folder or folder is at library root. Library scan will be used for ScanFolder");
|
||||
}
|
||||
}
|
||||
|
||||
// TODO: Figure out why we have the library type restriction here
|
||||
if (series != null && (series.Library.Type != LibraryType.Book || series.Library.Type != LibraryType.LightNovel))
|
||||
if (series != null && series.Library.Type is not (LibraryType.Book or LibraryType.LightNovel))
|
||||
{
|
||||
if (TaskScheduler.HasScanTaskRunningForSeries(series.Id))
|
||||
{
|
||||
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder);
|
||||
return;
|
||||
}
|
||||
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder}, Series matched to folder and ScanSeries enqueued for 1 minute", folder);
|
||||
BackgroundJob.Schedule(() => ScanSeries(series.Id, true), TimeSpan.FromMinutes(1));
|
||||
return;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -181,7 +181,9 @@ public class StatsService : IStatsService
|
|||
{
|
||||
InstallId = serverSettings.InstallId,
|
||||
KavitaVersion = serverSettings.InstallVersion,
|
||||
IsDocker = OsInfo.IsDocker
|
||||
IsDocker = OsInfo.IsDocker,
|
||||
FirstInstallDate = serverSettings.FirstInstallDate,
|
||||
FirstInstallVersion = serverSettings.FirstInstallVersion
|
||||
};
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue