AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

More Bugfixes (#2989)

Browse source
This commit is contained in:
Joe Milazzo 2024-06-09 13:16:11 -05:00 • committed by GitHub
parent 1ae723b405
commit a3e020fe17
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
49 changed files with 579 additions and 272 deletions
Download patch file Download diff file Expand all files Collapse all files

107
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -114,7 +114,6 @@ public class ParseScannedFiles
_eventHub = eventHub;
}
/// <summary>
/// This will Scan all files in a folder path. For each folder within the folderPath, FolderAction will be invoked for all files contained
/// </summary>
@ -122,48 +121,53 @@ public class ParseScannedFiles
/// <param name="seriesPaths">A dictionary mapping a normalized path to a list of <see cref="SeriesModified"/> to help scanner skip I/O</param>
/// <param name="folderPath">A library folder or series folder</param>
/// <param name="forceCheck">If we should bypass any folder last write time checks on the scan and force I/O</param>
public IList<ScanResult> ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
public async Task<IList<ScanResult>> ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
IDictionary<string, IList<SeriesModified>> seriesPaths, Library library, bool forceCheck = false)
{
string normalizedPath;
var result = new List<ScanResult>();
var fileExtensions = string.Join("|", library.LibraryFileTypes.Select(l => l.FileTypeGroup.GetRegex()));
var matcher = BuildMatcher(library);
var result = new List<ScanResult>();
if (scanDirectoryByDirectory)
{
// This is used in library scan, so we should check first for a ignore file and use that here as well
var matcher = new GlobMatcher();
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
{
matcher.AddExclude(pattern.Pattern);
}
var directories = _directoryService.GetDirectories(folderPath, matcher).ToList();
var directories = _directoryService.GetDirectories(folderPath, matcher).Select(Parser.Parser.NormalizePath);
foreach (var directory in directories)
{
// Since this is a loop, we need a list return
normalizedPath = Parser.Parser.NormalizePath(directory);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(directory, library.Name, ProgressEventType.Updated));
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, directory, forceCheck))
{
result.Add(new ScanResult()
if (result.Exists(r => r.Folder == directory))
{
Files = ArraySegment<string>.Empty,
Folder = directory,
LibraryRoot = folderPath,
HasChanged = false
});
continue;
}
result.Add(CreateScanResult(directory, folderPath, false, ArraySegment<string>.Empty));
}
else if (seriesPaths.TryGetValue(normalizedPath, out var series) && series.All(s => !string.IsNullOrEmpty(s.LowestFolderPath)))
else if (seriesPaths.TryGetValue(directory, out var series) && series.All(s => !string.IsNullOrEmpty(s.LowestFolderPath)))
{
// If there are multiple series inside this path, let's check each of them to see which was modified and only scan those
// This is very helpful for ComicVine libraries by Publisher
_logger.LogDebug("[ProcessFiles] {Directory} is dirty and has multiple series folders, checking if we can avoid a full scan", directory);
foreach (var seriesModified in series)
{
if (HasSeriesFolderNotChangedSinceLastScan(seriesModified, seriesModified.LowestFolderPath!))
var hasFolderChangedSinceLastScan = library.LastScanned.Truncate(TimeSpan.TicksPerSecond) <
_directoryService
.GetLastWriteTime(seriesModified.LowestFolderPath!)
.Truncate(TimeSpan.TicksPerSecond);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(seriesModified.LowestFolderPath!, library.Name, ProgressEventType.Updated));
if (!hasFolderChangedSinceLastScan)
{
result.Add(CreateScanResult(directory, folderPath, false, ArraySegment<string>.Empty));
_logger.LogDebug("[ProcessFiles] {Directory} subfolder {Folder} did not change since last scan, adding entry to skip", directory, seriesModified.LowestFolderPath);
result.Add(CreateScanResult(seriesModified.LowestFolderPath!, folderPath, false, ArraySegment<string>.Empty));
}
else
{
_logger.LogDebug("[ProcessFiles] {Directory} subfolder {Folder} changed, adding folders", directory, seriesModified.LowestFolderPath);
result.Add(CreateScanResult(directory, folderPath, true,
_directoryService.ScanFiles(seriesModified.LowestFolderPath!, fileExtensions, matcher)));
}
@ -173,19 +177,22 @@ public class ParseScannedFiles
{
// For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication
result.Add(CreateScanResult(directory, folderPath, true,
_directoryService.ScanFiles(directory, fileExtensions)));
_directoryService.ScanFiles(directory, fileExtensions, matcher)));
}
}
return result;
}
normalizedPath = Parser.Parser.NormalizePath(folderPath);
var normalizedPath = Parser.Parser.NormalizePath(folderPath);
var libraryRoot =
library.Folders.FirstOrDefault(f =>
Parser.Parser.NormalizePath(folderPath).Contains(Parser.Parser.NormalizePath(f.Path)))?.Path ??
normalizedPath.Contains(Parser.Parser.NormalizePath(f.Path)))?.Path ??
folderPath;
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(normalizedPath, library.Name, ProgressEventType.Updated));
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
{
result.Add(CreateScanResult(folderPath, libraryRoot, false, ArraySegment<string>.Empty));
@ -193,13 +200,24 @@ public class ParseScannedFiles
else
{
result.Add(CreateScanResult(folderPath, libraryRoot, true,
_directoryService.ScanFiles(folderPath, fileExtensions)));
_directoryService.ScanFiles(folderPath, fileExtensions, matcher)));
}
return result;
}
private static GlobMatcher BuildMatcher(Library library)
{
var matcher = new GlobMatcher();
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
{
matcher.AddExclude(pattern.Pattern);
}
return matcher;
}
private static ScanResult CreateScanResult(string folderPath, string libraryRoot, bool hasChanged,
IList<string> files)
{
@ -243,7 +261,7 @@ public class ParseScannedFiles
NormalizedName = normalizedSeries
};
scannedSeries.AddOrUpdate(existingKey, new List<ParserInfo>() {info}, (_, oldValue) =>
scannedSeries.AddOrUpdate(existingKey, [info], (_, oldValue) =>
{
oldValue ??= new List<ParserInfo>();
if (!oldValue.Contains(info))
@ -338,7 +356,7 @@ public class ParseScannedFiles
{
try
{
var scanResults = ProcessFiles(folderPath, isLibraryScan, seriesPaths, library, forceCheck);
var scanResults = await ProcessFiles(folderPath, isLibraryScan, seriesPaths, library, forceCheck);
foreach (var scanResult in scanResults)
{
@ -414,15 +432,19 @@ public class ParseScannedFiles
/// <param name="library"></param>
private async Task ProcessScanResult(ScanResult result, IDictionary<string, IList<SeriesModified>> seriesPaths, Library library)
{
// TODO: This should return the result as we are modifying it as a side effect
// If the folder hasn't changed, generate fake ParserInfos for the Series that were in that folder.
var normalizedFolder = Parser.Parser.NormalizePath(result.Folder);
if (!result.HasChanged)
{
var normalizedFolder = Parser.Parser.NormalizePath(result.Folder);
result.ParserInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
{
Series = fp.SeriesName,
Format = fp.Format,
}).ToList();
result.ParserInfos = seriesPaths[normalizedFolder]
.Select(fp => new ParserInfo()
{
Series = fp.SeriesName,
Format = fp.Format,
})
.ToList();
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", normalizedFolder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
@ -431,25 +453,24 @@ public class ParseScannedFiles
}
var files = result.Files;
var folder = result.Folder;
var libraryRoot = result.LibraryRoot;
// When processing files for a folder and we do enter, we need to parse the information and combine parser infos
// NOTE: We might want to move the merge step later in the process, like return and combine.
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
if (files.Count == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", folder);
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", normalizedFolder);
result.ParserInfos = ArraySegment<ParserInfo>.Empty;
return;
}
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, normalizedFolder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{files.Count} files in {normalizedFolder}", library.Name, ProgressEventType.Updated));
// Multiple Series can exist within a folder. We should instead put these infos on the result and perform merging above
IList<ParserInfo> infos = files
.Select(file => _readingItemService.ParseFile(file, folder, libraryRoot, library.Type))
.Select(file => _readingItemService.ParseFile(file, normalizedFolder, result.LibraryRoot, library.Type))
.Where(info => info != null)
.ToList()!;