From 3fe5933358d04afea7dd3a6dbaa94cd483d30cd1 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Sat, 3 May 2025 17:40:11 -0500 Subject: [PATCH 1/5] Start of the metadata/filename on/off stuff. --- API.Tests/Services/ScannerServiceTests.cs | 5 +++++ API/Controllers/LibraryController.cs | 3 +++ API/DTOs/LibraryDto.cs | 10 ++++++++++ API/DTOs/UpdateLibraryDto.cs | 4 ++++ API/Entities/Library.cs | 10 ++++++++++ API/Helpers/Builders/LibraryBuilder.cs | 12 ++++++++++++ API/Services/Tasks/Scanner/Parser/BasicParser.cs | 1 + API/Services/Tasks/Scanner/ProcessSeries.cs | 5 +---- 8 files changed, 46 insertions(+), 4 deletions(-) diff --git a/API.Tests/Services/ScannerServiceTests.cs b/API.Tests/Services/ScannerServiceTests.cs index 2e812647b..9b0271fc2 100644 --- a/API.Tests/Services/ScannerServiceTests.cs +++ b/API.Tests/Services/ScannerServiceTests.cs @@ -938,4 +938,9 @@ public class ScannerServiceTests : AbstractDbTest Assert.True(sortedChapters[1].SortOrder.Is(4f)); Assert.True(sortedChapters[2].SortOrder.Is(5f)); } + + #region Scanner Overhaul + + + #endregion } diff --git a/API/Controllers/LibraryController.cs b/API/Controllers/LibraryController.cs index 2f12aa1fe..4f3b6c832 100644 --- a/API/Controllers/LibraryController.cs +++ b/API/Controllers/LibraryController.cs @@ -623,6 +623,9 @@ public class LibraryController : BaseApiController library.ManageReadingLists = dto.ManageReadingLists; library.AllowScrobbling = dto.AllowScrobbling; library.AllowMetadataMatching = dto.AllowMetadataMatching; + library.AllowFilenameParsing = dto.AllowFilenameParsing; + library.AllowMetadataParsing = dto.AllowMetadataParsing; + library.LibraryFileTypes = dto.FileGroupTypes .Select(t => new LibraryFileTypeGroup() {FileTypeGroup = t, LibraryId = library.Id}) .Distinct() diff --git a/API/DTOs/LibraryDto.cs b/API/DTOs/LibraryDto.cs index 18dea9434..7ddf36926 100644 --- a/API/DTOs/LibraryDto.cs +++ b/API/DTOs/LibraryDto.cs @@ -67,4 +67,14 @@ public class LibraryDto /// This does not exclude the library from being linked to wrt Series Relationships /// Requires a valid LicenseKey public bool AllowMetadataMatching { get; set; } = true; + /// + /// Allow Kavita to parse Metadata from Files based on Filename + /// + /// Cannot be false if is false + public bool AllowFilenameParsing { get; set; } = true; + /// + /// Allow Kavita to parse Metadata from files (ComicInfo/Epub/Pdf) + /// + /// Cannot be false if is false + public bool AllowMetadataParsing { get; set; } = true; } diff --git a/API/DTOs/UpdateLibraryDto.cs b/API/DTOs/UpdateLibraryDto.cs index de02f304d..118c2d6bc 100644 --- a/API/DTOs/UpdateLibraryDto.cs +++ b/API/DTOs/UpdateLibraryDto.cs @@ -28,6 +28,10 @@ public class UpdateLibraryDto public bool AllowScrobbling { get; init; } [Required] public bool AllowMetadataMatching { get; init; } + [Required] + public bool AllowFilenameParsing { get; set; } = true; + [Required] + public bool AllowMetadataParsing { get; set; } = true; /// /// What types of files to allow the scanner to pickup /// diff --git a/API/Entities/Library.cs b/API/Entities/Library.cs index abab81378..be69a2363 100644 --- a/API/Entities/Library.cs +++ b/API/Entities/Library.cs @@ -48,6 +48,16 @@ public class Library : IEntityDate, IHasCoverImage /// This does not exclude the library from being linked to wrt Series Relationships /// Requires a valid LicenseKey public bool AllowMetadataMatching { get; set; } = true; + /// + /// Allow Kavita to parse Metadata from Files based on Filename + /// + /// Cannot be false if is false + public bool AllowFilenameParsing { get; set; } = true; + /// + /// Allow Kavita to parse Metadata from files (ComicInfo/Epub/Pdf) + /// + /// Cannot be false if is false + public bool AllowMetadataParsing { get; set; } = true; public DateTime Created { get; set; } diff --git a/API/Helpers/Builders/LibraryBuilder.cs b/API/Helpers/Builders/LibraryBuilder.cs index 30e6136a5..3ebe8315a 100644 --- a/API/Helpers/Builders/LibraryBuilder.cs +++ b/API/Helpers/Builders/LibraryBuilder.cs @@ -115,4 +115,16 @@ public class LibraryBuilder : IEntityBuilder _library.AllowScrobbling = allowScrobbling; return this; } + + public LibraryBuilder WithAllowFilenameParsing(bool allow) + { + _library.AllowFilenameParsing = allow; + return this; + } + + public LibraryBuilder WithAllowMetadataParsing(bool allow) + { + _library.AllowMetadataParsing = allow; + return this; + } } diff --git a/API/Services/Tasks/Scanner/Parser/BasicParser.cs b/API/Services/Tasks/Scanner/Parser/BasicParser.cs index 1462ab3d3..154760e36 100644 --- a/API/Services/Tasks/Scanner/Parser/BasicParser.cs +++ b/API/Services/Tasks/Scanner/Parser/BasicParser.cs @@ -16,6 +16,7 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag { var fileName = directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath); // TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this. + // NOTE: This may no longer be needed as we have file type group support now, thus an image wouldn't come for a Series if (type != LibraryType.Image && Parser.IsCoverImage(directoryService.FileSystem.Path.GetFileName(filePath))) return null; if (Parser.IsImage(filePath)) diff --git a/API/Services/Tasks/Scanner/ProcessSeries.cs b/API/Services/Tasks/Scanner/ProcessSeries.cs index 454c72733..336bbdba0 100644 --- a/API/Services/Tasks/Scanner/ProcessSeries.cs +++ b/API/Services/Tasks/Scanner/ProcessSeries.cs @@ -193,10 +193,6 @@ public class ProcessSeries : IProcessSeries if (seriesAdded) { - // See if any recommendations can link up to the series and pre-fetch external metadata for the series - // BackgroundJob.Enqueue(() => - // _externalMetadataService.FetchSeriesMetadata(series.Id, series.Library.Type)); - await _eventHub.SendMessageAsync(MessageFactory.SeriesAdded, MessageFactory.SeriesAddedEvent(series.Id, series.Name, series.LibraryId), false); } @@ -216,6 +212,7 @@ public class ProcessSeries : IProcessSeries if (seriesAdded) { + // Prefetch metadata if applicable await _externalMetadataService.FetchSeriesMetadata(series.Id, series.Library.Type); } await _metadataService.GenerateCoversForSeries(series.LibraryId, series.Id, false, false); From 85b3187f3faa497b1e51e1eac138cff901974f46 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Wed, 7 May 2025 16:58:39 -0500 Subject: [PATCH 2/5] Added some notes --- API/Services/Plus/ExternalMetadataService.cs | 1 + API/Services/SeriesService.cs | 1 + 2 files changed, 2 insertions(+) diff --git a/API/Services/Plus/ExternalMetadataService.cs b/API/Services/Plus/ExternalMetadataService.cs index f9af923a2..3f44f56d3 100644 --- a/API/Services/Plus/ExternalMetadataService.cs +++ b/API/Services/Plus/ExternalMetadataService.cs @@ -752,6 +752,7 @@ public class ExternalMetadataService : IExternalMetadataService { Name = w.Name, AniListId = ScrobblingService.ExtractId(w.Url, ScrobblingService.AniListCharacterWebsite), + // Can I tag links to resolve favicon? Need to parse html, rewrite the urls, then have a Description = StringHelper.CorrectUrls(StringHelper.RemoveSourceInDescription(StringHelper.SquashBreaklines(w.Description))), }) .Concat(series.Metadata.People diff --git a/API/Services/SeriesService.cs b/API/Services/SeriesService.cs index 805b3b06f..34c55e6ce 100644 --- a/API/Services/SeriesService.cs +++ b/API/Services/SeriesService.cs @@ -633,6 +633,7 @@ public class SeriesService : ISeriesService public async Task FormatChapterTitle(int userId, bool isSpecial, LibraryType libraryType, string chapterRange, string? chapterTitle, bool withHash) { + // TODO: Refactor so this is unit testable if (string.IsNullOrEmpty(chapterTitle) && (isSpecial || libraryType == LibraryType.Book)) throw new ArgumentException("Chapter Title cannot be null"); if (isSpecial) From 16498d4b4056a418ed37825712ddeba723379197 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Wed, 7 May 2025 19:10:54 -0500 Subject: [PATCH 3/5] Added a bunch of basic json files for the different structures Kavita supports. Unit tests to come. --- .../Services/Scanner/FileSystemParserTests.cs | 54 +++++++++++++++++++ .../ScannerService/TestCases/Base/ideal.json | 5 ++ .../TestCases/Base/localised-name-merge.json | 6 +++ .../TestCases/Base/loose-books.json | 4 ++ .../Base/loose-images-chapter-folders.json | 5 ++ .../Base/loose-images-mixed-folders.json | 5 ++ .../Base/loose-images-nested-folders.json | 5 ++ .../TestCases/Base/loose-images-pages.json | 4 ++ .../Base/loose-images-volume-folders.json | 5 ++ .../TestCases/Base/loose-series.json | 5 ++ .../TestCases/Base/publisher.json | 6 +++ .../TestCases/Base/special-folder.json | 6 +++ .../TestCases/Base/special-marker.json | 6 +++ .../TestCases/Exclude/exclude-file.json | 6 +++ .../Exclude/exclude-subfolder-all.json | 6 +++ .../TestCases/Exclude/exclude-subfolder.json | 6 +++ 16 files changed, 134 insertions(+) create mode 100644 API.Tests/Services/Scanner/FileSystemParserTests.cs create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/ideal.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/localised-name-merge.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-books.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-chapter-folders.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-mixed-folders.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-nested-folders.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-pages.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-volume-folders.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-series.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/publisher.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/special-folder.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Base/special-marker.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-file.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-subfolder-all.json create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-subfolder.json diff --git a/API.Tests/Services/Scanner/FileSystemParserTests.cs b/API.Tests/Services/Scanner/FileSystemParserTests.cs new file mode 100644 index 000000000..c2067e5d9 --- /dev/null +++ b/API.Tests/Services/Scanner/FileSystemParserTests.cs @@ -0,0 +1,54 @@ +using System.IO; +using System.Threading.Tasks; +using API.Data.Repositories; +using API.Tests.Helpers; +using Hangfire; +using Xunit; +using Xunit.Abstractions; + +namespace API.Tests.Services.Scanner; + +/// +/// Responsible for testing Change Detection, Exclude Patterns, +/// +public class FileSystemParserTests : AbstractDbTest +{ + + private readonly ITestOutputHelper _testOutputHelper; + private readonly ScannerHelper _scannerHelper; + private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/ScanTests"); + + public FileSystemParserTests(ITestOutputHelper testOutputHelper) + { + _testOutputHelper = testOutputHelper; + + // Set up Hangfire to use in-memory storage for testing + GlobalConfiguration.Configuration.UseInMemoryStorage(); + _scannerHelper = new ScannerHelper(UnitOfWork, testOutputHelper); + } + + protected override async Task ResetDb() + { + Context.Library.RemoveRange(Context.Library); + await Context.SaveChangesAsync(); + } + + + #region Validate Change Detection + + + [Fact] + public async Task ScanLibrary_ComicVine_PublisherFolder() + { + var testcase = "Publisher - ComicVine.json"; + var library = await _scannerHelper.GenerateScannerData(testcase); + var scanner = _scannerHelper.CreateServices(); + await scanner.ScanLibrary(library.Id); + var postLib = await UnitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series); + + Assert.NotNull(postLib); + Assert.Equal(4, postLib.Series.Count); + } + + #endregion +} diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/ideal.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/ideal.json new file mode 100644 index 000000000..df3b1bbbd --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/ideal.json @@ -0,0 +1,5 @@ +[ + "Root 1/Series A/Series A 01.cbz", + "Root 1/Series B/Series B 01.cbz", + "Root 2/Series C/Series C 01.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/localised-name-merge.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/localised-name-merge.json new file mode 100644 index 000000000..77addadef --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/localised-name-merge.json @@ -0,0 +1,6 @@ +[ + "Root 1/Series A/Series A/Series A 01.cbz", + "Root 1/Series A/Series A1/Series A1 01.cbz", + "Root 1/Series B/Series B 01.cbz", + "Root 2/Series C/Series C 01.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-books.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-books.json new file mode 100644 index 000000000..b2b274081 --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-books.json @@ -0,0 +1,4 @@ +[ + "Root 1/Books/book v1.pdf", + "Root 1/Books/book v2.pdf" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-chapter-folders.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-chapter-folders.json new file mode 100644 index 000000000..3c1cd10de --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-chapter-folders.json @@ -0,0 +1,5 @@ +[ + "Root 1/Books/Series A/ch 1/001.png", + "Root 1/Books/Series A/ch 1/002.png", + "Root 1/Books/Series A/ch 2/001.png" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-mixed-folders.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-mixed-folders.json new file mode 100644 index 000000000..74e288162 --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-mixed-folders.json @@ -0,0 +1,5 @@ +[ + "Root 1/Books/Series A/vol 1 ch 1/001.png", + "Root 1/Books/Series A/vol 1 ch 2/002.png", + "Root 1/Books/Series A/vol 2/001.png" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-nested-folders.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-nested-folders.json new file mode 100644 index 000000000..8fed641e6 --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-nested-folders.json @@ -0,0 +1,5 @@ +[ + "Root 1/Books/Series A/vol 1/ch 1/001.png", + "Root 1/Books/Series A/vol 1/ch 2/002.png", + "Root 1/Books/Series A/vol 2/001.png" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-pages.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-pages.json new file mode 100644 index 000000000..87b70eba0 --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-pages.json @@ -0,0 +1,4 @@ +[ + "Root 1/Books/Series A [Digital]/001.png", + "Root 1/Books/Series A [Digital]/002.png" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-volume-folders.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-volume-folders.json new file mode 100644 index 000000000..2055a276e --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-images-volume-folders.json @@ -0,0 +1,5 @@ +[ + "Root 1/Books/Series A/vol 1/001.png", + "Root 1/Books/Series A/vol 1/002.png", + "Root 1/Books/Series A/vol 2/001.png" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-series.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-series.json new file mode 100644 index 000000000..d4a749afe --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/loose-series.json @@ -0,0 +1,5 @@ +[ + "Root 1/Genre/Series A 01.cbz", + "Root 1/Genre/Series B 01.cbz", + "Root 1/Genre/Series C 01.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/publisher.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/publisher.json new file mode 100644 index 000000000..8abfa3728 --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/publisher.json @@ -0,0 +1,6 @@ +[ + "Root 1/Publisher 1/Series A/Series A #1.cbz", + "Root 1/Publisher 1/Series B/Series B #1.cbz", + "Root 1/Publisher 2/Series C/Series C #1.cbz", + "Root 1/Publisher 2/Series D/Series D #1.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/special-folder.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/special-folder.json new file mode 100644 index 000000000..fec7691a3 --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/special-folder.json @@ -0,0 +1,6 @@ +[ + "Root 1/Series A/Series A 01.cbz", + "Root 1/Series A/Series A 02.cbz", + "Root 1/Series A/Specials/Series A - Title.cbz", + "Root 1/Series A/Specials/Title Two.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Base/special-marker.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/special-marker.json new file mode 100644 index 000000000..e79249c78 --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Base/special-marker.json @@ -0,0 +1,6 @@ +[ + "Root 1/Series A/Series A 01.cbz", + "Root 1/Series A/Series A 02.cbz", + "Root 1/Series A/Series A - Title SP01.cbz", + "Root 1/Series A/Series A - Title SP02.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-file.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-file.json new file mode 100644 index 000000000..9b93ebfff --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-file.json @@ -0,0 +1,6 @@ +[ + "Root 1/Series A/Series A 01.cbz", + "Root 1/Series B/Series B 01.cbz", + "Root 1/Series B/Series B 02.cbz", + "Root 2/Series C/Series C 01.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-subfolder-all.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-subfolder-all.json new file mode 100644 index 000000000..ea3744f6f --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-subfolder-all.json @@ -0,0 +1,6 @@ +[ + "Root 1/Series A/Series A 01.cbz", + "Root 1/Series B/Series B 01.cbz", + "Root 1/Series B/Ignore/Series B 02.cbz", + "Root 2/Ignore/Series C/Series C 01.cbz" +] diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-subfolder.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-subfolder.json new file mode 100644 index 000000000..a85e342fc --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Exclude/exclude-subfolder.json @@ -0,0 +1,6 @@ +[ + "Root 1/Series A/Series A 01.cbz", + "Root 1/Series B/Series B 01.cbz", + "Root 1/Series B/Ignore/Series B 02.cbz", + "Root 2/Series C/Series C 01.cbz" +] From 4372d09ee49533461fcc00cc38bfd8e30b6108a9 Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Thu, 8 May 2025 16:49:33 -0500 Subject: [PATCH 4/5] Added the first part of the new scanner - file scanner. Responsible for walking all directories and finding all files. --- API.Tests/Helpers/ScannerHelper.cs | 20 ++- API.Tests/Services/FileScannerTests.cs | 156 ++++++++++++++++++ API.Tests/Services/ScannerServiceTests.cs | 3 - .../TestCases/Mixed Formats - Manga.json | 8 + API/DTOs/Internal/Scanner/ScannedDirectory.cs | 22 +++ API/DTOs/Internal/Scanner/ScannedFile.cs | 14 ++ API/DTOs/Internal/Scanner/ScannerOption.cs | 25 +++ API/Services/Tasks/Scanner/FileScanner.cs | 145 ++++++++++++++++ 8 files changed, 381 insertions(+), 12 deletions(-) create mode 100644 API.Tests/Services/FileScannerTests.cs create mode 100644 API.Tests/Services/Test Data/ScannerService/TestCases/Mixed Formats - Manga.json create mode 100644 API/DTOs/Internal/Scanner/ScannedDirectory.cs create mode 100644 API/DTOs/Internal/Scanner/ScannedFile.cs create mode 100644 API/DTOs/Internal/Scanner/ScannerOption.cs create mode 100644 API/Services/Tasks/Scanner/FileScanner.cs diff --git a/API.Tests/Helpers/ScannerHelper.cs b/API.Tests/Helpers/ScannerHelper.cs index 653efebb1..150850f99 100644 --- a/API.Tests/Helpers/ScannerHelper.cs +++ b/API.Tests/Helpers/ScannerHelper.cs @@ -35,7 +35,7 @@ public class ScannerHelper private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/ScanTests"); private readonly string _testcasesDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/TestCases"); private readonly string _imagePath = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/1x1.png"); - private static readonly string[] ComicInfoExtensions = new[] { ".cbz", ".cbr", ".zip", ".rar" }; + private static readonly string[] ComicInfoExtensions = [".cbz", ".cbr", ".zip", ".rar"]; public ScannerHelper(IUnitOfWork unitOfWork, ITestOutputHelper testOutputHelper) { @@ -43,7 +43,7 @@ public class ScannerHelper _testOutputHelper = testOutputHelper; } - public async Task GenerateScannerData(string testcase, Dictionary comicInfos = null) + public async Task GenerateScannerData(string testcase, Dictionary? comicInfos = null) { var testDirectoryPath = await GenerateTestDirectory(Path.Join(_testcasesDirectory, testcase), comicInfos); @@ -64,7 +64,7 @@ public class ScannerHelper return library; } - public ScannerService CreateServices(DirectoryService ds = null, IFileSystem fs = null) + public ScannerService CreateServices(DirectoryService? ds = null, IFileSystem? fs = null) { fs ??= new FileSystem(); ds ??= new DirectoryService(Substitute.For>(), fs); @@ -113,7 +113,7 @@ public class ScannerHelper - private async Task GenerateTestDirectory(string mapPath, Dictionary comicInfos = null) + private async Task GenerateTestDirectory(string mapPath, Dictionary? comicInfos = null) { // Read the map file var mapContent = await File.ReadAllTextAsync(mapPath); @@ -130,7 +130,7 @@ public class ScannerHelper Directory.CreateDirectory(testDirectory); // Generate the files and folders - await Scaffold(testDirectory, filePaths, comicInfos); + await Scaffold(testDirectory, filePaths ?? [], comicInfos); _testOutputHelper.WriteLine($"Test Directory Path: {testDirectory}"); @@ -138,18 +138,20 @@ public class ScannerHelper } - public async Task Scaffold(string testDirectory, List filePaths, Dictionary comicInfos = null) + public async Task Scaffold(string testDirectory, List filePaths, Dictionary? comicInfos = null) { foreach (var relativePath in filePaths) { var fullPath = Path.Combine(testDirectory, relativePath); var fileDir = Path.GetDirectoryName(fullPath); + if (string.IsNullOrEmpty(fileDir)) continue; + // Create the directory if it doesn't exist if (!Directory.Exists(fileDir)) { Directory.CreateDirectory(fileDir); - Console.WriteLine($"Created directory: {fileDir}"); + _testOutputHelper.WriteLine($"Created directory: {fileDir}"); } var ext = Path.GetExtension(fullPath).ToLower(); @@ -161,7 +163,7 @@ public class ScannerHelper { // Create an empty file await File.Create(fullPath).DisposeAsync(); - Console.WriteLine($"Created empty file: {fullPath}"); + _testOutputHelper.WriteLine($"Created empty file: {fullPath}"); } } } @@ -188,7 +190,7 @@ public class ScannerHelper } } - Console.WriteLine($"Created minimal CBZ archive: {filePath} with{(comicInfo != null ? "" : "out")} metadata."); + _testOutputHelper.WriteLine($"Created minimal CBZ archive: {filePath} with{(comicInfo != null ? "" : "out")} metadata."); } diff --git a/API.Tests/Services/FileScannerTests.cs b/API.Tests/Services/FileScannerTests.cs new file mode 100644 index 000000000..103c46c7c --- /dev/null +++ b/API.Tests/Services/FileScannerTests.cs @@ -0,0 +1,156 @@ +using System.IO; +using System.IO.Abstractions; +using System.Linq; +using System.Threading.Tasks; +using API.DTOs.Internal.Scanner; +using API.Entities.Enums; +using API.Services; +using API.Services.Tasks.Scanner; +using API.Services.Tasks.Scanner.Parser; +using API.Tests.Helpers; +using Microsoft.Extensions.Logging; +using NSubstitute; +using Xunit; +using Xunit.Abstractions; + +namespace API.Tests.Services; + +public class FileScannerTests : AbstractDbTest +{ + private readonly FileScanner _fileScanner; + private readonly IDirectoryService _directoryService; + private readonly ScannerHelper _scannerHelper; + private readonly string _outputDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/ScanTests"); + private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/TestCases"); + + public FileScannerTests(ITestOutputHelper testOutputHelper) + { + _directoryService = new DirectoryService(Substitute.For>(), new FileSystem()); + _fileScanner = new FileScanner(_directoryService, UnitOfWork); + _scannerHelper = new ScannerHelper(UnitOfWork, testOutputHelper); + } + + #region ScanFiles - Basic Tests + + /// + /// Validates that FileTypePattern works + /// + [Fact] + public async Task ScanFiles_ShouldIncludeOnlyArchiveTypes() + { + const string testcase = "Flat Series - Manga.json"; + var library = await _scannerHelper.GenerateScannerData(testcase); + var folder = library.Folders.First().Path; + + var options = new ScannerOption + { + FolderPaths = [folder], + FileTypePattern = [FileTypeGroup.Archive], + ExcludePatterns = [] + }; + + var result = _fileScanner.ScanFiles(options); + + Assert.Single(result); // One folder + var scanned = result[0]; + Assert.Equal(Parser.NormalizePath(Path.Join(folder, "My Dress-Up Darling")), scanned.DirectoryPath); + Assert.All(scanned.Files, file => + { + Assert.EndsWith(".cbz", file.FilePath); + }); + } + + [Fact] + public async Task ScanFiles_ShouldIncludeMultipleTypes() + { + const string testcase = "Mixed Formats - Manga.json"; + var library = await _scannerHelper.GenerateScannerData(testcase); + var folder = library.Folders.First().Path; + + var options = new ScannerOption + { + FolderPaths = [folder], + FileTypePattern = [FileTypeGroup.Archive, FileTypeGroup.Epub], + ExcludePatterns = [] + }; + + var result = _fileScanner.ScanFiles(options); + + Assert.Single(result); // One folder + var scanned = result[0]; + Assert.Equal(Parser.NormalizePath(Path.Join(folder, "My Dress-Up Darling")), scanned.DirectoryPath); + var validExtensions = new[] { ".cbz", ".epub" }; + Assert.All(scanned.Files, file => + { + Assert.Contains(Path.GetExtension(file.FilePath)?.ToLowerInvariant(), validExtensions); + }); + } + + + + + #endregion + + #region ScannFiles - Exclude Patterns + + + [Fact] + public async Task ScanFiles_ShouldExcludeMatchingPattern() + { + const string testcase = "Flat Series - Manga.json"; + var library = await _scannerHelper.GenerateScannerData(testcase); + var folder = library.Folders.First().Path; + + var options = new ScannerOption + { + FolderPaths = [folder], + FileTypePattern = [FileTypeGroup.Archive], + ExcludePatterns = ["*ch 10.cbz"] // Exclude chapter 10 + }; + + var result = _fileScanner.ScanFiles(options); + + var scannedFiles = result.SelectMany(d => d.Files).ToList(); + Assert.DoesNotContain(scannedFiles, f => f.FilePath.Contains("ch 10.cbz")); + Assert.Contains(scannedFiles, f => f.FilePath.Contains("v01.cbz")); + Assert.Contains(scannedFiles, f => f.FilePath.Contains("v02.cbz")); + } + + #endregion + + #region ScannFiles - Change Detection + + [Fact] + public async Task ScanFiles_ShouldHaveAccurateLastModifiedUtc() + { + const string testcase = "Flat Series - Manga.json"; + var library = await _scannerHelper.GenerateScannerData(testcase); + var folder = library.Folders.First().Path; + + var options = new ScannerOption + { + FolderPaths = [folder], + FileTypePattern = [FileTypeGroup.Archive], + ExcludePatterns = [] + }; + + var result = _fileScanner.ScanFiles(options); + + Assert.Single(result); + var scannedDir = result[0]; + var file = scannedDir.Files[0]; + + var expected = _directoryService.GetLastWriteTime(file.FilePath).ToUniversalTime(); + Assert.Equal(expected, file.LastModifiedUtc); + } + + #endregion + + + protected override async Task ResetDb() + { + Context.Series.RemoveRange(Context.Series); + Context.Library.RemoveRange(Context.Library); + await Context.SaveChangesAsync(); + } +} diff --git a/API.Tests/Services/ScannerServiceTests.cs b/API.Tests/Services/ScannerServiceTests.cs index 9b0271fc2..bc9b36843 100644 --- a/API.Tests/Services/ScannerServiceTests.cs +++ b/API.Tests/Services/ScannerServiceTests.cs @@ -18,14 +18,11 @@ namespace API.Tests.Services; public class ScannerServiceTests : AbstractDbTest { - private readonly ITestOutputHelper _testOutputHelper; private readonly ScannerHelper _scannerHelper; private readonly string _testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/ScanTests"); public ScannerServiceTests(ITestOutputHelper testOutputHelper) { - _testOutputHelper = testOutputHelper; - // Set up Hangfire to use in-memory storage for testing GlobalConfiguration.Configuration.UseInMemoryStorage(); _scannerHelper = new ScannerHelper(UnitOfWork, testOutputHelper); diff --git a/API.Tests/Services/Test Data/ScannerService/TestCases/Mixed Formats - Manga.json b/API.Tests/Services/Test Data/ScannerService/TestCases/Mixed Formats - Manga.json new file mode 100644 index 000000000..942c015f4 --- /dev/null +++ b/API.Tests/Services/Test Data/ScannerService/TestCases/Mixed Formats - Manga.json @@ -0,0 +1,8 @@ +[ + "My Dress-Up Darling/My Dress-Up Darling v01.cbz", + "My Dress-Up Darling/My Dress-Up Darling v02.cbz", + "My Dress-Up Darling/My Dress-Up Darling ch 10.cbz", + "My Dress-Up Darling/My Dress-Up Darling ch 11.epub", + "My Dress-Up Darling/My Dress-Up Darling ch 12.png", + "My Dress-Up Darling/My Dress-Up Darling ch 13.pdf" +] \ No newline at end of file diff --git a/API/DTOs/Internal/Scanner/ScannedDirectory.cs b/API/DTOs/Internal/Scanner/ScannedDirectory.cs new file mode 100644 index 000000000..1f7df5643 --- /dev/null +++ b/API/DTOs/Internal/Scanner/ScannedDirectory.cs @@ -0,0 +1,22 @@ +using System; +using System.Collections.Generic; +using API.Entities.Enums; +using API.Services.Tasks.Scanner.Parser; + +namespace API.DTOs.Internal.Scanner; + +/// +/// Represents a Directory on disk and metadata information for the Scan +/// +public sealed record ScannedDirectory +{ + /// + /// Normalized Directory Path + /// + public required string DirectoryPath { get => _directoryPath; set => _directoryPath = Parser.NormalizePath(value); } + private string _directoryPath; + + public required DateTime LastModifiedUtc { get; set; } + + public List Files { get; set; } = []; +} diff --git a/API/DTOs/Internal/Scanner/ScannedFile.cs b/API/DTOs/Internal/Scanner/ScannedFile.cs new file mode 100644 index 000000000..61c7c60d1 --- /dev/null +++ b/API/DTOs/Internal/Scanner/ScannedFile.cs @@ -0,0 +1,14 @@ +using System; +using API.Entities.Enums; +using API.Services.Tasks.Scanner.Parser; + +namespace API.DTOs.Internal.Scanner; + +public sealed record ScannedFile +{ + public required string FilePath { get => _filePath; set => _filePath = Parser.NormalizePath(value); } + private string _filePath; + + public required DateTime LastModifiedUtc { get; set; } + public required MangaFormat Format { get; set; } +} diff --git a/API/DTOs/Internal/Scanner/ScannerOption.cs b/API/DTOs/Internal/Scanner/ScannerOption.cs new file mode 100644 index 000000000..556f9ae06 --- /dev/null +++ b/API/DTOs/Internal/Scanner/ScannerOption.cs @@ -0,0 +1,25 @@ +using System.Collections.Generic; +using API.Entities.Enums; + +namespace API.DTOs.Internal.Scanner; + +public sealed record ScannerOption +{ + /// + /// A list of File Type Patterns to search files for. If empty, scan will abort + /// + public List FileTypePattern { get; set; } = [FileTypeGroup.Archive, FileTypeGroup.Epub, FileTypeGroup.Images, FileTypeGroup.Pdf]; + /// + /// Folders to scan + /// + public List FolderPaths { get; set; } + + /// + /// Glob syntax to exclude from scan results + /// + public List ExcludePatterns { get; set; } = []; + /// + /// Skip LastModified checks + /// + public bool ForceScan { get; set; } +} diff --git a/API/Services/Tasks/Scanner/FileScanner.cs b/API/Services/Tasks/Scanner/FileScanner.cs new file mode 100644 index 000000000..f29e57d61 --- /dev/null +++ b/API/Services/Tasks/Scanner/FileScanner.cs @@ -0,0 +1,145 @@ +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using API.Data; +using API.Data.Repositories; +using API.DTOs.Internal.Scanner; +using API.Entities.Enums; +using API.Extensions; +using Kavita.Common.Helpers; + +namespace API.Services.Tasks.Scanner; + +public interface IFileScanner +{ + // TODO: Move this to the scanner service + //Task ScanLibrary(int libraryId, bool forceScan = false); + List ScanFiles(ScannerOption options); +} + + +public class FileScanner : IFileScanner +{ + private readonly IDirectoryService _directoryService; + private readonly IUnitOfWork _unitOfWork; + + public FileScanner(IDirectoryService directoryService, IUnitOfWork unitOfWork) + { + _directoryService = directoryService; + _unitOfWork = unitOfWork; + } + + + public async Task ScanLibrary(int libraryId, bool forceScan = false) + { + var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, + LibraryIncludes.Folders | LibraryIncludes.ExcludePatterns | LibraryIncludes.FileTypes); + + if (library == null) + { + return; + } + + // Create a ScannerOption + var options = new ScannerOption() + { + FileTypePattern = library.LibraryFileTypes.Select(s => s.FileTypeGroup).ToList(), + ForceScan = forceScan, + ExcludePatterns = [.. library.LibraryExcludePatterns.Select(s => s.Pattern)], + FolderPaths = [.. library.Folders.Select(f => Parser.Parser.NormalizePath(f.Path))] + }; + + + // Find all the information about the directories and their files + var files = ScanFiles(options); + + // Parse said information + + + return; + } + + public List ScanFiles(ScannerOption options) + { + // Validate input options + if (options == null || options.FolderPaths.Count == 0 || options.FileTypePattern.Count == 0) + { + return []; + } + + // Build the file extensions regex from the file type patterns + var fileExtensions = string.Join("|", options.FileTypePattern.Select(l => l.GetRegex())); + if (string.IsNullOrWhiteSpace(fileExtensions)) + { + return []; + } + + + var matcher = BuildMatcher(options.ExcludePatterns); + var scannedDirectories = new List(); + + foreach (var folderPath in options.FolderPaths) + { + var normalizedFolderPath = Parser.Parser.NormalizePath(folderPath); + + var allDirectories = _directoryService.GetAllDirectories(normalizedFolderPath, matcher) + .Select(Parser.Parser.NormalizePath) + .OrderByDescending(d => d.Length) + .ToList(); + + // TODO: Optimization: If allDirectories is large, split into Parallel tasks + + foreach (var directory in allDirectories) + { + var files = _directoryService.ScanFiles(directory, fileExtensions, matcher) + .Select(filePath => + { + // Gather metadata for each file + var lastModifiedUtc = _directoryService.GetLastWriteTime(filePath).ToUniversalTime(); + var format = Parser.Parser.ParseFormat(filePath); + return new ScannedFile + { + FilePath = filePath, + LastModifiedUtc = lastModifiedUtc, + Format = format + }; + }) + .ToList(); + + // Skip directories with no valid files + if (files.Count == 0) + { + continue; + } + + // Get directory's metadata (TODO: Replace with _directoryService.GetLastWriteTime(folder).Truncate(TimeSpan.TicksPerSecond);) + //var directoryLastModifiedUtc = files.Max(f => f.LastModifiedUtc); + var directoryLastModifiedUtc = _directoryService.GetLastWriteTime(normalizedFolderPath).Truncate(TimeSpan.TicksPerSecond); + + // Add the directory and its files to the result + scannedDirectories.Add(new ScannedDirectory + { + DirectoryPath = directory, + LastModifiedUtc = directoryLastModifiedUtc, + Files = files + }); + } + } + + return scannedDirectories; + } + + + + private static GlobMatcher BuildMatcher(List excludePatterns) + { + var matcher = new GlobMatcher(); + foreach (var pattern in excludePatterns.Where(p => !string.IsNullOrEmpty(p))) + { + matcher.AddExclude(pattern); + } + + return matcher; + } +} From 2e53987dcacb770c447d42cada12b99f1c8becfb Mon Sep 17 00:00:00 2001 From: Joseph Milazzo Date: Fri, 9 May 2025 06:35:08 -0500 Subject: [PATCH 5/5] Started working on the parser step - still a bit rough in my head. --- API/Controllers/LibraryController.cs | 6 + API/DTOs/Internal/Scanner/ParsedFile.cs | 12 ++ API/DTOs/Internal/Scanner/ScannedDirectory.cs | 11 +- API/DTOs/Internal/Scanner/ScannedFile.cs | 1 + API/DTOs/Internal/Scanner/ScannerOption.cs | 8 ++ API/Services/Tasks/Scanner/FileParser.cs | 125 ++++++++++++++++++ API/Services/Tasks/Scanner/FileScanner.cs | 57 ++++---- 7 files changed, 190 insertions(+), 30 deletions(-) create mode 100644 API/DTOs/Internal/Scanner/ParsedFile.cs create mode 100644 API/Services/Tasks/Scanner/FileParser.cs diff --git a/API/Controllers/LibraryController.cs b/API/Controllers/LibraryController.cs index 4f3b6c832..557ef8fb3 100644 --- a/API/Controllers/LibraryController.cs +++ b/API/Controllers/LibraryController.cs @@ -623,6 +623,12 @@ public class LibraryController : BaseApiController library.ManageReadingLists = dto.ManageReadingLists; library.AllowScrobbling = dto.AllowScrobbling; library.AllowMetadataMatching = dto.AllowMetadataMatching; + + if (!dto.AllowFilenameParsing && !dto.AllowMetadataParsing) + { + throw new InvalidOperationException("At least one of UseFilenameParsing or UseInternalMetadataParsing must be true."); + } + library.AllowFilenameParsing = dto.AllowFilenameParsing; library.AllowMetadataParsing = dto.AllowMetadataParsing; diff --git a/API/DTOs/Internal/Scanner/ParsedFile.cs b/API/DTOs/Internal/Scanner/ParsedFile.cs new file mode 100644 index 000000000..14499adc8 --- /dev/null +++ b/API/DTOs/Internal/Scanner/ParsedFile.cs @@ -0,0 +1,12 @@ +using API.Data.Metadata; +using API.Services.Tasks.Scanner.Parser; + +namespace API.DTOs.Internal.Scanner; +#nullable enable + +public sealed record ParsedFile +{ + public int Pages { get; set; } + public ComicInfo? Metadata { get; set; } + public ParserInfo? ParsedInformation { get; set; } +} diff --git a/API/DTOs/Internal/Scanner/ScannedDirectory.cs b/API/DTOs/Internal/Scanner/ScannedDirectory.cs index 1f7df5643..2eef705be 100644 --- a/API/DTOs/Internal/Scanner/ScannedDirectory.cs +++ b/API/DTOs/Internal/Scanner/ScannedDirectory.cs @@ -16,7 +16,14 @@ public sealed record ScannedDirectory public required string DirectoryPath { get => _directoryPath; set => _directoryPath = Parser.NormalizePath(value); } private string _directoryPath; - public required DateTime LastModifiedUtc { get; set; } + /// + /// Root where the directory resides + /// + /// Library Root + public required string FolderRoot { get => _folderRoot; set => _folderRoot = Parser.NormalizePath(value); } + private string _folderRoot; - public List Files { get; set; } = []; + public required DateTime LastModifiedUtc { get; init; } + + public List Files { get; init; } = []; } diff --git a/API/DTOs/Internal/Scanner/ScannedFile.cs b/API/DTOs/Internal/Scanner/ScannedFile.cs index 61c7c60d1..abf40ba8a 100644 --- a/API/DTOs/Internal/Scanner/ScannedFile.cs +++ b/API/DTOs/Internal/Scanner/ScannedFile.cs @@ -10,5 +10,6 @@ public sealed record ScannedFile private string _filePath; public required DateTime LastModifiedUtc { get; set; } + public required string FolderRoot { get; set; } public required MangaFormat Format { get; set; } } diff --git a/API/DTOs/Internal/Scanner/ScannerOption.cs b/API/DTOs/Internal/Scanner/ScannerOption.cs index 556f9ae06..a5dfc2f16 100644 --- a/API/DTOs/Internal/Scanner/ScannerOption.cs +++ b/API/DTOs/Internal/Scanner/ScannerOption.cs @@ -22,4 +22,12 @@ public sealed record ScannerOption /// Skip LastModified checks /// public bool ForceScan { get; set; } + /// + /// Allow use of Filename Parsing + /// + public bool UseFilenameParsing { get; set; } + /// + /// Allow use of Internal Metadata + /// + public bool UseInternalMetadataParsing { get; set; } } diff --git a/API/Services/Tasks/Scanner/FileParser.cs b/API/Services/Tasks/Scanner/FileParser.cs new file mode 100644 index 000000000..91dec5269 --- /dev/null +++ b/API/Services/Tasks/Scanner/FileParser.cs @@ -0,0 +1,125 @@ +using System; +using API.Data.Metadata; +using API.DTOs.Internal.Scanner; +using API.Entities.Enums; +using API.Services.Tasks.Scanner.Parser; +using Microsoft.Extensions.Logging; + +namespace API.Services.Tasks.Scanner; +#nullable enable + +public interface IFileParser +{ + ParsedFile? Parse(ScannedFile file); +} + +public class FileParser : IFileParser +{ + private readonly IArchiveService _archiveService; + private readonly IBookService _bookService; + private readonly IImageService _imageService; + private readonly ILogger _logger; + private readonly BasicParser _basicParser; + private readonly ComicVineParser _comicVineParser; + private readonly ImageParser _imageParser; + private readonly BookParser _bookParser; + private readonly PdfParser _pdfParser; + + public FileParser(IArchiveService archiveService, IDirectoryService directoryService, + IBookService bookService, IImageService imageService, ILogger logger) + { + _archiveService = archiveService; + _bookService = bookService; + _imageService = imageService; + _logger = logger; + + _imageParser = new ImageParser(directoryService); + _basicParser = new BasicParser(directoryService, _imageParser); + _bookParser = new BookParser(directoryService, bookService, _basicParser); + _comicVineParser = new ComicVineParser(directoryService); + _pdfParser = new PdfParser(directoryService); + } + + + + + /// + /// Processes files found during a library scan. + /// + /// Path of a file + /// + /// Library type to determine parsing to perform + // public ParserInfo? ParseFile(string path, string rootPath, string libraryRoot, LibraryType type) + // { + // try + // { + // var info = Parse(path, rootPath, libraryRoot, type); + // if (info == null) + // { + // _logger.LogError("Unable to parse any meaningful information out of file {FilePath}", path); + // return null; + // } + // + // return info; + // } + // catch (Exception ex) + // { + // _logger.LogError(ex, "There was an exception when parsing file {FilePath}", path); + // return null; + // } + // } + + + public ParsedFile? Parse(ScannedFile file, string folderRoot, LibraryType type) + { + var path = file.FilePath; + var rootPath = file.FolderRoot; + + ParserInfo? parserInfo = null; + if (_comicVineParser.IsApplicable(path, type)) + { + parserInfo = _comicVineParser.Parse(path, rootPath, folderRoot, type, GetComicInfo(path)); + } + if (_imageParser.IsApplicable(path, type)) + { + parserInfo = _imageParser.Parse(path, rootPath, folderRoot, type, GetComicInfo(path)); + } + if (_bookParser.IsApplicable(path, type)) + { + parserInfo = _bookParser.Parse(path, rootPath, folderRoot, type, GetComicInfo(path)); + } + if (_pdfParser.IsApplicable(path, type)) + { + parserInfo = _pdfParser.Parse(path, rootPath, folderRoot, type, GetComicInfo(path)); + } + if (_basicParser.IsApplicable(path, type)) + { + parserInfo = _basicParser.Parse(path, rootPath, folderRoot, type, GetComicInfo(path)); + } + + if (parserInfo == null) return null; + + return null; + } + + + /// + /// Gets the ComicInfo for the file if it exists. Null otherwise. + /// + /// Fully qualified path of file + /// + private ComicInfo? GetComicInfo(string filePath) + { + if (Parser.Parser.IsEpub(filePath) || Parser.Parser.IsPdf(filePath)) + { + return _bookService.GetComicInfo(filePath); + } + + if (Parser.Parser.IsComicInfoExtension(filePath)) + { + return _archiveService.GetComicInfo(filePath); + } + + return null; + } +} diff --git a/API/Services/Tasks/Scanner/FileScanner.cs b/API/Services/Tasks/Scanner/FileScanner.cs index f29e57d61..62dce3891 100644 --- a/API/Services/Tasks/Scanner/FileScanner.cs +++ b/API/Services/Tasks/Scanner/FileScanner.cs @@ -31,34 +31,34 @@ public class FileScanner : IFileScanner } - public async Task ScanLibrary(int libraryId, bool forceScan = false) - { - var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, - LibraryIncludes.Folders | LibraryIncludes.ExcludePatterns | LibraryIncludes.FileTypes); - - if (library == null) - { - return; - } - - // Create a ScannerOption - var options = new ScannerOption() - { - FileTypePattern = library.LibraryFileTypes.Select(s => s.FileTypeGroup).ToList(), - ForceScan = forceScan, - ExcludePatterns = [.. library.LibraryExcludePatterns.Select(s => s.Pattern)], - FolderPaths = [.. library.Folders.Select(f => Parser.Parser.NormalizePath(f.Path))] - }; - - - // Find all the information about the directories and their files - var files = ScanFiles(options); - - // Parse said information - - - return; - } + // public async Task ScanLibrary(int libraryId, bool forceScan = false) + // { + // var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, + // LibraryIncludes.Folders | LibraryIncludes.ExcludePatterns | LibraryIncludes.FileTypes); + // + // if (library == null) + // { + // return; + // } + // + // // Create a ScannerOption + // var options = new ScannerOption() + // { + // FileTypePattern = library.LibraryFileTypes.Select(s => s.FileTypeGroup).ToList(), + // ForceScan = forceScan, + // ExcludePatterns = [.. library.LibraryExcludePatterns.Select(s => s.Pattern)], + // FolderPaths = [.. library.Folders.Select(f => Parser.Parser.NormalizePath(f.Path))] + // }; + // + // + // // Find all the information about the directories and their files + // var files = ScanFiles(options); + // + // // Parse said information + // + // + // return; + // } public List ScanFiles(ScannerOption options) { @@ -120,6 +120,7 @@ public class FileScanner : IFileScanner // Add the directory and its files to the result scannedDirectories.Add(new ScannedDirectory { + FolderRoot = folderPath, DirectoryPath = directory, LastModifiedUtc = directoryLastModifiedUtc, Files = files