AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

New Scanner + People Pages (#3286)

Browse source 
Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>
This commit is contained in:
Joe Milazzo 2024-10-23 15:11:18 -07:00 • committed by GitHub
parent 1ed0eae22d
commit ba20ad4ecc
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
142 changed files with 17529 additions and 3038 deletions
Download patch file Download diff file Expand all files Collapse all files

21
API.Tests/Services/DirectoryServiceTests.cs
View file

@ -6,6 +6,7 @@ using System.Linq;
using System.Text;
using System.Threading.Tasks;
using API.Services;
using Kavita.Common.Helpers;
using Microsoft.Extensions.Logging;
using NSubstitute;
using Xunit;
@ -745,6 +746,12 @@ public class DirectoryServiceTests
[InlineData(new [] {"/manga"},
new [] {"/manga/Love Hina/Vol. 01.cbz", "/manga/Love Hina/Specials/Sp01.cbz"},
"/manga/Love Hina")]
[InlineData(new [] {"/manga"},
new [] {"/manga/Love Hina/Hina/Vol. 01.cbz", "/manga/Love Hina/Specials/Sp01.cbz"},
"/manga/Love Hina")]
[InlineData(new [] {"/manga"},
new [] {"/manga/Dress Up Darling/Dress Up Darling Ch 01.cbz", "/manga/Dress Up Darling/Dress Up Darling/Dress Up Darling Vol 01.cbz"},
"/manga/Dress Up Darling")]
public void FindLowestDirectoriesFromFilesTest(string[] rootDirectories, string[] files, string expectedDirectory)
{
var fileSystem = new MockFileSystem();
@ -920,8 +927,9 @@ public class DirectoryServiceTests
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var allFiles = ds.ScanFiles("C:/Data/", API.Services.Tasks.Scanner.Parser.Parser.SupportedExtensions);
var globMatcher = new GlobMatcher();
globMatcher.AddExclude("*.*");
var allFiles = ds.ScanFiles("C:/Data/", API.Services.Tasks.Scanner.Parser.Parser.SupportedExtensions, globMatcher);
Assert.Empty(allFiles);
@ -945,7 +953,9 @@ public class DirectoryServiceTests
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var allFiles = ds.ScanFiles("C:/Data/", API.Services.Tasks.Scanner.Parser.Parser.SupportedExtensions);
var globMatcher = new GlobMatcher();
globMatcher.AddExclude("**/Accel World/*");
var allFiles = ds.ScanFiles("C:/Data/", API.Services.Tasks.Scanner.Parser.Parser.SupportedExtensions, globMatcher);
Assert.Single(allFiles); // Ignore files are not counted in files, only valid extensions
@ -974,7 +984,10 @@ public class DirectoryServiceTests
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fileSystem);
var allFiles = ds.ScanFiles("C:/Data/", API.Services.Tasks.Scanner.Parser.Parser.SupportedExtensions);
var globMatcher = new GlobMatcher();
globMatcher.AddExclude("**/Accel World/*");
globMatcher.AddExclude("**/ArtBooks/*");
var allFiles = ds.ScanFiles("C:/Data/", API.Services.Tasks.Scanner.Parser.Parser.SupportedExtensions, globMatcher);
Assert.Equal(2, allFiles.Count); // Ignore files are not counted in files, only valid extensions

26
API.Tests/Services/ParseScannedFilesTests.cs
View file

@ -206,24 +206,6 @@ public class ParseScannedFilesTests : AbstractDbTest
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(ds, Substitute.For<IBookService>()), Substitute.For<IEventHub>());
// var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
//
// Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
// {
// var skippedScan = parsedInfo.Item1;
// var parsedFiles = parsedInfo.Item2;
// if (parsedFiles.Count == 0) return Task.CompletedTask;
//
// var foundParsedSeries = new ParsedSeries()
// {
// Name = parsedFiles.First().Series,
// NormalizedName = parsedFiles.First().Series.ToNormalized(),
// Format = parsedFiles.First().Format
// };
//
// parsedSeries.Add(foundParsedSeries, parsedFiles);
// return Task.CompletedTask;
// }
var library =
await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
@ -273,7 +255,7 @@ public class ParseScannedFilesTests : AbstractDbTest
var directoriesSeen = new HashSet<string>();
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
var scanResults = await psf.ProcessFiles("C:/Data/", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
var scanResults = await psf.ScanFiles("C:/Data/", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
foreach (var scanResult in scanResults)
{
directoriesSeen.Add(scanResult.Folder);
@ -295,7 +277,7 @@ public class ParseScannedFilesTests : AbstractDbTest
Assert.NotNull(library);
var directoriesSeen = new HashSet<string>();
var scanResults = await psf.ProcessFiles("C:/Data/", false,
var scanResults = await psf.ScanFiles("C:/Data/", false,
await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
foreach (var scanResult in scanResults)
@ -328,7 +310,7 @@ public class ParseScannedFilesTests : AbstractDbTest
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
Assert.NotNull(library);
var scanResults = await psf.ProcessFiles("C:/Data", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
var scanResults = await psf.ScanFiles("C:/Data", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
Assert.Equal(2, scanResults.Count);
}
@ -357,7 +339,7 @@ public class ParseScannedFilesTests : AbstractDbTest
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
Assert.NotNull(library);
var scanResults = await psf.ProcessFiles("C:/Data", false,
var scanResults = await psf.ScanFiles("C:/Data", false,
await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
Assert.Single(scanResults);

131
API.Tests/Services/ScannerServiceTests.cs
View file

@ -50,65 +50,14 @@ public class ScannerServiceTests : AbstractDbTest
await _context.SaveChangesAsync();
}
[Fact]
public void FindSeriesNotOnDisk_Should_Remove1()
{
var infos = new Dictionary<ParsedSeries, IList<ParserInfo>>();
ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Volumes = "1", Format = MangaFormat.Archive});
//AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Volumes = "1", Format = MangaFormat.Epub});
var existingSeries = new List<Series>
{
new SeriesBuilder("Darker Than Black")
.WithFormat(MangaFormat.Epub)
.WithVolume(new VolumeBuilder("1")
.WithName("1")
.Build())
.WithLocalizedName("Darker Than Black")
.Build()
};
Assert.Single(ScannerService.FindSeriesNotOnDisk(existingSeries, infos));
}
[Fact]
public void FindSeriesNotOnDisk_Should_RemoveNothing_Test()
{
var infos = new Dictionary<ParsedSeries, IList<ParserInfo>>();
ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Darker than Black", Format = MangaFormat.Archive});
ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Cage of Eden", Volumes = "1", Format = MangaFormat.Archive});
ParserInfoFactory.AddToParsedInfo(infos, new ParserInfo() {Series = "Cage of Eden", Volumes = "10", Format = MangaFormat.Archive});
var existingSeries = new List<Series>
{
new SeriesBuilder("Cage of Eden")
.WithFormat(MangaFormat.Archive)
.WithVolume(new VolumeBuilder("1")
.WithName("1")
.Build())
.WithLocalizedName("Darker Than Black")
.Build(),
new SeriesBuilder("Darker Than Black")
.WithFormat(MangaFormat.Archive)
.WithVolume(new VolumeBuilder("1")
.WithName("1")
.Build())
.WithLocalizedName("Darker Than Black")
.Build(),
};
Assert.Empty(ScannerService.FindSeriesNotOnDisk(existingSeries, infos));
}
[Fact]
public async Task ScanLibrary_ComicVine_PublisherFolder()
{
var testcase = "Publisher - ComicVine.json";
var postLib = await GenerateScannerData(testcase);
var library = await GenerateScannerData(testcase);
var scanner = CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Equal(4, postLib.Series.Count);
@ -118,18 +67,67 @@ public class ScannerServiceTests : AbstractDbTest
public async Task ScanLibrary_ShouldCombineNestedFolder()
{
var testcase = "Series and Series-Series Combined - Manga.json";
var postLib = await GenerateScannerData(testcase);
var library = await GenerateScannerData(testcase);
var scanner = CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Single(postLib.Series);
Assert.Single(postLib.Series);
Assert.Equal(2, postLib.Series.First().Volumes.Count);
}
[Fact]
public async Task ScanLibrary_FlatSeries()
{
var testcase = "Flat Series - Manga.json";
var library = await GenerateScannerData(testcase);
var scanner = CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Single(postLib.Series);
Assert.Equal(3, postLib.Series.First().Volumes.Count);
// TODO: Trigger a deletion of ch 10
}
[Fact]
public async Task ScanLibrary_FlatSeriesWithSpecialFolder()
{
var testcase = "Flat Series with Specials Folder - Manga.json";
var library = await GenerateScannerData(testcase);
var scanner = CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Single(postLib.Series);
Assert.Equal(4, postLib.Series.First().Volumes.Count);
Assert.NotNull(postLib.Series.First().Volumes.FirstOrDefault(v => v.Chapters.FirstOrDefault(c => c.IsSpecial) != null));
}
[Fact]
public async Task ScanLibrary_FlatSeriesWithSpecial()
{
const string testcase = "Flat Special - Manga.json";
var library = await GenerateScannerData(testcase);
var scanner = CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Single(postLib.Series);
Assert.Equal(3, postLib.Series.First().Volumes.Count);
Assert.NotNull(postLib.Series.First().Volumes.FirstOrDefault(v => v.Chapters.FirstOrDefault(c => c.IsSpecial) != null));
}
private async Task<Library> GenerateScannerData(string testcase)
{
var testDirectoryPath = await GenerateTestDirectory(Path.Join(_testcasesDirectory, testcase));
_testOutputHelper.WriteLine($"Test Directory Path: {testDirectoryPath}");
var (publisher, type) = SplitPublisherAndLibraryType(Path.GetFileNameWithoutExtension(testcase));
@ -145,25 +143,26 @@ public class ScannerServiceTests : AbstractDbTest
_unitOfWork.LibraryRepository.Add(library);
await _unitOfWork.CommitAsync();
return library;
}
private ScannerService CreateServices()
{
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), new FileSystem());
var mockReadingService = new MockReadingItemService(ds, Substitute.For<IBookService>());
var processSeries = new ProcessSeries(_unitOfWork, Substitute.For<ILogger<ProcessSeries>>(),
Substitute.For<IEventHub>(),
ds, Substitute.For<ICacheHelper>(), mockReadingService, Substitute.For<IFileService>(),
Substitute.For<IMetadataService>(),
Substitute.For<IWordCountAnalyzerService>(), Substitute.For<ICollectionTagService>(),
Substitute.For<IWordCountAnalyzerService>(),
Substitute.For<IReadingListService>(),
Substitute.For<IExternalMetadataService>(), new TagManagerService(_unitOfWork, Substitute.For<ILogger<TagManagerService>>()));
Substitute.For<IExternalMetadataService>());
var scanner = new ScannerService(_unitOfWork, Substitute.For<ILogger<ScannerService>>(),
Substitute.For<IMetadataService>(),
Substitute.For<ICacheService>(), Substitute.For<IEventHub>(), ds,
mockReadingService, processSeries, Substitute.For<IWordCountAnalyzerService>());
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
return postLib;
return scanner;
}
private static (string Publisher, LibraryType Type) SplitPublisherAndLibraryType(string input)
@ -209,6 +208,8 @@ public class ScannerServiceTests : AbstractDbTest
// Generate the files and folders
await Scaffold(testDirectory, filePaths);
_testOutputHelper.WriteLine($"Test Directory Path: {testDirectory}");
return testDirectory;
}

31
API.Tests/Services/SeriesServiceTests.cs
View file

@ -817,12 +817,17 @@ public class SeriesServiceTests : AbstractDbTest
public async Task UpdateSeriesMetadata_ShouldAddNewPerson_NoExistingPeople()
{
await ResetDb();
var g = new PersonBuilder("Existing Person").Build();
await _context.SaveChangesAsync();
var s = new SeriesBuilder("Test")
.WithMetadata(new SeriesMetadataBuilder().Build())
.WithMetadata(new SeriesMetadataBuilder()
.WithPerson(g, PersonRole.Publisher)
.Build())
.Build();
s.Library = new LibraryBuilder("Test LIb", LibraryType.Book).Build();
var g = new PersonBuilder("Existing Person", PersonRole.Publisher).Build();
_context.Series.Add(s);
_context.Person.Add(g);
@ -833,7 +838,7 @@ public class SeriesServiceTests : AbstractDbTest
SeriesMetadata = new SeriesMetadataDto
{
SeriesId = 1,
Publishers = new List<PersonDto> {new () {Id = 0, Name = "Existing Person", Role = PersonRole.Publisher}},
Publishers = new List<PersonDto> {new () {Id = 0, Name = "Existing Person"}},
},
});
@ -842,7 +847,7 @@ public class SeriesServiceTests : AbstractDbTest
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(1);
Assert.NotNull(series.Metadata);
Assert.True(series.Metadata.People.Select(g => g.Name).All(g => g == "Existing Person"));
Assert.True(series.Metadata.People.Select(g => g.Person.Name).All(personName => personName == "Existing Person"));
Assert.False(series.Metadata.PublisherLocked); // PublisherLocked is false unless the UI Explicitly says it should be locked
}
@ -854,10 +859,14 @@ public class SeriesServiceTests : AbstractDbTest
.WithMetadata(new SeriesMetadataBuilder().Build())
.Build();
s.Library = new LibraryBuilder("Test LIb", LibraryType.Book).Build();
var g = new PersonBuilder("Existing Person", PersonRole.Publisher).Build();
s.Metadata.People = new List<Person>
{new PersonBuilder("Existing Writer", PersonRole.Writer).Build(),
new PersonBuilder("Existing Translator", PersonRole.Translator).Build(), new PersonBuilder("Existing Publisher 2", PersonRole.Publisher).Build()};
var g = new PersonBuilder("Existing Person").Build();
s.Metadata.People = new List<SeriesMetadataPeople>
{
new SeriesMetadataPeople() {Person = new PersonBuilder("Existing Writer").Build(), Role = PersonRole.Writer},
new SeriesMetadataPeople() {Person = new PersonBuilder("Existing Translator").Build(), Role = PersonRole.Translator},
new SeriesMetadataPeople() {Person = new PersonBuilder("Existing Publisher 2").Build(), Role = PersonRole.Publisher}
};
_context.Series.Add(s);
_context.Person.Add(g);
@ -868,7 +877,7 @@ public class SeriesServiceTests : AbstractDbTest
SeriesMetadata = new SeriesMetadataDto
{
SeriesId = 1,
Publishers = new List<PersonDto> {new () {Id = 0, Name = "Existing Person", Role = PersonRole.Publisher}},
Publishers = new List<PersonDto> {new () {Id = 0, Name = "Existing Person"}},
PublisherLocked = true
},
@ -878,7 +887,7 @@ public class SeriesServiceTests : AbstractDbTest
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(1);
Assert.NotNull(series.Metadata);
Assert.True(series.Metadata.People.Select(g => g.Name).All(g => g == "Existing Person"));
Assert.True(series.Metadata.People.Select(g => g.Person.Name).All(personName => personName == "Existing Person"));
Assert.True(series.Metadata.PublisherLocked);
}
@ -891,7 +900,7 @@ public class SeriesServiceTests : AbstractDbTest
.WithMetadata(new SeriesMetadataBuilder().Build())
.Build();
s.Library = new LibraryBuilder("Test LIb", LibraryType.Book).Build();
var g = new PersonBuilder("Existing Person", PersonRole.Publisher).Build();
var g = new PersonBuilder("Existing Person").Build();
_context.Series.Add(s);
_context.Person.Add(g);

5
API.Tests/Services/Test Data/ScannerService/TestCases/Flat Series - Manga.json Normal file
View file

@ -0,0 +1,5 @@
[
"My Dress-Up Darling/My Dress-Up Darling v01.cbz",
"My Dress-Up Darling/My Dress-Up Darling v02.cbz",
"My Dress-Up Darling/My Dress-Up Darling ch 10.cbz"
]

6
API.Tests/Services/Test Data/ScannerService/TestCases/Flat Series with Specials Folder - Manga.json Normal file
View file

@ -0,0 +1,6 @@
[
"My Dress-Up Darling/My Dress-Up Darling v01.cbz",
"My Dress-Up Darling/My Dress-Up Darling v02.cbz",
"My Dress-Up Darling/My Dress-Up Darling ch 10.cbz",
"My Dress-Up Darling/Specials/Official Anime Fanbook SP05 (2024) (Digital).cbz"
]

5
API.Tests/Services/Test Data/ScannerService/TestCases/Flat Special - Manga.json Normal file
View file

@ -0,0 +1,5 @@
[
"Uzaki-chan Wants to Hang Out!\\Uzaki-chan Wants to Hang Out! - 2022 New Years Special SP01.cbz",
"Uzaki-chan Wants to Hang Out!\\Uzaki-chan Wants to Hang Out! - Ch. 103 - Kouhai and Control.cbz",
"Uzaki-chan Wants to Hang Out!\\Uzaki-chan Wants to Hang Out! v01 (2019) (Digital) (danke-Empire).cbz"
]

4
API.Tests/Services/Test Data/ScannerService/TestCases/Nested Chapters - Manga.json Normal file
View file

@ -0,0 +1,4 @@
[
"My Dress-Up Darling/Chapter 1/01.cbz",
"My Dress-Up Darling/Chapter 2/02.cbz"
]