AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Ability to turn off Metadata Parsing (#3872)

Browse source
This commit is contained in:
Joe Milazzo 2025-06-23 18:57:14 -05:00 • committed by GitHub
parent fa8d778c8d
commit 36aa5f5c85
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
63 changed files with 4257 additions and 186 deletions
Download patch file Download diff file Expand all files Collapse all files

6
API.Tests/API.Tests.csproj
View file

@ -6,13 +6,13 @@
</PropertyGroup>
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="9.0.4" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.13.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="9.0.6" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.14.1" />
<PackageReference Include="NSubstitute" Version="5.3.0" />
<PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="22.0.14" />
<PackageReference Include="TestableIO.System.IO.Abstractions.Wrappers" Version="22.0.14" />
<PackageReference Include="xunit" Version="2.9.3" />
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.0">
<PackageReference Include="xunit.runner.visualstudio" Version="3.1.1">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
<PrivateAssets>all</PrivateAssets>
</PackageReference>

8
API.Tests/Parsers/ComicVineParserTests.cs
View file

@ -36,7 +36,7 @@ public class ComicVineParserTests
public void Parse_SeriesWithComicInfo()
{
var actual = _parser.Parse("C:/Comics/Birds of Prey (2002)/Birds of Prey 001 (2002).cbz", "C:/Comics/Birds of Prey (2002)/",
RootDirectory, LibraryType.ComicVine, new ComicInfo()
RootDirectory, LibraryType.ComicVine, true, new ComicInfo()
{
Series = "Birds of Prey",
Volume = "2002"
@ -54,7 +54,7 @@ public class ComicVineParserTests
public void Parse_SeriesWithDirectoryNameAsSeriesYear()
{
var actual = _parser.Parse("C:/Comics/Birds of Prey (2002)/Birds of Prey 001 (2002).cbz", "C:/Comics/Birds of Prey (2002)/",
RootDirectory, LibraryType.ComicVine, null);
RootDirectory, LibraryType.ComicVine, true, null);
Assert.NotNull(actual);
Assert.Equal("Birds of Prey (2002)", actual.Series);
@ -69,7 +69,7 @@ public class ComicVineParserTests
public void Parse_SeriesWithADirectoryNameAsSeriesYear()
{
var actual = _parser.Parse("C:/Comics/DC Comics/Birds of Prey (1999)/Birds of Prey 001 (1999).cbz", "C:/Comics/DC Comics/",
RootDirectory, LibraryType.ComicVine, null);
RootDirectory, LibraryType.ComicVine, true, null);
Assert.NotNull(actual);
Assert.Equal("Birds of Prey (1999)", actual.Series);
@ -84,7 +84,7 @@ public class ComicVineParserTests
public void Parse_FallbackToDirectoryNameOnly()
{
var actual = _parser.Parse("C:/Comics/DC Comics/Blood Syndicate/Blood Syndicate 001 (1999).cbz", "C:/Comics/DC Comics/",
RootDirectory, LibraryType.ComicVine, null);
RootDirectory, LibraryType.ComicVine, true, null);
Assert.NotNull(actual);
Assert.Equal("Blood Syndicate", actual.Series);

20
API.Tests/Parsers/DefaultParserTests.cs
View file

@ -33,7 +33,7 @@ public class DefaultParserTests
[InlineData("C:/", "C:/Something Random/Mujaki no Rakuen SP01.cbz", "Something Random")]
public void ParseFromFallbackFolders_FallbackShouldParseSeries(string rootDir, string inputPath, string expectedSeries)
{
var actual = _defaultParser.Parse(inputPath, rootDir, rootDir, LibraryType.Manga, null);
var actual = _defaultParser.Parse(inputPath, rootDir, rootDir, LibraryType.Manga, true, null);
if (actual == null)
{
Assert.NotNull(actual);
@ -74,7 +74,7 @@ public class DefaultParserTests
fs.AddFile(inputFile, new MockFileData(""));
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fs);
var parser = new BasicParser(ds, new ImageParser(ds));
var actual = parser.Parse(inputFile, rootDirectory, rootDirectory, LibraryType.Manga, null);
var actual = parser.Parse(inputFile, rootDirectory, rootDirectory, LibraryType.Manga, true, null);
_defaultParser.ParseFromFallbackFolders(inputFile, rootDirectory, LibraryType.Manga, ref actual);
Assert.Equal(expectedParseInfo, actual.Series);
}
@ -90,7 +90,7 @@ public class DefaultParserTests
fs.AddFile(inputFile, new MockFileData(""));
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fs);
var parser = new BasicParser(ds, new ImageParser(ds));
var actual = parser.Parse(inputFile, rootDirectory, rootDirectory, LibraryType.Manga, null);
var actual = parser.Parse(inputFile, rootDirectory, rootDirectory, LibraryType.Manga, true, null);
_defaultParser.ParseFromFallbackFolders(inputFile, rootDirectory, LibraryType.Manga, ref actual);
Assert.Equal(expectedParseInfo, actual.Series);
}
@ -251,7 +251,7 @@ public class DefaultParserTests
foreach (var file in expected.Keys)
{
var expectedInfo = expected[file];
var actual = _defaultParser.Parse(file, rootPath, rootPath, LibraryType.Manga, null);
var actual = _defaultParser.Parse(file, rootPath, rootPath, LibraryType.Manga, true, null);
if (expectedInfo == null)
{
Assert.Null(actual);
@ -289,7 +289,7 @@ public class DefaultParserTests
Chapters = "8", Filename = "13.jpg", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
};
var actual2 = _defaultParser.Parse(filepath, @"E:/Manga/Monster #8", "E:/Manga", LibraryType.Manga, null);
var actual2 = _defaultParser.Parse(filepath, @"E:/Manga/Monster #8", "E:/Manga", LibraryType.Manga, true, null);
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
@ -315,7 +315,7 @@ public class DefaultParserTests
FullFilePath = filepath, IsSpecial = false
};
actual2 = _defaultParser.Parse(filepath, @"E:/Manga/Extra layer for no reason/", "E:/Manga",LibraryType.Manga, null);
actual2 = _defaultParser.Parse(filepath, @"E:/Manga/Extra layer for no reason/", "E:/Manga",LibraryType.Manga, true, null);
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
@ -341,7 +341,7 @@ public class DefaultParserTests
FullFilePath = filepath, IsSpecial = false
};
actual2 = _defaultParser.Parse(filepath, @"E:/Manga/Extra layer for no reason/", "E:/Manga", LibraryType.Manga, null);
actual2 = _defaultParser.Parse(filepath, @"E:/Manga/Extra layer for no reason/", "E:/Manga", LibraryType.Manga, true, null);
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
@ -383,7 +383,7 @@ public class DefaultParserTests
FullFilePath = filepath
};
var actual = parser.Parse(filepath, rootPath, rootPath, LibraryType.Manga, null);
var actual = parser.Parse(filepath, rootPath, rootPath, LibraryType.Manga, true, null);
Assert.NotNull(actual);
_testOutputHelper.WriteLine($"Validating {filepath}");
@ -412,7 +412,7 @@ public class DefaultParserTests
FullFilePath = filepath
};
actual = parser.Parse(filepath, rootPath, rootPath, LibraryType.Manga, null);
actual = parser.Parse(filepath, rootPath, rootPath, LibraryType.Manga, true, null);
Assert.NotNull(actual);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expected.Format, actual.Format);
@ -475,7 +475,7 @@ public class DefaultParserTests
foreach (var file in expected.Keys)
{
var expectedInfo = expected[file];
var actual = _defaultParser.Parse(file, rootPath, rootPath, LibraryType.Comic, null);
var actual = _defaultParser.Parse(file, rootPath, rootPath, LibraryType.Comic, true, null);
if (expectedInfo == null)
{
Assert.Null(actual);

6
API.Tests/Parsers/ImageParserTests.cs
View file

@ -34,7 +34,7 @@ public class ImageParserTests
public void Parse_SeriesWithDirectoryName()
{
var actual = _parser.Parse("C:/Comics/Birds of Prey/Chapter 01/01.jpg", "C:/Comics/Birds of Prey/",
RootDirectory, LibraryType.Image, null);
RootDirectory, LibraryType.Image, true, null);
Assert.NotNull(actual);
Assert.Equal("Birds of Prey", actual.Series);
@ -48,7 +48,7 @@ public class ImageParserTests
public void Parse_SeriesWithNoNestedChapter()
{
var actual = _parser.Parse("C:/Comics/Birds of Prey/Chapter 01 page 01.jpg", "C:/Comics/",
RootDirectory, LibraryType.Image, null);
RootDirectory, LibraryType.Image, true, null);
Assert.NotNull(actual);
Assert.Equal("Birds of Prey", actual.Series);
@ -62,7 +62,7 @@ public class ImageParserTests
public void Parse_SeriesWithLooseImages()
{
var actual = _parser.Parse("C:/Comics/Birds of Prey/page 01.jpg", "C:/Comics/",
RootDirectory, LibraryType.Image, null);
RootDirectory, LibraryType.Image, true, null);
Assert.NotNull(actual);
Assert.Equal("Birds of Prey", actual.Series);

2
API.Tests/Parsers/PdfParserTests.cs
View file

@ -35,7 +35,7 @@ public class PdfParserTests
{
var actual = _parser.Parse("C:/Books/A Dictionary of Japanese Food - Ingredients and Culture/A Dictionary of Japanese Food - Ingredients and Culture.pdf",
"C:/Books/A Dictionary of Japanese Food - Ingredients and Culture/",
RootDirectory, LibraryType.Book, null);
RootDirectory, LibraryType.Book, true, null);
Assert.NotNull(actual);
Assert.Equal("A Dictionary of Japanese Food - Ingredients and Culture", actual.Series);

6
API.Tests/Parsing/ImageParsingTests.cs
View file

@ -34,7 +34,7 @@ public class ImageParsingTests
Chapters = "8", Filename = "13.jpg", Format = MangaFormat.Image,
FullFilePath = filepath, IsSpecial = false
};
var actual2 = _parser.Parse(filepath, @"E:\Manga\Monster #8", "E:/Manga", LibraryType.Image, null);
var actual2 = _parser.Parse(filepath, @"E:\Manga\Monster #8", "E:/Manga", LibraryType.Image, true, null);
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
@ -60,7 +60,7 @@ public class ImageParsingTests
FullFilePath = filepath, IsSpecial = false
};
actual2 = _parser.Parse(filepath, @"E:\Manga\Extra layer for no reason\", "E:/Manga", LibraryType.Image, null);
actual2 = _parser.Parse(filepath, @"E:\Manga\Extra layer for no reason\", "E:/Manga", LibraryType.Image, true, null);
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);
@ -86,7 +86,7 @@ public class ImageParsingTests
FullFilePath = filepath, IsSpecial = false
};
actual2 = _parser.Parse(filepath, @"E:\Manga\Extra layer for no reason\", "E:/Manga", LibraryType.Image, null);
actual2 = _parser.Parse(filepath, @"E:\Manga\Extra layer for no reason\", "E:/Manga", LibraryType.Image, true, null);
Assert.NotNull(actual2);
_testOutputHelper.WriteLine($"Validating {filepath}");
Assert.Equal(expectedInfo2.Format, actual2.Format);

2
API.Tests/Parsing/MangaParsingTests.cs
View file

@ -68,10 +68,8 @@ public class MangaParsingTests
[InlineData("Манга Тома 1-4", "1-4")]
[InlineData("Манга Том 1-4", "1-4")]
[InlineData("조선왕조실톡 106화", "106")]
[InlineData("죽음 13회", "13")]
[InlineData("동의보감 13장", "13")]
[InlineData("몰?루 아카이브 7.5권", "7.5")]
[InlineData("주술회전 1.5권", "1.5")]
[InlineData("63권#200", "63")]
[InlineData("시즌34삽화2", "34")]
[InlineData("Accel World Chapter 001 Volume 002", "2")]

2
API.Tests/Services/BookServiceTests.cs
View file

@ -137,7 +137,7 @@ public class BookServiceTests
var comicInfo = _bookService.GetComicInfo(filePath);
Assert.NotNull(comicInfo);
var parserInfo = pdfParser.Parse(filePath, testDirectory, ds.GetParentDirectoryName(testDirectory), LibraryType.Book, comicInfo);
var parserInfo = pdfParser.Parse(filePath, testDirectory, ds.GetParentDirectoryName(testDirectory), LibraryType.Book, true, comicInfo);
Assert.NotNull(parserInfo);
Assert.Equal(parserInfo.Title, comicInfo.Title);
Assert.Equal(parserInfo.Series, comicInfo.Title);

4
API.Tests/Services/CacheServiceTests.cs
View file

@ -50,12 +50,12 @@ internal class MockReadingItemServiceForCacheService : IReadingItemService
throw new System.NotImplementedException();
}
public ParserInfo Parse(string path, string rootPath, string libraryRoot, LibraryType type)
public ParserInfo Parse(string path, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true)
{
throw new System.NotImplementedException();
}
public ParserInfo ParseFile(string path, string rootPath, string libraryRoot, LibraryType type)
public ParserInfo ParseFile(string path, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true)
{
throw new System.NotImplementedException();
}

2
API.Tests/Services/ExternalMetadataServiceTests.cs
View file

@ -42,7 +42,7 @@ public class ExternalMetadataServiceTests : AbstractDbTest
_externalMetadataService = new ExternalMetadataService(UnitOfWork, Substitute.For<ILogger<ExternalMetadataService>>(),
Mapper, Substitute.For<ILicenseService>(), Substitute.For<IScrobblingService>(), Substitute.For<IEventHub>(),
Substitute.For<ICoverDbService>());
Substitute.For<ICoverDbService>(), Substitute.For<IKavitaPlusApiService>());
}
#region Gloabl

16
API.Tests/Services/ParseScannedFilesTests.cs
View file

@ -58,35 +58,35 @@ public class MockReadingItemService : IReadingItemService
throw new NotImplementedException();
}
public ParserInfo Parse(string path, string rootPath, string libraryRoot, LibraryType type)
public ParserInfo Parse(string path, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata)
{
if (_comicVineParser.IsApplicable(path, type))
{
return _comicVineParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _comicVineParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
if (_imageParser.IsApplicable(path, type))
{
return _imageParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _imageParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
if (_bookParser.IsApplicable(path, type))
{
return _bookParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _bookParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
if (_pdfParser.IsApplicable(path, type))
{
return _pdfParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _pdfParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
if (_basicParser.IsApplicable(path, type))
{
return _basicParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _basicParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
return null;
}
public ParserInfo ParseFile(string path, string rootPath, string libraryRoot, LibraryType type)
public ParserInfo ParseFile(string path, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata)
{
return Parse(path, rootPath, libraryRoot, type);
return Parse(path, rootPath, libraryRoot, type, enableMetadata);
}
}

38
API.Tests/Services/ScannerServiceTests.cs
View file

@ -483,7 +483,7 @@ public class ScannerServiceTests : AbstractDbTest
var infos = new Dictionary<string, ComicInfo>();
var library = await _scannerHelper.GenerateScannerData(testcase, infos);
library.LibraryExcludePatterns = [new LibraryExcludePattern() {Pattern = "**/Extra/*"}];
library.LibraryExcludePatterns = [new LibraryExcludePattern() { Pattern = "**/Extra/*" }];
UnitOfWork.LibraryRepository.Update(library);
await UnitOfWork.CommitAsync();
@ -507,7 +507,7 @@ public class ScannerServiceTests : AbstractDbTest
var infos = new Dictionary<string, ComicInfo>();
var library = await _scannerHelper.GenerateScannerData(testcase, infos);
library.LibraryExcludePatterns = [new LibraryExcludePattern() {Pattern = "**\\Extra\\*"}];
library.LibraryExcludePatterns = [new LibraryExcludePattern() { Pattern = "**\\Extra\\*" }];
UnitOfWork.LibraryRepository.Update(library);
await UnitOfWork.CommitAsync();
@ -938,4 +938,38 @@ public class ScannerServiceTests : AbstractDbTest
Assert.True(sortedChapters[1].SortOrder.Is(4f));
Assert.True(sortedChapters[2].SortOrder.Is(5f));
}
[Fact]
public async Task ScanLibrary_MetadataDisabled_NoOverrides()
{
const string testcase = "Series with Localized No Metadata - Manga.json";
// Get the first file and generate a ComicInfo
var infos = new Dictionary<string, ComicInfo>();
infos.Add("Immoral Guild v01.cbz", new ComicInfo()
{
Series = "Immoral Guild",
LocalizedSeries = "Futoku no Guild" // Filename has a capital N and localizedSeries has lowercase
});
var library = await _scannerHelper.GenerateScannerData(testcase, infos);
// Disable metadata
library.EnableMetadata = false;
UnitOfWork.LibraryRepository.Update(library);
await UnitOfWork.CommitAsync();
var scanner = _scannerHelper.CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await UnitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
// Validate that there are 2 series
Assert.NotNull(postLib);
Assert.Equal(2, postLib.Series.Count);
Assert.Contains(postLib.Series, x => x.Name == "Immoral Guild");
Assert.Contains(postLib.Series, x => x.Name == "Futoku No Guild");
}
}

5
API.Tests/Services/Test Data/ScannerService/TestCases/Series with Localized No Metadata - Manga.json Normal file
View file

@ -0,0 +1,5 @@
[
"Immoral Guild/Immoral Guild v01.cbz",
"Immoral Guild/Immoral Guild v02.cbz",
"Immoral Guild/Futoku No Guild - Vol. 12 Ch. 67 - Take Responsibility.cbz"
]