Ability to turn off Metadata Parsing (#3872)

This commit is contained in:
Joe Milazzo 2025-06-23 18:57:14 -05:00 committed by GitHub
parent fa8d778c8d
commit 36aa5f5c85
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
63 changed files with 4257 additions and 186 deletions

View file

@ -67,6 +67,7 @@ public class ExternalMetadataService : IExternalMetadataService
private readonly IScrobblingService _scrobblingService;
private readonly IEventHub _eventHub;
private readonly ICoverDbService _coverDbService;
private readonly IKavitaPlusApiService _kavitaPlusApiService;
private readonly TimeSpan _externalSeriesMetadataCache = TimeSpan.FromDays(30);
public static readonly HashSet<LibraryType> NonEligibleLibraryTypes =
[LibraryType.Comic, LibraryType.Book, LibraryType.Image];
@ -82,7 +83,8 @@ public class ExternalMetadataService : IExternalMetadataService
private static bool IsRomanCharacters(string input) => Regex.IsMatch(input, @"^[\p{IsBasicLatin}\p{IsLatin-1Supplement}]+$");
public ExternalMetadataService(IUnitOfWork unitOfWork, ILogger<ExternalMetadataService> logger, IMapper mapper,
ILicenseService licenseService, IScrobblingService scrobblingService, IEventHub eventHub, ICoverDbService coverDbService)
ILicenseService licenseService, IScrobblingService scrobblingService, IEventHub eventHub, ICoverDbService coverDbService,
IKavitaPlusApiService kavitaPlusApiService)
{
_unitOfWork = unitOfWork;
_logger = logger;
@ -91,6 +93,7 @@ public class ExternalMetadataService : IExternalMetadataService
_scrobblingService = scrobblingService;
_eventHub = eventHub;
_coverDbService = coverDbService;
_kavitaPlusApiService = kavitaPlusApiService;
FlurlConfiguration.ConfigureClientForUrl(Configuration.KavitaPlusApiUrl);
}
@ -179,9 +182,7 @@ public class ExternalMetadataService : IExternalMetadataService
_logger.LogDebug("Fetching Kavita+ for MAL Stacks for user {UserName}", user.MalUserName);
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var result = await ($"{Configuration.KavitaPlusApiUrl}/api/metadata/v2/stacks?username={user.MalUserName}")
.WithKavitaPlusHeaders(license)
.GetJsonAsync<IList<MalStackDto>>();
var result = await _kavitaPlusApiService.GetMalStacks(user.MalUserName, license);
if (result == null)
{
@ -207,7 +208,7 @@ public class ExternalMetadataService : IExternalMetadataService
/// <returns></returns>
public async Task<IList<ExternalSeriesMatchDto>> MatchSeries(MatchSeriesDto dto)
{
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(dto.SeriesId,
SeriesIncludes.Metadata | SeriesIncludes.ExternalMetadata | SeriesIncludes.Library);
if (series == null) return [];
@ -239,14 +240,9 @@ public class ExternalMetadataService : IExternalMetadataService
MalId = potentialMalId ?? ScrobblingService.GetMalId(series)
};
var token = (await _unitOfWork.UserRepository.GetDefaultAdminUser()).AniListAccessToken;
try
{
var results = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/match-series")
.WithKavitaPlusHeaders(license, token)
.PostJsonAsync(matchRequest)
.ReceiveJson<IList<ExternalSeriesMatchDto>>();
var results = await _kavitaPlusApiService.MatchSeries(matchRequest);
// Some summaries can contain multiple <br/>s, we need to ensure it's only 1
foreach (var result in results)
@ -287,9 +283,7 @@ public class ExternalMetadataService : IExternalMetadataService
}
// This is for the Series drawer. We can get this extra information during the initial SeriesDetail call so it's all coming from the DB
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var details = await GetSeriesDetail(license, aniListId, malId, seriesId);
var details = await GetSeriesDetail(aniListId, malId, seriesId);
return details;
@ -392,6 +386,9 @@ public class ExternalMetadataService : IExternalMetadataService
{
// We can't rethrow because Fix match is done in a background thread and Hangfire will requeue multiple times
_logger.LogInformation(ex, "Rate limit hit for matching {SeriesName} with Kavita+", series.Name);
// Fire SignalR event about this
await _eventHub.SendMessageAsync(MessageFactory.ExternalMatchRateLimitError,
MessageFactory.ExternalMatchRateLimitErrorEvent(series.Id, series.Name));
}
}
@ -442,16 +439,12 @@ public class ExternalMetadataService : IExternalMetadataService
try
{
_logger.LogDebug("Fetching Kavita+ Series Detail data for {SeriesName}", string.IsNullOrEmpty(data.SeriesName) ? data.AniListId : data.SeriesName);
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var token = (await _unitOfWork.UserRepository.GetDefaultAdminUser()).AniListAccessToken;
SeriesDetailPlusApiDto? result = null;
try
{
result = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-detail")
.WithKavitaPlusHeaders(license, token)
.PostJsonAsync(data)
.ReceiveJson<SeriesDetailPlusApiDto>(); // This returns an AniListSeries and Match returns ExternalSeriesDto
// This returns an AniListSeries and Match returns ExternalSeriesDto
result = await _kavitaPlusApiService.GetSeriesDetail(data);
}
catch (FlurlHttpException ex)
{
@ -466,11 +459,7 @@ public class ExternalMetadataService : IExternalMetadataService
_logger.LogDebug("Hit rate limit, will retry in 3 seconds");
await Task.Delay(3000);
result = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-detail")
.WithKavitaPlusHeaders(license, token)
.PostJsonAsync(data)
.ReceiveJson<
SeriesDetailPlusApiDto>();
result = await _kavitaPlusApiService.GetSeriesDetail(data);
}
else if (errorMessage.Contains("Unknown Series"))
{
@ -1777,7 +1766,7 @@ public class ExternalMetadataService : IExternalMetadataService
/// <param name="malId"></param>
/// <param name="seriesId"></param>
/// <returns></returns>
private async Task<ExternalSeriesDetailDto?> GetSeriesDetail(string license, int? aniListId, long? malId, int? seriesId)
private async Task<ExternalSeriesDetailDto?> GetSeriesDetail(int? aniListId, long? malId, int? seriesId)
{
var payload = new ExternalMetadataIdsDto()
{
@ -1809,11 +1798,7 @@ public class ExternalMetadataService : IExternalMetadataService
}
try
{
var token = (await _unitOfWork.UserRepository.GetDefaultAdminUser()).AniListAccessToken;
var ret = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-by-ids")
.WithKavitaPlusHeaders(license, token)
.PostJsonAsync(payload)
.ReceiveJson<ExternalSeriesDetailDto>();
var ret = await _kavitaPlusApiService.GetSeriesDetailById(payload);
ret.Summary = StringHelper.RemoveSourceInDescription(StringHelper.SquashBreaklines(ret.Summary));

View file

@ -1,6 +1,13 @@
#nullable enable
using System.Collections.Generic;
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Collection;
using API.DTOs.KavitaPlus.ExternalMetadata;
using API.DTOs.KavitaPlus.Metadata;
using API.DTOs.Metadata.Matching;
using API.DTOs.Scrobbling;
using API.Entities.Enums;
using API.Extensions;
using Flurl.Http;
using Kavita.Common;
@ -17,9 +24,13 @@ public interface IKavitaPlusApiService
Task<bool> HasTokenExpired(string license, string token, ScrobbleProvider provider);
Task<int> GetRateLimit(string license, string token);
Task<ScrobbleResponseDto> PostScrobbleUpdate(ScrobbleDto data, string license);
Task<IList<MalStackDto>> GetMalStacks(string malUsername, string license);
Task<IList<ExternalSeriesMatchDto>> MatchSeries(MatchSeriesRequestDto request);
Task<SeriesDetailPlusApiDto> GetSeriesDetail(PlusSeriesRequestDto request);
Task<ExternalSeriesDetailDto> GetSeriesDetailById(ExternalMetadataIdsDto request);
}
public class KavitaPlusApiService(ILogger<KavitaPlusApiService> logger): IKavitaPlusApiService
public class KavitaPlusApiService(ILogger<KavitaPlusApiService> logger, IUnitOfWork unitOfWork): IKavitaPlusApiService
{
private const string ScrobblingPath = "/api/scrobbling/";
@ -42,6 +53,46 @@ public class KavitaPlusApiService(ILogger<KavitaPlusApiService> logger): IKavita
return await PostAndReceive<ScrobbleResponseDto>(ScrobblingPath + "update", data, license);
}
public async Task<IList<MalStackDto>> GetMalStacks(string malUsername, string license)
{
return await $"{Configuration.KavitaPlusApiUrl}/api/metadata/v2/stacks?username={malUsername}"
.WithKavitaPlusHeaders(license)
.GetJsonAsync<IList<MalStackDto>>();
}
public async Task<IList<ExternalSeriesMatchDto>> MatchSeries(MatchSeriesRequestDto request)
{
var license = (await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var token = (await unitOfWork.UserRepository.GetDefaultAdminUser()).AniListAccessToken;
return await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/match-series")
.WithKavitaPlusHeaders(license, token)
.PostJsonAsync(request)
.ReceiveJson<IList<ExternalSeriesMatchDto>>();
}
public async Task<SeriesDetailPlusApiDto> GetSeriesDetail(PlusSeriesRequestDto request)
{
var license = (await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var token = (await unitOfWork.UserRepository.GetDefaultAdminUser()).AniListAccessToken;
return await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-detail")
.WithKavitaPlusHeaders(license, token)
.PostJsonAsync(request)
.ReceiveJson<SeriesDetailPlusApiDto>();
}
public async Task<ExternalSeriesDetailDto> GetSeriesDetailById(ExternalMetadataIdsDto request)
{
var license = (await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var token = (await unitOfWork.UserRepository.GetDefaultAdminUser()).AniListAccessToken;
return await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-by-ids")
.WithKavitaPlusHeaders(license, token)
.PostJsonAsync(request)
.ReceiveJson<ExternalSeriesDetailDto>();
}
/// <summary>
/// Send a GET request to K+
/// </summary>

View file

@ -12,7 +12,7 @@ public interface IReadingItemService
int GetNumberOfPages(string filePath, MangaFormat format);
string GetCoverImage(string filePath, string fileName, MangaFormat format, EncodeFormat encodeFormat, CoverImageSize size = CoverImageSize.Default);
void Extract(string fileFilePath, string targetDirectory, MangaFormat format, int imageCount = 1);
ParserInfo? ParseFile(string path, string rootPath, string libraryRoot, LibraryType type);
ParserInfo? ParseFile(string path, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata);
}
public class ReadingItemService : IReadingItemService
@ -71,11 +71,12 @@ public class ReadingItemService : IReadingItemService
/// <param name="path">Path of a file</param>
/// <param name="rootPath"></param>
/// <param name="type">Library type to determine parsing to perform</param>
public ParserInfo? ParseFile(string path, string rootPath, string libraryRoot, LibraryType type)
/// <param name="enableMetadata">Enable Metadata parsing overriding filename parsing</param>
public ParserInfo? ParseFile(string path, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata)
{
try
{
var info = Parse(path, rootPath, libraryRoot, type);
var info = Parse(path, rootPath, libraryRoot, type, enableMetadata);
if (info == null)
{
_logger.LogError("Unable to parse any meaningful information out of file {FilePath}", path);
@ -174,28 +175,29 @@ public class ReadingItemService : IReadingItemService
/// <param name="path"></param>
/// <param name="rootPath"></param>
/// <param name="type"></param>
/// <param name="enableMetadata"></param>
/// <returns></returns>
private ParserInfo? Parse(string path, string rootPath, string libraryRoot, LibraryType type)
private ParserInfo? Parse(string path, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata)
{
if (_comicVineParser.IsApplicable(path, type))
{
return _comicVineParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _comicVineParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
if (_imageParser.IsApplicable(path, type))
{
return _imageParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _imageParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
if (_bookParser.IsApplicable(path, type))
{
return _bookParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _bookParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
if (_pdfParser.IsApplicable(path, type))
{
return _pdfParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _pdfParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
if (_basicParser.IsApplicable(path, type))
{
return _basicParser.Parse(path, rootPath, libraryRoot, type, GetComicInfo(path));
return _basicParser.Parse(path, rootPath, libraryRoot, type, enableMetadata, GetComicInfo(path));
}
return null;

View file

@ -804,7 +804,7 @@ public class ParseScannedFiles
{
// Process files sequentially
result.ParserInfos = files
.Select(file => _readingItemService.ParseFile(file, normalizedFolder, result.LibraryRoot, library.Type))
.Select(file => _readingItemService.ParseFile(file, normalizedFolder, result.LibraryRoot, library.Type, library.EnableMetadata))
.Where(info => info != null)
.ToList()!;
}
@ -812,7 +812,7 @@ public class ParseScannedFiles
{
// Process files in parallel
var tasks = files.Select(file => Task.Run(() =>
_readingItemService.ParseFile(file, normalizedFolder, result.LibraryRoot, library.Type)));
_readingItemService.ParseFile(file, normalizedFolder, result.LibraryRoot, library.Type, library.EnableMetadata)));
var infos = await Task.WhenAll(tasks);
result.ParserInfos = infos.Where(info => info != null).ToList()!;

View file

@ -12,7 +12,7 @@ namespace API.Services.Tasks.Scanner.Parser;
/// </summary>
public class BasicParser(IDirectoryService directoryService, IDefaultParser imageParser) : DefaultParser(directoryService)
{
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null)
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true, ComicInfo? comicInfo = null)
{
var fileName = directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this.
@ -20,7 +20,7 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag
if (Parser.IsImage(filePath))
{
return imageParser.Parse(filePath, rootPath, libraryRoot, LibraryType.Image, comicInfo);
return imageParser.Parse(filePath, rootPath, libraryRoot, LibraryType.Image, enableMetadata, comicInfo);
}
var ret = new ParserInfo()
@ -101,7 +101,12 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag
}
// Patch in other information from ComicInfo
UpdateFromComicInfo(ret);
if (enableMetadata)
{
UpdateFromComicInfo(ret);
}
if (ret.Volumes == Parser.LooseLeafVolume && ret.Chapters == Parser.DefaultChapter)
{

View file

@ -5,7 +5,7 @@ namespace API.Services.Tasks.Scanner.Parser;
public class BookParser(IDirectoryService directoryService, IBookService bookService, BasicParser basicParser) : DefaultParser(directoryService)
{
public override ParserInfo Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo comicInfo = null)
public override ParserInfo Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true, ComicInfo comicInfo = null)
{
var info = bookService.ParseInfo(filePath);
if (info == null) return null;
@ -35,7 +35,7 @@ public class BookParser(IDirectoryService directoryService, IBookService bookSer
}
else
{
var info2 = basicParser.Parse(filePath, rootPath, libraryRoot, LibraryType.Book, comicInfo);
var info2 = basicParser.Parse(filePath, rootPath, libraryRoot, LibraryType.Book, enableMetadata, comicInfo);
info.Merge(info2);
if (hasVolumeInSeries && info2 != null && Parser.ParseVolume(info2.Series, type)
.Equals(Parser.LooseLeafVolume))

View file

@ -19,7 +19,7 @@ public class ComicVineParser(IDirectoryService directoryService) : DefaultParser
/// <param name="rootPath"></param>
/// <param name="type"></param>
/// <returns></returns>
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null)
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true, ComicInfo? comicInfo = null)
{
if (type != LibraryType.ComicVine) return null;
@ -81,7 +81,10 @@ public class ComicVineParser(IDirectoryService directoryService) : DefaultParser
info.IsSpecial = Parser.IsSpecial(info.Filename, type) || Parser.IsSpecial(info.ComicInfo?.Format, type);
// Patch in other information from ComicInfo
UpdateFromComicInfo(info);
if (enableMetadata)
{
UpdateFromComicInfo(info);
}
if (string.IsNullOrEmpty(info.Series))
{

View file

@ -8,7 +8,7 @@ namespace API.Services.Tasks.Scanner.Parser;
public interface IDefaultParser
{
ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null);
ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true, ComicInfo? comicInfo = null);
void ParseFromFallbackFolders(string filePath, string rootPath, LibraryType type, ref ParserInfo ret);
bool IsApplicable(string filePath, LibraryType type);
}
@ -26,8 +26,9 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
/// <param name="filePath"></param>
/// <param name="rootPath">Root folder</param>
/// <param name="type">Allows different Regex to be used for parsing.</param>
/// <param name="enableMetadata">Allows overriding data from metadata (ComicInfo/pdf/epub)</param>
/// <returns><see cref="ParserInfo"/> or null if Series was empty</returns>
public abstract ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null);
public abstract ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true, ComicInfo? comicInfo = null);
/// <summary>
/// Fills out <see cref="ParserInfo"/> by trying to parse volume, chapters, and series from folders

View file

@ -7,7 +7,7 @@ namespace API.Services.Tasks.Scanner.Parser;
public class ImageParser(IDirectoryService directoryService) : DefaultParser(directoryService)
{
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null)
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true, ComicInfo? comicInfo = null)
{
if (!IsApplicable(filePath, type)) return null;

View file

@ -165,9 +165,9 @@ public static partial class Parser
new Regex(
@"(卷|册)(?<Volume>\d+)",
MatchOptions, RegexTimeout),
// Korean Volume: 제n화|권|회|장 -> Volume n, n화|권|회|장 -> Volume n, 63권#200.zip -> Volume 63 (no chapter, #200 is just files inside)
// Korean Volume: 제n화|회|장 -> Volume n, n화|권|장 -> Volume n, 63권#200.zip -> Volume 63 (no chapter, #200 is just files inside)
new Regex(
@"제?(?<Volume>\d+(\.\d+)?)(권|회|화|장)",
@"제?(?<Volume>\d+(\.\d+)?)(권|화|장)",
MatchOptions, RegexTimeout),
// Korean Season: 시즌n -> Season n,
new Regex(

View file

@ -6,7 +6,7 @@ namespace API.Services.Tasks.Scanner.Parser;
public class PdfParser(IDirectoryService directoryService) : DefaultParser(directoryService)
{
public override ParserInfo Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo comicInfo = null)
public override ParserInfo Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, bool enableMetadata = true, ComicInfo comicInfo = null)
{
var fileName = directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
var ret = new ParserInfo
@ -68,14 +68,18 @@ public class PdfParser(IDirectoryService directoryService) : DefaultParser(direc
ParseFromFallbackFolders(filePath, tempRootPath, type, ref ret);
}
// Patch in other information from ComicInfo
UpdateFromComicInfo(ret);
if (comicInfo != null && !string.IsNullOrEmpty(comicInfo.Title))
if (enableMetadata)
{
ret.Title = comicInfo.Title.Trim();
// Patch in other information from ComicInfo
UpdateFromComicInfo(ret);
if (comicInfo != null && !string.IsNullOrEmpty(comicInfo.Title))
{
ret.Title = comicInfo.Title.Trim();
}
}
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.LooseLeafVolume && type == LibraryType.Book)
{
ret.IsSpecial = true;

View file

@ -521,6 +521,11 @@ public class ScannerService : IScannerService
// Validations are done, now we can start actual scan
_logger.LogInformation("[ScannerService] Beginning file scan on {LibraryName}", library.Name);
if (!library.EnableMetadata)
{
_logger.LogInformation("[ScannerService] Warning! {LibraryName} has metadata turned off", library.Name);
}
// This doesn't work for something like M:/Manga/ and a series has library folder as root
var shouldUseLibraryScan = !(await _unitOfWork.LibraryRepository.DoAnySeriesFoldersMatch(libraryFolderPaths));
if (!shouldUseLibraryScan)