AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Stats & More Polish on Metadata Matching (#3538)

Browse source
This commit is contained in:
Joe Milazzo 2025-02-09 14:39:43 -06:00 • committed by GitHub
parent 6f3ba0948b
commit 5d6a5f0987
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 178 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

4
API/Controllers/SettingsController.cs
View file

@ -572,8 +572,8 @@ public class SettingsController : BaseApiController
existingMetadataSetting.AgeRatingMappings = dto.AgeRatingMappings ?? [];
existingMetadataSetting.Blacklist = dto.Blacklist.DistinctBy(d => d.ToNormalized()).ToList() ?? [];
existingMetadataSetting.Whitelist = dto.Whitelist.DistinctBy(d => d.ToNormalized()).ToList() ?? [];
existingMetadataSetting.Blacklist = dto.Blacklist.Where(s => !string.IsNullOrWhiteSpace(s)).DistinctBy(d => d.ToNormalized()).ToList() ?? [];
existingMetadataSetting.Whitelist = dto.Whitelist.Where(s => !string.IsNullOrWhiteSpace(s)).DistinctBy(d => d.ToNormalized()).ToList() ?? [];
existingMetadataSetting.Overrides = dto.Overrides.ToList() ?? [];
// Handle Field Mappings

5
API/DTOs/Stats/V3/ServerInfoV3Dto.cs
View file

@ -55,6 +55,11 @@ public class ServerInfoV3Dto
/// </summary>
/// <remarks>This pings a health check and does not capture any IP Information</remarks>
public long TimeToPingKavitaStatsApi { get; set; }
/// <summary>
/// If using the downloading metadata feature
/// </summary>
/// <remarks>Kavita+ Only</remarks>
public bool MatchedMetadataEnabled { get; set; }

8
API/Data/Repositories/ExternalSeriesMetadataRepository.cs
View file

@ -36,7 +36,7 @@ public interface IExternalSeriesMetadataRepository
Task<SeriesDetailPlusDto?> GetSeriesDetailPlusDto(int seriesId);
Task LinkRecommendationsToSeries(Series series);
Task<bool> IsBlacklistedSeries(int seriesId);
Task<IList<int>> GetAllSeriesIdsWithoutMetadata(int limit);
Task<IList<int>> GetSeriesThatNeedExternalMetadata(int limit, bool includeStaleData = false);
Task<IList<ManageMatchSeriesDto>> GetAllSeries(ManageMatchFilterDto filter);
}
@ -209,11 +209,13 @@ public class ExternalSeriesMetadataRepository : IExternalSeriesMetadataRepositor
}
public async Task<IList<int>> GetAllSeriesIdsWithoutMetadata(int limit)
public async Task<IList<int>> GetSeriesThatNeedExternalMetadata(int limit, bool includeStaleData = false)
{
return await _context.Series
.Where(s => !ExternalMetadataService.NonEligibleLibraryTypes.Contains(s.Library.Type))
.Where(s => s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc < DateTime.UtcNow)
.WhereIf(includeStaleData, s => s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc < DateTime.UtcNow)
.Where(s => s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc == DateTime.MinValue)
.Where(s => s.Library.AllowMetadataMatching)
.OrderByDescending(s => s.Library.Type)
.ThenBy(s => s.NormalizedName)
.Select(s => s.Id)

12
API/Extensions/FlurlExtensions.cs
View file

@ -18,4 +18,16 @@ public static class FlurlExtensions
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(Configuration.DefaultTimeOutSecs));
}
public static IFlurlRequest WithBasicHeaders(this string request, string apiKey)
{
return request
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", apiKey)
.WithHeader("x-installId", HashUtil.ServerToken())
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(Configuration.DefaultTimeOutSecs));
}
}

2
API/Extensions/QueryExtensions/QueryableExtensions.cs
View file

@ -294,7 +294,7 @@ public static class QueryableExtensions
MatchStateOption.NotMatched => query.
Include(s => s.ExternalSeriesMetadata)
.Where(s => (s.ExternalSeriesMetadata == null || s.ExternalSeriesMetadata.ValidUntilUtc == DateTime.MinValue) && !s.IsBlacklisted && !s.DontMatch),
MatchStateOption.Error => query.Where(s => s.IsBlacklisted),
MatchStateOption.Error => query.Where(s => s.IsBlacklisted && !s.DontMatch),
MatchStateOption.DontMatch => query.Where(s => s.DontMatch),
_ => query
};

47
API/Services/Plus/ExternalMetadataService.cs
View file

@ -44,8 +44,8 @@ public interface IExternalMetadataService
/// </summary>
/// <param name="seriesId"></param>
/// <param name="libraryType"></param>
/// <returns></returns>
Task FetchSeriesMetadata(int seriesId, LibraryType libraryType);
/// <returns>If the fetch was made</returns>
Task<bool> FetchSeriesMetadata(int seriesId, LibraryType libraryType);
Task<IList<MalStackDto>> GetStacksForUser(int userId);
Task<IList<ExternalSeriesMatchDto>> MatchSeries(MatchSeriesDto dto);
@ -73,6 +73,7 @@ public class ExternalMetadataService : IExternalMetadataService
};
// Allow 50 requests per 24 hours
private static readonly RateLimiter RateLimiter = new RateLimiter(50, TimeSpan.FromHours(24), false);
static bool IsRomanCharacters(string input) => Regex.IsMatch(input, @"^[\p{IsBasicLatin}\p{IsLatin-1Supplement}]+$");
public ExternalMetadataService(IUnitOfWork unitOfWork, ILogger<ExternalMetadataService> logger, IMapper mapper,
ILicenseService licenseService, IScrobblingService scrobblingService, IEventHub eventHub, ICoverDbService coverDbService)
@ -109,7 +110,7 @@ public class ExternalMetadataService : IExternalMetadataService
public async Task FetchExternalDataTask()
{
// Find all Series that are eligible and limit
var ids = await _unitOfWork.ExternalSeriesMetadataRepository.GetAllSeriesIdsWithoutMetadata(25);
var ids = await _unitOfWork.ExternalSeriesMetadataRepository.GetSeriesThatNeedExternalMetadata(25);
if (ids.Count == 0) return;
_logger.LogInformation("[Kavita+ Data Refresh] Started Refreshing {Count} series data from Kavita+", ids.Count);
@ -118,9 +119,9 @@ public class ExternalMetadataService : IExternalMetadataService
foreach (var seriesId in ids)
{
var libraryType = libTypes[seriesId];
await FetchSeriesMetadata(seriesId, libraryType);
var success = await FetchSeriesMetadata(seriesId, libraryType);
if (success) count++;
await Task.Delay(1500);
count++;
}
_logger.LogInformation("[Kavita+ Data Refresh] Finished Refreshing {Count} series data from Kavita+", count);
}
@ -131,10 +132,10 @@ public class ExternalMetadataService : IExternalMetadataService
/// </summary>
/// <param name="seriesId"></param>
/// <param name="libraryType"></param>
public async Task FetchSeriesMetadata(int seriesId, LibraryType libraryType)
public async Task<bool> FetchSeriesMetadata(int seriesId, LibraryType libraryType)
{
if (!IsPlusEligible(libraryType)) return;
if (!await _licenseService.HasActiveLicense()) return;
if (!IsPlusEligible(libraryType)) return false;
if (!await _licenseService.HasActiveLicense()) return false;
// Generate key based on seriesId and libraryType or any unique identifier for the request
// Check if the request is allowed based on the rate limit
@ -142,14 +143,14 @@ public class ExternalMetadataService : IExternalMetadataService
{
// Request not allowed due to rate limit
_logger.LogDebug("Rate Limit hit for Kavita+ prefetch");
return;
return false;
}
_logger.LogDebug("Prefetching Kavita+ data for Series {SeriesId}", seriesId);
// Prefetch SeriesDetail data
await GetSeriesDetailPlus(seriesId, libraryType);
return true;
}
public async Task<IList<MalStackDto>> GetStacksForUser(int userId)
@ -512,31 +513,43 @@ public class ExternalMetadataService : IExternalMetadataService
var madeModification = false;
if (settings.EnableLocalizedName && (!series.LocalizedNameLocked || settings.HasOverride(MetadataSettingField.LocalizedName)))
if (settings.EnableLocalizedName && (settings.HasOverride(MetadataSettingField.LocalizedName)
|| !series.LocalizedNameLocked && !string.IsNullOrWhiteSpace(series.LocalizedName)))
{
// We need to make the best appropriate guess
if (externalMetadata.Name == series.Name)
{
// Choose closest (usually last) synonym
series.LocalizedName = externalMetadata.Synonyms.Last();
var validSynonyms = externalMetadata.Synonyms
.Where(IsRomanCharacters)
.Where(s => s.ToNormalized() != series.Name.ToNormalized())
.ToList();
if (validSynonyms.Count != 0)
{
series.LocalizedName = validSynonyms[^1];
series.LocalizedNameLocked = true;
}
}
else
else if (IsRomanCharacters(externalMetadata.Name))
{
series.LocalizedName = externalMetadata.Name;
series.LocalizedNameLocked = true;
}
madeModification = true;
}
if (settings.EnableSummary && (!series.Metadata.SummaryLocked ||
settings.HasOverride(MetadataSettingField.Summary)))
if (settings.EnableSummary && (settings.HasOverride(MetadataSettingField.Summary) ||
(!series.Metadata.SummaryLocked && !string.IsNullOrWhiteSpace(series.Metadata.Summary))))
{
series.Metadata.Summary = CleanSummary(externalMetadata.Summary);
madeModification = true;
}
if (settings.EnableStartDate && externalMetadata.StartDate.HasValue && (!series.Metadata.ReleaseYearLocked ||
settings.HasOverride(MetadataSettingField.StartDate)))
if (settings.EnableStartDate && externalMetadata.StartDate.HasValue && (settings.HasOverride(MetadataSettingField.StartDate) ||
(!series.Metadata.ReleaseYearLocked &&
series.Metadata.ReleaseYear == 0)))
{
series.Metadata.ReleaseYear = externalMetadata.StartDate.Value.Year;
madeModification = true;

2
API/Services/SeriesService.cs
View file

@ -382,10 +382,12 @@ public class SeriesService : ISeriesService
// Check if the person exists in the dictionary
if (existingPeopleDictionary.TryGetValue(normalizedPersonName, out var p))
{
// TODO: Should I add more controls here to map back?
if (personDto.AniListId > 0 && p.AniListId <= 0 && p.AniListId != personDto.AniListId)
{
p.AniListId = personDto.AniListId;
}
p.Description = string.IsNullOrEmpty(p.Description) ? personDto.Description : p.Description;
continue; // If we ever want to update metadata for existing people, we'd do it here
}

38
API/Services/Tasks/StatsService.cs
View file

@ -13,14 +13,17 @@ using API.DTOs.Stats;
using API.DTOs.Stats.V3;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Services.Plus;
using API.Services.Tasks.Scanner.Parser;
using Flurl.Http;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Kavita.Common.Helpers;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace API.Services.Tasks;
@ -45,12 +48,12 @@ public class StatsService : IStatsService
private readonly UserManager<AppUser> _userManager;
private readonly IEmailService _emailService;
private readonly ICacheService _cacheService;
private const string ApiUrl = "https://stats.kavitareader.com";
private readonly string _apiUrl = "";
private const string ApiKey = "MsnvA2DfQqxSK5jh"; // It's not important this is public, just a way to keep bots from hitting the API willy-nilly
public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork, DataContext context,
ILicenseService licenseService, UserManager<AppUser> userManager, IEmailService emailService,
ICacheService cacheService)
ICacheService cacheService, IHostEnvironment environment)
{
_logger = logger;
_unitOfWork = unitOfWork;
@ -60,7 +63,9 @@ public class StatsService : IStatsService
_emailService = emailService;
_cacheService = cacheService;
FlurlConfiguration.ConfigureClientForUrl(ApiUrl);
FlurlConfiguration.ConfigureClientForUrl(Configuration.StatsApiUrl);
_apiUrl = environment.IsDevelopment() ? "http://localhost:5001" : Configuration.StatsApiUrl;
}
/// <summary>
@ -98,13 +103,8 @@ public class StatsService : IStatsService
try
{
var response = await (ApiUrl + "/api/v3/stats")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", ApiKey)
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(30))
var response = await (_apiUrl + "/api/v3/stats")
.WithBasicHeaders(ApiKey)
.PostJsonAsync(data);
if (response.StatusCode != StatusCodes.Status200OK)
@ -151,12 +151,8 @@ public class StatsService : IStatsService
try
{
var response = await (ApiUrl + "/api/v2/stats/opt-out?installId=" + installId)
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", ApiKey)
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
var response = await (_apiUrl + "/api/v2/stats/opt-out?installId=" + installId)
.WithBasicHeaders(ApiKey)
.WithTimeout(TimeSpan.FromSeconds(30))
.PostAsync();
@ -180,12 +176,8 @@ public class StatsService : IStatsService
try
{
var sw = Stopwatch.StartNew();
var response = await (ApiUrl + "/api/health/")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", ApiKey)
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
var response = await (Configuration.StatsApiUrl + "/api/health/")
.WithBasicHeaders(ApiKey)
.WithTimeout(TimeSpan.FromSeconds(30))
.GetAsync();
@ -244,6 +236,7 @@ public class StatsService : IStatsService
private async Task<ServerInfoV3Dto> GetStatV3Payload()
{
var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
var mediaSettings = await _unitOfWork.SettingsRepository.GetMetadataSettings();
var dto = new ServerInfoV3Dto()
{
InstallId = serverSettings.InstallId,
@ -256,6 +249,7 @@ public class StatsService : IStatsService
DotnetVersion = Environment.Version.ToString(),
OpdsEnabled = serverSettings.EnableOpds,
EncodeMediaAs = serverSettings.EncodeMediaAs,
MatchedMetadataEnabled = mediaSettings.Enabled
};
dto.OsLocale = CultureInfo.CurrentCulture.EnglishName;