AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Kavita+ Overhaul & New Changelog (#3507)

Browse source
This commit is contained in:
Joe Milazzo 2025-01-20 08:14:57 -06:00 • committed by GitHub
parent d880c1690c
commit a5707617f2
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
249 changed files with 14775 additions and 2300 deletions
Download patch file Download diff file Expand all files Collapse all files

302
API/Services/Plus/ExternalMetadataService.cs
View file

@ -1,12 +1,13 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
using API.DTOs;
using API.DTOs.Collection;
using API.DTOs.KavitaPlus.ExternalMetadata;
using API.DTOs.Metadata.Matching;
using API.DTOs.Recommendation;
using API.DTOs.Scrobbling;
using API.DTOs.SeriesDetail;
@ -15,44 +16,24 @@ using API.Entities.Enums;
using API.Entities.Metadata;
using API.Extensions;
using API.Helpers;
using API.SignalR;
using AutoMapper;
using Flurl.Http;
using Hangfire;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Kavita.Common.Helpers;
using Microsoft.Extensions.Logging;
namespace API.Services.Plus;
#nullable enable
/// <summary>
/// Used for matching and fetching metadata on a series
/// </summary>
internal class ExternalMetadataIdsDto
{
public long? MalId { get; set; }
public int? AniListId { get; set; }
public string? SeriesName { get; set; }
public string? LocalizedSeriesName { get; set; }
public MediaFormat? PlusMediaFormat { get; set; } = MediaFormat.Unknown;
}
internal class SeriesDetailPlusApiDto
{
public IEnumerable<MediaRecommendationDto> Recommendations { get; set; }
public IEnumerable<UserReviewDto> Reviews { get; set; }
public IEnumerable<RatingDto> Ratings { get; set; }
public int? AniListId { get; set; }
public long? MalId { get; set; }
}
public interface IExternalMetadataService
{
Task<ExternalSeriesDetailDto?> GetExternalSeriesDetail(int? aniListId, long? malId, int? seriesId);
Task<SeriesDetailPlusDto> GetSeriesDetailPlus(int seriesId, LibraryType libraryType);
Task ForceKavitaPlusRefresh(int seriesId);
Task<SeriesDetailPlusDto?> GetSeriesDetailPlus(int seriesId, LibraryType libraryType);
//Task ForceKavitaPlusRefresh(int seriesId);
Task FetchExternalDataTask();
/// <summary>
/// This is an entry point and provides a level of protection against calling upstream API. Will only allow 100 new
@ -64,6 +45,9 @@ public interface IExternalMetadataService
Task GetNewSeriesData(int seriesId, LibraryType libraryType);
Task<IList<MalStackDto>> GetStacksForUser(int userId);
Task<IList<ExternalSeriesMatchDto>> MatchSeries(MatchSeriesDto dto);
Task FixSeriesMatch(int seriesId, ExternalSeriesDetailDto dto);
Task UpdateSeriesDontMatch(int seriesId, bool dontMatch);
}
public class ExternalMetadataService : IExternalMetadataService
@ -72,9 +56,11 @@ public class ExternalMetadataService : IExternalMetadataService
private readonly ILogger<ExternalMetadataService> _logger;
private readonly IMapper _mapper;
private readonly ILicenseService _licenseService;
private readonly IScrobblingService _scrobblingService;
private readonly IEventHub _eventHub;
private readonly TimeSpan _externalSeriesMetadataCache = TimeSpan.FromDays(30);
public static readonly ImmutableArray<LibraryType> NonEligibleLibraryTypes = ImmutableArray.Create
(LibraryType.Comic, LibraryType.Book, LibraryType.Image, LibraryType.ComicVine);
public static readonly HashSet<LibraryType> NonEligibleLibraryTypes =
[LibraryType.Comic, LibraryType.Book, LibraryType.Image, LibraryType.ComicVine];
private readonly SeriesDetailPlusDto _defaultReturn = new()
{
Recommendations = null,
@ -84,16 +70,17 @@ public class ExternalMetadataService : IExternalMetadataService
// Allow 50 requests per 24 hours
private static readonly RateLimiter RateLimiter = new RateLimiter(50, TimeSpan.FromHours(24), false);
public ExternalMetadataService(IUnitOfWork unitOfWork, ILogger<ExternalMetadataService> logger, IMapper mapper, ILicenseService licenseService)
public ExternalMetadataService(IUnitOfWork unitOfWork, ILogger<ExternalMetadataService> logger, IMapper mapper,
ILicenseService licenseService, IScrobblingService scrobblingService, IEventHub eventHub)
{
_unitOfWork = unitOfWork;
_logger = logger;
_mapper = mapper;
_licenseService = licenseService;
_scrobblingService = scrobblingService;
_eventHub = eventHub;
FlurlHttp.ConfigureClient(Configuration.KavitaPlusApiUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
FlurlConfiguration.ConfigureClientForUrl(Configuration.KavitaPlusApiUrl);
}
/// <summary>
@ -110,7 +97,7 @@ public class ExternalMetadataService : IExternalMetadataService
/// This is a task that runs on a schedule and slowly fetches data from Kavita+ to keep
/// data in the DB non-stale and fetched.
/// </summary>
/// <remarks>To avoid blasting Kavita+ API, this only processes a few records. The goal is to slowly build </remarks>
/// <remarks>To avoid blasting Kavita+ API, this only processes 25 records. The goal is to slowly build out/refresh the data</remarks>
/// <returns></returns>
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
@ -138,21 +125,24 @@ public class ExternalMetadataService : IExternalMetadataService
/// </summary>
/// <param name="seriesId"></param>
/// <returns></returns>
public async Task ForceKavitaPlusRefresh(int seriesId)
{
if (!await _licenseService.HasActiveLicense()) return;
var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeBySeriesIdAsync(seriesId);
if (!IsPlusEligible(libraryType)) return;
// Remove from Blacklist if applicable
await _unitOfWork.ExternalSeriesMetadataRepository.RemoveFromBlacklist(seriesId);
var metadata = await _unitOfWork.ExternalSeriesMetadataRepository.GetExternalSeriesMetadata(seriesId);
if (metadata == null) return;
metadata.ValidUntilUtc = DateTime.UtcNow.Subtract(_externalSeriesMetadataCache);
await _unitOfWork.CommitAsync();
}
// public async Task ForceKavitaPlusRefresh(int seriesId)
// {
// // TODO: I think we can remove this now
// if (!await _licenseService.HasActiveLicense()) return;
// var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeBySeriesIdAsync(seriesId);
// if (!IsPlusEligible(libraryType)) return;
//
// // Remove from Blacklist if applicable
// var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId);
// series!.IsBlacklisted = false;
// _unitOfWork.SeriesRepository.Update(series);
//
// var metadata = await _unitOfWork.ExternalSeriesMetadataRepository.GetExternalSeriesMetadata(seriesId);
// if (metadata == null) return;
//
// metadata.ValidUntilUtc = DateTime.UtcNow.Subtract(_externalSeriesMetadataCache);
// await _unitOfWork.CommitAsync();
// }
/// <summary>
/// Fetches data from Kavita+
@ -198,13 +188,7 @@ public class ExternalMetadataService : IExternalMetadataService
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var result = await ($"{Configuration.KavitaPlusApiUrl}/api/metadata/v2/stacks?username={user.MalUserName}")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-license-key", license)
.WithHeader("x-installId", HashUtil.ServerToken())
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(Configuration.DefaultTimeOutSecs))
.WithKavitaPlusHeaders(license)
.GetJsonAsync<IList<MalStackDto>>();
if (result == null)
@ -221,6 +205,72 @@ public class ExternalMetadataService : IExternalMetadataService
}
}
/// <summary>
/// Returns the match results for a Series from UI Flow
/// </summary>
/// <param name="dto"></param>
/// <returns></returns>
public async Task<IList<ExternalSeriesMatchDto>> MatchSeries(MatchSeriesDto dto)
{
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(dto.SeriesId,
SeriesIncludes.Metadata | SeriesIncludes.ExternalMetadata);
var potentialAnilistId = ScrobblingService.ExtractId<int?>(dto.Query, ScrobblingService.AniListWeblinkWebsite);
var potentialMalId = ScrobblingService.ExtractId<long?>(dto.Query, ScrobblingService.MalWeblinkWebsite);
List<string> altNames = [series.LocalizedName, series.OriginalName];
if (potentialAnilistId == null && potentialMalId == null && !string.IsNullOrEmpty(dto.Query))
{
altNames.Add(dto.Query);
}
var matchRequest = new MatchSeriesRequestDto()
{
Format = series.Format == MangaFormat.Epub ? PlusMediaFormat.LightNovel : PlusMediaFormat.Manga,
Query = dto.Query,
SeriesName = series.Name,
AlternativeNames = altNames,
Year = series.Metadata.ReleaseYear,
AniListId = potentialAnilistId ?? ScrobblingService.GetAniListId(series),
MalId = potentialMalId ?? ScrobblingService.GetMalId(series),
};
try
{
var results = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/match-series")
.WithKavitaPlusHeaders(license)
.PostJsonAsync(matchRequest)
.ReceiveJson<IList<ExternalSeriesMatchDto>>();
// Some summaries can contain multiple <br/>s, we need to ensure it's only 1
foreach (var result in results)
{
result.Series.Summary = CleanSummary(result.Series.Summary);
}
return results;
}
catch (Exception ex)
{
_logger.LogError(ex, "An error happened during the request to Kavita+ API");
}
return ArraySegment<ExternalSeriesMatchDto>.Empty;
}
private static string CleanSummary(string? summary)
{
if (string.IsNullOrWhiteSpace(summary))
{
return string.Empty; // Return as is if null, empty, or whitespace.
}
return summary.Replace("<br/>", string.Empty);
}
/// <summary>
/// Retrieves Metadata about a Recommended External Series
/// </summary>
@ -249,16 +299,18 @@ public class ExternalMetadataService : IExternalMetadataService
/// Returns Series Detail data from Kavita+ - Review, Recs, Ratings
/// </summary>
/// <param name="seriesId"></param>
/// <param name="libraryType"></param>
/// <returns></returns>
public async Task<SeriesDetailPlusDto> GetSeriesDetailPlus(int seriesId, LibraryType libraryType)
public async Task<SeriesDetailPlusDto?> GetSeriesDetailPlus(int seriesId, LibraryType libraryType)
{
if (!IsPlusEligible(libraryType) || !await _licenseService.HasActiveLicense()) return _defaultReturn;
// Check blacklist (bad matches)
if (await _unitOfWork.ExternalSeriesMetadataRepository.IsBlacklistedSeries(seriesId)) return _defaultReturn;
// Check blacklist (bad matches) or if there is a don't match
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId);
if (series == null || !series.WillScrobble()) return _defaultReturn;
var needsRefresh =
await _unitOfWork.ExternalSeriesMetadataRepository.ExternalSeriesMetadataNeedsRefresh(seriesId);
await _unitOfWork.ExternalSeriesMetadataRepository.NeedsDataRefresh(seriesId);
if (!needsRefresh)
{
@ -266,28 +318,105 @@ public class ExternalMetadataService : IExternalMetadataService
return await _unitOfWork.ExternalSeriesMetadataRepository.GetSeriesDetailPlusDto(seriesId);
}
var data = await _unitOfWork.SeriesRepository.GetPlusSeriesDto(seriesId);
if (data == null) return _defaultReturn;
// Get from Kavita+ API the Full Series metadata with rec/rev and cache to ExternalMetadata tables
return await FetchExternalMetadataForSeries(seriesId, libraryType, data);
}
/// <summary>
/// This will override any sort of matching that was done prior and force it to be what the user Selected
/// </summary>
/// <param name="seriesId"></param>
/// <param name="dto"></param>
public async Task FixSeriesMatch(int seriesId, ExternalSeriesDetailDto dto)
{
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId, SeriesIncludes.Library);
if (series == null) return;
// Remove from Blacklist
series.IsBlacklisted = false;
series.DontMatch = false;
_unitOfWork.SeriesRepository.Update(series);
// Refetch metadata with a Direct lookup
await FetchExternalMetadataForSeries(seriesId, series.Library.Type, new PlusSeriesDto()
{
SeriesName = dto.Name,
AniListId = dto.AniListId,
MalId = dto.MALId,
MediaFormat = dto.PlusMediaFormat,
});
// Find all scrobble events and rewrite them to be the correct
var events = await _unitOfWork.ScrobbleRepository.GetAllEventsForSeries(seriesId);
_unitOfWork.ScrobbleRepository.Remove(events);
await _unitOfWork.CommitAsync();
// Regenerate all events for the series for all users
BackgroundJob.Enqueue(() => _scrobblingService.CreateEventsFromExistingHistoryForSeries(seriesId));
await _eventHub.SendMessageAsync(MessageFactory.Info,
MessageFactory.InfoEvent($"Fix Match: {series.Name}", "Scrobble Events are regenerating with the new match"));
_logger.LogInformation("Matched {SeriesName} with Kavita+ Series {MatchSeriesName}", series.Name, dto.Name);
}
/// <summary>
/// Sets a series to Dont Match and removes all previously cached
/// </summary>
/// <param name="seriesId"></param>
public async Task UpdateSeriesDontMatch(int seriesId, bool dontMatch)
{
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId, SeriesIncludes.ExternalMetadata);
if (series == null) return;
_logger.LogInformation("User has asked Kavita to stop matching/scrobbling on {SeriesName}", series.Name);
series.DontMatch = dontMatch;
if (dontMatch)
{
// When we set as DontMatch, we will clear existing External Metadata
var externalSeriesMetadata = await GetOrCreateExternalSeriesMetadataForSeries(seriesId, series!);
_unitOfWork.ExternalSeriesMetadataRepository.Remove(series.ExternalSeriesMetadata);
_unitOfWork.ExternalSeriesMetadataRepository.Remove(externalSeriesMetadata.ExternalReviews);
_unitOfWork.ExternalSeriesMetadataRepository.Remove(externalSeriesMetadata.ExternalRatings);
_unitOfWork.ExternalSeriesMetadataRepository.Remove(externalSeriesMetadata.ExternalRecommendations);
}
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
}
/// <summary>
/// Requests the full SeriesDetail (rec, review, metadata) data for a Series. Will save to ExternalMetadata tables.
/// </summary>
/// <param name="seriesId"></param>
/// <param name="libraryType"></param>
/// <param name="data"></param>
/// <returns></returns>
private async Task<SeriesDetailPlusDto> FetchExternalMetadataForSeries(int seriesId, LibraryType libraryType, PlusSeriesDto data)
{
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId);
if (series == null) return _defaultReturn;
try
{
var data = await _unitOfWork.SeriesRepository.GetPlusSeriesDto(seriesId);
if (data == null) return _defaultReturn;
_logger.LogDebug("Fetching Kavita+ Series Detail data for {SeriesName}", data.SeriesName);
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var result = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-detail")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-license-key", license)
.WithHeader("x-installId", HashUtil.ServerToken())
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(Configuration.DefaultTimeOutSecs))
.WithKavitaPlusHeaders(license)
.PostJsonAsync(data)
.ReceiveJson<SeriesDetailPlusApiDto>();
// Clear out existing results
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId);
var externalSeriesMetadata = await GetExternalSeriesMetadataForSeries(seriesId, series!);
var externalSeriesMetadata = await GetOrCreateExternalSeriesMetadataForSeries(seriesId, series!);
_unitOfWork.ExternalSeriesMetadataRepository.Remove(externalSeriesMetadata.ExternalReviews);
_unitOfWork.ExternalSeriesMetadataRepository.Remove(externalSeriesMetadata.ExternalRatings);
_unitOfWork.ExternalSeriesMetadataRepository.Remove(externalSeriesMetadata.ExternalRecommendations);
@ -339,17 +468,24 @@ public class ExternalMetadataService : IExternalMetadataService
}
catch (Exception ex)
{
_logger.LogError(ex, "An error happened during the request to Kavita+ API");
_logger.LogError(ex, "Unable to fetch external series metadata from Kavita+");
}
// Blacklist the series as it wasn't found in Kavita+
await _unitOfWork.ExternalSeriesMetadataRepository.CreateBlacklistedSeries(seriesId);
series.IsBlacklisted = true;
await _unitOfWork.CommitAsync();
return _defaultReturn;
}
private async Task<ExternalSeriesMetadata> GetExternalSeriesMetadataForSeries(int seriesId, Series series)
/// <summary>
/// Gets from DB or creates a new one with just SeriesId
/// </summary>
/// <param name="seriesId"></param>
/// <param name="series"></param>
/// <returns></returns>
private async Task<ExternalSeriesMetadata> GetOrCreateExternalSeriesMetadataForSeries(int seriesId, Series series)
{
var externalSeriesMetadata = await _unitOfWork.ExternalSeriesMetadataRepository.GetExternalSeriesMetadata(seriesId);
if (externalSeriesMetadata != null) return externalSeriesMetadata;
@ -454,20 +590,14 @@ public class ExternalMetadataService : IExternalMetadataService
}
payload.SeriesName = series.Name;
payload.LocalizedSeriesName = series.LocalizedName;
payload.PlusMediaFormat = ConvertToMediaFormat(series.Library.Type, series.Format);
payload.PlusMediaFormat = series.Library.Type.ConvertToPlusMediaFormat(series.Format);
}
}
try
{
return await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-by-ids")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-license-key", license)
.WithHeader("x-installId", HashUtil.ServerToken())
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(Configuration.DefaultTimeOutSecs))
.WithKavitaPlusHeaders(license)
.PostJsonAsync(payload)
.ReceiveJson<ExternalSeriesDetailDto>();
@ -479,16 +609,4 @@ public class ExternalMetadataService : IExternalMetadataService
return null;
}
private static MediaFormat ConvertToMediaFormat(LibraryType libraryType, MangaFormat seriesFormat)
{
return libraryType switch
{
LibraryType.Manga => seriesFormat == MangaFormat.Epub ? MediaFormat.LightNovel : MediaFormat.Manga,
LibraryType.Comic => MediaFormat.Comic,
LibraryType.Book => MediaFormat.Book,
LibraryType.LightNovel => MediaFormat.LightNovel,
_ => MediaFormat.Unknown
};
}
}