AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Last PR before Release (#2692)

Browse source
This commit is contained in:
Joe Milazzo 2024-02-05 18:58:03 -06:00 • committed by GitHub
parent 07e96389fb
commit 5cf6077dfd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 3801 additions and 2044 deletions
Download patch file Download diff file Expand all files Collapse all files

53
API/Services/Plus/ExternalMetadataService.cs
View file

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
@ -14,6 +15,7 @@ using API.Entities.Metadata;
using API.Extensions;
using AutoMapper;
using Flurl.Http;
using Hangfire;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Kavita.Common.Helpers;
@ -49,6 +51,15 @@ public interface IExternalMetadataService
Task<ExternalSeriesDetailDto?> GetExternalSeriesDetail(int? aniListId, long? malId, int? seriesId);
Task<SeriesDetailPlusDto> GetSeriesDetailPlus(int seriesId, LibraryType libraryType);
Task ForceKavitaPlusRefresh(int seriesId);
Task FetchExternalDataTask();
/// <summary>
/// This is an entry point and provides a level of protection against calling upstream API. Will only allow 100 new
/// series to fetch data within a day and enqueues background jobs at certain times to fetch that data.
/// </summary>
/// <param name="seriesId"></param>
/// <param name="libraryType"></param>
/// <returns></returns>
Task GetNewSeriesData(int seriesId, LibraryType libraryType);
}
public class ExternalMetadataService : IExternalMetadataService
@ -58,6 +69,7 @@ public class ExternalMetadataService : IExternalMetadataService
private readonly IMapper _mapper;
private readonly ILicenseService _licenseService;
private readonly TimeSpan _externalSeriesMetadataCache = TimeSpan.FromDays(30);
public static readonly ImmutableArray<LibraryType> NonEligibleLibraryTypes = ImmutableArray.Create<LibraryType>(LibraryType.Comic);
private readonly SeriesDetailPlusDto _defaultReturn = new()
{
Recommendations = null,
@ -72,6 +84,8 @@ public class ExternalMetadataService : IExternalMetadataService
_mapper = mapper;
_licenseService = licenseService;
FlurlHttp.ConfigureClient(Configuration.KavitaPlusApiUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
}
@ -83,7 +97,34 @@ public class ExternalMetadataService : IExternalMetadataService
/// <returns></returns>
public static bool IsPlusEligible(LibraryType type)
{
return type != LibraryType.Comic;
return !NonEligibleLibraryTypes.Contains(type);
}
/// <summary>
/// This is a task that runs on a schedule and slowly fetches data from Kavita+ to keep
/// data in the DB non-stale and fetched.
/// </summary>
/// <remarks>To avoid blasting Kavita+ API, this only processes a few records. The goal is to slowly build </remarks>
/// <returns></returns>
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task FetchExternalDataTask()
{
// Find all Series that are eligible and limit
var ids = await _unitOfWork.ExternalSeriesMetadataRepository.GetAllSeriesIdsWithoutMetadata(25);
if (ids.Count == 0) return;
_logger.LogInformation("Started Refreshing {Count} series data from Kavita+", ids.Count);
var count = 0;
foreach (var seriesId in ids)
{
// TODO: Rewrite this so it's streamlined and not multiple DB calls
var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeBySeriesIdAsync(seriesId);
await GetSeriesDetailPlus(seriesId, libraryType);
await Task.Delay(1500);
count++;
}
_logger.LogInformation("Finished Refreshing {Count} series data from Kavita+", count);
}
/// <summary>
@ -104,6 +145,15 @@ public class ExternalMetadataService : IExternalMetadataService
await _unitOfWork.CommitAsync();
}
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public Task GetNewSeriesData(int seriesId, LibraryType libraryType)
{
// TODO: Implement this task
if (!IsPlusEligible(libraryType)) return Task.CompletedTask;
return Task.CompletedTask;
}
/// <summary>
/// Retrieves Metadata about a Recommended External Series
/// </summary>
@ -153,6 +203,7 @@ public class ExternalMetadataService : IExternalMetadataService
{
var data = await _unitOfWork.SeriesRepository.GetPlusSeriesDto(seriesId);
if (data == null) return _defaultReturn;
_logger.LogDebug("Fetching Kavita+ Series Detail data for {SeriesName}", data.SeriesName);
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var result = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-detail")

53
API/Services/Plus/ScrobblingService.cs
View file

@ -89,6 +89,9 @@ public class ScrobblingService : IScrobblingService
ScrobbleProvider.AniList
};
private const string UnknownSeriesErrorMessage = "Series cannot be matched for Scrobbling";
private const string AccessTokenErrorMessage = "Access Token needs to be rotated to continue scrobbling";
public ScrobblingService(IUnitOfWork unitOfWork, ITokenService tokenService,
IEventHub eventHub, ILogger<ScrobblingService> logger, ILicenseService licenseService,
@ -374,7 +377,7 @@ public class ScrobblingService : IScrobblingService
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId);
if (library is not {AllowScrobbling: true}) return true;
if (library.Type == LibraryType.Comic) return true;
if (!ExternalMetadataService.IsPlusEligible(library.Type)) return true;
return false;
}
@ -424,14 +427,18 @@ public class ScrobblingService : IScrobblingService
if (response.ErrorMessage != null && response.ErrorMessage.Contains("Too Many Requests"))
{
_logger.LogInformation("Hit Too many requests, sleeping to regain requests");
await Task.Delay(TimeSpan.FromMinutes(5));
await Task.Delay(TimeSpan.FromMinutes(10));
} else if (response.ErrorMessage != null && response.ErrorMessage.Contains("Unauthorized"))
{
_logger.LogInformation("Kavita+ responded with Unauthorized. Please check your subscription");
_logger.LogCritical("Kavita+ responded with Unauthorized. Please check your subscription");
await _licenseService.HasActiveLicense(true);
evt.IsErrored = true;
evt.ErrorDetails = "Kavita+ subscription no longer active";
throw new KavitaException("Kavita+ responded with Unauthorized. Please check your subscription");
} else if (response.ErrorMessage != null && response.ErrorMessage.Contains("Access token is invalid"))
{
evt.IsErrored = true;
evt.ErrorDetails = AccessTokenErrorMessage;
throw new KavitaException("Access token is invalid");
}
else if (response.ErrorMessage != null && response.ErrorMessage.Contains("Unknown Series"))
@ -442,12 +449,16 @@ public class ScrobblingService : IScrobblingService
{
_unitOfWork.ScrobbleRepository.Attach(new ScrobbleError()
{
Comment = "Unknown Series",
Comment = UnknownSeriesErrorMessage,
Details = data.SeriesName,
LibraryId = evt.LibraryId,
SeriesId = evt.SeriesId
});
await _unitOfWork.ExternalSeriesMetadataRepository.CreateBlacklistedSeries(evt.SeriesId, false);
}
evt.IsErrored = true;
evt.ErrorDetails = UnknownSeriesErrorMessage;
} else if (response.ErrorMessage != null && response.ErrorMessage.StartsWith("Review"))
{
// Log the Series name and Id in ScrobbleErrors
@ -462,8 +473,11 @@ public class ScrobblingService : IScrobblingService
SeriesId = evt.SeriesId
});
}
evt.IsErrored = true;
evt.ErrorDetails = "Review was unable to be saved due to upstream requirements";
}
evt.IsErrored = true;
_logger.LogError("Scrobbling failed due to {ErrorMessage}: {SeriesName}", response.ErrorMessage, data.SeriesName);
throw new KavitaException($"Scrobbling failed due to {response.ErrorMessage}: {data.SeriesName}");
}
@ -479,12 +493,14 @@ public class ScrobblingService : IScrobblingService
{
_unitOfWork.ScrobbleRepository.Attach(new ScrobbleError()
{
Comment = "Unknown Series",
Comment = UnknownSeriesErrorMessage,
Details = data.SeriesName,
LibraryId = evt.LibraryId,
SeriesId = evt.SeriesId
});
}
evt.IsErrored = true;
evt.ErrorDetails = "Bad payload from Scrobble Provider";
throw new KavitaException("Bad payload from Scrobble Provider");
}
throw;
@ -602,11 +618,10 @@ public class ScrobblingService : IScrobblingService
.ToImmutableHashSet();
var errors = (await _unitOfWork.ScrobbleRepository.GetScrobbleErrors())
.Where(e => e.Comment == "Unknown Series")
.Where(e => e.Comment == "Unknown Series" || e.Comment == UnknownSeriesErrorMessage || e.Comment == AccessTokenErrorMessage)
.Select(e => e.SeriesId)
.ToList();
var readEvents = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.ChapterRead))
.Where(e => librariesWithScrobbling.Contains(e.LibraryId))
.Where(e => !errors.Contains(e.SeriesId))
@ -674,7 +689,7 @@ public class ScrobblingService : IScrobblingService
MALId = (int?) evt.MalId,
ScrobbleEventType = evt.ScrobbleEventType,
ChapterNumber = evt.ChapterNumber,
VolumeNumber = evt.VolumeNumber,
VolumeNumber = (int?) evt.VolumeNumber,
AniListToken = evt.AppUser.AniListAccessToken,
SeriesName = evt.Series.Name,
LocalizedSeriesName = evt.Series.LocalizedName,
@ -706,7 +721,7 @@ public class ScrobblingService : IScrobblingService
MALId = (int?) evt.MalId,
ScrobbleEventType = evt.ScrobbleEventType,
ChapterNumber = evt.ChapterNumber,
VolumeNumber = evt.VolumeNumber,
VolumeNumber = (int?) evt.VolumeNumber,
AniListToken = evt.AppUser.AniListAccessToken,
SeriesName = evt.Series.Name,
LocalizedSeriesName = evt.Series.LocalizedName,
@ -770,6 +785,23 @@ public class ScrobblingService : IScrobblingService
return 0;
}
if (await _unitOfWork.ExternalSeriesMetadataRepository.IsBlacklistedSeries(evt.SeriesId))
{
_unitOfWork.ScrobbleRepository.Attach(new ScrobbleError()
{
Comment = UnknownSeriesErrorMessage,
Details = $"User: {evt.AppUser.UserName} Series: {evt.Series.Name}",
LibraryId = evt.LibraryId,
SeriesId = evt.SeriesId
});
evt.IsErrored = true;
evt.ErrorDetails = "Series cannot be matched for Scrobbling";
evt.ProcessDateUtc = DateTime.UtcNow;
_unitOfWork.ScrobbleRepository.Update(evt);
await _unitOfWork.CommitAsync();
return 0;
}
var count = await SetAndCheckRateLimit(userRateLimits, evt.AppUser, license.Value);
userRateLimits[evt.AppUserId] = count;
if (count == 0)
@ -796,6 +828,9 @@ public class ScrobblingService : IScrobblingService
if (ex.Message.Contains("Access token is invalid"))
{
_logger.LogCritical("Access Token for UserId: {UserId} needs to be rotated to continue scrobbling", evt.AppUser.Id);
evt.IsErrored = true;
evt.ErrorDetails = AccessTokenErrorMessage;
_unitOfWork.ScrobbleRepository.Update(evt);
return progressCounter;
}
}