AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Last PR before Release (#2692)

Browse source
This commit is contained in:
Joe Milazzo 2024-02-05 18:58:03 -06:00 • committed by GitHub
parent 07e96389fb
commit 5cf6077dfd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
38 changed files with 3801 additions and 2044 deletions
Download patch file Download diff file Expand all files Collapse all files

7
API/Services/EmailService.cs
View file

@ -246,10 +246,15 @@ public class EmailService : IEmailService
};
email.From.Add(new MailboxAddress(smtpConfig.SenderDisplayName, smtpConfig.SenderAddress));
// Inject the body into the base template
var fullBody = UpdatePlaceHolders(await GetEmailBody("base"), new List<KeyValuePair<string, string>>()
{
new ("{{Body}}", userEmailOptions.Body)
});
var body = new BodyBuilder
{
HtmlBody = userEmailOptions.Body
HtmlBody = fullBody
};
if (userEmailOptions.Attachments != null)

53
API/Services/Plus/ExternalMetadataService.cs
View file

@ -1,5 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
@ -14,6 +15,7 @@ using API.Entities.Metadata;
using API.Extensions;
using AutoMapper;
using Flurl.Http;
using Hangfire;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Kavita.Common.Helpers;
@ -49,6 +51,15 @@ public interface IExternalMetadataService
Task<ExternalSeriesDetailDto?> GetExternalSeriesDetail(int? aniListId, long? malId, int? seriesId);
Task<SeriesDetailPlusDto> GetSeriesDetailPlus(int seriesId, LibraryType libraryType);
Task ForceKavitaPlusRefresh(int seriesId);
Task FetchExternalDataTask();
/// <summary>
/// This is an entry point and provides a level of protection against calling upstream API. Will only allow 100 new
/// series to fetch data within a day and enqueues background jobs at certain times to fetch that data.
/// </summary>
/// <param name="seriesId"></param>
/// <param name="libraryType"></param>
/// <returns></returns>
Task GetNewSeriesData(int seriesId, LibraryType libraryType);
}
public class ExternalMetadataService : IExternalMetadataService
@ -58,6 +69,7 @@ public class ExternalMetadataService : IExternalMetadataService
private readonly IMapper _mapper;
private readonly ILicenseService _licenseService;
private readonly TimeSpan _externalSeriesMetadataCache = TimeSpan.FromDays(30);
public static readonly ImmutableArray<LibraryType> NonEligibleLibraryTypes = ImmutableArray.Create<LibraryType>(LibraryType.Comic);
private readonly SeriesDetailPlusDto _defaultReturn = new()
{
Recommendations = null,
@ -72,6 +84,8 @@ public class ExternalMetadataService : IExternalMetadataService
_mapper = mapper;
_licenseService = licenseService;
FlurlHttp.ConfigureClient(Configuration.KavitaPlusApiUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
}
@ -83,7 +97,34 @@ public class ExternalMetadataService : IExternalMetadataService
/// <returns></returns>
public static bool IsPlusEligible(LibraryType type)
{
return type != LibraryType.Comic;
return !NonEligibleLibraryTypes.Contains(type);
}
/// <summary>
/// This is a task that runs on a schedule and slowly fetches data from Kavita+ to keep
/// data in the DB non-stale and fetched.
/// </summary>
/// <remarks>To avoid blasting Kavita+ API, this only processes a few records. The goal is to slowly build </remarks>
/// <returns></returns>
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task FetchExternalDataTask()
{
// Find all Series that are eligible and limit
var ids = await _unitOfWork.ExternalSeriesMetadataRepository.GetAllSeriesIdsWithoutMetadata(25);
if (ids.Count == 0) return;
_logger.LogInformation("Started Refreshing {Count} series data from Kavita+", ids.Count);
var count = 0;
foreach (var seriesId in ids)
{
// TODO: Rewrite this so it's streamlined and not multiple DB calls
var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeBySeriesIdAsync(seriesId);
await GetSeriesDetailPlus(seriesId, libraryType);
await Task.Delay(1500);
count++;
}
_logger.LogInformation("Finished Refreshing {Count} series data from Kavita+", count);
}
/// <summary>
@ -104,6 +145,15 @@ public class ExternalMetadataService : IExternalMetadataService
await _unitOfWork.CommitAsync();
}
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public Task GetNewSeriesData(int seriesId, LibraryType libraryType)
{
// TODO: Implement this task
if (!IsPlusEligible(libraryType)) return Task.CompletedTask;
return Task.CompletedTask;
}
/// <summary>
/// Retrieves Metadata about a Recommended External Series
/// </summary>
@ -153,6 +203,7 @@ public class ExternalMetadataService : IExternalMetadataService
{
var data = await _unitOfWork.SeriesRepository.GetPlusSeriesDto(seriesId);
if (data == null) return _defaultReturn;
_logger.LogDebug("Fetching Kavita+ Series Detail data for {SeriesName}", data.SeriesName);
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
var result = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-detail")

53
API/Services/Plus/ScrobblingService.cs
View file

@ -89,6 +89,9 @@ public class ScrobblingService : IScrobblingService
ScrobbleProvider.AniList
};
private const string UnknownSeriesErrorMessage = "Series cannot be matched for Scrobbling";
private const string AccessTokenErrorMessage = "Access Token needs to be rotated to continue scrobbling";
public ScrobblingService(IUnitOfWork unitOfWork, ITokenService tokenService,
IEventHub eventHub, ILogger<ScrobblingService> logger, ILicenseService licenseService,
@ -374,7 +377,7 @@ public class ScrobblingService : IScrobblingService
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId);
if (library is not {AllowScrobbling: true}) return true;
if (library.Type == LibraryType.Comic) return true;
if (!ExternalMetadataService.IsPlusEligible(library.Type)) return true;
return false;
}
@ -424,14 +427,18 @@ public class ScrobblingService : IScrobblingService
if (response.ErrorMessage != null && response.ErrorMessage.Contains("Too Many Requests"))
{
_logger.LogInformation("Hit Too many requests, sleeping to regain requests");
await Task.Delay(TimeSpan.FromMinutes(5));
await Task.Delay(TimeSpan.FromMinutes(10));
} else if (response.ErrorMessage != null && response.ErrorMessage.Contains("Unauthorized"))
{
_logger.LogInformation("Kavita+ responded with Unauthorized. Please check your subscription");
_logger.LogCritical("Kavita+ responded with Unauthorized. Please check your subscription");
await _licenseService.HasActiveLicense(true);
evt.IsErrored = true;
evt.ErrorDetails = "Kavita+ subscription no longer active";
throw new KavitaException("Kavita+ responded with Unauthorized. Please check your subscription");
} else if (response.ErrorMessage != null && response.ErrorMessage.Contains("Access token is invalid"))
{
evt.IsErrored = true;
evt.ErrorDetails = AccessTokenErrorMessage;
throw new KavitaException("Access token is invalid");
}
else if (response.ErrorMessage != null && response.ErrorMessage.Contains("Unknown Series"))
@ -442,12 +449,16 @@ public class ScrobblingService : IScrobblingService
{
_unitOfWork.ScrobbleRepository.Attach(new ScrobbleError()
{
Comment = "Unknown Series",
Comment = UnknownSeriesErrorMessage,
Details = data.SeriesName,
LibraryId = evt.LibraryId,
SeriesId = evt.SeriesId
});
await _unitOfWork.ExternalSeriesMetadataRepository.CreateBlacklistedSeries(evt.SeriesId, false);
}
evt.IsErrored = true;
evt.ErrorDetails = UnknownSeriesErrorMessage;
} else if (response.ErrorMessage != null && response.ErrorMessage.StartsWith("Review"))
{
// Log the Series name and Id in ScrobbleErrors
@ -462,8 +473,11 @@ public class ScrobblingService : IScrobblingService
SeriesId = evt.SeriesId
});
}
evt.IsErrored = true;
evt.ErrorDetails = "Review was unable to be saved due to upstream requirements";
}
evt.IsErrored = true;
_logger.LogError("Scrobbling failed due to {ErrorMessage}: {SeriesName}", response.ErrorMessage, data.SeriesName);
throw new KavitaException($"Scrobbling failed due to {response.ErrorMessage}: {data.SeriesName}");
}
@ -479,12 +493,14 @@ public class ScrobblingService : IScrobblingService
{
_unitOfWork.ScrobbleRepository.Attach(new ScrobbleError()
{
Comment = "Unknown Series",
Comment = UnknownSeriesErrorMessage,
Details = data.SeriesName,
LibraryId = evt.LibraryId,
SeriesId = evt.SeriesId
});
}
evt.IsErrored = true;
evt.ErrorDetails = "Bad payload from Scrobble Provider";
throw new KavitaException("Bad payload from Scrobble Provider");
}
throw;
@ -602,11 +618,10 @@ public class ScrobblingService : IScrobblingService
.ToImmutableHashSet();
var errors = (await _unitOfWork.ScrobbleRepository.GetScrobbleErrors())
.Where(e => e.Comment == "Unknown Series")
.Where(e => e.Comment == "Unknown Series" || e.Comment == UnknownSeriesErrorMessage || e.Comment == AccessTokenErrorMessage)
.Select(e => e.SeriesId)
.ToList();
var readEvents = (await _unitOfWork.ScrobbleRepository.GetByEvent(ScrobbleEventType.ChapterRead))
.Where(e => librariesWithScrobbling.Contains(e.LibraryId))
.Where(e => !errors.Contains(e.SeriesId))
@ -674,7 +689,7 @@ public class ScrobblingService : IScrobblingService
MALId = (int?) evt.MalId,
ScrobbleEventType = evt.ScrobbleEventType,
ChapterNumber = evt.ChapterNumber,
VolumeNumber = evt.VolumeNumber,
VolumeNumber = (int?) evt.VolumeNumber,
AniListToken = evt.AppUser.AniListAccessToken,
SeriesName = evt.Series.Name,
LocalizedSeriesName = evt.Series.LocalizedName,
@ -706,7 +721,7 @@ public class ScrobblingService : IScrobblingService
MALId = (int?) evt.MalId,
ScrobbleEventType = evt.ScrobbleEventType,
ChapterNumber = evt.ChapterNumber,
VolumeNumber = evt.VolumeNumber,
VolumeNumber = (int?) evt.VolumeNumber,
AniListToken = evt.AppUser.AniListAccessToken,
SeriesName = evt.Series.Name,
LocalizedSeriesName = evt.Series.LocalizedName,
@ -770,6 +785,23 @@ public class ScrobblingService : IScrobblingService
return 0;
}
if (await _unitOfWork.ExternalSeriesMetadataRepository.IsBlacklistedSeries(evt.SeriesId))
{
_unitOfWork.ScrobbleRepository.Attach(new ScrobbleError()
{
Comment = UnknownSeriesErrorMessage,
Details = $"User: {evt.AppUser.UserName} Series: {evt.Series.Name}",
LibraryId = evt.LibraryId,
SeriesId = evt.SeriesId
});
evt.IsErrored = true;
evt.ErrorDetails = "Series cannot be matched for Scrobbling";
evt.ProcessDateUtc = DateTime.UtcNow;
_unitOfWork.ScrobbleRepository.Update(evt);
await _unitOfWork.CommitAsync();
return 0;
}
var count = await SetAndCheckRateLimit(userRateLimits, evt.AppUser, license.Value);
userRateLimits[evt.AppUserId] = count;
if (count == 0)
@ -796,6 +828,9 @@ public class ScrobblingService : IScrobblingService
if (ex.Message.Contains("Access token is invalid"))
{
_logger.LogCritical("Access Token for UserId: {UserId} needs to be rotated to continue scrobbling", evt.AppUser.Id);
evt.IsErrored = true;
evt.ErrorDetails = AccessTokenErrorMessage;
_unitOfWork.ScrobbleRepository.Update(evt);
return progressCounter;
}
}

49
API/Services/TaskScheduler.cs
View file

@ -56,6 +56,7 @@ public class TaskScheduler : ITaskScheduler
private readonly IMediaConversionService _mediaConversionService;
private readonly IScrobblingService _scrobblingService;
private readonly ILicenseService _licenseService;
private readonly IExternalMetadataService _externalMetadataService;
public static BackgroundJobServer Client => new ();
public const string ScanQueue = "scan";
@ -68,10 +69,11 @@ public class TaskScheduler : ITaskScheduler
public const string BackupTaskId = "backup";
public const string ScanLibrariesTaskId = "scan-libraries";
public const string ReportStatsTaskId = "report-stats";
public const string CheckScrobblingTokens = "check-scrobbling-tokens";
public const string ProcessScrobblingEvents = "process-scrobbling-events";
public const string ProcessProcessedScrobblingEvents = "process-processed-scrobbling-events";
public const string LicenseCheck = "license-check";
public const string CheckScrobblingTokensId = "check-scrobbling-tokens";
public const string ProcessScrobblingEventsId = "process-scrobbling-events";
public const string ProcessProcessedScrobblingEventsId = "process-processed-scrobbling-events";
public const string LicenseCheckId = "license-check";
public const string KavitaPlusDataRefreshId = "kavita+-data-refresh";
private static readonly ImmutableArray<string> ScanTasks =
["ScannerService", "ScanLibrary", "ScanLibraries", "ScanFolder", "ScanSeries"];
@ -88,7 +90,8 @@ public class TaskScheduler : ITaskScheduler
IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService,
ICleanupService cleanupService, IStatsService statsService, IVersionUpdaterService versionUpdaterService,
IThemeService themeService, IWordCountAnalyzerService wordCountAnalyzerService, IStatisticService statisticService,
IMediaConversionService mediaConversionService, IScrobblingService scrobblingService, ILicenseService licenseService)
IMediaConversionService mediaConversionService, IScrobblingService scrobblingService, ILicenseService licenseService,
IExternalMetadataService externalMetadataService)
{
_cacheService = cacheService;
_logger = logger;
@ -105,6 +108,7 @@ public class TaskScheduler : ITaskScheduler
_mediaConversionService = mediaConversionService;
_scrobblingService = scrobblingService;
_licenseService = licenseService;
_externalMetadataService = externalMetadataService;
}
public async Task ScheduleTasks()
@ -121,7 +125,8 @@ public class TaskScheduler : ITaskScheduler
}
else
{
RecurringJob.AddOrUpdate(ScanLibrariesTaskId, () => ScanLibraries(false), Cron.Daily, RecurringJobOptions);
RecurringJob.AddOrUpdate(ScanLibrariesTaskId, () => ScanLibraries(false),
Cron.Daily, RecurringJobOptions);
}
setting = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskBackup)).Value;
@ -134,19 +139,24 @@ public class TaskScheduler : ITaskScheduler
// Override daily and make 2am so that everything on system has cleaned up and no blocking
schedule = Cron.Daily(2);
}
RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), () => schedule, RecurringJobOptions);
RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(),
() => schedule, RecurringJobOptions);
}
else
{
RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), Cron.Weekly, RecurringJobOptions);
RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(),
Cron.Weekly, RecurringJobOptions);
}
setting = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskCleanup)).Value;
_logger.LogDebug("Scheduling Cleanup Task for {Setting}", setting);
RecurringJob.AddOrUpdate(CleanupTaskId, () => _cleanupService.Cleanup(), CronConverter.ConvertToCronNotation(setting), RecurringJobOptions);
RecurringJob.AddOrUpdate(CleanupTaskId, () => _cleanupService.Cleanup(),
CronConverter.ConvertToCronNotation(setting), RecurringJobOptions);
RecurringJob.AddOrUpdate(RemoveFromWantToReadTaskId, () => _cleanupService.CleanupWantToRead(), Cron.Daily, RecurringJobOptions);
RecurringJob.AddOrUpdate(UpdateYearlyStatsTaskId, () => _statisticService.UpdateServerStatistics(), Cron.Monthly, RecurringJobOptions);
RecurringJob.AddOrUpdate(RemoveFromWantToReadTaskId, () => _cleanupService.CleanupWantToRead(),
Cron.Daily, RecurringJobOptions);
RecurringJob.AddOrUpdate(UpdateYearlyStatsTaskId, () => _statisticService.UpdateServerStatistics(),
Cron.Monthly, RecurringJobOptions);
await ScheduleKavitaPlusTasks();
}
@ -159,14 +169,23 @@ public class TaskScheduler : ITaskScheduler
{
return;
}
RecurringJob.AddOrUpdate(CheckScrobblingTokens, () => _scrobblingService.CheckExternalAccessTokens(), Cron.Daily, RecurringJobOptions);
RecurringJob.AddOrUpdate(CheckScrobblingTokensId, () => _scrobblingService.CheckExternalAccessTokens(),
Cron.Daily, RecurringJobOptions);
BackgroundJob.Enqueue(() => _scrobblingService.CheckExternalAccessTokens()); // We also kick off an immediate check on startup
RecurringJob.AddOrUpdate(LicenseCheck, () => _licenseService.HasActiveLicense(true), LicenseService.Cron, RecurringJobOptions);
RecurringJob.AddOrUpdate(LicenseCheckId, () => _licenseService.HasActiveLicense(true),
LicenseService.Cron, RecurringJobOptions);
BackgroundJob.Enqueue(() => _licenseService.HasActiveLicense(true));
// KavitaPlus Scrobbling (every 4 hours)
RecurringJob.AddOrUpdate(ProcessScrobblingEvents, () => _scrobblingService.ProcessUpdatesSinceLastSync(), "0 */4 * * *", RecurringJobOptions);
RecurringJob.AddOrUpdate(ProcessProcessedScrobblingEvents, () => _scrobblingService.ClearProcessedEvents(), Cron.Daily, RecurringJobOptions);
RecurringJob.AddOrUpdate(ProcessScrobblingEventsId, () => _scrobblingService.ProcessUpdatesSinceLastSync(),
"0 */4 * * *", RecurringJobOptions);
RecurringJob.AddOrUpdate(ProcessProcessedScrobblingEventsId, () => _scrobblingService.ClearProcessedEvents(),
Cron.Daily, RecurringJobOptions);
// Backfilling/Freshening Reviews/Rating/Recommendations (TODO: This will come in v0.8.x)
// RecurringJob.AddOrUpdate(KavitaPlusDataRefreshId,
// () => _externalMetadataService.FetchExternalDataTask(), Cron.Hourly(Rnd.Next(0, 59)),
// RecurringJobOptions);
}
#region StatsTasks

8
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -13,6 +13,7 @@ using API.Entities.Enums;
using API.Extensions;
using API.Helpers;
using API.Helpers.Builders;
using API.Services.Plus;
using API.Services.Tasks.Metadata;
using API.Services.Tasks.Scanner.Parser;
using API.SignalR;
@ -57,6 +58,7 @@ public class ProcessSeries : IProcessSeries
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly ICollectionTagService _collectionTagService;
private readonly IReadingListService _readingListService;
private readonly IExternalMetadataService _externalMetadataService;
private Dictionary<string, Genre> _genres;
private IList<Person> _people;
@ -66,7 +68,7 @@ public class ProcessSeries : IProcessSeries
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
IFileService fileService, IMetadataService metadataService, IWordCountAnalyzerService wordCountAnalyzerService,
ICollectionTagService collectionTagService, IReadingListService readingListService)
ICollectionTagService collectionTagService, IReadingListService readingListService, IExternalMetadataService externalMetadataService)
{
_unitOfWork = unitOfWork;
_logger = logger;
@ -79,6 +81,7 @@ public class ProcessSeries : IProcessSeries
_wordCountAnalyzerService = wordCountAnalyzerService;
_collectionTagService = collectionTagService;
_readingListService = readingListService;
_externalMetadataService = externalMetadataService;
_genres = new Dictionary<string, Genre>();
@ -236,8 +239,9 @@ public class ProcessSeries : IProcessSeries
if (seriesAdded)
{
// See if any recommendations can link up to the series
// See if any recommendations can link up to the series and pre-fetch external metadata for the series
_logger.LogInformation("Linking up External Recommendations new series (if applicable)");
await _externalMetadataService.GetNewSeriesData(series.Id, series.Library.Type);
await _unitOfWork.ExternalSeriesMetadataRepository.LinkRecommendationsToSeries(series);
await _eventHub.SendMessageAsync(MessageFactory.SeriesAdded,
MessageFactory.SeriesAddedEvent(series.Id, series.Name, series.LibraryId), false);

21
API/Services/Tasks/VersionUpdaterService.cs
View file

@ -47,7 +47,7 @@ public interface IVersionUpdaterService
{
Task<UpdateNotificationDto?> CheckForUpdate();
Task PushUpdate(UpdateNotificationDto update);
Task<IEnumerable<UpdateNotificationDto>> GetAllReleases();
Task<IList<UpdateNotificationDto>> GetAllReleases();
Task<int> GetNumberOfReleasesBehind();
}
@ -82,10 +82,21 @@ public class VersionUpdaterService : IVersionUpdaterService
return CreateDto(update);
}
public async Task<IEnumerable<UpdateNotificationDto>> GetAllReleases()
public async Task<IList<UpdateNotificationDto>> GetAllReleases()
{
var updates = await GetGithubReleases();
return updates.Select(CreateDto).Where(d => d != null)!;
var updateDtos = updates.Select(CreateDto)
.Where(d => d != null)
.OrderByDescending(d => d!.PublishDate)
.Select(d => d!)
.ToList();
// Find the latest dto
var latestRelease = updateDtos[0]!;
var isNightly = BuildInfo.Version > new Version(latestRelease.UpdateVersion);
latestRelease.IsOnNightlyInRelease = isNightly;
return updateDtos;
}
public async Task<int> GetNumberOfReleasesBehind()
@ -108,7 +119,9 @@ public class VersionUpdaterService : IVersionUpdaterService
UpdateTitle = update.Name,
UpdateUrl = update.Html_Url,
IsDocker = OsInfo.IsDocker,
PublishDate = update.Published_At
PublishDate = update.Published_At,
IsReleaseEqual = BuildInfo.Version == updateVersion,
IsReleaseNewer = BuildInfo.Version < updateVersion,
};
}

3
API/Services/TokenService.cs
View file

@ -142,8 +142,9 @@ public class TokenService : ITokenService
return jwtClaim?.Value;
}
public bool HasTokenExpired(string token)
public bool HasTokenExpired(string? token)
{
if (string.IsNullOrEmpty(token)) return true;
var tokenHandler = new JwtSecurityTokenHandler();
var tokenContent = tokenHandler.ReadJwtToken(token);
return tokenContent.ValidTo <= DateTime.UtcNow;