AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Background Prefetching for Kavita+ (#2707)

Browse source
This commit is contained in:
Joe Milazzo 2024-02-10 09:43:17 -06:00 • committed by GitHub
parent f616b99585
commit 5dc5029a75
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 3300 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

1
API/Services/DeviceService.cs
View file

@ -126,6 +126,7 @@ public class DeviceService : IDeviceService
device.UpdateLastUsed();
_unitOfWork.DeviceRepository.Update(device);
await _unitOfWork.CommitAsync();
var success = await _emailService.SendFilesToEmail(new SendToDto()
{
DestinationEmail = device.EmailAddress!,

2
API/Services/EmailService.cs
View file

@ -224,7 +224,7 @@ public class EmailService : IEmailService
public async Task<bool> SendFilesToEmail(SendToDto data)
{
var serverSetting = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
if (!serverSetting.IsEmailSetup()) return false;
if (!serverSetting.IsEmailSetupForSendToDevice()) return false;
var emailOptions = new EmailOptionsDto()
{

43
API/Services/Plus/ExternalMetadataService.cs
View file

@ -13,6 +13,7 @@ using API.Entities;
using API.Entities.Enums;
using API.Entities.Metadata;
using API.Extensions;
using API.Helpers;
using AutoMapper;
using Flurl.Http;
using Hangfire;
@ -76,6 +77,8 @@ public class ExternalMetadataService : IExternalMetadataService
Ratings = ArraySegment<RatingDto>.Empty,
Reviews = ArraySegment<UserReviewDto>.Empty
};
// Allow 50 requests per 24 hours
private static readonly RateLimiter RateLimiter = new RateLimiter(50, TimeSpan.FromHours(12), false);
public ExternalMetadataService(IUnitOfWork unitOfWork, ILogger<ExternalMetadataService> logger, IMapper mapper, ILicenseService licenseService)
{
@ -85,7 +88,6 @@ public class ExternalMetadataService : IExternalMetadataService
_licenseService = licenseService;
FlurlHttp.ConfigureClient(Configuration.KavitaPlusApiUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
}
@ -114,17 +116,17 @@ public class ExternalMetadataService : IExternalMetadataService
var ids = await _unitOfWork.ExternalSeriesMetadataRepository.GetAllSeriesIdsWithoutMetadata(25);
if (ids.Count == 0) return;
_logger.LogInformation("Started Refreshing {Count} series data from Kavita+", ids.Count);
_logger.LogInformation("[Kavita+ Data Refresh] Started Refreshing {Count} series data from Kavita+", ids.Count);
var count = 0;
var libTypes = await _unitOfWork.LibraryRepository.GetLibraryTypesBySeriesIdsAsync(ids);
foreach (var seriesId in ids)
{
// TODO: Rewrite this so it's streamlined and not multiple DB calls
var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeBySeriesIdAsync(seriesId);
await GetSeriesDetailPlus(seriesId, libraryType);
var libraryType = libTypes[seriesId];
await GetNewSeriesData(seriesId, libraryType);
await Task.Delay(1500);
count++;
}
_logger.LogInformation("Finished Refreshing {Count} series data from Kavita+", count);
_logger.LogInformation("[Kavita+ Data Refresh] Finished Refreshing {Count} series data from Kavita+", count);
}
/// <summary>
@ -145,13 +147,30 @@ public class ExternalMetadataService : IExternalMetadataService
await _unitOfWork.CommitAsync();
}
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public Task GetNewSeriesData(int seriesId, LibraryType libraryType)
/// <summary>
/// Fetches data from Kavita+
/// </summary>
/// <param name="seriesId"></param>
/// <param name="libraryType"></param>
public async Task GetNewSeriesData(int seriesId, LibraryType libraryType)
{
// TODO: Implement this task
if (!IsPlusEligible(libraryType)) return Task.CompletedTask;
return Task.CompletedTask;
if (!IsPlusEligible(libraryType)) return;
// Generate key based on seriesId and libraryType or any unique identifier for the request
// Check if the request is allowed based on the rate limit
if (!RateLimiter.TryAcquire(string.Empty))
{
// Request not allowed due to rate limit
_logger.LogDebug("Rate Limit hit for Kavita+ prefetch");
return;
}
_logger.LogDebug("Prefetching Kavita+ data for Series {SeriesId}", seriesId);
// Prefetch SeriesDetail data
await GetSeriesDetailPlus(seriesId, libraryType);
// TODO: Fetch Series Metadata
}
/// <summary>

11
API/Services/SeriesService.cs
View file

@ -427,6 +427,9 @@ public class SeriesService : ISeriesService
}
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdsAsync(seriesIds);
_unitOfWork.SeriesRepository.Remove(series);
var libraryIds = series.Select(s => s.LibraryId);
var libraries = await _unitOfWork.LibraryRepository.GetLibraryForIdsAsync(libraryIds);
foreach (var library in libraries)
@ -434,11 +437,8 @@ public class SeriesService : ISeriesService
library.UpdateLastModified();
_unitOfWork.LibraryRepository.Update(library);
}
await _unitOfWork.CommitAsync();
_unitOfWork.SeriesRepository.Remove(series);
if (!_unitOfWork.HasChanges() || !await _unitOfWork.CommitAsync()) return true;
foreach (var s in series)
{
@ -449,14 +449,13 @@ public class SeriesService : ISeriesService
await _unitOfWork.AppUserProgressRepository.CleanupAbandonedChapters();
await _unitOfWork.CollectionTagRepository.RemoveTagsWithoutSeries();
_taskScheduler.CleanupChapters(allChapterIds.ToArray());
return true;
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an issue when trying to delete multiple series");
return false;
}
return true;
}
/// <summary>

8
API/Services/TaskScheduler.cs
View file

@ -182,10 +182,10 @@ public class TaskScheduler : ITaskScheduler
RecurringJob.AddOrUpdate(ProcessProcessedScrobblingEventsId, () => _scrobblingService.ClearProcessedEvents(),
Cron.Daily, RecurringJobOptions);
// Backfilling/Freshening Reviews/Rating/Recommendations (TODO: This will come in v0.8.x)
// RecurringJob.AddOrUpdate(KavitaPlusDataRefreshId,
// () => _externalMetadataService.FetchExternalDataTask(), Cron.Hourly(Rnd.Next(0, 59)),
// RecurringJobOptions);
// Backfilling/Freshening Reviews/Rating/Recommendations
RecurringJob.AddOrUpdate(KavitaPlusDataRefreshId,
() => _externalMetadataService.FetchExternalDataTask(), Cron.Daily(Rnd.Next(1, 4)),
RecurringJobOptions);
}
#region StatsTasks