AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Background Prefetching for Kavita+ (#2707)

Browse source
This commit is contained in:
Joe Milazzo 2024-02-10 09:43:17 -06:00 • committed by GitHub
parent f616b99585
commit 5dc5029a75
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
35 changed files with 3300 additions and 100 deletions
Download patch file Download diff file Expand all files Collapse all files

4
API/Controllers/DeviceController.cs
View file

@ -103,7 +103,7 @@ public class DeviceController : BaseApiController
if (dto.ChapterIds.Any(i => i < 0)) return BadRequest(await _localizationService.Translate(User.GetUserId(), "greater-0", "ChapterIds"));
if (dto.DeviceId < 0) return BadRequest(await _localizationService.Translate(User.GetUserId(), "greater-0", "DeviceId"));
var isEmailSetup = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).IsEmailSetup();
var isEmailSetup = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).IsEmailSetupForSendToDevice();
if (!isEmailSetup)
return BadRequest(await _localizationService.Translate(User.GetUserId(), "send-to-kavita-email"));
@ -142,7 +142,7 @@ public class DeviceController : BaseApiController
if (dto.SeriesId <= 0) return BadRequest(await _localizationService.Translate(User.GetUserId(), "greater-0", "SeriesId"));
if (dto.DeviceId < 0) return BadRequest(await _localizationService.Translate(User.GetUserId(), "greater-0", "DeviceId"));
var isEmailSetup = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).IsEmailSetup();
var isEmailSetup = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).IsEmailSetupForSendToDevice();
if (!isEmailSetup)
return BadRequest(await _localizationService.Translate(User.GetUserId(), "send-to-kavita-email"));

7
API/Controllers/SeriesController.cs
View file

@ -122,6 +122,11 @@ public class SeriesController : BaseApiController
return Ok(series);
}
/// <summary>
/// Deletes a series from Kavita
/// </summary>
/// <param name="seriesId"></param>
/// <returns>If the series was deleted or not</returns>
[Authorize(Policy = "RequireAdminRole")]
[HttpDelete("{seriesId}")]
public async Task<ActionResult<bool>> DeleteSeries(int seriesId)
@ -139,7 +144,7 @@ public class SeriesController : BaseApiController
var username = User.GetUsername();
_logger.LogInformation("Series {@SeriesId} is being deleted by {UserName}", dto.SeriesIds, username);
if (await _seriesService.DeleteMultipleSeries(dto.SeriesIds)) return Ok();
if (await _seriesService.DeleteMultipleSeries(dto.SeriesIds)) return Ok(true);
return BadRequest(await _localizationService.Translate(User.GetUserId(), "generic-series-delete"));
}

11
API/DTOs/Settings/ServerSettingDTO.cs
View file

@ -95,9 +95,18 @@ public class ServerSettingDto
/// <returns></returns>
public bool IsEmailSetup()
{
//return false;
return !string.IsNullOrEmpty(SmtpConfig.Host)
&& !string.IsNullOrEmpty(SmtpConfig.UserName)
&& !string.IsNullOrEmpty(HostName);
}
/// <summary>
/// Are at least some basics filled in, but not hostname as not required for Send to Device
/// </summary>
/// <returns></returns>
public bool IsEmailSetupForSendToDevice()
{
return !string.IsNullOrEmpty(SmtpConfig.Host)
&& !string.IsNullOrEmpty(SmtpConfig.UserName);
}
}

6
API/Data/DataContext.cs
View file

@ -143,6 +143,12 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
builder.Entity<AppUserSideNavStream>()
.HasIndex(e => e.Visible)
.IsUnique(false);
builder.Entity<ExternalSeriesMetadata>()
.HasOne(em => em.Series)
.WithOne(s => s.ExternalSeriesMetadata)
.HasForeignKey<ExternalSeriesMetadata>(em => em.SeriesId)
.OnDelete(DeleteBehavior.Cascade);
}
#nullable enable

2871
API/Data/Migrations/20240209224347_DBTweaks.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

29
API/Data/Migrations/20240209224347_DBTweaks.cs Normal file
View file

@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class DBTweaks : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_ExternalRecommendation_Series_SeriesId",
table: "ExternalRecommendation");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddForeignKey(
name: "FK_ExternalRecommendation_Series_SeriesId",
table: "ExternalRecommendation",
column: "SeriesId",
principalTable: "Series",
principalColumn: "Id");
}
}
}

9
API/Data/Migrations/DataContextModelSnapshot.cs
View file

@ -2398,15 +2398,6 @@ namespace API.Data.Migrations
b.Navigation("Chapter");
});
modelBuilder.Entity("API.Entities.Metadata.ExternalRecommendation", b =>
{
b.HasOne("API.Entities.Series", "Series")
.WithMany()
.HasForeignKey("SeriesId");
b.Navigation("Series");
});
modelBuilder.Entity("API.Entities.Metadata.ExternalSeriesMetadata", b =>
{
b.HasOne("API.Entities.Series", "Series")

12
API/Data/Repositories/ExternalSeriesMetadataRepository.cs
View file

@ -19,6 +19,7 @@ using Microsoft.AspNetCore.Identity;
using Microsoft.EntityFrameworkCore;
namespace API.Data.Repositories;
#nullable enable
public interface IExternalSeriesMetadataRepository
{
@ -28,6 +29,7 @@ public interface IExternalSeriesMetadataRepository
void Remove(IEnumerable<ExternalReview>? reviews);
void Remove(IEnumerable<ExternalRating>? ratings);
void Remove(IEnumerable<ExternalRecommendation>? recommendations);
void Remove(ExternalSeriesMetadata metadata);
Task<ExternalSeriesMetadata?> GetExternalSeriesMetadata(int seriesId);
Task<bool> ExternalSeriesMetadataNeedsRefresh(int seriesId);
Task<SeriesDetailPlusDto> GetSeriesDetailPlusDto(int seriesId);
@ -70,18 +72,24 @@ public class ExternalSeriesMetadataRepository : IExternalSeriesMetadataRepositor
_context.ExternalReview.RemoveRange(reviews);
}
public void Remove(IEnumerable<ExternalRating> ratings)
public void Remove(IEnumerable<ExternalRating>? ratings)
{
if (ratings == null) return;
_context.ExternalRating.RemoveRange(ratings);
}
public void Remove(IEnumerable<ExternalRecommendation> recommendations)
public void Remove(IEnumerable<ExternalRecommendation>? recommendations)
{
if (recommendations == null) return;
_context.ExternalRecommendation.RemoveRange(recommendations);
}
public void Remove(ExternalSeriesMetadata? metadata)
{
if (metadata == null) return;
_context.ExternalSeriesMetadata.Remove(metadata);
}
/// <summary>
/// Returns the ExternalSeriesMetadata entity for the given Series including all linked tables
/// </summary>

13
API/Data/Repositories/LibraryRepository.cs
View file

@ -56,6 +56,7 @@ public interface ILibraryRepository
Task<IList<Library>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat);
Task<bool> GetAllowsScrobblingBySeriesId(int seriesId);
Task<IDictionary<int, LibraryType>> GetLibraryTypesBySeriesIdsAsync(IList<int> seriesIds);
}
public class LibraryRepository : ILibraryRepository
@ -352,4 +353,16 @@ public class LibraryRepository : ILibraryRepository
.Select(s => s.Library.AllowScrobbling)
.SingleOrDefaultAsync();
}
public async Task<IDictionary<int, LibraryType>> GetLibraryTypesBySeriesIdsAsync(IList<int> seriesIds)
{
return await _context.Series
.Where(series => seriesIds.Contains(series.Id))
.Select(series => new
{
series.Id,
series.Library.Type
})
.ToDictionaryAsync(entity => entity.Id, entity => entity.Type);
}
}

16
API/Data/Repositories/SeriesRepository.cs
View file

@ -49,6 +49,7 @@ public enum SeriesIncludes
ExternalReviews = 64,
ExternalRatings = 128,
ExternalRecommendations = 256,
ExternalMetadata = 512
}
@ -551,7 +552,7 @@ public class SeriesRepository : ISeriesRepository
}
/// <summary>
/// Returns Volumes, Metadata, and Collection Tags
/// Returns Full Series including all external links
/// </summary>
/// <param name="seriesIds"></param>
/// <returns></returns>
@ -559,9 +560,20 @@ public class SeriesRepository : ISeriesRepository
{
return await _context.Series
.Include(s => s.Volumes)
.Include(s => s.Relations)
.Include(s => s.Metadata)
.ThenInclude(m => m.CollectionTags)
.Include(s => s.Relations)
.Include(s => s.ExternalSeriesMetadata)
.Include(s => s.ExternalSeriesMetadata)
.ThenInclude(e => e.ExternalRatings)
.Include(s => s.ExternalSeriesMetadata)
.ThenInclude(e => e.ExternalReviews)
.Include(s => s.ExternalSeriesMetadata)
.ThenInclude(e => e.ExternalRecommendations)
.Where(s => seriesIds.Contains(s.Id))
.AsSplitQuery()
.ToListAsync();

3
API/Data/Seed.cs
View file

@ -1,5 +1,4 @@
using System;
using System.Collections.Generic;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.IO;
using System.Linq;

5
API/Entities/Metadata/ExternalRecommendation.cs
View file

@ -1,8 +1,11 @@
using System.Collections.Generic;
using API.Services.Plus;
using Microsoft.EntityFrameworkCore;
namespace API.Entities.Metadata;
[Index(nameof(SeriesId), IsUnique = false)]
public class ExternalRecommendation
{
public int Id { get; set; }
@ -19,7 +22,7 @@ public class ExternalRecommendation
/// When null, represents an external series. When set, it is a Series
/// </summary>
public int? SeriesId { get; set; }
public virtual Series Series { get; set; }
//public virtual Series? Series { get; set; }
// Relationships
public ICollection<ExternalSeriesMetadata> ExternalSeriesMetadatas { get; set; } = null!;

6
API/Extensions/QueryExtensions/IncludesExtensions.cs
View file

@ -80,6 +80,12 @@ public static class IncludesExtensions
.ThenInclude(s => s.ExternalRatings);
}
if (includeFlags.HasFlag(SeriesIncludes.ExternalMetadata))
{
query = query
.Include(s => s.ExternalSeriesMetadata);
}
if (includeFlags.HasFlag(SeriesIncludes.ExternalRecommendations))
{
query = query

59
API/Helpers/RateLimiter.cs Normal file
View file

@ -0,0 +1,59 @@
using System;
using System.Collections.Generic;
namespace API.Helpers;
public class RateLimiter(int maxRequests, TimeSpan duration, bool refillBetween = true)
{
private readonly Dictionary<string, (int Tokens, DateTime LastRefill)> _tokenBuckets = new();
private readonly object _lock = new();
public bool TryAcquire(string key)
{
lock (_lock)
{
if (!_tokenBuckets.TryGetValue(key, out var bucket))
{
bucket = (Tokens: maxRequests, LastRefill: DateTime.UtcNow);
_tokenBuckets[key] = bucket;
}
RefillTokens(key);
lock (_lock)
{
if (_tokenBuckets[key].Tokens > 0)
{
_tokenBuckets[key] = (Tokens: _tokenBuckets[key].Tokens - 1, LastRefill: _tokenBuckets[key].LastRefill);
return true;
}
}
return false;
}
}
private void RefillTokens(string key)
{
lock (_lock)
{
var now = DateTime.UtcNow;
var timeSinceLastRefill = now - _tokenBuckets[key].LastRefill;
var tokensToAdd = (int) (timeSinceLastRefill.TotalSeconds / duration.TotalSeconds);
// Refill the bucket if the elapsed time is greater than or equal to the duration
if (timeSinceLastRefill >= duration)
{
_tokenBuckets[key] = (Tokens: maxRequests, LastRefill: now);
Console.WriteLine($"Tokens Refilled to Max: {maxRequests}");
}
else if (tokensToAdd > 0 && refillBetween)
{
_tokenBuckets[key] = (Tokens: Math.Min(maxRequests, _tokenBuckets[key].Tokens + tokensToAdd), LastRefill: now);
Console.WriteLine($"Tokens Refilled: {_tokenBuckets[key].Tokens}");
}
}
}
}

1
API/Services/DeviceService.cs
View file

@ -126,6 +126,7 @@ public class DeviceService : IDeviceService
device.UpdateLastUsed();
_unitOfWork.DeviceRepository.Update(device);
await _unitOfWork.CommitAsync();
var success = await _emailService.SendFilesToEmail(new SendToDto()
{
DestinationEmail = device.EmailAddress!,

2
API/Services/EmailService.cs
View file

@ -224,7 +224,7 @@ public class EmailService : IEmailService
public async Task<bool> SendFilesToEmail(SendToDto data)
{
var serverSetting = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
if (!serverSetting.IsEmailSetup()) return false;
if (!serverSetting.IsEmailSetupForSendToDevice()) return false;
var emailOptions = new EmailOptionsDto()
{

43
API/Services/Plus/ExternalMetadataService.cs
View file

@ -13,6 +13,7 @@ using API.Entities;
using API.Entities.Enums;
using API.Entities.Metadata;
using API.Extensions;
using API.Helpers;
using AutoMapper;
using Flurl.Http;
using Hangfire;
@ -76,6 +77,8 @@ public class ExternalMetadataService : IExternalMetadataService
Ratings = ArraySegment<RatingDto>.Empty,
Reviews = ArraySegment<UserReviewDto>.Empty
};
// Allow 50 requests per 24 hours
private static readonly RateLimiter RateLimiter = new RateLimiter(50, TimeSpan.FromHours(12), false);
public ExternalMetadataService(IUnitOfWork unitOfWork, ILogger<ExternalMetadataService> logger, IMapper mapper, ILicenseService licenseService)
{
@ -85,7 +88,6 @@ public class ExternalMetadataService : IExternalMetadataService
_licenseService = licenseService;
FlurlHttp.ConfigureClient(Configuration.KavitaPlusApiUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
}
@ -114,17 +116,17 @@ public class ExternalMetadataService : IExternalMetadataService
var ids = await _unitOfWork.ExternalSeriesMetadataRepository.GetAllSeriesIdsWithoutMetadata(25);
if (ids.Count == 0) return;
_logger.LogInformation("Started Refreshing {Count} series data from Kavita+", ids.Count);
_logger.LogInformation("[Kavita+ Data Refresh] Started Refreshing {Count} series data from Kavita+", ids.Count);
var count = 0;
var libTypes = await _unitOfWork.LibraryRepository.GetLibraryTypesBySeriesIdsAsync(ids);
foreach (var seriesId in ids)
{
// TODO: Rewrite this so it's streamlined and not multiple DB calls
var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeBySeriesIdAsync(seriesId);
await GetSeriesDetailPlus(seriesId, libraryType);
var libraryType = libTypes[seriesId];
await GetNewSeriesData(seriesId, libraryType);
await Task.Delay(1500);
count++;
}
_logger.LogInformation("Finished Refreshing {Count} series data from Kavita+", count);
_logger.LogInformation("[Kavita+ Data Refresh] Finished Refreshing {Count} series data from Kavita+", count);
}
/// <summary>
@ -145,13 +147,30 @@ public class ExternalMetadataService : IExternalMetadataService
await _unitOfWork.CommitAsync();
}
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public Task GetNewSeriesData(int seriesId, LibraryType libraryType)
/// <summary>
/// Fetches data from Kavita+
/// </summary>
/// <param name="seriesId"></param>
/// <param name="libraryType"></param>
public async Task GetNewSeriesData(int seriesId, LibraryType libraryType)
{
// TODO: Implement this task
if (!IsPlusEligible(libraryType)) return Task.CompletedTask;
return Task.CompletedTask;
if (!IsPlusEligible(libraryType)) return;
// Generate key based on seriesId and libraryType or any unique identifier for the request
// Check if the request is allowed based on the rate limit
if (!RateLimiter.TryAcquire(string.Empty))
{
// Request not allowed due to rate limit
_logger.LogDebug("Rate Limit hit for Kavita+ prefetch");
return;
}
_logger.LogDebug("Prefetching Kavita+ data for Series {SeriesId}", seriesId);
// Prefetch SeriesDetail data
await GetSeriesDetailPlus(seriesId, libraryType);
// TODO: Fetch Series Metadata
}
/// <summary>

11
API/Services/SeriesService.cs
View file

@ -427,6 +427,9 @@ public class SeriesService : ISeriesService
}
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdsAsync(seriesIds);
_unitOfWork.SeriesRepository.Remove(series);
var libraryIds = series.Select(s => s.LibraryId);
var libraries = await _unitOfWork.LibraryRepository.GetLibraryForIdsAsync(libraryIds);
foreach (var library in libraries)
@ -434,11 +437,8 @@ public class SeriesService : ISeriesService
library.UpdateLastModified();
_unitOfWork.LibraryRepository.Update(library);
}
await _unitOfWork.CommitAsync();
_unitOfWork.SeriesRepository.Remove(series);
if (!_unitOfWork.HasChanges() || !await _unitOfWork.CommitAsync()) return true;
foreach (var s in series)
{
@ -449,14 +449,13 @@ public class SeriesService : ISeriesService
await _unitOfWork.AppUserProgressRepository.CleanupAbandonedChapters();
await _unitOfWork.CollectionTagRepository.RemoveTagsWithoutSeries();
_taskScheduler.CleanupChapters(allChapterIds.ToArray());
return true;
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an issue when trying to delete multiple series");
return false;
}
return true;
}
/// <summary>

8
API/Services/TaskScheduler.cs
View file

@ -182,10 +182,10 @@ public class TaskScheduler : ITaskScheduler
RecurringJob.AddOrUpdate(ProcessProcessedScrobblingEventsId, () => _scrobblingService.ClearProcessedEvents(),
Cron.Daily, RecurringJobOptions);
// Backfilling/Freshening Reviews/Rating/Recommendations (TODO: This will come in v0.8.x)
// RecurringJob.AddOrUpdate(KavitaPlusDataRefreshId,
// () => _externalMetadataService.FetchExternalDataTask(), Cron.Hourly(Rnd.Next(0, 59)),
// RecurringJobOptions);
// Backfilling/Freshening Reviews/Rating/Recommendations
RecurringJob.AddOrUpdate(KavitaPlusDataRefreshId,
() => _externalMetadataService.FetchExternalDataTask(), Cron.Daily(Rnd.Next(1, 4)),
RecurringJobOptions);
}
#region StatsTasks