Better Caching & Global Downloads (#1372)

* Fixed a bug where cache TTL was using a field which always was 0.

* Updated Scan Series task (from UI) to always re-calculate what's on file and not rely on last update. This leads to more reliable results, despite extra overhead.

* Added image range processing on images for the reader, for slower networks or large files

* On manga (single) try to use prefetched image, rather than re-requesting an image on pagination

* Reduced some more latency when rendering first page of next chapter via continuous reading mode

* Fixed a bug where metadata filter, after updating a typeahead, collapsing filter area then re-opening, the filter would still be applied, but the typeahead wouldn't show the modification.

* Coded an idea around download reporting, commiting for history, might not go with it.

* Refactored the download indicator into it's own component. Cleaning up some code for download within card component

* Another throw away commit. Put in some temp code, not working but not sure if I'm ditching entirely.

* Updated download service to enable range processing (so downloads can resume) and to reduce re-zipping if we've just downloaded something.

* Refactored events widget download indicator to the correct design. I will be moving forward with this new functionality.

* Added Required fields to ProgressDTO

* Cleaned up the event widget and updated existing download progress to indicate preparing the download, rather than the download itself.

* Updated dependencies for security alerts

* Refactored all download code to be streamlined and globally handled

* Updated ScanSeries to find the highest folder path before library, not just within the files. This could lead to scan series missing files due to nested folders on same parent level.

* Updated the caching code to use a builtin annotation. Images are now caching correctly.

* Fixed a bad redirect on an auth guard

* Tweaked how long we allow cache for, as the cover update now doesn't work well.

* Fixed a bug on downloading bookmarks from multiple series, where it would just choose the first series id for the temp file.

* Added an extra check for downloading bookmarks

* UI Security updates, Fixed a bug on bookmark reader, the reader on last page would throw some errors and not show No Next Chapter toast.

* After scan, clear temp

* Code smells
This commit is contained in:
Joseph Milazzo 2022-07-13 10:45:14 -04:00 committed by GitHub
parent 45bbf422be
commit af4f35da5b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
58 changed files with 2309 additions and 624 deletions

View file

@ -94,6 +94,7 @@ namespace API.Controllers
/// <param name="file"></param>
/// <returns></returns>
[HttpGet("{chapterId}/book-resources")]
[ResponseCache(Duration = 60 * 1, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetBookPageResources(int chapterId, [FromQuery] string file)
{
var chapter = await _unitOfWork.ChapterRepository.GetChapterAsync(chapterId);
@ -105,7 +106,6 @@ namespace API.Controllers
var bookFile = book.Content.AllFiles[key];
var content = await bookFile.ReadContentAsBytesAsync();
Response.AddCacheHeader(content);
var contentType = BookService.GetContentType(bookFile.ContentType);
return File(content, contentType, $"{chapterId}-{file}");
}

View file

@ -83,7 +83,7 @@ namespace API.Controllers
/// <summary>
/// Downloads all chapters within a volume.
/// Downloads all chapters within a volume. If the chapters are multiple zips, they will all be zipped up.
/// </summary>
/// <param name="volumeId"></param>
/// <returns></returns>
@ -112,12 +112,17 @@ namespace API.Controllers
return await _downloadService.HasDownloadPermission(user);
}
private async Task<ActionResult> GetFirstFileDownload(IEnumerable<MangaFile> files)
private ActionResult GetFirstFileDownload(IEnumerable<MangaFile> files)
{
var (bytes, contentType, fileDownloadName) = await _downloadService.GetFirstFileDownload(files);
return File(bytes, contentType, fileDownloadName);
var (zipFile, contentType, fileDownloadName) = _downloadService.GetFirstFileDownload(files);
return PhysicalFile(zipFile, contentType, fileDownloadName, true);
}
/// <summary>
/// Returns the zip for a single chapter. If the chapter contains multiple files, they will be zipped.
/// </summary>
/// <param name="chapterId"></param>
/// <returns></returns>
[HttpGet("chapter")]
public async Task<ActionResult> DownloadChapter(int chapterId)
{
@ -148,15 +153,14 @@ namespace API.Controllers
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.DownloadProgressEvent(User.GetUsername(),
Path.GetFileNameWithoutExtension(downloadName), 1F, "ended"));
return await GetFirstFileDownload(files);
return GetFirstFileDownload(files);
}
var (fileBytes, _) = await _archiveService.CreateZipForDownload(files.Select(c => c.FilePath),
tempFolder);
var filePath = _archiveService.CreateZipForDownload(files.Select(c => c.FilePath), tempFolder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.DownloadProgressEvent(User.GetUsername(),
Path.GetFileNameWithoutExtension(downloadName), 1F, "ended"));
return File(fileBytes, DefaultContentType, downloadName);
return PhysicalFile(filePath, DefaultContentType, downloadName, true);
}
catch (Exception ex)
{
@ -184,10 +188,16 @@ namespace API.Controllers
}
}
/// <summary>
/// Downloads all bookmarks in a zip for
/// </summary>
/// <param name="downloadBookmarkDto"></param>
/// <returns></returns>
[HttpPost("bookmarks")]
public async Task<ActionResult> DownloadBookmarkPages(DownloadBookmarkDto downloadBookmarkDto)
{
if (!await HasDownloadPermission()) return BadRequest("You do not have permission");
if (!downloadBookmarkDto.Bookmarks.Any()) return BadRequest("Bookmarks cannot be empty");
// We know that all bookmarks will be for one single seriesId
var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername());
@ -198,13 +208,14 @@ namespace API.Controllers
var filename = $"{series.Name} - Bookmarks.zip";
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.DownloadProgressEvent(User.GetUsername(), Path.GetFileNameWithoutExtension(filename), 0F));
var (fileBytes, _) = await _archiveService.CreateZipForDownload(files,
$"download_{user.Id}_{series.Id}_bookmarks");
var seriesIds = string.Join("_", downloadBookmarkDto.Bookmarks.Select(b => b.SeriesId).Distinct());
var filePath = _archiveService.CreateZipForDownload(files,
$"download_{user.Id}_{seriesIds}_bookmarks");
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.DownloadProgressEvent(User.GetUsername(), Path.GetFileNameWithoutExtension(filename), 1F));
return File(fileBytes, DefaultContentType, filename);
return PhysicalFile(filePath, DefaultContentType, filename, true);
}
}

View file

@ -2,7 +2,6 @@
using System.Threading.Tasks;
using API.Data;
using API.Entities.Enums;
using API.Extensions;
using API.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
@ -16,6 +15,7 @@ namespace API.Controllers
{
private readonly IUnitOfWork _unitOfWork;
private readonly IDirectoryService _directoryService;
private const int ImageCacheSeconds = 1 * 60;
/// <inheritdoc />
public ImageController(IUnitOfWork unitOfWork, IDirectoryService directoryService)
@ -30,13 +30,13 @@ namespace API.Controllers
/// <param name="chapterId"></param>
/// <returns></returns>
[HttpGet("chapter-cover")]
[ResponseCache(Duration = ImageCacheSeconds, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetChapterCoverImage(int chapterId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.ChapterRepository.GetChapterCoverImageAsync(chapterId));
if (string.IsNullOrEmpty(path) || !_directoryService.FileSystem.File.Exists(path)) return BadRequest($"No cover image");
var format = _directoryService.FileSystem.Path.GetExtension(path).Replace(".", "");
Response.AddCacheHeader(path);
return PhysicalFile(path, "image/" + format, _directoryService.FileSystem.Path.GetFileName(path));
}
@ -46,13 +46,13 @@ namespace API.Controllers
/// <param name="volumeId"></param>
/// <returns></returns>
[HttpGet("volume-cover")]
[ResponseCache(Duration = ImageCacheSeconds, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetVolumeCoverImage(int volumeId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.VolumeRepository.GetVolumeCoverImageAsync(volumeId));
if (string.IsNullOrEmpty(path) || !_directoryService.FileSystem.File.Exists(path)) return BadRequest($"No cover image");
var format = _directoryService.FileSystem.Path.GetExtension(path).Replace(".", "");
Response.AddCacheHeader(path);
return PhysicalFile(path, "image/" + format, _directoryService.FileSystem.Path.GetFileName(path));
}
@ -61,6 +61,7 @@ namespace API.Controllers
/// </summary>
/// <param name="seriesId">Id of Series</param>
/// <returns></returns>
[ResponseCache(Duration = ImageCacheSeconds, Location = ResponseCacheLocation.Client, NoStore = false)]
[HttpGet("series-cover")]
public async Task<ActionResult> GetSeriesCoverImage(int seriesId)
{
@ -68,7 +69,6 @@ namespace API.Controllers
if (string.IsNullOrEmpty(path) || !_directoryService.FileSystem.File.Exists(path)) return BadRequest($"No cover image");
var format = _directoryService.FileSystem.Path.GetExtension(path).Replace(".", "");
Response.AddCacheHeader(path);
return PhysicalFile(path, "image/" + format, _directoryService.FileSystem.Path.GetFileName(path));
}
@ -78,13 +78,13 @@ namespace API.Controllers
/// <param name="collectionTagId"></param>
/// <returns></returns>
[HttpGet("collection-cover")]
[ResponseCache(Duration = ImageCacheSeconds, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetCollectionCoverImage(int collectionTagId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.CollectionTagRepository.GetCoverImageAsync(collectionTagId));
if (string.IsNullOrEmpty(path) || !_directoryService.FileSystem.File.Exists(path)) return BadRequest($"No cover image");
var format = _directoryService.FileSystem.Path.GetExtension(path).Replace(".", "");
Response.AddCacheHeader(path);
return PhysicalFile(path, "image/" + format, _directoryService.FileSystem.Path.GetFileName(path));
}
@ -94,13 +94,13 @@ namespace API.Controllers
/// <param name="readingListId"></param>
/// <returns></returns>
[HttpGet("readinglist-cover")]
[ResponseCache(Duration = ImageCacheSeconds, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetReadingListCoverImage(int readingListId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.ReadingListRepository.GetCoverImageAsync(readingListId));
if (string.IsNullOrEmpty(path) || !_directoryService.FileSystem.File.Exists(path)) return BadRequest($"No cover image");
var format = _directoryService.FileSystem.Path.GetExtension(path).Replace(".", "");
Response.AddCacheHeader(path);
return PhysicalFile(path, "image/" + format, _directoryService.FileSystem.Path.GetFileName(path));
}
@ -113,6 +113,7 @@ namespace API.Controllers
/// <param name="apiKey">API Key for user. Needed to authenticate request</param>
/// <returns></returns>
[HttpGet("bookmark")]
[ResponseCache(Duration = ImageCacheSeconds, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetBookmarkImage(int chapterId, int pageNum, string apiKey)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey);
@ -124,7 +125,6 @@ namespace API.Controllers
var file = new FileInfo(Path.Join(bookmarkDirectory, bookmark.FileName));
var format = Path.GetExtension(file.FullName).Replace(".", "");
Response.AddCacheHeader(file.FullName);
return PhysicalFile(file.FullName, "image/" + format, Path.GetFileName(file.FullName));
}
@ -135,13 +135,13 @@ namespace API.Controllers
/// <returns></returns>
[AllowAnonymous]
[HttpGet("cover-upload")]
[ResponseCache(Duration = ImageCacheSeconds, Location = ResponseCacheLocation.Client, NoStore = false)]
public ActionResult GetCoverUploadImage(string filename)
{
var path = Path.Join(_directoryService.TempDirectory, filename);
if (string.IsNullOrEmpty(path) || !_directoryService.FileSystem.File.Exists(path)) return BadRequest($"File does not exist");
var format = _directoryService.FileSystem.Path.GetExtension(path).Replace(".", "");
Response.AddCacheHeader(path);
return PhysicalFile(path, "image/" + format, _directoryService.FileSystem.Path.GetFileName(path));
}
}

View file

@ -622,8 +622,8 @@ public class OpdsController : BaseApiController
}
var files = await _unitOfWork.ChapterRepository.GetFilesForChapterAsync(chapterId);
var (bytes, contentType, fileDownloadName) = await _downloadService.GetFirstFileDownload(files);
return File(bytes, contentType, fileDownloadName);
var (zipFile, contentType, fileDownloadName) = _downloadService.GetFirstFileDownload(files);
return PhysicalFile(zipFile, contentType, fileDownloadName, true);
}
private static ContentResult CreateXmlResult(string xml)
@ -830,6 +830,7 @@ public class OpdsController : BaseApiController
}
[HttpGet("{apiKey}/favicon")]
[ResponseCache(Duration = 60 * 60, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetFavicon(string apiKey)
{
var files = _directoryService.GetFilesWithExtension(Path.Join(Directory.GetCurrentDirectory(), ".."), @"\.ico");
@ -838,9 +839,6 @@ public class OpdsController : BaseApiController
var content = await _directoryService.ReadFileAsync(path);
var format = Path.GetExtension(path).Replace(".", "");
// Calculates SHA1 Hash for byte[]
Response.AddCacheHeader(content);
return File(content, "image/" + format);
}

View file

@ -47,6 +47,7 @@ namespace API.Controllers
/// <param name="chapterId"></param>
/// <returns></returns>
[HttpGet("pdf")]
[ResponseCache(Duration = 60 * 10, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetPdf(int chapterId)
{
var chapter = await _cacheService.Ensure(chapterId);
@ -57,7 +58,6 @@ namespace API.Controllers
var path = _cacheService.GetCachedFile(chapter);
if (string.IsNullOrEmpty(path) || !System.IO.File.Exists(path)) return BadRequest($"Pdf doesn't exist when it should.");
Response.AddCacheHeader(path, TimeSpan.FromMinutes(60).Seconds);
return PhysicalFile(path, "application/pdf", Path.GetFileName(path), true);
}
catch (Exception)
@ -74,6 +74,7 @@ namespace API.Controllers
/// <param name="page"></param>
/// <returns></returns>
[HttpGet("image")]
[ResponseCache(Duration = 60 * 10, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetImage(int chapterId, int page)
{
if (page < 0) page = 0;
@ -86,8 +87,7 @@ namespace API.Controllers
if (string.IsNullOrEmpty(path) || !System.IO.File.Exists(path)) return BadRequest($"No such image for page {page}");
var format = Path.GetExtension(path).Replace(".", "");
Response.AddCacheHeader(path, TimeSpan.FromMinutes(10).Seconds);
return PhysicalFile(path, "image/" + format, Path.GetFileName(path));
return PhysicalFile(path, "image/" + format, Path.GetFileName(path), true);
}
catch (Exception)
{
@ -105,6 +105,7 @@ namespace API.Controllers
/// <remarks>We must use api key as bookmarks could be leaked to other users via the API</remarks>
/// <returns></returns>
[HttpGet("bookmark-image")]
[ResponseCache(Duration = 60 * 10, Location = ResponseCacheLocation.Client, NoStore = false)]
public async Task<ActionResult> GetBookmarkImage(int seriesId, string apiKey, int page)
{
if (page < 0) page = 0;
@ -121,7 +122,6 @@ namespace API.Controllers
if (string.IsNullOrEmpty(path) || !System.IO.File.Exists(path)) return BadRequest($"No such image for page {page}");
var format = Path.GetExtension(path).Replace(".", "");
Response.AddCacheHeader(path, TimeSpan.FromMinutes(10).Seconds);
return PhysicalFile(path, "image/" + format, Path.GetFileName(path));
}
catch (Exception)

View file

@ -253,30 +253,50 @@ namespace API.Controllers
return Ok(pagedList);
}
/// <summary>
/// Runs a Cover Image Generation task
/// </summary>
/// <param name="refreshSeriesDto"></param>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[HttpPost("refresh-metadata")]
public ActionResult RefreshSeriesMetadata(RefreshSeriesDto refreshSeriesDto)
{
_taskScheduler.RefreshSeriesMetadata(refreshSeriesDto.LibraryId, refreshSeriesDto.SeriesId, true);
_taskScheduler.RefreshSeriesMetadata(refreshSeriesDto.LibraryId, refreshSeriesDto.SeriesId, refreshSeriesDto.ForceUpdate);
return Ok();
}
/// <summary>
/// Scan a series and force each file to be updated. This should be invoked via the User, hence why we force.
/// </summary>
/// <param name="refreshSeriesDto"></param>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[HttpPost("scan")]
public ActionResult ScanSeries(RefreshSeriesDto refreshSeriesDto)
{
_taskScheduler.ScanSeries(refreshSeriesDto.LibraryId, refreshSeriesDto.SeriesId);
_taskScheduler.ScanSeries(refreshSeriesDto.LibraryId, refreshSeriesDto.SeriesId, refreshSeriesDto.ForceUpdate);
return Ok();
}
/// <summary>
/// Run a file analysis on the series.
/// </summary>
/// <param name="refreshSeriesDto"></param>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[HttpPost("analyze")]
public ActionResult AnalyzeSeries(RefreshSeriesDto refreshSeriesDto)
{
_taskScheduler.AnalyzeFilesForSeries(refreshSeriesDto.LibraryId, refreshSeriesDto.SeriesId, true);
_taskScheduler.AnalyzeFilesForSeries(refreshSeriesDto.LibraryId, refreshSeriesDto.SeriesId, refreshSeriesDto.ForceUpdate);
return Ok();
}
/// <summary>
/// Returns metadata for a given series
/// </summary>
/// <param name="seriesId"></param>
/// <returns></returns>
[HttpGet("metadata")]
public async Task<ActionResult<SeriesMetadataDto>> GetSeriesMetadata(int seriesId)
{
@ -284,6 +304,11 @@ namespace API.Controllers
return Ok(metadata);
}
/// <summary>
/// Update series metadata
/// </summary>
/// <param name="updateSeriesMetadataDto"></param>
/// <returns></returns>
[HttpPost("metadata")]
public async Task<ActionResult> UpdateSeriesMetadata(UpdateSeriesMetadataDto updateSeriesMetadataDto)
{
@ -331,6 +356,11 @@ namespace API.Controllers
return Ok(await _unitOfWork.SeriesRepository.GetSeriesDtoForIdsAsync(dto.SeriesIds, userId));
}
/// <summary>
/// Get the age rating for the <see cref="AgeRating"/> enum value
/// </summary>
/// <param name="ageRating"></param>
/// <returns></returns>
[HttpGet("age-rating")]
public ActionResult<string> GetAgeRating(int ageRating)
{
@ -339,6 +369,12 @@ namespace API.Controllers
return Ok(val.ToDescription());
}
/// <summary>
/// Get a special DTO for Series Detail page.
/// </summary>
/// <param name="seriesId"></param>
/// <returns></returns>
/// <remarks>Do not rely on this API externally. May change without hesitation. </remarks>
[HttpGet("series-detail")]
public async Task<ActionResult<SeriesDetailDto>> GetSeriesDetailBreakdown(int seriesId)
{
@ -386,16 +422,24 @@ namespace API.Controllers
return Ok(await _unitOfWork.SeriesRepository.GetSeriesForRelationKind(userId, seriesId, relation));
}
/// <summary>
/// Returns all related series against the passed series Id
/// </summary>
/// <param name="seriesId"></param>
/// <returns></returns>
[HttpGet("all-related")]
public async Task<ActionResult<RelatedSeriesDto>> GetAllRelatedSeries(int seriesId)
{
// Send back a custom DTO with each type or maybe sorted in some way
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
return Ok(await _unitOfWork.SeriesRepository.GetRelatedSeries(userId, seriesId));
}
/// <summary>
/// Update the relations attached to the Series. Does not generate associated Sequel/Prequel pairs on target series.
/// </summary>
/// <param name="dto"></param>
/// <returns></returns>
[Authorize(Policy="RequireAdminRole")]
[HttpPost("update-related")]
public async Task<ActionResult> UpdateRelatedSeries(UpdateRelatedSeriesDto dto)
@ -421,7 +465,8 @@ namespace API.Controllers
return BadRequest("There was an issue updating relationships");
}
private void UpdateRelationForKind(IList<int> dtoTargetSeriesIds, IEnumerable<SeriesRelation> adaptations, Series series, RelationKind kind)
// TODO: Move this to a Service and Unit Test it
private void UpdateRelationForKind(ICollection<int> dtoTargetSeriesIds, IEnumerable<SeriesRelation> adaptations, Series series, RelationKind kind)
{
foreach (var adaptation in adaptations.Where(adaptation => !dtoTargetSeriesIds.Contains(adaptation.TargetSeriesId)))
{

View file

@ -112,13 +112,13 @@ namespace API.Controllers
}
[HttpGet("logs")]
public async Task<ActionResult> GetLogs()
public ActionResult GetLogs()
{
var files = _backupService.GetLogFiles(_config.GetMaxRollingFiles(), _config.GetLoggingFileName());
try
{
var (fileBytes, zipPath) = await _archiveService.CreateZipForDownload(files, "logs");
return File(fileBytes, "application/zip", Path.GetFileName(zipPath), true);
var zipPath = _archiveService.CreateZipForDownload(files, "logs");
return PhysicalFile(zipPath, "application/zip", Path.GetFileName(zipPath), true);
}
catch (KavitaException ex)
{

View file

@ -98,6 +98,7 @@ namespace API.Controllers
existingPreferences.BlurUnreadSummaries = preferencesDto.BlurUnreadSummaries;
existingPreferences.Theme = await _unitOfWork.SiteThemeRepository.GetThemeById(preferencesDto.Theme.Id);
existingPreferences.LayoutMode = preferencesDto.LayoutMode;
existingPreferences.PromptForDownloadSize = preferencesDto.PromptForDownloadSize;
_unitOfWork.UserRepository.Update(existingPreferences);