Basic Stats (#1673)

* Refactored ResponseCache profiles into consts

* Refactored code to use an extension method for getting user library ids.

* Started server statistics, added a charting library, and added a table sort column (not finished)

* Refactored code and have a fully working example of sortable headers. Still doesn't work with default sorting state, will work on that later.

* Implemented file size, but it's too expensive, so commented out.

* Added a migration to provide extension and length/size information in the DB to allow for faster stat apis.

* Added the ability to force a library scan from library settings.

* Refactored some apis to provide more of a file breakdown rather than just file size.

* Working on visualization of file breakdown

* Fixed the file breakdown visual

* Fixed up 2 visualizations

* Added back an api for member names, started work on top reads

* Hooked up the other library types and username/days.

* Preparing to remove top reads and refactor into Top users

* Added LibraryId to AppUserProgress to help with complex lookups.

* Added the new libraryId hook into some stats methods

* Updated api methods to use libraryId for progress

* More places where LibraryId is needed

* Added some high level server stats

* Got a ton done on server stats

* Updated default theme (dark) to be the default root variables. This will allow user themes to override just what they want, rather than maintain their own css variables.

* Implemented a monster query for top users by reading time. It's very slow and can be cleaned up likely.

* Hooked up top reads. Code needs a big refactor. Handing off for Robbie treatment and I'll switch to User stats.

* Implemented last 5 recently read series (broken) and added some basic css

* Fixed recently read query

* Cleanup the css a bit, Robbie we need you

* More css love

* Cleaned up DTOs that aren't needed anymore

* Fixed top readers query

* When calculating top readers, don't include read events where nothing is read (0 pages)

* Hooked up the date into GetTopUsers

* Hooked top readers up with days and refactored and cleaned up componets not used

* Fixed up query

* Started on a day by day breakdown, but going to take a break from stats.

* Added a temp task to run some migration manually for stats to work

* Ensure OPDS-PS uses new libraryId for progress reporting

* Fixed a code smell

* Adding some styling

* adding more styles

* Removed some debug stuff from user stats

* Bump qs from 6.5.2 to 6.5.3 in /UI/Web

Bumps [qs](https://github.com/ljharb/qs) from 6.5.2 to 6.5.3.
- [Release notes](https://github.com/ljharb/qs/releases)
- [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ljharb/qs/compare/v6.5.2...v6.5.3)

---
updated-dependencies:
- dependency-name: qs
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

* Tweaked some code for bad data cases

* Refactored a chapter lookup to remove un-needed Volume join in 5 places across the code.

* API push

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
Joe Milazzo 2022-12-07 08:01:49 -06:00 committed by GitHub
parent 4724dc5a76
commit c361e66b35
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
106 changed files with 6898 additions and 170 deletions

View file

@ -1,5 +1,6 @@
using System.IO;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
using API.Entities.Enums;
using API.Extensions;
@ -31,7 +32,7 @@ public class ImageController : BaseApiController
/// <param name="chapterId"></param>
/// <returns></returns>
[HttpGet("chapter-cover")]
[ResponseCache(CacheProfileName = "Images")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Images)]
public async Task<ActionResult> GetChapterCoverImage(int chapterId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.ChapterRepository.GetChapterCoverImageAsync(chapterId));
@ -47,7 +48,7 @@ public class ImageController : BaseApiController
/// <param name="libraryId"></param>
/// <returns></returns>
[HttpGet("library-cover")]
[ResponseCache(CacheProfileName = "Images")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Images)]
public async Task<ActionResult> GetLibraryCoverImage(int libraryId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.LibraryRepository.GetLibraryCoverImageAsync(libraryId));
@ -63,7 +64,7 @@ public class ImageController : BaseApiController
/// <param name="volumeId"></param>
/// <returns></returns>
[HttpGet("volume-cover")]
[ResponseCache(CacheProfileName = "Images")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Images)]
public async Task<ActionResult> GetVolumeCoverImage(int volumeId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.VolumeRepository.GetVolumeCoverImageAsync(volumeId));
@ -78,7 +79,7 @@ public class ImageController : BaseApiController
/// </summary>
/// <param name="seriesId">Id of Series</param>
/// <returns></returns>
[ResponseCache(CacheProfileName = "Images")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Images)]
[HttpGet("series-cover")]
public async Task<ActionResult> GetSeriesCoverImage(int seriesId)
{
@ -97,7 +98,7 @@ public class ImageController : BaseApiController
/// <param name="collectionTagId"></param>
/// <returns></returns>
[HttpGet("collection-cover")]
[ResponseCache(CacheProfileName = "Images")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Images)]
public async Task<ActionResult> GetCollectionCoverImage(int collectionTagId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.CollectionTagRepository.GetCoverImageAsync(collectionTagId));
@ -113,7 +114,7 @@ public class ImageController : BaseApiController
/// <param name="readingListId"></param>
/// <returns></returns>
[HttpGet("readinglist-cover")]
[ResponseCache(CacheProfileName = "Images")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Images)]
public async Task<ActionResult> GetReadingListCoverImage(int readingListId)
{
var path = Path.Join(_directoryService.CoverImageDirectory, await _unitOfWork.ReadingListRepository.GetCoverImageAsync(readingListId));
@ -132,7 +133,7 @@ public class ImageController : BaseApiController
/// <param name="apiKey">API Key for user. Needed to authenticate request</param>
/// <returns></returns>
[HttpGet("bookmark")]
[ResponseCache(CacheProfileName = "Images")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Images)]
public async Task<ActionResult> GetBookmarkImage(int chapterId, int pageNum, string apiKey)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey);
@ -154,7 +155,7 @@ public class ImageController : BaseApiController
/// <returns></returns>
[Authorize(Policy="RequireAdminRole")]
[HttpGet("cover-upload")]
[ResponseCache(CacheProfileName = "Images")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Images)]
public ActionResult GetCoverUploadImage(string filename)
{
if (filename.Contains("..")) return BadRequest("Invalid Filename");

View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.Globalization;
using System.Linq;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
using API.DTOs;
using API.DTOs.Filtering;
@ -84,7 +85,7 @@ public class MetadataController : BaseApiController
/// <param name="libraryIds">String separated libraryIds or null for all ratings</param>
/// <remarks>This API is cached for 1 hour, varying by libraryIds</remarks>
/// <returns></returns>
[ResponseCache(CacheProfileName = "5Minute", VaryByQueryKeys = new [] {"libraryIds"})]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.FiveMinute, VaryByQueryKeys = new [] {"libraryIds"})]
[HttpGet("age-ratings")]
public async Task<ActionResult<IList<AgeRatingDto>>> GetAllAgeRatings(string? libraryIds)
{
@ -107,7 +108,7 @@ public class MetadataController : BaseApiController
/// <param name="libraryIds">String separated libraryIds or null for all publication status</param>
/// <remarks>This API is cached for 1 hour, varying by libraryIds</remarks>
/// <returns></returns>
[ResponseCache(CacheProfileName = "5Minute", VaryByQueryKeys = new [] {"libraryIds"})]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.FiveMinute, VaryByQueryKeys = new [] {"libraryIds"})]
[HttpGet("publication-status")]
public ActionResult<IList<AgeRatingDto>> GetAllPublicationStatus(string? libraryIds)
{

View file

@ -787,7 +787,7 @@ public class OpdsController : BaseApiController
CreateLink(FeedLinkRelation.Thumbnail, FeedLinkType.Image, $"/api/image/chapter-cover?chapterId={chapterId}"),
// We can't not include acc link in the feed, panels doesn't work with just page streaming option. We have to block download directly
accLink,
CreatePageStreamLink(seriesId, volumeId, chapterId, mangaFile, apiKey)
CreatePageStreamLink(series.LibraryId,seriesId, volumeId, chapterId, mangaFile, apiKey)
},
Content = new FeedEntryContent()
{
@ -800,7 +800,7 @@ public class OpdsController : BaseApiController
}
[HttpGet("{apiKey}/image")]
public async Task<ActionResult> GetPageStreamedImage(string apiKey, [FromQuery] int seriesId, [FromQuery] int volumeId,[FromQuery] int chapterId, [FromQuery] int pageNumber)
public async Task<ActionResult> GetPageStreamedImage(string apiKey, [FromQuery] int libraryId, [FromQuery] int seriesId, [FromQuery] int volumeId,[FromQuery] int chapterId, [FromQuery] int pageNumber)
{
if (pageNumber < 0) return BadRequest("Page cannot be less than 0");
var chapter = await _cacheService.Ensure(chapterId);
@ -823,7 +823,8 @@ public class OpdsController : BaseApiController
ChapterId = chapterId,
PageNum = pageNumber,
SeriesId = seriesId,
VolumeId = volumeId
VolumeId = volumeId,
LibraryId =libraryId
}, await GetUser(apiKey));
return File(content, "image/" + format);
@ -866,9 +867,9 @@ public class OpdsController : BaseApiController
throw new KavitaException("User does not exist");
}
private static FeedLink CreatePageStreamLink(int seriesId, int volumeId, int chapterId, MangaFile mangaFile, string apiKey)
private static FeedLink CreatePageStreamLink(int libraryId, int seriesId, int volumeId, int chapterId, MangaFile mangaFile, string apiKey)
{
var link = CreateLink(FeedLinkRelation.Stream, "image/jpeg", $"{Prefix}{apiKey}/image?seriesId={seriesId}&volumeId={volumeId}&chapterId={chapterId}&pageNumber=" + "{pageNumber}");
var link = CreateLink(FeedLinkRelation.Stream, "image/jpeg", $"{Prefix}{apiKey}/image?libraryId={libraryId}&seriesId={seriesId}&volumeId={volumeId}&chapterId={chapterId}&pageNumber=" + "{pageNumber}");
link.TotalPages = mangaFile.Pages;
return link;
}

View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
using API.Data.Repositories;
using API.DTOs;
@ -56,7 +57,7 @@ public class ReaderController : BaseApiController
/// <param name="chapterId"></param>
/// <returns></returns>
[HttpGet("pdf")]
[ResponseCache(CacheProfileName = "Hour")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Hour)]
public async Task<ActionResult> GetPdf(int chapterId)
{
var chapter = await _cacheService.Ensure(chapterId);
@ -90,7 +91,7 @@ public class ReaderController : BaseApiController
/// <param name="page"></param>
/// <returns></returns>
[HttpGet("image")]
[ResponseCache(CacheProfileName = "Hour")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Hour)]
[AllowAnonymous]
public async Task<ActionResult> GetImage(int chapterId, int page)
{
@ -122,7 +123,7 @@ public class ReaderController : BaseApiController
/// <remarks>We must use api key as bookmarks could be leaked to other users via the API</remarks>
/// <returns></returns>
[HttpGet("bookmark-image")]
[ResponseCache(CacheProfileName = "Hour")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Hour)]
[AllowAnonymous]
public async Task<ActionResult> GetBookmarkImage(int seriesId, string apiKey, int page)
{

View file

@ -2,6 +2,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
using API.Data.Repositories;
using API.DTOs;
@ -383,7 +384,7 @@ public class SeriesController : BaseApiController
/// <param name="seriesId"></param>
/// <returns></returns>
/// <remarks>Do not rely on this API externally. May change without hesitation. </remarks>
[ResponseCache(CacheProfileName = "5Minute", VaryByQueryKeys = new [] {"seriesId"})]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.FiveMinute, VaryByQueryKeys = new [] {"seriesId"})]
[HttpGet("series-detail")]
public async Task<ActionResult<SeriesDetailDto>> GetSeriesDetailBreakdown(int seriesId)
{

View file

@ -35,10 +35,11 @@ public class ServerController : BaseApiController
private readonly ICleanupService _cleanupService;
private readonly IEmailService _emailService;
private readonly IBookmarkService _bookmarkService;
private readonly IScannerService _scannerService;
public ServerController(IHostApplicationLifetime applicationLifetime, ILogger<ServerController> logger,
IBackupService backupService, IArchiveService archiveService, IVersionUpdaterService versionUpdaterService, IStatsService statsService,
ICleanupService cleanupService, IEmailService emailService, IBookmarkService bookmarkService)
ICleanupService cleanupService, IEmailService emailService, IBookmarkService bookmarkService, IScannerService scannerService)
{
_applicationLifetime = applicationLifetime;
_logger = logger;
@ -49,6 +50,7 @@ public class ServerController : BaseApiController
_cleanupService = cleanupService;
_emailService = emailService;
_bookmarkService = bookmarkService;
_scannerService = scannerService;
}
/// <summary>
@ -85,7 +87,7 @@ public class ServerController : BaseApiController
public ActionResult CleanupWantToRead()
{
_logger.LogInformation("{UserName} is clearing running want to read cleanup from admin dashboard", User.GetUsername());
RecurringJob.TriggerJob(API.Services.TaskScheduler.RemoveFromWantToReadTaskId);
RecurringJob.TriggerJob(TaskScheduler.RemoveFromWantToReadTaskId);
return Ok();
}
@ -98,7 +100,23 @@ public class ServerController : BaseApiController
public ActionResult BackupDatabase()
{
_logger.LogInformation("{UserName} is backing up database of server from admin dashboard", User.GetUsername());
RecurringJob.TriggerJob(API.Services.TaskScheduler.BackupTaskId);
RecurringJob.TriggerJob(TaskScheduler.BackupTaskId);
return Ok();
}
/// <summary>
/// This is a one time task that needs to be ran for v0.7 statistics to work
/// </summary>
/// <returns></returns>
[HttpPost("analyze-files")]
public ActionResult AnalyzeFiles()
{
_logger.LogInformation("{UserName} is performing file analysis from admin dashboard", User.GetUsername());
if (TaskScheduler.HasAlreadyEnqueuedTask(ScannerService.Name, "AnalyzeFiles",
Array.Empty<object>(), TaskScheduler.DefaultQueue, true))
return Ok("Job already running");
BackgroundJob.Enqueue(() => _scannerService.AnalyzeFiles());
return Ok();
}

View file

@ -0,0 +1,112 @@
using System;
using System.Collections.Generic;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
using API.DTOs.Statistics;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc;
namespace API.Controllers;
public class StatsController : BaseApiController
{
private readonly IStatisticService _statService;
private readonly IUnitOfWork _unitOfWork;
private readonly UserManager<AppUser> _userManager;
public StatsController(IStatisticService statService, IUnitOfWork unitOfWork, UserManager<AppUser> userManager)
{
_statService = statService;
_unitOfWork = unitOfWork;
_userManager = userManager;
}
[HttpGet("user/{userId}/read")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<UserReadStatistics>> GetUserReadStatistics(int userId)
{
var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername());
if (user.Id != userId && !await _userManager.IsInRoleAsync(user, PolicyConstants.AdminRole))
return Unauthorized("You are not authorized to view another user's statistics");
return Ok(await _statService.GetUserReadStatistics(userId, new List<int>()));
}
[Authorize("RequireAdminRole")]
[HttpGet("server/stats")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<ServerStatistics>> GetHighLevelStats()
{
return Ok(await _statService.GetServerStatistics());
}
[Authorize("RequireAdminRole")]
[HttpGet("server/count/year")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<IEnumerable<StatCount<int>>>> GetYearStatistics()
{
return Ok(await _statService.GetYearCount());
}
[Authorize("RequireAdminRole")]
[HttpGet("server/count/publication-status")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<IEnumerable<StatCount<PublicationStatus>>>> GetPublicationStatus()
{
return Ok(await _statService.GetPublicationCount());
}
[Authorize("RequireAdminRole")]
[HttpGet("server/count/manga-format")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<IEnumerable<StatCount<MangaFormat>>>> GetMangaFormat()
{
return Ok(await _statService.GetMangaFormatCount());
}
[Authorize("RequireAdminRole")]
[HttpGet("server/top/years")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<IEnumerable<StatCount<int>>>> GetTopYears()
{
return Ok(await _statService.GetTopYears());
}
/// <summary>
/// Returns
/// </summary>
/// <param name="days"></param>
/// <returns></returns>
[Authorize("RequireAdminRole")]
[HttpGet("server/top/users")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<IEnumerable<TopReadDto>>> GetTopReads(int days = 0)
{
return Ok(await _statService.GetTopUsers(days));
}
[Authorize("RequireAdminRole")]
[HttpGet("server/file-breakdown")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<IEnumerable<FileExtensionBreakdownDto>>> GetFileSize()
{
return Ok(await _statService.GetFileBreakdown());
}
[HttpGet("user/reading-history")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult<IEnumerable<ReadHistoryEvent>>> GetReadingHistory(int userId)
{
// TODO: Put a check in if the calling user is said userId or has admin
return Ok(await _statService.GetReadingHistory(userId));
}
}

View file

@ -60,7 +60,7 @@ public class UsersController : BaseApiController
public async Task<ActionResult<bool>> HasReadingProgress(int libraryId)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.None);
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId);
return Ok(await _unitOfWork.AppUserProgressRepository.UserHasProgress(library.Type, userId));
}
@ -115,6 +115,10 @@ public class UsersController : BaseApiController
return BadRequest("There was an issue saving preferences.");
}
/// <summary>
/// Returns the preferences of the user
/// </summary>
/// <returns></returns>
[HttpGet("get-preferences")]
public async Task<ActionResult<UserPreferencesDto>> GetPreferences()
{
@ -122,4 +126,15 @@ public class UsersController : BaseApiController
await _unitOfWork.UserRepository.GetPreferencesAsync(User.GetUsername()));
}
/// <summary>
/// Returns a list of the user names within the system
/// </summary>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[HttpGet("names")]
public async Task<ActionResult<IEnumerable<string>>> GetUserNames()
{
return Ok((await _unitOfWork.UserRepository.GetAllUsersAsync()).Select(u => u.UserName));
}
}