AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Basic Stats (#1673)

Browse source 
* Refactored ResponseCache profiles into consts

* Refactored code to use an extension method for getting user library ids.

* Started server statistics, added a charting library, and added a table sort column (not finished)

* Refactored code and have a fully working example of sortable headers. Still doesn't work with default sorting state, will work on that later.

* Implemented file size, but it's too expensive, so commented out.

* Added a migration to provide extension and length/size information in the DB to allow for faster stat apis.

* Added the ability to force a library scan from library settings.

* Refactored some apis to provide more of a file breakdown rather than just file size.

* Working on visualization of file breakdown

* Fixed the file breakdown visual

* Fixed up 2 visualizations

* Added back an api for member names, started work on top reads

* Hooked up the other library types and username/days.

* Preparing to remove top reads and refactor into Top users

* Added LibraryId to AppUserProgress to help with complex lookups.

* Added the new libraryId hook into some stats methods

* Updated api methods to use libraryId for progress

* More places where LibraryId is needed

* Added some high level server stats

* Got a ton done on server stats

* Updated default theme (dark) to be the default root variables. This will allow user themes to override just what they want, rather than maintain their own css variables.

* Implemented a monster query for top users by reading time. It's very slow and can be cleaned up likely.

* Hooked up top reads. Code needs a big refactor. Handing off for Robbie treatment and I'll switch to User stats.

* Implemented last 5 recently read series (broken) and added some basic css

* Fixed recently read query

* Cleanup the css a bit, Robbie we need you

* More css love

* Cleaned up DTOs that aren't needed anymore

* Fixed top readers query

* When calculating top readers, don't include read events where nothing is read (0 pages)

* Hooked up the date into GetTopUsers

* Hooked top readers up with days and refactored and cleaned up componets not used

* Fixed up query

* Started on a day by day breakdown, but going to take a break from stats.

* Added a temp task to run some migration manually for stats to work

* Ensure OPDS-PS uses new libraryId for progress reporting

* Fixed a code smell

* Adding some styling

* adding more styles

* Removed some debug stuff from user stats

* Bump qs from 6.5.2 to 6.5.3 in /UI/Web

Bumps [qs](https://github.com/ljharb/qs) from 6.5.2 to 6.5.3.
- [Release notes](https://github.com/ljharb/qs/releases)
- [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ljharb/qs/compare/v6.5.2...v6.5.3)

---
updated-dependencies:
- dependency-name: qs
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

* Tweaked some code for bad data cases

* Refactored a chapter lookup to remove un-needed Volume join in 5 places across the code.

* API push

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
Joe Milazzo 2022-12-07 08:01:49 -06:00 • committed by GitHub
parent 4724dc5a76
commit c361e66b35
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
106 changed files with 6898 additions and 170 deletions
Download patch file Download diff file Expand all files Collapse all files

16
API/Data/Repositories/AppUserProgressRepository.cs
View file

@ -14,7 +14,13 @@ public interface IAppUserProgressRepository
Task<bool> UserHasProgress(LibraryType libraryType, int userId);
Task<AppUserProgress> GetUserProgressAsync(int chapterId, int userId);
Task<bool> HasAnyProgressOnSeriesAsync(int seriesId, int userId);
/// <summary>
/// This is built exclusively for <see cref="MigrateUserProgressLibraryId"/>
/// </summary>
/// <returns></returns>
Task<AppUserProgress> GetAnyProgress();
Task<IEnumerable<AppUserProgress>> GetUserProgressForSeriesAsync(int seriesId, int userId);
Task<IEnumerable<AppUserProgress>> GetAllProgress();
}
public class AppUserProgressRepository : IAppUserProgressRepository
@ -85,6 +91,11 @@ public class AppUserProgressRepository : IAppUserProgressRepository
.AnyAsync(aup => aup.PagesRead > 0 && aup.AppUserId == userId && aup.SeriesId == seriesId);
}
public async Task<AppUserProgress> GetAnyProgress()
{
return await _context.AppUserProgresses.FirstOrDefaultAsync();
}
/// <summary>
/// This will return any user progress. This filters out progress rows that have no pages read.
/// </summary>
@ -98,6 +109,11 @@ public class AppUserProgressRepository : IAppUserProgressRepository
.ToListAsync();
}
public async Task<IEnumerable<AppUserProgress>> GetAllProgress()
{
return await _context.AppUserProgresses.ToListAsync();
}
public async Task<AppUserProgress> GetUserProgressAsync(int chapterId, int userId)
{
return await _context.AppUserProgresses

17
API/Data/Repositories/ChapterRepository.cs
View file

@ -1,20 +1,29 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using API.DTOs;
using API.DTOs.Metadata;
using API.DTOs.Reader;
using API.Entities;
using API.Extensions;
using AutoMapper;
using AutoMapper.QueryableExtensions;
using Microsoft.EntityFrameworkCore;
namespace API.Data.Repositories;
[Flags]
public enum ChapterIncludes
{
None = 1,
Volumes = 2,
}
public interface IChapterRepository
{
void Update(Chapter chapter);
Task<IEnumerable<Chapter>> GetChaptersByIdsAsync(IList<int> chapterIds);
Task<IEnumerable<Chapter>> GetChaptersByIdsAsync(IList<int> chapterIds, ChapterIncludes includes = ChapterIncludes.None);
Task<IChapterInfoDto> GetChapterInfoDtoAsync(int chapterId);
Task<int> GetChapterTotalPagesAsync(int chapterId);
Task<Chapter> GetChapterAsync(int chapterId);
@ -43,11 +52,11 @@ public class ChapterRepository : IChapterRepository
_context.Entry(chapter).State = EntityState.Modified;
}
public async Task<IEnumerable<Chapter>> GetChaptersByIdsAsync(IList<int> chapterIds)
public async Task<IEnumerable<Chapter>> GetChaptersByIdsAsync(IList<int> chapterIds, ChapterIncludes includes)
{
return await _context.Chapter
.Where(c => chapterIds.Contains(c.Id))
.Include(c => c.Volume)
.Includes(includes)
.AsSplitQuery()
.ToListAsync();
}

19
API/Data/Repositories/MangaFileRepository.cs
View file

@ -1,4 +1,7 @@
using API.Entities;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using AutoMapper;
using Microsoft.EntityFrameworkCore;
@ -7,6 +10,8 @@ namespace API.Data.Repositories;
public interface IMangaFileRepository
{
void Update(MangaFile file);
Task<bool> AnyMissingExtension();
Task<IList<MangaFile>> GetAllWithMissingExtension();
}
public class MangaFileRepository : IMangaFileRepository
@ -24,4 +29,16 @@ public class MangaFileRepository : IMangaFileRepository
{
_context.Entry(file).State = EntityState.Modified;
}
public async Task<bool> AnyMissingExtension()
{
return (await _context.MangaFile.CountAsync(f => string.IsNullOrEmpty(f.Extension))) > 0;
}
public async Task<IList<MangaFile>> GetAllWithMissingExtension()
{
return await _context.MangaFile
.Where(f => string.IsNullOrEmpty(f.Extension))
.ToListAsync();
}
}

88
API/Data/Repositories/SeriesRepository.cs
View file

@ -119,6 +119,11 @@ public interface ISeriesRepository
Task<IList<Series>> RemoveSeriesNotInList(IList<ParsedSeries> seenSeries, int libraryId);
Task<IDictionary<string, IList<SeriesModified>>> GetFolderPathMap(int libraryId);
Task<AgeRating> GetMaxAgeRatingFromSeriesAsync(IEnumerable<int> seriesIds);
/// <summary>
/// This is only used for <see cref="MigrateUserProgressLibraryId"/>
/// </summary>
/// <returns></returns>
Task<IDictionary<int, int>> GetLibraryIdsForSeriesAsync();
}
public class SeriesRepository : ISeriesRepository
@ -283,14 +288,7 @@ public class SeriesRepository : ISeriesRepository
{
if (libraryId == 0)
{
return await _context.Library
.Include(l => l.AppUsers)
.Where(library => library.AppUsers.Any(user => user.Id == userId))
.IsRestricted(queryContext)
.AsNoTracking()
.AsSplitQuery()
.Select(library => library.Id)
.ToListAsync();
return await _context.Library.GetUserLibraries(userId, queryContext).ToListAsync();
}
return new List<int>()
@ -513,6 +511,21 @@ public class SeriesRepository : ISeriesRepository
return seriesChapters;
}
public async Task<IDictionary<int, int>> GetLibraryIdsForSeriesAsync()
{
var seriesChapters = new Dictionary<int, int>();
var series = await _context.Series.Select(s => new
{
Id = s.Id, LibraryId = s.LibraryId
}).ToListAsync();
foreach (var s in series)
{
seriesChapters.Add(s.Id, s.LibraryId);
}
return seriesChapters;
}
public async Task AddSeriesModifiers(int userId, List<SeriesDto> series)
{
var userProgress = await _context.AppUserProgresses
@ -672,7 +685,8 @@ public class SeriesRepository : ISeriesRepository
var cutoffProgressPoint = DateTime.Now - TimeSpan.FromDays(30);
var cutoffLastAddedPoint = DateTime.Now - TimeSpan.FromDays(7);
var libraryIds = GetLibraryIdsForUser(userId, libraryId, QueryContext.Dashboard);
var libraryIds = _context.Library.GetUserLibraries(userId, QueryContext.Dashboard)
.Where(id => libraryId == 0 || id == libraryId);
var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds);
@ -1046,7 +1060,7 @@ public class SeriesRepository : ISeriesRepository
public async Task<IEnumerable<SeriesDto>> GetSeriesForRelationKind(int userId, int seriesId, RelationKind kind)
{
var libraryIds = GetLibraryIdsForUser(userId);
var libraryIds = _context.Library.GetUserLibraries(userId);
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
var usersSeriesIds = _context.Series
@ -1073,7 +1087,8 @@ public class SeriesRepository : ISeriesRepository
public async Task<PagedList<SeriesDto>> GetMoreIn(int userId, int libraryId, int genreId, UserParams userParams)
{
var libraryIds = GetLibraryIdsForUser(userId, libraryId, QueryContext.Recommended);
var libraryIds = _context.Library.GetUserLibraries(userId, QueryContext.Recommended)
.Where(id => libraryId == 0 || id == libraryId);
var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds);
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
@ -1100,7 +1115,8 @@ public class SeriesRepository : ISeriesRepository
/// <returns></returns>
public async Task<PagedList<SeriesDto>> GetRediscover(int userId, int libraryId, UserParams userParams)
{
var libraryIds = GetLibraryIdsForUser(userId, libraryId, QueryContext.Recommended);
var libraryIds = _context.Library.GetUserLibraries(userId, QueryContext.Recommended)
.Where(id => libraryId == 0 || id == libraryId);
var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds);
var distinctSeriesIdsWithProgress = _context.AppUserProgresses
.Where(s => usersSeriesIds.Contains(s.SeriesId))
@ -1119,7 +1135,7 @@ public class SeriesRepository : ISeriesRepository
public async Task<SeriesDto> GetSeriesForMangaFile(int mangaFileId, int userId)
{
var libraryIds = GetLibraryIdsForUser(userId, 0, QueryContext.Search);
var libraryIds = _context.Library.GetUserLibraries(userId, QueryContext.Search);
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
return await _context.MangaFile
@ -1136,7 +1152,7 @@ public class SeriesRepository : ISeriesRepository
public async Task<SeriesDto> GetSeriesForChapter(int chapterId, int userId)
{
var libraryIds = GetLibraryIdsForUser(userId);
var libraryIds = _context.Library.GetUserLibraries(userId);
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
return await _context.Chapter
.Where(m => m.Id == chapterId)
@ -1278,7 +1294,8 @@ public class SeriesRepository : ISeriesRepository
public async Task<PagedList<SeriesDto>> GetHighlyRated(int userId, int libraryId, UserParams userParams)
{
var libraryIds = GetLibraryIdsForUser(userId, libraryId, QueryContext.Recommended);
var libraryIds = _context.Library.GetUserLibraries(userId, QueryContext.Recommended)
.Where(id => libraryId == 0 || id == libraryId);
var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds);
var distinctSeriesIdsWithHighRating = _context.AppUserRating
.Where(s => usersSeriesIds.Contains(s.SeriesId) && s.Rating > 4)
@ -1299,7 +1316,8 @@ public class SeriesRepository : ISeriesRepository
public async Task<PagedList<SeriesDto>> GetQuickReads(int userId, int libraryId, UserParams userParams)
{
var libraryIds = GetLibraryIdsForUser(userId, libraryId, QueryContext.Recommended);
var libraryIds = _context.Library.GetUserLibraries(userId, QueryContext.Recommended)
.Where(id => libraryId == 0 || id == libraryId);
var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds);
var distinctSeriesIdsWithProgress = _context.AppUserProgresses
.Where(s => usersSeriesIds.Contains(s.SeriesId))
@ -1325,7 +1343,8 @@ public class SeriesRepository : ISeriesRepository
public async Task<PagedList<SeriesDto>> GetQuickCatchupReads(int userId, int libraryId, UserParams userParams)
{
var libraryIds = GetLibraryIdsForUser(userId, libraryId, QueryContext.Recommended);
var libraryIds = _context.Library.GetUserLibraries(userId, QueryContext.Recommended)
.Where(id => libraryId == 0 || id == libraryId);
var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds);
var distinctSeriesIdsWithProgress = _context.AppUserProgresses
.Where(s => usersSeriesIds.Contains(s.SeriesId))
@ -1350,37 +1369,9 @@ public class SeriesRepository : ISeriesRepository
return await PagedList<SeriesDto>.CreateAsync(query, userParams.PageNumber, userParams.PageSize);
}
/// <summary>
/// Returns all library ids for a user
/// </summary>
/// <param name="userId"></param>
/// <param name="libraryId">0 for no library filter</param>
/// <param name="queryContext">Defaults to None - The context behind this query, so appropriate restrictions can be placed</param>
/// <returns></returns>
private IQueryable<int> GetLibraryIdsForUser(int userId, int libraryId = 0, QueryContext queryContext = QueryContext.None)
{
var user = _context.AppUser
.AsSplitQuery()
.AsNoTracking()
.Where(u => u.Id == userId)
.AsSingleQuery();
if (libraryId == 0)
{
return user.SelectMany(l => l.Libraries)
.IsRestricted(queryContext)
.Select(lib => lib.Id);
}
return user.SelectMany(l => l.Libraries)
.Where(lib => lib.Id == libraryId)
.IsRestricted(queryContext)
.Select(lib => lib.Id);
}
public async Task<RelatedSeriesDto> GetRelatedSeries(int userId, int seriesId)
{
var libraryIds = GetLibraryIdsForUser(userId);
var libraryIds = _context.Library.GetUserLibraries(userId);
var usersSeriesIds = GetSeriesIdsForLibraryIds(libraryIds);
var userRating = await _context.AppUser.GetUserAgeRestriction(userId);
@ -1486,7 +1477,7 @@ public class SeriesRepository : ISeriesRepository
public async Task<PagedList<SeriesDto>> GetWantToReadForUserAsync(int userId, UserParams userParams, FilterDto filter)
{
var libraryIds = GetLibraryIdsForUser(userId);
var libraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync();
var query = _context.AppUser
.Where(user => user.Id == userId)
.SelectMany(u => u.WantToRead)
@ -1501,8 +1492,7 @@ public class SeriesRepository : ISeriesRepository
public async Task<bool> IsSeriesInWantToRead(int userId, int seriesId)
{
// BUG: This is always returning true for any series
var libraryIds = GetLibraryIdsForUser(userId);
var libraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync();
return await _context.AppUser
.Where(user => user.Id == userId)
.SelectMany(u => u.WantToRead.Where(s => s.Id == seriesId && libraryIds.Contains(s.LibraryId)))

10
API/Data/Repositories/UserRepository.cs
View file

@ -59,10 +59,9 @@ public interface IUserRepository
Task<int> GetUserIdByUsernameAsync(string username);
Task<IList<AppUserBookmark>> GetAllBookmarksByIds(IList<int> bookmarkIds);
Task<AppUser> GetUserByEmailAsync(string email);
Task<IEnumerable<AppUser>> GetAllUsers();
Task<IEnumerable<AppUserPreferences>> GetAllPreferencesByThemeAsync(int themeId);
Task<bool> HasAccessToLibrary(int libraryId, int userId);
Task<IEnumerable<AppUser>> GetAllUsersAsync(AppUserIncludes includeFlags);
Task<IEnumerable<AppUser>> GetAllUsersAsync(AppUserIncludes includeFlags = AppUserIncludes.None);
Task<AppUser> GetUserByConfirmationToken(string token);
}
@ -241,11 +240,6 @@ public class UserRepository : IUserRepository
return await _context.AppUser.SingleOrDefaultAsync(u => u.Email.ToLower().Equals(lowerEmail));
}
public async Task<IEnumerable<AppUser>> GetAllUsers()
{
return await _context.AppUser
.ToListAsync();
}
public async Task<IEnumerable<AppUserPreferences>> GetAllPreferencesByThemeAsync(int themeId)
{
@ -264,7 +258,7 @@ public class UserRepository : IUserRepository
.AnyAsync(library => library.AppUsers.Any(user => user.Id == userId));
}
public async Task<IEnumerable<AppUser>> GetAllUsersAsync(AppUserIncludes includeFlags)
public async Task<IEnumerable<AppUser>> GetAllUsersAsync(AppUserIncludes includeFlags = AppUserIncludes.None)
{
var query = AddIncludesToQuery(_context.Users.AsQueryable(), includeFlags);
return await query.ToListAsync();