Basic Stats (#1673)
* Refactored ResponseCache profiles into consts * Refactored code to use an extension method for getting user library ids. * Started server statistics, added a charting library, and added a table sort column (not finished) * Refactored code and have a fully working example of sortable headers. Still doesn't work with default sorting state, will work on that later. * Implemented file size, but it's too expensive, so commented out. * Added a migration to provide extension and length/size information in the DB to allow for faster stat apis. * Added the ability to force a library scan from library settings. * Refactored some apis to provide more of a file breakdown rather than just file size. * Working on visualization of file breakdown * Fixed the file breakdown visual * Fixed up 2 visualizations * Added back an api for member names, started work on top reads * Hooked up the other library types and username/days. * Preparing to remove top reads and refactor into Top users * Added LibraryId to AppUserProgress to help with complex lookups. * Added the new libraryId hook into some stats methods * Updated api methods to use libraryId for progress * More places where LibraryId is needed * Added some high level server stats * Got a ton done on server stats * Updated default theme (dark) to be the default root variables. This will allow user themes to override just what they want, rather than maintain their own css variables. * Implemented a monster query for top users by reading time. It's very slow and can be cleaned up likely. * Hooked up top reads. Code needs a big refactor. Handing off for Robbie treatment and I'll switch to User stats. * Implemented last 5 recently read series (broken) and added some basic css * Fixed recently read query * Cleanup the css a bit, Robbie we need you * More css love * Cleaned up DTOs that aren't needed anymore * Fixed top readers query * When calculating top readers, don't include read events where nothing is read (0 pages) * Hooked up the date into GetTopUsers * Hooked top readers up with days and refactored and cleaned up componets not used * Fixed up query * Started on a day by day breakdown, but going to take a break from stats. * Added a temp task to run some migration manually for stats to work * Ensure OPDS-PS uses new libraryId for progress reporting * Fixed a code smell * Adding some styling * adding more styles * Removed some debug stuff from user stats * Bump qs from 6.5.2 to 6.5.3 in /UI/Web Bumps [qs](https://github.com/ljharb/qs) from 6.5.2 to 6.5.3. - [Release notes](https://github.com/ljharb/qs/releases) - [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md) - [Commits](https://github.com/ljharb/qs/compare/v6.5.2...v6.5.3) --- updated-dependencies: - dependency-name: qs dependency-type: indirect ... Signed-off-by: dependabot[bot] <support@github.com> * Tweaked some code for bad data cases * Refactored a chapter lookup to remove un-needed Volume join in 5 places across the code. * API push Signed-off-by: dependabot[bot] <support@github.com> Co-authored-by: Robbie Davis <robbie@therobbiedavis.com> Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
Joe Milazzo
GitHub
parent
4724dc5a76
commit
c361e66b35
106 changed files with 6898 additions and 170 deletions
417
API/Services/StatisticService.cs
Normal file
417
API/Services/StatisticService.cs
Normal file
|
|
@ -0,0 +1,417 @@
|
|||
using System;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using API.Data;
|
||||
using API.DTOs;
|
||||
using API.DTOs.Statistics;
|
||||
using API.Entities.Enums;
|
||||
using API.Extensions;
|
||||
using AutoMapper;
|
||||
using AutoMapper.QueryableExtensions;
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace API.Services;
|
||||
|
||||
public interface IStatisticService
|
||||
{
|
||||
Task<ServerStatistics> GetServerStatistics();
|
||||
Task<UserReadStatistics> GetUserReadStatistics(int userId, IList<int> libraryIds);
|
||||
Task<IEnumerable<StatCount<int>>> GetYearCount();
|
||||
Task<IEnumerable<StatCount<int>>> GetTopYears();
|
||||
Task<IEnumerable<StatCount<PublicationStatus>>> GetPublicationCount();
|
||||
Task<IEnumerable<StatCount<MangaFormat>>> GetMangaFormatCount();
|
||||
Task<FileExtensionBreakdownDto> GetFileBreakdown();
|
||||
Task<IEnumerable<TopReadDto>> GetTopUsers(int days);
|
||||
Task<IEnumerable<ReadHistoryEvent>> GetReadingHistory(int userId);
|
||||
Task<IEnumerable<ReadHistoryEvent>> GetHistory();
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Responsible for computing statistics for the server
|
||||
/// </summary>
|
||||
/// <remarks>This performs raw queries and does not use a repository</remarks>
|
||||
public class StatisticService : IStatisticService
|
||||
{
|
||||
private readonly DataContext _context;
|
||||
private readonly IMapper _mapper;
|
||||
private readonly IUnitOfWork _unitOfWork;
|
||||
|
||||
public StatisticService(DataContext context, IMapper mapper, IUnitOfWork unitOfWork)
|
||||
{
|
||||
_context = context;
|
||||
_mapper = mapper;
|
||||
_unitOfWork = unitOfWork;
|
||||
}
|
||||
|
||||
public async Task<UserReadStatistics> GetUserReadStatistics(int userId, IList<int> libraryIds)
|
||||
{
|
||||
if (libraryIds.Count == 0)
|
||||
libraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync();
|
||||
|
||||
|
||||
// Total Pages Read
|
||||
var totalPagesRead = await _context.AppUserProgresses
|
||||
.Where(p => p.AppUserId == userId)
|
||||
.Where(p => libraryIds.Contains(p.LibraryId))
|
||||
.SumAsync(p => p.PagesRead);
|
||||
|
||||
var ids = await _context.AppUserProgresses
|
||||
.Where(p => p.AppUserId == userId)
|
||||
.Where(p => libraryIds.Contains(p.LibraryId))
|
||||
.Where(p => p.PagesRead > 0)
|
||||
.Select(p => new {p.ChapterId, p.SeriesId})
|
||||
.ToListAsync();
|
||||
|
||||
var chapterIds = ids.Select(id => id.ChapterId);
|
||||
|
||||
var timeSpentReading = await _context.Chapter
|
||||
.Where(c => chapterIds.Contains(c.Id))
|
||||
.SumAsync(c => c.AvgHoursToRead);
|
||||
|
||||
// Maybe make this top 5 genres? But usually there are 3-5 genres that are always common...
|
||||
// Maybe use rating to calculate top genres?
|
||||
// var genres = await _context.Series
|
||||
// .Where(s => seriesIds.Contains(s.Id))
|
||||
// .Select(s => s.Metadata)
|
||||
// .SelectMany(sm => sm.Genres)
|
||||
// //.DistinctBy(g => g.NormalizedTitle)
|
||||
// .ToListAsync();
|
||||
|
||||
// How many series of each format have you read? (Epub, Archive, etc)
|
||||
|
||||
// Percentage of libraries read. For each library, get the total pages vs read
|
||||
//var allLibraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync();
|
||||
|
||||
var chaptersRead = await _context.AppUserProgresses
|
||||
.Where(p => p.AppUserId == userId)
|
||||
.Where(p => libraryIds.Contains(p.LibraryId))
|
||||
.Where(p => p.PagesRead >= _context.Chapter.Single(c => c.Id == p.ChapterId).Pages)
|
||||
.CountAsync();
|
||||
|
||||
var lastActive = await _context.AppUserProgresses
|
||||
.OrderByDescending(p => p.LastModified)
|
||||
.Select(p => p.LastModified)
|
||||
.FirstOrDefaultAsync();
|
||||
|
||||
//var
|
||||
|
||||
return new UserReadStatistics()
|
||||
{
|
||||
TotalPagesRead = totalPagesRead,
|
||||
TimeSpentReading = timeSpentReading,
|
||||
ChaptersRead = chaptersRead,
|
||||
LastActive = lastActive,
|
||||
};
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Returns the Release Years and their count
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public async Task<IEnumerable<StatCount<int>>> GetYearCount()
|
||||
{
|
||||
return await _context.SeriesMetadata
|
||||
.Where(sm => sm.ReleaseYear != 0)
|
||||
.AsSplitQuery()
|
||||
.GroupBy(sm => sm.ReleaseYear)
|
||||
.Select(sm => new StatCount<int>
|
||||
{
|
||||
Value = sm.Key,
|
||||
Count = _context.SeriesMetadata.Where(sm2 => sm2.ReleaseYear == sm.Key).Distinct().Count()
|
||||
})
|
||||
.OrderByDescending(d => d.Value)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<StatCount<int>>> GetTopYears()
|
||||
{
|
||||
return await _context.SeriesMetadata
|
||||
.Where(sm => sm.ReleaseYear != 0)
|
||||
.AsSplitQuery()
|
||||
.GroupBy(sm => sm.ReleaseYear)
|
||||
.Select(sm => new StatCount<int>
|
||||
{
|
||||
Value = sm.Key,
|
||||
Count = _context.SeriesMetadata.Where(sm2 => sm2.ReleaseYear == sm.Key).Distinct().Count()
|
||||
})
|
||||
.OrderByDescending(d => d.Count)
|
||||
.Take(5)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
|
||||
|
||||
public async Task<IEnumerable<StatCount<PublicationStatus>>> GetPublicationCount()
|
||||
{
|
||||
return await _context.SeriesMetadata
|
||||
.AsSplitQuery()
|
||||
.GroupBy(sm => sm.PublicationStatus)
|
||||
.Select(sm => new StatCount<PublicationStatus>
|
||||
{
|
||||
Value = sm.Key,
|
||||
Count = _context.SeriesMetadata.Where(sm2 => sm2.PublicationStatus == sm.Key).Distinct().Count()
|
||||
})
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<StatCount<MangaFormat>>> GetMangaFormatCount()
|
||||
{
|
||||
return await _context.MangaFile
|
||||
.AsSplitQuery()
|
||||
.GroupBy(sm => sm.Format)
|
||||
.Select(mf => new StatCount<MangaFormat>
|
||||
{
|
||||
Value = mf.Key,
|
||||
Count = _context.MangaFile.Where(mf2 => mf2.Format == mf.Key).Distinct().Count()
|
||||
})
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
|
||||
public async Task<ServerStatistics> GetServerStatistics()
|
||||
{
|
||||
var mostActiveUsers = _context.AppUserProgresses
|
||||
.AsSplitQuery()
|
||||
.AsEnumerable()
|
||||
.GroupBy(sm => sm.AppUserId)
|
||||
.Select(sm => new StatCount<UserDto>
|
||||
{
|
||||
Value = _context.AppUser.Where(u => u.Id == sm.Key).ProjectTo<UserDto>(_mapper.ConfigurationProvider)
|
||||
.Single(),
|
||||
Count = _context.AppUserProgresses.Where(u => u.AppUserId == sm.Key).Distinct().Count()
|
||||
})
|
||||
.OrderByDescending(d => d.Count)
|
||||
.Take(5);
|
||||
|
||||
var mostActiveLibrary = _context.AppUserProgresses
|
||||
.AsSplitQuery()
|
||||
.AsEnumerable()
|
||||
.Where(sm => sm.LibraryId > 0)
|
||||
.GroupBy(sm => sm.LibraryId)
|
||||
.Select(sm => new StatCount<LibraryDto>
|
||||
{
|
||||
Value = _context.Library.Where(u => u.Id == sm.Key).ProjectTo<LibraryDto>(_mapper.ConfigurationProvider)
|
||||
.Single(),
|
||||
Count = _context.AppUserProgresses.Where(u => u.LibraryId == sm.Key).Distinct().Count()
|
||||
})
|
||||
.OrderByDescending(d => d.Count)
|
||||
.Take(5);
|
||||
|
||||
var mostPopularSeries = _context.AppUserProgresses
|
||||
.AsSplitQuery()
|
||||
.AsEnumerable()
|
||||
.GroupBy(sm => sm.SeriesId)
|
||||
.Select(sm => new StatCount<SeriesDto>
|
||||
{
|
||||
Value = _context.Series.Where(u => u.Id == sm.Key).ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
|
||||
.Single(),
|
||||
Count = _context.AppUserProgresses.Where(u => u.SeriesId == sm.Key).Distinct().Count()
|
||||
})
|
||||
.OrderByDescending(d => d.Count)
|
||||
.Take(5);
|
||||
|
||||
var mostReadSeries = _context.AppUserProgresses
|
||||
.AsSplitQuery()
|
||||
.AsEnumerable()
|
||||
.GroupBy(sm => sm.SeriesId)
|
||||
.Select(sm => new StatCount<SeriesDto>
|
||||
{
|
||||
Value = _context.Series.Where(u => u.Id == sm.Key).ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
|
||||
.Single(),
|
||||
Count = _context.AppUserProgresses.Where(u => u.SeriesId == sm.Key).AsEnumerable().DistinctBy(p => p.AppUserId).Count()
|
||||
})
|
||||
.OrderByDescending(d => d.Count)
|
||||
.Take(5);
|
||||
|
||||
var seriesIds = (await _context.AppUserProgresses
|
||||
.AsSplitQuery()
|
||||
.OrderByDescending(d => d.LastModified)
|
||||
.Select(d => d.SeriesId)
|
||||
.ToListAsync())
|
||||
.Distinct()
|
||||
.Take(5);
|
||||
|
||||
var recentlyRead = _context.Series
|
||||
.AsSplitQuery()
|
||||
.Where(s => seriesIds.Contains(s.Id))
|
||||
.ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
|
||||
.AsEnumerable();
|
||||
|
||||
return new ServerStatistics()
|
||||
{
|
||||
ChapterCount = await _context.Chapter.CountAsync(),
|
||||
SeriesCount = await _context.Series.CountAsync(),
|
||||
TotalFiles = await _context.MangaFile.CountAsync(),
|
||||
TotalGenres = await _context.Genre.CountAsync(),
|
||||
TotalPeople = await _context.Person.CountAsync(),
|
||||
TotalSize = await _context.MangaFile.SumAsync(m => m.Bytes),
|
||||
TotalTags = await _context.Tag.CountAsync(),
|
||||
VolumeCount = await _context.Volume.Where(v => v.Number != 0).CountAsync(),
|
||||
MostActiveUsers = mostActiveUsers,
|
||||
MostActiveLibraries = mostActiveLibrary,
|
||||
MostPopularSeries = mostPopularSeries,
|
||||
MostReadSeries = mostReadSeries,
|
||||
RecentlyRead = recentlyRead
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<FileExtensionBreakdownDto> GetFileBreakdown()
|
||||
{
|
||||
return new FileExtensionBreakdownDto()
|
||||
{
|
||||
FileBreakdown = await _context.MangaFile
|
||||
.AsSplitQuery()
|
||||
.AsNoTracking()
|
||||
.GroupBy(sm => sm.Extension)
|
||||
.Select(mf => new FileExtensionDto()
|
||||
{
|
||||
Extension = mf.Key,
|
||||
Format =_context.MangaFile.Where(mf2 => mf2.Extension == mf.Key).Select(mf2 => mf2.Format).Single(),
|
||||
TotalSize = _context.MangaFile.Where(mf2 => mf2.Extension == mf.Key).Distinct().Sum(mf2 => mf2.Bytes),
|
||||
TotalFiles = _context.MangaFile.Where(mf2 => mf2.Extension == mf.Key).Distinct().Count()
|
||||
})
|
||||
.ToListAsync(),
|
||||
TotalFileSize = await _context.MangaFile
|
||||
.AsNoTracking()
|
||||
.AsSplitQuery()
|
||||
.SumAsync(f => f.Bytes)
|
||||
};
|
||||
}
|
||||
|
||||
public async Task<IEnumerable<ReadHistoryEvent>> GetReadingHistory(int userId)
|
||||
{
|
||||
return await _context.AppUserProgresses
|
||||
.Where(u => u.AppUserId == userId)
|
||||
.AsNoTracking()
|
||||
.AsSplitQuery()
|
||||
.Select(u => new ReadHistoryEvent
|
||||
{
|
||||
UserId = u.AppUserId,
|
||||
UserName = _context.AppUser.Single(u => u.Id == userId).UserName,
|
||||
SeriesName = _context.Series.Single(s => s.Id == u.SeriesId).Name,
|
||||
SeriesId = u.SeriesId,
|
||||
LibraryId = u.LibraryId,
|
||||
ReadDate = u.LastModified,
|
||||
ChapterId = u.ChapterId,
|
||||
ChapterNumber = _context.Chapter.Single(c => c.Id == u.ChapterId).Number
|
||||
})
|
||||
.OrderByDescending(d => d.ReadDate)
|
||||
.ToListAsync();
|
||||
}
|
||||
|
||||
public Task<IEnumerable<ReadHistoryEvent>> GetHistory()
|
||||
{
|
||||
// _context.AppUserProgresses
|
||||
// .AsSplitQuery()
|
||||
// .AsEnumerable()
|
||||
// .GroupBy(sm => sm.LastModified)
|
||||
// .Select(sm => new
|
||||
// {
|
||||
// User = _context.AppUser.Single(u => u.Id == sm.Key),
|
||||
// Chapters = _context.Chapter.Where(c => _context.AppUserProgresses
|
||||
// .Where(u => u.AppUserId == sm.Key)
|
||||
// .Where(p => p.PagesRead > 0)
|
||||
// .Select(p => p.ChapterId)
|
||||
// .Distinct()
|
||||
// .Contains(c.Id))
|
||||
// })
|
||||
// .OrderByDescending(d => d.Chapters.Sum(c => c.AvgHoursToRead))
|
||||
// .Take(5)
|
||||
// .ToList();
|
||||
|
||||
var firstOfWeek = DateTime.Now.StartOfWeek(DayOfWeek.Monday);
|
||||
var groupedReadingDays = _context.AppUserProgresses
|
||||
.Where(x => x.LastModified >= firstOfWeek)
|
||||
.GroupBy(x => x.LastModified.Day)
|
||||
.Select(g => new StatCount<int>()
|
||||
{
|
||||
Value = g.Key,
|
||||
Count = _context.AppUserProgresses.Where(p => p.LastModified.Day == g.Key).Select(p => p.ChapterId).Distinct().Count()
|
||||
})
|
||||
.AsEnumerable();
|
||||
|
||||
// var records = firstOfWeek.Range(7)
|
||||
// .GroupJoin(groupedReadingDays, wd => wd.Day, lg => lg.Key, (_, lg) => lg.Any() ? lg.First().Count() : 0).ToArray();
|
||||
return Task.FromResult<IEnumerable<ReadHistoryEvent>>(null);
|
||||
}
|
||||
|
||||
|
||||
public async Task<IEnumerable<TopReadDto>> GetTopUsers(int days)
|
||||
{
|
||||
var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();
|
||||
var users = (await _unitOfWork.UserRepository.GetAllUsersAsync()).ToList();
|
||||
var minDate = DateTime.Now.Subtract(TimeSpan.FromDays(days));
|
||||
|
||||
var topUsersAndReadChapters = _context.AppUserProgresses
|
||||
.AsSplitQuery()
|
||||
.AsEnumerable()
|
||||
.GroupBy(sm => sm.AppUserId)
|
||||
.Select(sm => new
|
||||
{
|
||||
User = _context.AppUser.Single(u => u.Id == sm.Key),
|
||||
Chapters = _context.Chapter.Where(c => _context.AppUserProgresses
|
||||
.Where(u => u.AppUserId == sm.Key)
|
||||
.Where(p => p.PagesRead > 0)
|
||||
.Where(p => days == 0 || (p.Created >= minDate && p.LastModified >= minDate))
|
||||
.Select(p => p.ChapterId)
|
||||
.Distinct()
|
||||
.Contains(c.Id))
|
||||
})
|
||||
.OrderByDescending(d => d.Chapters.Sum(c => c.AvgHoursToRead))
|
||||
.Take(5)
|
||||
.ToList();
|
||||
|
||||
|
||||
// Need a mapping of Library to chapter ids
|
||||
var chapterIdWithLibraryId = topUsersAndReadChapters
|
||||
.SelectMany(u => u.Chapters
|
||||
.Select(c => c.Id)).Select(d => new
|
||||
{
|
||||
LibraryId = _context.Chapter.Where(c => c.Id == d).AsSplitQuery().Select(c => c.Volume).Select(v => v.Series).Select(s => s.LibraryId).Single(),
|
||||
ChapterId = d
|
||||
})
|
||||
.ToList();
|
||||
|
||||
var chapterLibLookup = new Dictionary<int, int>();
|
||||
foreach (var cl in chapterIdWithLibraryId)
|
||||
{
|
||||
if (chapterLibLookup.ContainsKey(cl.ChapterId)) continue;
|
||||
chapterLibLookup.Add(cl.ChapterId, cl.LibraryId);
|
||||
}
|
||||
|
||||
var user = new Dictionary<int, Dictionary<LibraryType, long>>();
|
||||
foreach (var userChapter in topUsersAndReadChapters)
|
||||
{
|
||||
if (!user.ContainsKey(userChapter.User.Id)) user.Add(userChapter.User.Id, new Dictionary<LibraryType, long>());
|
||||
var libraryTimes = user[userChapter.User.Id];
|
||||
|
||||
foreach (var chapter in userChapter.Chapters)
|
||||
{
|
||||
var library = libraries.First(l => l.Id == chapterLibLookup[chapter.Id]);
|
||||
if (!libraryTimes.ContainsKey(library.Type)) libraryTimes.Add(library.Type, 0L);
|
||||
var existingHours = libraryTimes[library.Type];
|
||||
libraryTimes[library.Type] = existingHours + chapter.AvgHoursToRead;
|
||||
}
|
||||
|
||||
user[userChapter.User.Id] = libraryTimes;
|
||||
}
|
||||
|
||||
var ret = new List<TopReadDto>();
|
||||
foreach (var userId in user.Keys)
|
||||
{
|
||||
ret.Add(new TopReadDto()
|
||||
{
|
||||
UserId = userId,
|
||||
Username = users.First(u => u.Id == userId).UserName,
|
||||
BooksTime = user[userId].ContainsKey(LibraryType.Book) ? user[userId][LibraryType.Book] : 0,
|
||||
ComicsTime = user[userId].ContainsKey(LibraryType.Comic) ? user[userId][LibraryType.Comic] : 0,
|
||||
MangaTime = user[userId].ContainsKey(LibraryType.Manga) ? user[userId][LibraryType.Manga] : 0,
|
||||
});
|
||||
}
|
||||
|
||||
return ret;
|
||||
}
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue