Better Themes, Stats, and bugfixes (#1740)

* Fixed a bug where when clicking on a series rating for first time, the rating wasn't populating in the modal.

* Fixed a bug on Scroll mode with immersive mode, the bottom bar could clip with the book body.

* Cleanup some uses of var

* Refactored text as json into a type so I don't have to copy/paste everywhere

* Theme styles now override the defaults and theme owners no longer need to maintain all the variables themselves.

Themes can now override the color of the header on mobile devices via --theme-color and Kavita will now update both theme color as well as color scheme.

* Fixed a bug where last active on user stats wasn't for the particular user.

* Added a more accurate word count calculation and the ability to see the word counts year over year.

* Added a new table for long term statistics, like number of files over the years. No views are present for this data, I will add them later.

API/Controllers/StatsController.cs

@@ -41,7 +41,7 @@ public class StatsController : BaseApiController
     [Authorize("RequireAdminRole")]
     [HttpGet("server/stats")]
     [ResponseCache(CacheProfileName = "Statistics")]
-    public async Task<ActionResult<ServerStatistics>> GetHighLevelStats()
+    public async Task<ActionResult<ServerStatisticsDto>> GetHighLevelStats()
     {
         return Ok(await _statService.GetServerStatistics());
     }
@@ -141,4 +141,34 @@ public class StatsController : BaseApiController
         return Ok(await _statService.GetReadingHistory(userId));
     }
 
+    /// <summary>
+    /// Returns a count of pages read per year for a given userId.
+    /// </summary>
+    /// <param name="userId">If userId is 0 and user is not an admin, API will default to userId</param>
+    /// <returns></returns>
+    [HttpGet("pages-per-year")]
+    [ResponseCache(CacheProfileName = "Statistics")]
+    public async Task<ActionResult<IEnumerable<StatCount<int>>>> GetPagesReadPerYear(int userId = 0)
+    {
+        var isAdmin = User.IsInRole(PolicyConstants.AdminRole);
+        if (!isAdmin) userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
+        return Ok(_statService.GetPagesReadCountByYear(userId));
+    }
+
+    /// <summary>
+    /// Returns a count of words read per year for a given userId.
+    /// </summary>
+    /// <param name="userId">If userId is 0 and user is not an admin, API will default to userId</param>
+    /// <returns></returns>
+    [HttpGet("words-per-year")]
+    [ResponseCache(CacheProfileName = "Statistics")]
+    public async Task<ActionResult<IEnumerable<StatCount<int>>>> GetWordsReadPerYear(int userId = 0)
+    {
+        var isAdmin = User.IsInRole(PolicyConstants.AdminRole);
+        if (!isAdmin) userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
+        return Ok(_statService.GetWordsReadCountByYear(userId));
+    }
+
+
+
 }

API/Controllers/WantToReadController.cs

@@ -42,7 +42,7 @@ public class WantToReadController : BaseApiController
     }
 
     [HttpGet]
-    public async Task<ActionResult<bool>> GetWantToRead([FromQuery] int seriesId)
+    public async Task<ActionResult<bool>> IsSeriesInWantToRead([FromQuery] int seriesId)
     {
         var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername());
         return Ok(await _unitOfWork.SeriesRepository.IsSeriesInWantToRead(user.Id, seriesId));

API/DTOs/Statistics/Count.cs

@@ -3,5 +3,5 @@
 public class StatCount<T> : ICount<T>
 {
     public T Value { get; set; }
-    public int Count { get; set; }
+    public long Count { get; set; }
 }

API/DTOs/Statistics/ICount.cs

@@ -3,5 +3,5 @@
 public interface ICount<T>
 {
     public T Value { get; set; }
-    public int Count { get; set; }
+    public long Count { get; set; }
 }

API/DTOs/Statistics/PagesReadOnADayCount.cs

@@ -12,10 +12,9 @@ public class PagesReadOnADayCount<T> : ICount<T>
     /// <summary>
     /// Number of pages read
     /// </summary>
-    public int Count { get; set; }
+    public long Count { get; set; }
     /// <summary>
     /// Format of those files
     /// </summary>
     public MangaFormat Format { get; set; }
-
 }

API/DTOs/Statistics/ServerStatisticsDto.cs

@@ -3,7 +3,7 @@ using System.Collections.Generic;
 
 namespace API.DTOs.Statistics;
 
-public class ServerStatistics
+public class ServerStatisticsDto
 {
     public long ChapterCount { get; set; }
     public long VolumeCount { get; set; }

API/Data/DataContext.cs

@@ -45,6 +45,7 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
     public DbSet<SeriesRelation> SeriesRelation { get; set; }
     public DbSet<FolderPath> FolderPath { get; set; }
     public DbSet<Device> Device { get; set; }
+    public DbSet<ServerStatistics> ServerStatistics { get; set; }
 
 
     protected override void OnModelCreating(ModelBuilder builder)

API/Data/Migrations/20230111014852_YearlyStats.cs

@@ -0,0 +1,39 @@
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace API.Data.Migrations
+{
+    public partial class YearlyStats : Migration
+    {
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.CreateTable(
+                name: "ServerStatistics",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "INTEGER", nullable: false)
+                        .Annotation("Sqlite:Autoincrement", true),
+                    Year = table.Column<int>(type: "INTEGER", nullable: false),
+                    SeriesCount = table.Column<long>(type: "INTEGER", nullable: false),
+                    VolumeCount = table.Column<long>(type: "INTEGER", nullable: false),
+                    ChapterCount = table.Column<long>(type: "INTEGER", nullable: false),
+                    FileCount = table.Column<long>(type: "INTEGER", nullable: false),
+                    UserCount = table.Column<long>(type: "INTEGER", nullable: false),
+                    GenreCount = table.Column<long>(type: "INTEGER", nullable: false),
+                    PersonCount = table.Column<long>(type: "INTEGER", nullable: false),
+                    TagCount = table.Column<long>(type: "INTEGER", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_ServerStatistics", x => x.Id);
+                });
+        }
+
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "ServerStatistics");
+        }
+    }
+}

API/Data/Migrations/DataContextModelSnapshot.cs

@@ -949,6 +949,44 @@ namespace API.Data.Migrations
                     b.ToTable("ServerSetting");
                 });
 
+            modelBuilder.Entity("API.Entities.ServerStatistics", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER");
+
+                    b.Property<long>("ChapterCount")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<long>("FileCount")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<long>("GenreCount")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<long>("PersonCount")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<long>("SeriesCount")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<long>("TagCount")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<long>("UserCount")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<long>("VolumeCount")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<int>("Year")
+                        .HasColumnType("INTEGER");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("ServerStatistics");
+                });
+
             modelBuilder.Entity("API.Entities.SiteTheme", b =>
                 {
                     b.Property<int>("Id")

API/Entities/ServerStatistics.cs

@@ -0,0 +1,15 @@
+namespace API.Entities;
+
+public class ServerStatistics
+{
+    public int Id { get; set; }
+    public int Year { get; set; }
+    public long SeriesCount { get; set; }
+    public long VolumeCount { get; set; }
+    public long ChapterCount { get; set; }
+    public long FileCount { get; set; }
+    public long UserCount { get; set; }
+    public long GenreCount { get; set; }
+    public long PersonCount { get; set; }
+    public long TagCount { get; set; }
+}

API/Services/StatisticService.cs

@@ -6,6 +6,7 @@ using System.Threading.Tasks;
 using API.Data;
 using API.DTOs;
 using API.DTOs.Statistics;
+using API.Entities;
 using API.Entities.Enums;
 using API.Extensions;
 using AutoMapper;
@@ -17,7 +18,7 @@ namespace API.Services;
 
 public interface IStatisticService
 {
-    Task<ServerStatistics> GetServerStatistics();
+    Task<ServerStatisticsDto> GetServerStatistics();
     Task<UserReadStatistics> GetUserReadStatistics(int userId, IList<int> libraryIds);
     Task<IEnumerable<StatCount<int>>> GetYearCount();
     Task<IEnumerable<StatCount<int>>> GetTopYears();
@@ -28,6 +29,9 @@ public interface IStatisticService
     Task<IEnumerable<ReadHistoryEvent>> GetReadingHistory(int userId);
     Task<IEnumerable<PagesReadOnADayCount<DateTime>>> ReadCountByDay(int userId = 0, int days = 0);
     IEnumerable<StatCount<DayOfWeek>> GetDayBreakdown();
+    IEnumerable<StatCount<int>> GetPagesReadCountByYear(int userId = 0);
+    IEnumerable<StatCount<int>> GetWordsReadCountByYear(int userId = 0);
+    Task UpdateServerStatistics();
 }
 
 /// <summary>
@@ -71,9 +75,12 @@ public class StatisticService : IStatisticService
             .Where(c => chapterIds.Contains(c.Id))
             .SumAsync(c => c.AvgHoursToRead);
 
-        var totalWordsRead = await _context.Chapter
-            .Where(c => chapterIds.Contains(c.Id))
-            .SumAsync(c => c.WordCount);
+        var totalWordsRead =  (long) Math.Round(await _context.AppUserProgresses
+            .Where(p => p.AppUserId == userId)
+            .Where(p => libraryIds.Contains(p.LibraryId))
+            .Join(_context.Chapter, p => p.ChapterId, c => c.Id, (progress, chapter) => new {chapter, progress})
+            .Where(p => p.chapter.WordCount > 0)
+            .SumAsync(p => p.chapter.WordCount * (p.progress.PagesRead / (1.0f * p.chapter.Pages))));
 
         var chaptersRead = await _context.AppUserProgresses
             .Where(p => p.AppUserId == userId)
@@ -83,10 +90,10 @@ public class StatisticService : IStatisticService
 
         var lastActive = await _context.AppUserProgresses
             .OrderByDescending(p => p.LastModified)
+            .Where(p => p.AppUserId == userId)
             .Select(p => p.LastModified)
             .FirstOrDefaultAsync();
 
-        // Reading Progress by Library Name
 
         // First get the total pages per library
         var totalPageCountByLibrary = _context.Chapter
@@ -190,7 +197,7 @@ public class StatisticService : IStatisticService
     }
 
 
-    public async Task<ServerStatistics> GetServerStatistics()
+    public async Task<ServerStatisticsDto> GetServerStatistics()
     {
         var mostActiveUsers = _context.AppUserProgresses
             .AsSplitQuery()
@@ -268,7 +275,7 @@ public class StatisticService : IStatisticService
             .Distinct()
             .Count();
 
-        return new ServerStatistics()
+        return new ServerStatisticsDto()
         {
             ChapterCount = await _context.Chapter.CountAsync(),
             SeriesCount = await _context.Series.CountAsync(),
@@ -397,6 +404,85 @@ public class StatisticService : IStatisticService
             .AsEnumerable();
     }
 
+    /// <summary>
+    /// Return a list of years for the given userId
+    /// </summary>
+    /// <param name="userId"></param>
+    /// <returns></returns>
+    public IEnumerable<StatCount<int>> GetPagesReadCountByYear(int userId = 0)
+    {
+        var query = _context.AppUserProgresses
+            .AsSplitQuery()
+            .AsNoTracking();
+
+        if (userId > 0)
+        {
+            query = query.Where(p => p.AppUserId == userId);
+        }
+
+        return query.GroupBy(p => p.LastModified.Year)
+            .OrderBy(g => g.Key)
+            .Select(g => new StatCount<int> {Value = g.Key, Count = g.Sum(x => x.PagesRead)})
+            .AsEnumerable();
+    }
+
+    public IEnumerable<StatCount<int>> GetWordsReadCountByYear(int userId = 0)
+    {
+        var query = _context.AppUserProgresses
+            .AsSplitQuery()
+            .AsNoTracking();
+
+        if (userId > 0)
+        {
+            query = query.Where(p => p.AppUserId == userId);
+        }
+
+        return query
+            .Join(_context.Chapter, p => p.ChapterId, c => c.Id, (progress, chapter) => new {chapter, progress})
+            .Where(p => p.chapter.WordCount > 0)
+            .GroupBy(p => p.progress.LastModified.Year)
+            .Select(g => new StatCount<int>{
+                Value = g.Key,
+                Count = (long) Math.Round(g.Sum(p => p.chapter.WordCount * ((1.0f * p.progress.PagesRead) / p.chapter.Pages)))
+            })
+            .AsEnumerable();
+    }
+
+    /// <summary>
+    /// Updates the ServerStatistics table for the current year
+    /// </summary>
+    /// <remarks>This commits</remarks>
+    /// <returns></returns>
+    public async Task UpdateServerStatistics()
+    {
+        var year = DateTime.Today.Year;
+
+        var existingRecord = await _context.ServerStatistics.SingleOrDefaultAsync(s => s.Year == year) ?? new ServerStatistics();
+
+        existingRecord.Year = year;
+        existingRecord.ChapterCount = await _context.Chapter.CountAsync();
+        existingRecord.VolumeCount = await _context.Volume.CountAsync();
+        existingRecord.FileCount = await _context.MangaFile.CountAsync();
+        existingRecord.SeriesCount = await _context.Series.CountAsync();
+        existingRecord.UserCount = await _context.Users.CountAsync();
+        existingRecord.GenreCount = await _context.Genre.CountAsync();
+        existingRecord.TagCount = await _context.Tag.CountAsync();
+        existingRecord.PersonCount =  _context.Person
+            .AsSplitQuery()
+            .AsEnumerable()
+            .GroupBy(sm => sm.NormalizedName)
+            .Select(sm => sm.Key)
+            .Distinct()
+            .Count();
+
+        _context.ServerStatistics.Attach(existingRecord);
+        if (existingRecord.Id > 0)
+        {
+            _context.Entry(existingRecord).State = EntityState.Modified;
+        }
+        await _unitOfWork.CommitAsync();
+    }
+
     public async Task<IEnumerable<TopReadDto>> GetTopUsers(int days)
     {
         var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();
@@ -434,9 +520,8 @@ public class StatisticService : IStatisticService
             .ToList();
 
         var chapterLibLookup = new Dictionary<int, int>();
-        foreach (var cl in chapterIdWithLibraryId)
+        foreach (var cl in chapterIdWithLibraryId.Where(cl => !chapterLibLookup.ContainsKey(cl.ChapterId)))
         {
-            if (chapterLibLookup.ContainsKey(cl.ChapterId)) continue;
             chapterLibLookup.Add(cl.ChapterId, cl.LibraryId);
         }
 
@@ -457,19 +542,14 @@ public class StatisticService : IStatisticService
             user[userChapter.User.Id] = libraryTimes;
         }
 
-        var ret = new List<TopReadDto>();
-        foreach (var userId in user.Keys)
-        {
-            ret.Add(new TopReadDto()
+        return user.Keys.Select(userId => new TopReadDto()
             {
                 UserId = userId,
                 Username = users.First(u => u.Id == userId).UserName,
                 BooksTime = user[userId].ContainsKey(LibraryType.Book) ? user[userId][LibraryType.Book] : 0,
                 ComicsTime = user[userId].ContainsKey(LibraryType.Comic) ? user[userId][LibraryType.Comic] : 0,
                 MangaTime = user[userId].ContainsKey(LibraryType.Manga) ? user[userId][LibraryType.Manga] : 0,
-            });
-        }
-
-        return ret;
+            })
+            .ToList();
     }
 }

API/Services/TaskScheduler.cs

@@ -46,11 +46,13 @@ public class TaskScheduler : ITaskScheduler
     private readonly IVersionUpdaterService _versionUpdaterService;
     private readonly IThemeService _themeService;
     private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
+    private readonly IStatisticService _statisticService;
 
     public static BackgroundJobServer Client => new BackgroundJobServer();
     public const string ScanQueue = "scan";
     public const string DefaultQueue = "default";
     public const string RemoveFromWantToReadTaskId = "remove-from-want-to-read";
+    public const string UpdateYearlyStatsTaskId = "update-yearly-stats";
     public const string CleanupDbTaskId = "cleanup-db";
     public const string CleanupTaskId = "cleanup";
     public const string BackupTaskId = "backup";
@@ -65,7 +67,7 @@ public class TaskScheduler : ITaskScheduler
     public TaskScheduler(ICacheService cacheService, ILogger<TaskScheduler> logger, IScannerService scannerService,
         IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService,
         ICleanupService cleanupService, IStatsService statsService, IVersionUpdaterService versionUpdaterService,
-        IThemeService themeService, IWordCountAnalyzerService wordCountAnalyzerService)
+        IThemeService themeService, IWordCountAnalyzerService wordCountAnalyzerService, IStatisticService statisticService)
     {
         _cacheService = cacheService;
         _logger = logger;
@@ -78,6 +80,7 @@ public class TaskScheduler : ITaskScheduler
         _versionUpdaterService = versionUpdaterService;
         _themeService = themeService;
         _wordCountAnalyzerService = wordCountAnalyzerService;
+        _statisticService = statisticService;
     }
 
     public async Task ScheduleTasks()
@@ -111,6 +114,7 @@ public class TaskScheduler : ITaskScheduler
         RecurringJob.AddOrUpdate(CleanupTaskId, () => _cleanupService.Cleanup(), Cron.Daily, TimeZoneInfo.Local);
         RecurringJob.AddOrUpdate(CleanupDbTaskId, () => _cleanupService.CleanupDbEntries(), Cron.Daily, TimeZoneInfo.Local);
         RecurringJob.AddOrUpdate(RemoveFromWantToReadTaskId, () => _cleanupService.CleanupWantToRead(), Cron.Daily, TimeZoneInfo.Local);
+        RecurringJob.AddOrUpdate(UpdateYearlyStatsTaskId, () => _statisticService.UpdateServerStatistics(), Cron.Monthly, TimeZoneInfo.Local);
     }
 
     #region StatsTasks