Basic Stats (#1673)

* Refactored ResponseCache profiles into consts

* Refactored code to use an extension method for getting user library ids.

* Started server statistics, added a charting library, and added a table sort column (not finished)

* Refactored code and have a fully working example of sortable headers. Still doesn't work with default sorting state, will work on that later.

* Implemented file size, but it's too expensive, so commented out.

* Added a migration to provide extension and length/size information in the DB to allow for faster stat apis.

* Added the ability to force a library scan from library settings.

* Refactored some apis to provide more of a file breakdown rather than just file size.

* Working on visualization of file breakdown

* Fixed the file breakdown visual

* Fixed up 2 visualizations

* Added back an api for member names, started work on top reads

* Hooked up the other library types and username/days.

* Preparing to remove top reads and refactor into Top users

* Added LibraryId to AppUserProgress to help with complex lookups.

* Added the new libraryId hook into some stats methods

* Updated api methods to use libraryId for progress

* More places where LibraryId is needed

* Added some high level server stats

* Got a ton done on server stats

* Updated default theme (dark) to be the default root variables. This will allow user themes to override just what they want, rather than maintain their own css variables.

* Implemented a monster query for top users by reading time. It's very slow and can be cleaned up likely.

* Hooked up top reads. Code needs a big refactor. Handing off for Robbie treatment and I'll switch to User stats.

* Implemented last 5 recently read series (broken) and added some basic css

* Fixed recently read query

* Cleanup the css a bit, Robbie we need you

* More css love

* Cleaned up DTOs that aren't needed anymore

* Fixed top readers query

* When calculating top readers, don't include read events where nothing is read (0 pages)

* Hooked up the date into GetTopUsers

* Hooked top readers up with days and refactored and cleaned up componets not used

* Fixed up query

* Started on a day by day breakdown, but going to take a break from stats.

* Added a temp task to run some migration manually for stats to work

* Ensure OPDS-PS uses new libraryId for progress reporting

* Fixed a code smell

* Adding some styling

* adding more styles

* Removed some debug stuff from user stats

* Bump qs from 6.5.2 to 6.5.3 in /UI/Web

Bumps [qs](https://github.com/ljharb/qs) from 6.5.2 to 6.5.3.
- [Release notes](https://github.com/ljharb/qs/releases)
- [Changelog](https://github.com/ljharb/qs/blob/main/CHANGELOG.md)
- [Commits](https://github.com/ljharb/qs/compare/v6.5.2...v6.5.3)

---
updated-dependencies:
- dependency-name: qs
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

* Tweaked some code for bad data cases

* Refactored a chapter lookup to remove un-needed Volume join in 5 places across the code.

* API push

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>

API/Services/ReaderService.cs

@@ -103,6 +103,7 @@ public class ReaderService : IReaderService
     public async Task MarkChaptersAsRead(AppUser user, int seriesId, IList<Chapter> chapters)
     {
         var seenVolume = new Dictionary<int, bool>();
+        var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId);
         foreach (var chapter in chapters)
         {
             var userProgress = GetUserProgressForChapter(user, chapter);
@@ -114,7 +115,8 @@ public class ReaderService : IReaderService
                     PagesRead = chapter.Pages,
                     VolumeId = chapter.VolumeId,
                     SeriesId = seriesId,
-                    ChapterId = chapter.Id
+                    ChapterId = chapter.Id,
+                    LibraryId = series.LibraryId
                 });
             }
             else
@@ -239,6 +241,7 @@ public class ReaderService : IReaderService
                     VolumeId = progressDto.VolumeId,
                     SeriesId = progressDto.SeriesId,
                     ChapterId = progressDto.ChapterId,
+                    LibraryId = progressDto.LibraryId,
                     BookScrollId = progressDto.BookScrollId,
                     LastModified = DateTime.Now
                 });
@@ -249,6 +252,7 @@ public class ReaderService : IReaderService
                 userProgress.PagesRead = progressDto.PageNum;
                 userProgress.SeriesId = progressDto.SeriesId;
                 userProgress.VolumeId = progressDto.VolumeId;
+                userProgress.LibraryId = progressDto.LibraryId;
                 userProgress.BookScrollId = progressDto.BookScrollId;
                 userProgress.LastModified = DateTime.Now;
                 _unitOfWork.AppUserProgressRepository.Update(userProgress);

API/Services/ReadingListService.cs

@@ -195,7 +195,7 @@ public class ReadingListService : IReadingListService
         }
 
         var existingChapterExists = readingList.Items.Select(rli => rli.ChapterId).ToHashSet();
-        var chaptersForSeries = (await _unitOfWork.ChapterRepository.GetChaptersByIdsAsync(chapterIds))
+        var chaptersForSeries = (await _unitOfWork.ChapterRepository.GetChaptersByIdsAsync(chapterIds, ChapterIncludes.Volumes))
             .OrderBy(c => Tasks.Scanner.Parser.Parser.MinNumberFromRange(c.Volume.Name))
             .ThenBy(x => double.Parse(x.Number), _chapterSortComparerForInChapterSorting)
             .ToList();

API/Services/StatisticService.cs

@@ -0,0 +1,417 @@
+using System;
+using System.Collections.Generic;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using API.Data;
+using API.DTOs;
+using API.DTOs.Statistics;
+using API.Entities.Enums;
+using API.Extensions;
+using AutoMapper;
+using AutoMapper.QueryableExtensions;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Services;
+
+public interface IStatisticService
+{
+    Task<ServerStatistics> GetServerStatistics();
+    Task<UserReadStatistics> GetUserReadStatistics(int userId, IList<int> libraryIds);
+    Task<IEnumerable<StatCount<int>>> GetYearCount();
+    Task<IEnumerable<StatCount<int>>> GetTopYears();
+    Task<IEnumerable<StatCount<PublicationStatus>>> GetPublicationCount();
+    Task<IEnumerable<StatCount<MangaFormat>>> GetMangaFormatCount();
+    Task<FileExtensionBreakdownDto> GetFileBreakdown();
+    Task<IEnumerable<TopReadDto>> GetTopUsers(int days);
+    Task<IEnumerable<ReadHistoryEvent>> GetReadingHistory(int userId);
+    Task<IEnumerable<ReadHistoryEvent>> GetHistory();
+}
+
+/// <summary>
+/// Responsible for computing statistics for the server
+/// </summary>
+/// <remarks>This performs raw queries and does not use a repository</remarks>
+public class StatisticService : IStatisticService
+{
+    private readonly DataContext _context;
+    private readonly IMapper _mapper;
+    private readonly IUnitOfWork _unitOfWork;
+
+    public StatisticService(DataContext context, IMapper mapper, IUnitOfWork unitOfWork)
+    {
+        _context = context;
+        _mapper = mapper;
+        _unitOfWork = unitOfWork;
+    }
+
+    public async Task<UserReadStatistics> GetUserReadStatistics(int userId, IList<int> libraryIds)
+    {
+        if (libraryIds.Count == 0)
+            libraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync();
+
+
+        // Total Pages Read
+        var totalPagesRead = await _context.AppUserProgresses
+            .Where(p => p.AppUserId == userId)
+            .Where(p => libraryIds.Contains(p.LibraryId))
+            .SumAsync(p => p.PagesRead);
+
+        var ids = await _context.AppUserProgresses
+            .Where(p => p.AppUserId == userId)
+            .Where(p => libraryIds.Contains(p.LibraryId))
+            .Where(p => p.PagesRead > 0)
+            .Select(p => new {p.ChapterId, p.SeriesId})
+            .ToListAsync();
+
+        var chapterIds = ids.Select(id => id.ChapterId);
+
+        var timeSpentReading = await _context.Chapter
+            .Where(c => chapterIds.Contains(c.Id))
+            .SumAsync(c => c.AvgHoursToRead);
+
+        // Maybe make this top 5 genres? But usually there are 3-5 genres that are always common...
+        // Maybe use rating to calculate top genres?
+        // var genres = await _context.Series
+        //     .Where(s => seriesIds.Contains(s.Id))
+        //     .Select(s => s.Metadata)
+        //     .SelectMany(sm => sm.Genres)
+        //     //.DistinctBy(g => g.NormalizedTitle)
+        //     .ToListAsync();
+
+        // How many series of each format have you read? (Epub, Archive, etc)
+
+        // Percentage of libraries read. For each library, get the total pages vs read
+        //var allLibraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync();
+
+        var chaptersRead = await _context.AppUserProgresses
+            .Where(p => p.AppUserId == userId)
+            .Where(p => libraryIds.Contains(p.LibraryId))
+            .Where(p => p.PagesRead >= _context.Chapter.Single(c => c.Id == p.ChapterId).Pages)
+            .CountAsync();
+
+        var lastActive = await _context.AppUserProgresses
+            .OrderByDescending(p => p.LastModified)
+            .Select(p => p.LastModified)
+            .FirstOrDefaultAsync();
+
+        //var
+
+        return new UserReadStatistics()
+        {
+            TotalPagesRead = totalPagesRead,
+            TimeSpentReading = timeSpentReading,
+            ChaptersRead = chaptersRead,
+            LastActive = lastActive,
+        };
+    }
+
+    /// <summary>
+    /// Returns the Release Years and their count
+    /// </summary>
+    /// <returns></returns>
+    public async Task<IEnumerable<StatCount<int>>> GetYearCount()
+    {
+        return await _context.SeriesMetadata
+            .Where(sm => sm.ReleaseYear != 0)
+            .AsSplitQuery()
+            .GroupBy(sm => sm.ReleaseYear)
+            .Select(sm => new StatCount<int>
+            {
+                Value = sm.Key,
+                Count = _context.SeriesMetadata.Where(sm2 => sm2.ReleaseYear == sm.Key).Distinct().Count()
+            })
+            .OrderByDescending(d => d.Value)
+            .ToListAsync();
+    }
+
+    public async Task<IEnumerable<StatCount<int>>> GetTopYears()
+    {
+        return await _context.SeriesMetadata
+            .Where(sm => sm.ReleaseYear != 0)
+            .AsSplitQuery()
+            .GroupBy(sm => sm.ReleaseYear)
+            .Select(sm => new StatCount<int>
+            {
+                Value = sm.Key,
+                Count = _context.SeriesMetadata.Where(sm2 => sm2.ReleaseYear == sm.Key).Distinct().Count()
+            })
+            .OrderByDescending(d => d.Count)
+            .Take(5)
+            .ToListAsync();
+    }
+
+
+
+    public async Task<IEnumerable<StatCount<PublicationStatus>>> GetPublicationCount()
+    {
+        return await _context.SeriesMetadata
+            .AsSplitQuery()
+            .GroupBy(sm => sm.PublicationStatus)
+            .Select(sm => new StatCount<PublicationStatus>
+            {
+                Value = sm.Key,
+                Count = _context.SeriesMetadata.Where(sm2 => sm2.PublicationStatus == sm.Key).Distinct().Count()
+            })
+            .ToListAsync();
+    }
+
+    public async Task<IEnumerable<StatCount<MangaFormat>>> GetMangaFormatCount()
+    {
+        return await _context.MangaFile
+            .AsSplitQuery()
+            .GroupBy(sm => sm.Format)
+            .Select(mf => new StatCount<MangaFormat>
+            {
+                Value = mf.Key,
+                Count = _context.MangaFile.Where(mf2 => mf2.Format == mf.Key).Distinct().Count()
+            })
+            .ToListAsync();
+    }
+
+
+    public async Task<ServerStatistics> GetServerStatistics()
+    {
+        var mostActiveUsers = _context.AppUserProgresses
+            .AsSplitQuery()
+            .AsEnumerable()
+            .GroupBy(sm => sm.AppUserId)
+            .Select(sm => new StatCount<UserDto>
+            {
+                Value = _context.AppUser.Where(u => u.Id == sm.Key).ProjectTo<UserDto>(_mapper.ConfigurationProvider)
+                    .Single(),
+                Count = _context.AppUserProgresses.Where(u => u.AppUserId == sm.Key).Distinct().Count()
+            })
+            .OrderByDescending(d => d.Count)
+            .Take(5);
+
+        var mostActiveLibrary = _context.AppUserProgresses
+            .AsSplitQuery()
+            .AsEnumerable()
+            .Where(sm => sm.LibraryId > 0)
+            .GroupBy(sm => sm.LibraryId)
+            .Select(sm => new StatCount<LibraryDto>
+            {
+                Value = _context.Library.Where(u => u.Id == sm.Key).ProjectTo<LibraryDto>(_mapper.ConfigurationProvider)
+                    .Single(),
+                Count = _context.AppUserProgresses.Where(u => u.LibraryId == sm.Key).Distinct().Count()
+            })
+            .OrderByDescending(d => d.Count)
+            .Take(5);
+
+        var mostPopularSeries = _context.AppUserProgresses
+            .AsSplitQuery()
+            .AsEnumerable()
+            .GroupBy(sm => sm.SeriesId)
+            .Select(sm => new StatCount<SeriesDto>
+            {
+                Value = _context.Series.Where(u => u.Id == sm.Key).ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
+                    .Single(),
+                Count = _context.AppUserProgresses.Where(u => u.SeriesId == sm.Key).Distinct().Count()
+            })
+            .OrderByDescending(d => d.Count)
+            .Take(5);
+
+        var mostReadSeries = _context.AppUserProgresses
+            .AsSplitQuery()
+            .AsEnumerable()
+            .GroupBy(sm => sm.SeriesId)
+            .Select(sm => new StatCount<SeriesDto>
+            {
+                Value = _context.Series.Where(u => u.Id == sm.Key).ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
+                    .Single(),
+                Count = _context.AppUserProgresses.Where(u => u.SeriesId == sm.Key).AsEnumerable().DistinctBy(p => p.AppUserId).Count()
+            })
+            .OrderByDescending(d => d.Count)
+            .Take(5);
+
+        var seriesIds = (await _context.AppUserProgresses
+            .AsSplitQuery()
+            .OrderByDescending(d => d.LastModified)
+            .Select(d => d.SeriesId)
+            .ToListAsync())
+            .Distinct()
+            .Take(5);
+
+        var recentlyRead = _context.Series
+            .AsSplitQuery()
+            .Where(s => seriesIds.Contains(s.Id))
+            .ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
+            .AsEnumerable();
+
+        return new ServerStatistics()
+        {
+            ChapterCount = await _context.Chapter.CountAsync(),
+            SeriesCount = await _context.Series.CountAsync(),
+            TotalFiles = await _context.MangaFile.CountAsync(),
+            TotalGenres = await _context.Genre.CountAsync(),
+            TotalPeople = await _context.Person.CountAsync(),
+            TotalSize = await _context.MangaFile.SumAsync(m => m.Bytes),
+            TotalTags = await _context.Tag.CountAsync(),
+            VolumeCount = await _context.Volume.Where(v => v.Number != 0).CountAsync(),
+            MostActiveUsers = mostActiveUsers,
+            MostActiveLibraries = mostActiveLibrary,
+            MostPopularSeries = mostPopularSeries,
+            MostReadSeries = mostReadSeries,
+            RecentlyRead = recentlyRead
+        };
+    }
+
+    public async Task<FileExtensionBreakdownDto> GetFileBreakdown()
+    {
+        return new FileExtensionBreakdownDto()
+        {
+            FileBreakdown = await _context.MangaFile
+                .AsSplitQuery()
+                .AsNoTracking()
+                .GroupBy(sm => sm.Extension)
+                .Select(mf => new FileExtensionDto()
+                {
+                    Extension = mf.Key,
+                    Format =_context.MangaFile.Where(mf2 => mf2.Extension == mf.Key).Select(mf2 => mf2.Format).Single(),
+                    TotalSize = _context.MangaFile.Where(mf2 => mf2.Extension == mf.Key).Distinct().Sum(mf2 => mf2.Bytes),
+                    TotalFiles = _context.MangaFile.Where(mf2 => mf2.Extension == mf.Key).Distinct().Count()
+                })
+                .ToListAsync(),
+            TotalFileSize = await _context.MangaFile
+                .AsNoTracking()
+                .AsSplitQuery()
+                .SumAsync(f => f.Bytes)
+        };
+    }
+
+    public async Task<IEnumerable<ReadHistoryEvent>> GetReadingHistory(int userId)
+    {
+        return await _context.AppUserProgresses
+            .Where(u => u.AppUserId == userId)
+            .AsNoTracking()
+            .AsSplitQuery()
+            .Select(u => new ReadHistoryEvent
+            {
+                UserId = u.AppUserId,
+                UserName = _context.AppUser.Single(u => u.Id == userId).UserName,
+                SeriesName = _context.Series.Single(s => s.Id == u.SeriesId).Name,
+                SeriesId = u.SeriesId,
+                LibraryId = u.LibraryId,
+                ReadDate = u.LastModified,
+                ChapterId = u.ChapterId,
+                ChapterNumber = _context.Chapter.Single(c => c.Id == u.ChapterId).Number
+            })
+            .OrderByDescending(d => d.ReadDate)
+            .ToListAsync();
+    }
+
+    public Task<IEnumerable<ReadHistoryEvent>> GetHistory()
+    {
+        // _context.AppUserProgresses
+        //     .AsSplitQuery()
+        //     .AsEnumerable()
+        //     .GroupBy(sm => sm.LastModified)
+        //     .Select(sm => new
+        //     {
+        //         User = _context.AppUser.Single(u => u.Id == sm.Key),
+        //         Chapters = _context.Chapter.Where(c => _context.AppUserProgresses
+        //             .Where(u => u.AppUserId == sm.Key)
+        //             .Where(p => p.PagesRead > 0)
+        //             .Select(p => p.ChapterId)
+        //             .Distinct()
+        //             .Contains(c.Id))
+        //     })
+        //     .OrderByDescending(d => d.Chapters.Sum(c => c.AvgHoursToRead))
+        //     .Take(5)
+        //     .ToList();
+
+        var firstOfWeek = DateTime.Now.StartOfWeek(DayOfWeek.Monday);
+        var groupedReadingDays = _context.AppUserProgresses
+            .Where(x => x.LastModified >= firstOfWeek)
+            .GroupBy(x => x.LastModified.Day)
+            .Select(g => new StatCount<int>()
+            {
+                Value = g.Key,
+                Count = _context.AppUserProgresses.Where(p => p.LastModified.Day == g.Key).Select(p => p.ChapterId).Distinct().Count()
+            })
+            .AsEnumerable();
+
+        // var records = firstOfWeek.Range(7)
+        //     .GroupJoin(groupedReadingDays, wd => wd.Day, lg => lg.Key, (_, lg) => lg.Any() ? lg.First().Count() : 0).ToArray();
+        return Task.FromResult<IEnumerable<ReadHistoryEvent>>(null);
+    }
+
+
+    public async Task<IEnumerable<TopReadDto>> GetTopUsers(int days)
+    {
+        var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();
+        var users = (await _unitOfWork.UserRepository.GetAllUsersAsync()).ToList();
+        var minDate = DateTime.Now.Subtract(TimeSpan.FromDays(days));
+
+        var topUsersAndReadChapters = _context.AppUserProgresses
+            .AsSplitQuery()
+            .AsEnumerable()
+            .GroupBy(sm => sm.AppUserId)
+            .Select(sm => new
+            {
+                User = _context.AppUser.Single(u => u.Id == sm.Key),
+                Chapters = _context.Chapter.Where(c => _context.AppUserProgresses
+                    .Where(u => u.AppUserId == sm.Key)
+                    .Where(p => p.PagesRead > 0)
+                    .Where(p => days == 0 || (p.Created >= minDate && p.LastModified >= minDate))
+                    .Select(p => p.ChapterId)
+                    .Distinct()
+                    .Contains(c.Id))
+            })
+            .OrderByDescending(d => d.Chapters.Sum(c => c.AvgHoursToRead))
+            .Take(5)
+            .ToList();
+
+
+        // Need a mapping of Library to chapter ids
+        var chapterIdWithLibraryId = topUsersAndReadChapters
+            .SelectMany(u => u.Chapters
+                .Select(c => c.Id)).Select(d => new
+                    {
+                        LibraryId = _context.Chapter.Where(c => c.Id == d).AsSplitQuery().Select(c => c.Volume).Select(v => v.Series).Select(s => s.LibraryId).Single(),
+                        ChapterId = d
+                    })
+            .ToList();
+
+        var chapterLibLookup = new Dictionary<int, int>();
+        foreach (var cl in chapterIdWithLibraryId)
+        {
+            if (chapterLibLookup.ContainsKey(cl.ChapterId)) continue;
+            chapterLibLookup.Add(cl.ChapterId, cl.LibraryId);
+        }
+
+        var user = new Dictionary<int, Dictionary<LibraryType, long>>();
+        foreach (var userChapter in topUsersAndReadChapters)
+        {
+            if (!user.ContainsKey(userChapter.User.Id)) user.Add(userChapter.User.Id, new Dictionary<LibraryType, long>());
+            var libraryTimes = user[userChapter.User.Id];
+
+            foreach (var chapter in userChapter.Chapters)
+            {
+                var library = libraries.First(l => l.Id == chapterLibLookup[chapter.Id]);
+                if (!libraryTimes.ContainsKey(library.Type)) libraryTimes.Add(library.Type, 0L);
+                var existingHours = libraryTimes[library.Type];
+                libraryTimes[library.Type] = existingHours + chapter.AvgHoursToRead;
+            }
+
+            user[userChapter.User.Id] = libraryTimes;
+        }
+
+        var ret = new List<TopReadDto>();
+        foreach (var userId in user.Keys)
+        {
+            ret.Add(new TopReadDto()
+            {
+                UserId = userId,
+                Username = users.First(u => u.Id == userId).UserName,
+                BooksTime = user[userId].ContainsKey(LibraryType.Book) ? user[userId][LibraryType.Book] : 0,
+                ComicsTime = user[userId].ContainsKey(LibraryType.Comic) ? user[userId][LibraryType.Comic] : 0,
+                MangaTime = user[userId].ContainsKey(LibraryType.Manga) ? user[userId][LibraryType.Manga] : 0,
+            });
+        }
+
+        return ret;
+    }
+}

API/Services/Tasks/Scanner/ProcessSeries.cs

@@ -27,7 +27,7 @@ public interface IProcessSeries
     /// </summary>
     /// <returns></returns>
     Task Prime();
-    Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library);
+    Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false);
     void EnqueuePostSeriesProcessTasks(int libraryId, int seriesId, bool forceUpdate = false);
 }
 
@@ -75,7 +75,7 @@ public class ProcessSeries : IProcessSeries
         _tags = await _unitOfWork.TagRepository.GetAllTagsAsync();
     }
 
-    public async Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library)
+    public async Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false)
     {
         if (!parsedInfos.Any()) return;
 
@@ -120,7 +120,7 @@ public class ProcessSeries : IProcessSeries
             // parsedInfos[0] is not the first volume or chapter. We need to find it using a ComicInfo check (as it uses firstParsedInfo for series sort)
             var firstParsedInfo = parsedInfos.FirstOrDefault(p => p.ComicInfo != null, firstInfo);
 
-            UpdateVolumes(series, parsedInfos);
+            UpdateVolumes(series, parsedInfos, forceUpdate);
             series.Pages = series.Volumes.Sum(v => v.Pages);
 
             series.NormalizedName = Parser.Parser.Normalize(series.Name);
@@ -430,7 +430,7 @@ public class ProcessSeries : IProcessSeries
             });
     }
 
-    private void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos)
+    private void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
     {
         var startingVolumeCount = series.Volumes.Count;
         // Add new volumes and update chapters per volume
@@ -465,7 +465,7 @@ public class ProcessSeries : IProcessSeries
 
             _logger.LogDebug("[ScannerService] Parsing {SeriesName} - Volume {VolumeNumber}", series.Name, volume.Name);
             var infos = parsedInfos.Where(p => p.Volumes == volumeNumber).ToArray();
-            UpdateChapters(series, volume, infos);
+            UpdateChapters(series, volume, infos, forceUpdate);
             volume.Pages = volume.Chapters.Sum(c => c.Pages);
 
             // Update all the metadata on the Chapters
@@ -512,7 +512,7 @@ public class ProcessSeries : IProcessSeries
             series.Name, startingVolumeCount, series.Volumes.Count);
     }
 
-    private void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos)
+    private void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
     {
         // Add new chapters
         foreach (var info in parsedInfos)
@@ -546,7 +546,7 @@ public class ProcessSeries : IProcessSeries
             if (chapter == null) continue;
             // Add files
             var specialTreatment = info.IsSpecialInfo();
-            AddOrUpdateFileForChapter(chapter, info);
+            AddOrUpdateFileForChapter(chapter, info, forceUpdate);
             chapter.Number = Parser.Parser.MinNumberFromRange(info.Chapters) + string.Empty;
             chapter.Range = specialTreatment ? info.Filename : info.Chapters;
         }
@@ -572,22 +572,26 @@ public class ProcessSeries : IProcessSeries
         }
     }
 
-    private void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info)
+    private void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info, bool forceUpdate = false)
     {
         chapter.Files ??= new List<MangaFile>();
         var existingFile = chapter.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
+        var fileInfo = _directoryService.FileSystem.FileInfo.FromFileName(info.FullFilePath);
         if (existingFile != null)
         {
             existingFile.Format = info.Format;
-            if (!_fileService.HasFileBeenModifiedSince(existingFile.FilePath, existingFile.LastModified) && existingFile.Pages != 0) return;
+            if (!forceUpdate && !_fileService.HasFileBeenModifiedSince(existingFile.FilePath, existingFile.LastModified) && existingFile.Pages != 0) return;
             existingFile.Pages = _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format);
+            existingFile.Extension = fileInfo.Extension.ToLowerInvariant();
+            existingFile.Bytes = fileInfo.Length;
             // We skip updating DB here with last modified time so that metadata refresh can do it
         }
         else
         {
             var file = DbFactory.MangaFile(info.FullFilePath, info.Format, _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format));
             if (file == null) return;
-
+            file.Extension = fileInfo.Extension.ToLowerInvariant();
+            file.Bytes = fileInfo.Length;
             chapter.Files.Add(file);
         }
     }

API/Services/Tasks/ScannerService.cs

@@ -42,6 +42,7 @@ public interface IScannerService
     Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true);
 
     Task ScanFolder(string folder);
+    Task AnalyzeFiles();
 
 }
 
@@ -97,6 +98,35 @@ public class ScannerService : IScannerService
         _wordCountAnalyzerService = wordCountAnalyzerService;
     }
 
+    /// <summary>
+    /// This is only used for v0.7 to get files analyzed
+    /// </summary>
+    public async Task AnalyzeFiles()
+    {
+        _logger.LogInformation("Starting Analyze Files task");
+        var missingExtensions = await _unitOfWork.MangaFileRepository.GetAllWithMissingExtension();
+        if (missingExtensions.Count == 0)
+        {
+            _logger.LogInformation("Nothing to do");
+            return;
+        }
+
+        var sw = Stopwatch.StartNew();
+
+        foreach (var file in missingExtensions)
+        {
+            var fileInfo = _directoryService.FileSystem.FileInfo.FromFileName(file.FilePath);
+            if (!fileInfo.Exists)continue;
+            file.Extension = fileInfo.Extension.ToLowerInvariant();
+            file.Bytes = fileInfo.Length;
+            _unitOfWork.MangaFileRepository.Update(file);
+        }
+
+        await _unitOfWork.CommitAsync();
+
+        _logger.LogInformation("Completed Analyze Files task in {ElapsedTime}", sw.Elapsed);
+    }
+
     /// <summary>
     /// Given a generic folder path, will invoke a Series scan or Library scan.
     /// </summary>
@@ -483,7 +513,7 @@ public class ScannerService : IScannerService
 
 
             seenSeries.Add(foundParsedSeries);
-            processTasks.Add(async () => await _processSeries.ProcessSeriesAsync(parsedFiles, library));
+            processTasks.Add(async () => await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate));
             return Task.CompletedTask;
         }
 

API/Services/Tasks/StatsService.cs

@@ -121,7 +121,7 @@ public class StatsService : IStatsService
             NumberOfCollections = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync()).Count(),
             NumberOfReadingLists = await _unitOfWork.ReadingListRepository.Count(),
             OPDSEnabled = serverSettings.EnableOpds,
-            NumberOfUsers = (await _unitOfWork.UserRepository.GetAllUsers()).Count(),
+            NumberOfUsers = (await _unitOfWork.UserRepository.GetAllUsersAsync()).Count(),
             TotalFiles = await _unitOfWork.LibraryRepository.GetTotalFiles(),
             TotalGenres = await _unitOfWork.GenreRepository.GetCountAsync(),
             TotalPeople = await _unitOfWork.PersonRepository.GetCountAsync(),