Sort by Average Rating and Big Want to Read fix (#2672)

API/Data/DataContext.cs

@@ -62,6 +62,7 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
     public DbSet<ExternalRating> ExternalRating { get; set; } = null!;
     public DbSet<ExternalSeriesMetadata> ExternalSeriesMetadata { get; set; } = null!;
     public DbSet<ExternalRecommendation> ExternalRecommendation { get; set; } = null!;
+    public DbSet<ManualMigrationHistory> ManualMigrationHistory { get; set; } = null!;
 
 
     protected override void OnModelCreating(ModelBuilder builder)

API/Data/ManualMigrations/MigrateManualHistory.cs

@@ -0,0 +1,86 @@
+using System;
+using System.Threading.Tasks;
+using API.Entities;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+/// <summary>
+/// Introduced in v0.7.14, will store history so that going forward, migrations can just check against the history
+/// and I don't need to remove old migrations
+/// </summary>
+public static class MigrateManualHistory
+{
+    public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
+    {
+        logger.LogCritical(
+            "Running MigrateManualHistory migration - Please be patient, this may take some time. This is not an error");
+
+        if (await dataContext.ManualMigrationHistory.AnyAsync())
+        {
+            logger.LogCritical(
+                "Running MigrateManualHistory migration - Completed. This is not an error");
+            return;
+        }
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateUserLibrarySideNavStream",
+            ProductVersion = "0.7.9.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateSmartFilterEncoding",
+            ProductVersion = "0.7.11.0",
+            RanAt = DateTime.UtcNow
+        });
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateLibrariesToHaveAllFileTypes",
+            ProductVersion = "0.7.11.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateEmailTemplates",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateVolumeNumber",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateWantToReadExport",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateWantToReadImport",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateManualHistory",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        await dataContext.SaveChangesAsync();
+
+        logger.LogCritical(
+            "Running MigrateManualHistory migration - Completed. This is not an error");
+    }
+}

API/Data/ManualMigrations/MigrateWantToReadExport.cs

@@ -0,0 +1,81 @@
+using System.Globalization;
+using System.IO;
+using System.Threading.Tasks;
+using API.Services;
+using CsvHelper;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+
+/// <summary>
+/// v0.7.13.12/v0.7.14 - Want to read is extracted and saved in a csv
+/// </summary>
+/// <remarks>This must run BEFORE any DB migrations</remarks>
+public static class MigrateWantToReadExport
+{
+    public static async Task Migrate(DataContext dataContext, IDirectoryService directoryService, ILogger<Program> logger)
+    {
+        logger.LogCritical(
+            "Running MigrateWantToReadExport migration - Please be patient, this may take some time. This is not an error");
+
+        var columnExists = false;
+        await using var command = dataContext.Database.GetDbConnection().CreateCommand();
+        command.CommandText = "PRAGMA table_info('Series')";
+
+        await dataContext.Database.OpenConnectionAsync();
+        await using var result = await command.ExecuteReaderAsync();
+        while (await result.ReadAsync())
+        {
+            var columnName = result["name"].ToString();
+            if (columnName != "AppUserId") continue;
+
+            logger.LogInformation("Column 'AppUserId' exists in the 'Series' table. Running migration...");
+            // Your migration logic here
+            columnExists = true;
+            break;
+        }
+
+        await result.CloseAsync();
+
+        if (!columnExists)
+        {
+            logger.LogCritical(
+                "Running MigrateWantToReadExport migration - Completed. This is not an error");
+            return;
+        }
+
+        await using var command2 = dataContext.Database.GetDbConnection().CreateCommand();
+        command.CommandText = "Select AppUserId, Id from Series WHERE AppUserId IS NOT NULL ORDER BY AppUserId;";
+
+        await dataContext.Database.OpenConnectionAsync();
+        await using var result2 = await command.ExecuteReaderAsync();
+
+        await using var writer = new StreamWriter(Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv"));
+        await using var csvWriter = new CsvWriter(writer, CultureInfo.InvariantCulture);
+
+        // Write header
+        csvWriter.WriteField("AppUserId");
+        csvWriter.WriteField("Id");
+        await csvWriter.NextRecordAsync();
+
+        // Write data
+        while (await result2.ReadAsync())
+        {
+            var appUserId = result2["AppUserId"].ToString();
+            var id = result2["Id"].ToString();
+
+            csvWriter.WriteField(appUserId);
+            csvWriter.WriteField(id);
+            await csvWriter.NextRecordAsync();
+        }
+
+
+        await result2.CloseAsync();
+        writer.Close();
+
+        logger.LogCritical(
+            "Running MigrateWantToReadExport migration - Completed. This is not an error");
+    }
+}

API/Data/ManualMigrations/MigrateWantToReadImport.cs

@@ -0,0 +1,60 @@
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using API.Data.Repositories;
+using API.Entities;
+using API.Services;
+using CsvHelper;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+/// <summary>
+/// v0.7.13.12/v0.7.14 - Want to read is imported from a csv
+/// </summary>
+public static class MigrateWantToReadImport
+{
+    public static async Task Migrate(IUnitOfWork unitOfWork, IDirectoryService directoryService, ILogger<Program> logger)
+    {
+        var importFile = Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv");
+        var outputFile = Path.Join(directoryService.ConfigDirectory, "imported-want-to-read-migration.csv");
+
+        logger.LogCritical(
+            "Running MigrateWantToReadImport migration - Please be patient, this may take some time. This is not an error");
+
+        if (!File.Exists(importFile) || File.Exists(outputFile))
+        {
+            logger.LogCritical(
+                "Running MigrateWantToReadImport migration - Completed. This is not an error");
+            return;
+        }
+
+        using var reader = new StreamReader(importFile);
+        using var csvReader = new CsvReader(reader, CultureInfo.InvariantCulture);
+        // Read the records from the CSV file
+        await csvReader.ReadAsync();
+        csvReader.ReadHeader(); // Skip the header row
+
+        while (await csvReader.ReadAsync())
+        {
+            // Read the values of AppUserId and Id columns
+            var appUserId = csvReader.GetField<int>("AppUserId");
+            var seriesId = csvReader.GetField<int>("Id");
+            var user = await unitOfWork.UserRepository.GetUserByIdAsync(appUserId, AppUserIncludes.WantToRead);
+            if (user == null || user.WantToRead.Any(w => w.SeriesId == seriesId)) continue;
+
+            user.WantToRead.Add(new AppUserWantToRead()
+            {
+                SeriesId = seriesId
+            });
+        }
+
+        await unitOfWork.CommitAsync();
+        reader.Close();
+
+        File.WriteAllLines(outputFile, await File.ReadAllLinesAsync(importFile));
+        logger.LogCritical(
+            "Running MigrateWantToReadImport migration - Completed. This is not an error");
+    }
+}

API/Data/Migrations/20240130190617_WantToReadFix.cs

@@ -0,0 +1,106 @@
+using System;
+using Microsoft.EntityFrameworkCore.Migrations;
+
+#nullable disable
+
+namespace API.Data.Migrations
+{
+    /// <inheritdoc />
+    public partial class WantToReadFix : Migration
+    {
+        /// <inheritdoc />
+        protected override void Up(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropForeignKey(
+                name: "FK_Series_AspNetUsers_AppUserId",
+                table: "Series");
+
+            migrationBuilder.DropIndex(
+                name: "IX_Series_AppUserId",
+                table: "Series");
+
+            migrationBuilder.DropColumn(
+                name: "AppUserId",
+                table: "Series");
+
+            migrationBuilder.CreateTable(
+                name: "AppUserWantToRead",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "INTEGER", nullable: false)
+                        .Annotation("Sqlite:Autoincrement", true),
+                    SeriesId = table.Column<int>(type: "INTEGER", nullable: false),
+                    AppUserId = table.Column<int>(type: "INTEGER", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_AppUserWantToRead", x => x.Id);
+                    table.ForeignKey(
+                        name: "FK_AppUserWantToRead_AspNetUsers_AppUserId",
+                        column: x => x.AppUserId,
+                        principalTable: "AspNetUsers",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                    table.ForeignKey(
+                        name: "FK_AppUserWantToRead_Series_SeriesId",
+                        column: x => x.SeriesId,
+                        principalTable: "Series",
+                        principalColumn: "Id",
+                        onDelete: ReferentialAction.Cascade);
+                });
+
+            migrationBuilder.CreateTable(
+                name: "ManualMigrationHistory",
+                columns: table => new
+                {
+                    Id = table.Column<int>(type: "INTEGER", nullable: false)
+                        .Annotation("Sqlite:Autoincrement", true),
+                    ProductVersion = table.Column<string>(type: "TEXT", nullable: true),
+                    Name = table.Column<string>(type: "TEXT", nullable: true),
+                    RanAt = table.Column<DateTime>(type: "TEXT", nullable: false)
+                },
+                constraints: table =>
+                {
+                    table.PrimaryKey("PK_ManualMigrationHistory", x => x.Id);
+                });
+
+            migrationBuilder.CreateIndex(
+                name: "IX_AppUserWantToRead_AppUserId",
+                table: "AppUserWantToRead",
+                column: "AppUserId");
+
+            migrationBuilder.CreateIndex(
+                name: "IX_AppUserWantToRead_SeriesId",
+                table: "AppUserWantToRead",
+                column: "SeriesId");
+        }
+
+        /// <inheritdoc />
+        protected override void Down(MigrationBuilder migrationBuilder)
+        {
+            migrationBuilder.DropTable(
+                name: "AppUserWantToRead");
+
+            migrationBuilder.DropTable(
+                name: "ManualMigrationHistory");
+
+            migrationBuilder.AddColumn<int>(
+                name: "AppUserId",
+                table: "Series",
+                type: "INTEGER",
+                nullable: true);
+
+            migrationBuilder.CreateIndex(
+                name: "IX_Series_AppUserId",
+                table: "Series",
+                column: "AppUserId");
+
+            migrationBuilder.AddForeignKey(
+                name: "FK_Series_AspNetUsers_AppUserId",
+                table: "Series",
+                column: "AppUserId",
+                principalTable: "AspNetUsers",
+                principalColumn: "Id");
+        }
+    }
+}

API/Data/Migrations/DataContextModelSnapshot.cs

@@ -602,6 +602,27 @@ namespace API.Data.Migrations
                     b.ToTable("AppUserTableOfContent");
                 });
 
+            modelBuilder.Entity("API.Entities.AppUserWantToRead", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER");
+
+                    b.Property<int>("AppUserId")
+                        .HasColumnType("INTEGER");
+
+                    b.Property<int>("SeriesId")
+                        .HasColumnType("INTEGER");
+
+                    b.HasKey("Id");
+
+                    b.HasIndex("AppUserId");
+
+                    b.HasIndex("SeriesId");
+
+                    b.ToTable("AppUserWantToRead");
+                });
+
             modelBuilder.Entity("API.Entities.Chapter", b =>
                 {
                     b.Property<int>("Id")
@@ -980,6 +1001,26 @@ namespace API.Data.Migrations
                     b.ToTable("MangaFile");
                 });
 
+            modelBuilder.Entity("API.Entities.ManualMigrationHistory", b =>
+                {
+                    b.Property<int>("Id")
+                        .ValueGeneratedOnAdd()
+                        .HasColumnType("INTEGER");
+
+                    b.Property<string>("Name")
+                        .HasColumnType("TEXT");
+
+                    b.Property<string>("ProductVersion")
+                        .HasColumnType("TEXT");
+
+                    b.Property<DateTime>("RanAt")
+                        .HasColumnType("TEXT");
+
+                    b.HasKey("Id");
+
+                    b.ToTable("ManualMigrationHistory");
+                });
+
             modelBuilder.Entity("API.Entities.MediaError", b =>
                 {
                     b.Property<int>("Id")
@@ -1551,9 +1592,6 @@ namespace API.Data.Migrations
                         .ValueGeneratedOnAdd()
                         .HasColumnType("INTEGER");
 
-                    b.Property<int?>("AppUserId")
-                        .HasColumnType("INTEGER");
-
                     b.Property<int>("AvgHoursToRead")
                         .HasColumnType("INTEGER");
 
@@ -1634,8 +1672,6 @@ namespace API.Data.Migrations
 
                     b.HasKey("Id");
 
-                    b.HasIndex("AppUserId");
-
                     b.HasIndex("LibraryId");
 
                     b.ToTable("Series");
@@ -2252,6 +2288,25 @@ namespace API.Data.Migrations
                     b.Navigation("Series");
                 });
 
+            modelBuilder.Entity("API.Entities.AppUserWantToRead", b =>
+                {
+                    b.HasOne("API.Entities.AppUser", "AppUser")
+                        .WithMany("WantToRead")
+                        .HasForeignKey("AppUserId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.HasOne("API.Entities.Series", "Series")
+                        .WithMany()
+                        .HasForeignKey("SeriesId")
+                        .OnDelete(DeleteBehavior.Cascade)
+                        .IsRequired();
+
+                    b.Navigation("AppUser");
+
+                    b.Navigation("Series");
+                });
+
             modelBuilder.Entity("API.Entities.Chapter", b =>
                 {
                     b.HasOne("API.Entities.Volume", "Volume")
@@ -2479,10 +2534,6 @@ namespace API.Data.Migrations
 
             modelBuilder.Entity("API.Entities.Series", b =>
                 {
-                    b.HasOne("API.Entities.AppUser", null)
-                        .WithMany("WantToRead")
-                        .HasForeignKey("AppUserId");
-
                     b.HasOne("API.Entities.Library", "Library")
                         .WithMany("Series")
                         .HasForeignKey("LibraryId")

API/Data/Repositories/SeriesRepository.cs

@@ -363,8 +363,8 @@ public class SeriesRepository : ISeriesRepository
             .Where(l => EF.Functions.Like(l.Name, $"%{searchQuery}%"))
             .IsRestricted(QueryContext.Search)
             .AsSplitQuery()
-            .Take(maxRecords)
             .OrderBy(l => l.Name.ToLower())
+            .Take(maxRecords)
             .ProjectTo<LibraryDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -383,8 +383,8 @@ public class SeriesRepository : ISeriesRepository
             .Include(s => s.Library)
             .AsNoTracking()
             .AsSplitQuery()
-            .Take(maxRecords)
             .OrderBy(s => s.SortName!.ToLower())
+            .Take(maxRecords)
             .ProjectTo<SearchResultDto>(_mapper.ConfigurationProvider)
             .AsEnumerable();
 
@@ -420,8 +420,8 @@ public class SeriesRepository : ISeriesRepository
             .Where(rl => EF.Functions.Like(rl.Title, $"%{searchQuery}%"))
             .RestrictAgainstAgeRestriction(userRating)
             .AsSplitQuery()
-            .Take(maxRecords)
             .OrderBy(r => r.NormalizedTitle)
+            .Take(maxRecords)
             .ProjectTo<ReadingListDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -431,7 +431,6 @@ public class SeriesRepository : ISeriesRepository
             .Where(c => c.Promoted || isAdmin)
             .RestrictAgainstAgeRestriction(userRating)
             .OrderBy(s => s.NormalizedTitle)
-            .AsNoTracking()
             .AsSplitQuery()
             .Take(maxRecords)
             .OrderBy(c => c.NormalizedTitle)
@@ -443,8 +442,8 @@ public class SeriesRepository : ISeriesRepository
             .SelectMany(sm => sm.People.Where(t => t.Name != null && EF.Functions.Like(t.Name, $"%{searchQuery}%")))
             .AsSplitQuery()
             .Distinct()
-            .Take(maxRecords)
             .OrderBy(p => p.NormalizedName)
+            .Take(maxRecords)
             .ProjectTo<PersonDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -453,8 +452,8 @@ public class SeriesRepository : ISeriesRepository
             .SelectMany(sm => sm.Genres.Where(t => EF.Functions.Like(t.Title, $"%{searchQuery}%")))
             .AsSplitQuery()
             .Distinct()
-            .Take(maxRecords)
             .OrderBy(t => t.NormalizedTitle)
+            .Take(maxRecords)
             .ProjectTo<GenreTagDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -463,8 +462,8 @@ public class SeriesRepository : ISeriesRepository
             .SelectMany(sm => sm.Tags.Where(t => EF.Functions.Like(t.Title, $"%{searchQuery}%")))
             .AsSplitQuery()
             .Distinct()
-            .Take(maxRecords)
             .OrderBy(t => t.NormalizedTitle)
+            .Take(maxRecords)
             .ProjectTo<TagDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -482,8 +481,8 @@ public class SeriesRepository : ISeriesRepository
             result.Files = await _context.MangaFile
                 .Where(m => EF.Functions.Like(m.FilePath, $"%{searchQuery}%") && fileIds.Contains(m.Id))
                 .AsSplitQuery()
-                .Take(maxRecords)
                 .OrderBy(f => f.FilePath)
+                .Take(maxRecords)
                 .ProjectTo<MangaFileDto>(_mapper.ConfigurationProvider)
                 .ToListAsync();
         }
@@ -499,8 +498,8 @@ public class SeriesRepository : ISeriesRepository
                 )
             .Where(c => c.Files.All(f => fileIds.Contains(f.Id)))
             .AsSplitQuery()
-            .Take(maxRecords)
             .OrderBy(c => c.TitleName)
+            .Take(maxRecords)
             .ProjectTo<ChapterDto>(_mapper.ConfigurationProvider)
             .ToListAsync();
 
@@ -991,6 +990,8 @@ public class SeriesRepository : ISeriesRepository
             SortField.TimeToRead => query.DoOrderBy(s => s.AvgHoursToRead, filter.SortOptions),
             SortField.ReleaseYear => query.DoOrderBy(s => s.Metadata.ReleaseYear, filter.SortOptions),
             SortField.ReadProgress => query.DoOrderBy(s => s.Progress.Where(p => p.SeriesId == s.Id).Select(p => p.LastModified).Max(), filter.SortOptions),
+            SortField.AverageRating => query.DoOrderBy(s => s.ExternalSeriesMetadata.ExternalRatings
+                .Where(p => p.SeriesId == s.Id).Average(p => p.AverageScore), filter.SortOptions),
             _ => query
         };
 
@@ -1043,7 +1044,9 @@ public class SeriesRepository : ISeriesRepository
         var wantToReadStmt = filter.Statements.FirstOrDefault(stmt => stmt.Field == FilterField.WantToRead);
         if (wantToReadStmt == null) return query;
 
-        var seriesIds = _context.AppUser.Where(u => u.Id == userId).SelectMany(u => u.WantToRead).Select(s => s.Id);
+        var seriesIds = _context.AppUser.Where(u => u.Id == userId)
+            .SelectMany(u => u.WantToRead)
+            .Select(s => s.SeriesId);
         if (bool.Parse(wantToReadStmt.Value))
         {
             query = query.Where(s => seriesIds.Contains(s.Id));
@@ -1869,7 +1872,8 @@ public class SeriesRepository : ISeriesRepository
         var query = _context.AppUser
             .Where(user => user.Id == userId)
             .SelectMany(u => u.WantToRead)
-            .Where(s => libraryIds.Contains(s.LibraryId))
+            .Where(s => libraryIds.Contains(s.Series.LibraryId))
+            .Select(w => w.Series)
             .AsSplitQuery()
             .AsNoTracking();
 
@@ -1884,7 +1888,8 @@ public class SeriesRepository : ISeriesRepository
         var query = _context.AppUser
             .Where(user => user.Id == userId)
             .SelectMany(u => u.WantToRead)
-            .Where(s => libraryIds.Contains(s.LibraryId))
+            .Where(s => libraryIds.Contains(s.Series.LibraryId))
+            .Select(w => w.Series)
             .AsSplitQuery()
             .AsNoTracking();
 
@@ -1899,7 +1904,8 @@ public class SeriesRepository : ISeriesRepository
         return await _context.AppUser
             .Where(user => user.Id == userId)
             .SelectMany(u => u.WantToRead)
-            .Where(s => libraryIds.Contains(s.LibraryId))
+            .Where(s => libraryIds.Contains(s.Series.LibraryId))
+            .Select(w => w.Series)
             .AsSplitQuery()
             .AsNoTracking()
             .ToListAsync();
@@ -1994,7 +2000,7 @@ public class SeriesRepository : ISeriesRepository
         var libraryIds = await _context.Library.GetUserLibraries(userId).ToListAsync();
         return await _context.AppUser
             .Where(user => user.Id == userId)
-            .SelectMany(u => u.WantToRead.Where(s => s.Id == seriesId && libraryIds.Contains(s.LibraryId)))
+            .SelectMany(u => u.WantToRead.Where(s => s.SeriesId == seriesId && libraryIds.Contains(s.Series.LibraryId)))
             .AsSplitQuery()
             .AsNoTracking()
             .AnyAsync();