Fixed Series Relations Schema (#1654)

* Bump loader-utils from 2.0.2 to 2.0.3 in /UI/Web

Bumps [loader-utils](https://github.com/webpack/loader-utils) from 2.0.2 to 2.0.3.
- [Release notes](https://github.com/webpack/loader-utils/releases)
- [Changelog](https://github.com/webpack/loader-utils/blob/v2.0.3/CHANGELOG.md)
- [Commits](https://github.com/webpack/loader-utils/compare/v2.0.2...v2.0.3)

---
updated-dependencies:
- dependency-name: loader-utils
  dependency-type: indirect
...

Signed-off-by: dependabot[bot] <support@github.com>

* Fixed is want to read coming back as a string and not working correctly.

* Changed from to Continue to be more explicit

* Added the first migration which exports data as a csv in temp/. This is the backup in case data is lost in the migration.

* Note for later

* Fixed the migration for the series relation so when deleting any series on any edge of the relationship, the SeriesRelation row deletes.

* Change buttons back to titles on series detail page

* Wrote the code to import relations from the backup.

* Added an additional version check to avoid file io on migration.

* Code cleanup

Signed-off-by: dependabot[bot] <support@github.com>
Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com>
This commit is contained in:
Joe Milazzo 2022-11-15 08:45:02 -06:00 committed by GitHub
parent d5a7c31c7d
commit 15e09a0cf1
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 2111 additions and 83 deletions

View file

@ -68,13 +68,15 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
.HasOne(pt => pt.Series)
.WithMany(p => p.Relations)
.HasForeignKey(pt => pt.SeriesId)
.OnDelete(DeleteBehavior.ClientCascade);
.OnDelete(DeleteBehavior.Cascade);
builder.Entity<SeriesRelation>()
.HasOne(pt => pt.TargetSeries)
.WithMany(t => t.RelationOf)
.HasForeignKey(pt => pt.TargetSeriesId)
.OnDelete(DeleteBehavior.ClientCascade);
.OnDelete(DeleteBehavior.Cascade);
builder.Entity<AppUserPreferences>()

View file

@ -0,0 +1,104 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using API.Entities.Enums;
using CsvHelper;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
internal sealed class SeriesRelationMigrationOutput
{
public string SeriesName { get; set; }
public int SeriesId { get; set; }
public string TargetSeriesName { get; set; }
public int TargetId { get; set; }
public RelationKind Relationship { get; set; }
}
/// <summary>
/// Introduced in v0.6.1.2 and v0.7, this exports to a temp file the existing series relationships. It is a 3 part migration.
/// This will run first, to export the data, then the DB migration will change the way the DB is constructed, then the last migration
/// will import said file and re-construct the relationships.
/// </summary>
public static class MigrateSeriesRelationsExport
{
private const string OutputFile = "config/relations.csv";
private const string CompleteOutputFile = "config/relations-imported.csv";
public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateSeriesRelationsExport migration - Please be patient, this may take some time. This is not an error");
if (BuildInfo.Version > new Version(0, 6, 1, 3)
|| new FileInfo(OutputFile).Exists
|| new FileInfo(CompleteOutputFile).Exists)
{
logger.LogCritical("Running MigrateSeriesRelationsExport migration - complete. Nothing to do");
return;
}
var seriesWithRelationships = await dataContext.Series
.Where(s => s.Relations.Any())
.Include(s => s.Relations)
.ThenInclude(r => r.TargetSeries)
.ToListAsync();
var records = new List<SeriesRelationMigrationOutput>();
var excludedRelationships = new List<RelationKind>()
{
RelationKind.Parent,
};
foreach (var series in seriesWithRelationships)
{
foreach (var relationship in series.Relations.Where(r => !excludedRelationships.Contains(r.RelationKind)))
{
records.Add(new SeriesRelationMigrationOutput()
{
SeriesId = series.Id,
SeriesName = series.Name,
Relationship = relationship.RelationKind,
TargetId = relationship.TargetSeriesId,
TargetSeriesName = relationship.TargetSeries.Name
});
}
}
await using var writer = new StreamWriter(OutputFile);
await using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
await csv.WriteRecordsAsync(records);
}
await writer.DisposeAsync();
logger.LogCritical("{OutputFile} has a backup of all data", OutputFile);
logger.LogCritical("Deleting all relationships in the DB. This is not an error");
var entities = await dataContext.SeriesRelation
.Include(s => s.Series)
.Include(s => s.TargetSeries)
.Select(s => s)
.ToListAsync();
foreach (var seriesWithRelationship in entities)
{
logger.LogCritical("Deleting {SeriesName} --{RelationshipKind}--> {TargetSeriesName}",
seriesWithRelationship.Series.Name, seriesWithRelationship.RelationKind, seriesWithRelationship.TargetSeries.Name);
dataContext.SeriesRelation.Remove(seriesWithRelationship);
await dataContext.SaveChangesAsync();
}
// In case of corrupted entities (where series were deleted but their Id still existed, we delete the rest of the table)
dataContext.SeriesRelation.RemoveRange(dataContext.SeriesRelation);
await dataContext.SaveChangesAsync();
logger.LogCritical("Running MigrateSeriesRelationsExport migration - Completed. This is not an error");
}
}

View file

@ -0,0 +1,64 @@
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using API.Entities.Enums;
using API.Entities.Metadata;
using CsvHelper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
/// <summary>
/// Introduced in v0.6.1.2 and v0.7, this imports to a temp file the existing series relationships. It is a 3 part migration.
/// This will run last, to import the data and re-construct the relationships.
/// </summary>
public static class MigrateSeriesRelationsImport
{
private const string OutputFile = "config/relations.csv";
private const string CompleteOutputFile = "config/relations-imported.csv";
public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateSeriesRelationsImport migration - Please be patient, this may take some time. This is not an error");
if (!new FileInfo(OutputFile).Exists)
{
logger.LogCritical("Running MigrateSeriesRelationsImport migration - complete. Nothing to do");
return;
}
logger.LogCritical("Loading backed up relationships into the DB");
List<SeriesRelationMigrationOutput> records;
using var reader = new StreamReader(OutputFile);
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
records = csv.GetRecords<SeriesRelationMigrationOutput>().ToList();
}
foreach (var relation in records)
{
logger.LogCritical("Importing {SeriesName} --{RelationshipKind}--> {TargetSeriesName}",
relation.SeriesName, relation.Relationship, relation.TargetSeriesName);
// Filter out series that don't exist
if (!await dataContext.Series.AnyAsync(s => s.Id == relation.SeriesId) ||
!await dataContext.Series.AnyAsync(s => s.Id == relation.TargetId))
continue;
await dataContext.SeriesRelation.AddAsync(new SeriesRelation()
{
SeriesId = relation.SeriesId,
TargetSeriesId = relation.TargetId,
RelationKind = relation.Relationship
});
}
await dataContext.SaveChangesAsync();
File.Move(OutputFile, CompleteOutputFile);
logger.LogCritical("Running MigrateSeriesRelationsImport migration - Completed. This is not an error");
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,61 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
public partial class SeriesRelationChange : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_SeriesRelation_Series_SeriesId",
table: "SeriesRelation");
migrationBuilder.DropForeignKey(
name: "FK_SeriesRelation_Series_TargetSeriesId",
table: "SeriesRelation");
migrationBuilder.AddForeignKey(
name: "FK_SeriesRelation_Series_SeriesId",
table: "SeriesRelation",
column: "SeriesId",
principalTable: "Series",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
migrationBuilder.AddForeignKey(
name: "FK_SeriesRelation_Series_TargetSeriesId",
table: "SeriesRelation",
column: "TargetSeriesId",
principalTable: "Series",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropForeignKey(
name: "FK_SeriesRelation_Series_SeriesId",
table: "SeriesRelation");
migrationBuilder.DropForeignKey(
name: "FK_SeriesRelation_Series_TargetSeriesId",
table: "SeriesRelation");
migrationBuilder.AddForeignKey(
name: "FK_SeriesRelation_Series_SeriesId",
table: "SeriesRelation",
column: "SeriesId",
principalTable: "Series",
principalColumn: "Id");
migrationBuilder.AddForeignKey(
name: "FK_SeriesRelation_Series_TargetSeriesId",
table: "SeriesRelation",
column: "TargetSeriesId",
principalTable: "Series",
principalColumn: "Id");
}
}
}

View file

@ -15,7 +15,7 @@ namespace API.Data.Migrations
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "6.0.9");
modelBuilder.HasAnnotation("ProductVersion", "6.0.10");
modelBuilder.Entity("API.Entities.AppRole", b =>
{
@ -165,7 +165,7 @@ namespace API.Data.Migrations
b.HasIndex("AppUserId");
b.ToTable("AppUserBookmark", (string)null);
b.ToTable("AppUserBookmark");
});
modelBuilder.Entity("API.Entities.AppUserPreferences", b =>
@ -256,7 +256,7 @@ namespace API.Data.Migrations
b.HasIndex("ThemeId");
b.ToTable("AppUserPreferences", (string)null);
b.ToTable("AppUserPreferences");
});
modelBuilder.Entity("API.Entities.AppUserProgress", b =>
@ -295,7 +295,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesId");
b.ToTable("AppUserProgresses", (string)null);
b.ToTable("AppUserProgresses");
});
modelBuilder.Entity("API.Entities.AppUserRating", b =>
@ -322,7 +322,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesId");
b.ToTable("AppUserRating", (string)null);
b.ToTable("AppUserRating");
});
modelBuilder.Entity("API.Entities.AppUserRole", b =>
@ -413,7 +413,7 @@ namespace API.Data.Migrations
b.HasIndex("VolumeId");
b.ToTable("Chapter", (string)null);
b.ToTable("Chapter");
});
modelBuilder.Entity("API.Entities.CollectionTag", b =>
@ -448,7 +448,7 @@ namespace API.Data.Migrations
b.HasIndex("Id", "Promoted")
.IsUnique();
b.ToTable("CollectionTag", (string)null);
b.ToTable("CollectionTag");
});
modelBuilder.Entity("API.Entities.Device", b =>
@ -485,7 +485,7 @@ namespace API.Data.Migrations
b.HasIndex("AppUserId");
b.ToTable("Device", (string)null);
b.ToTable("Device");
});
modelBuilder.Entity("API.Entities.FolderPath", b =>
@ -507,7 +507,7 @@ namespace API.Data.Migrations
b.HasIndex("LibraryId");
b.ToTable("FolderPath", (string)null);
b.ToTable("FolderPath");
});
modelBuilder.Entity("API.Entities.Genre", b =>
@ -530,7 +530,7 @@ namespace API.Data.Migrations
b.HasIndex("NormalizedTitle", "ExternalTag")
.IsUnique();
b.ToTable("Genre", (string)null);
b.ToTable("Genre");
});
modelBuilder.Entity("API.Entities.Library", b =>
@ -559,7 +559,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.ToTable("Library", (string)null);
b.ToTable("Library");
});
modelBuilder.Entity("API.Entities.MangaFile", b =>
@ -593,7 +593,7 @@ namespace API.Data.Migrations
b.HasIndex("ChapterId");
b.ToTable("MangaFile", (string)null);
b.ToTable("MangaFile");
});
modelBuilder.Entity("API.Entities.Metadata.SeriesMetadata", b =>
@ -689,7 +689,7 @@ namespace API.Data.Migrations
b.HasIndex("Id", "SeriesId")
.IsUnique();
b.ToTable("SeriesMetadata", (string)null);
b.ToTable("SeriesMetadata");
});
modelBuilder.Entity("API.Entities.Metadata.SeriesRelation", b =>
@ -713,7 +713,7 @@ namespace API.Data.Migrations
b.HasIndex("TargetSeriesId");
b.ToTable("SeriesRelation", (string)null);
b.ToTable("SeriesRelation");
});
modelBuilder.Entity("API.Entities.Person", b =>
@ -733,7 +733,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.ToTable("Person", (string)null);
b.ToTable("Person");
});
modelBuilder.Entity("API.Entities.ReadingList", b =>
@ -776,7 +776,7 @@ namespace API.Data.Migrations
b.HasIndex("AppUserId");
b.ToTable("ReadingList", (string)null);
b.ToTable("ReadingList");
});
modelBuilder.Entity("API.Entities.ReadingListItem", b =>
@ -810,7 +810,7 @@ namespace API.Data.Migrations
b.HasIndex("VolumeId");
b.ToTable("ReadingListItem", (string)null);
b.ToTable("ReadingListItem");
});
modelBuilder.Entity("API.Entities.Series", b =>
@ -897,7 +897,7 @@ namespace API.Data.Migrations
b.HasIndex("LibraryId");
b.ToTable("Series", (string)null);
b.ToTable("Series");
});
modelBuilder.Entity("API.Entities.ServerSetting", b =>
@ -914,7 +914,7 @@ namespace API.Data.Migrations
b.HasKey("Key");
b.ToTable("ServerSetting", (string)null);
b.ToTable("ServerSetting");
});
modelBuilder.Entity("API.Entities.SiteTheme", b =>
@ -946,7 +946,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.ToTable("SiteTheme", (string)null);
b.ToTable("SiteTheme");
});
modelBuilder.Entity("API.Entities.Tag", b =>
@ -969,7 +969,7 @@ namespace API.Data.Migrations
b.HasIndex("NormalizedTitle", "ExternalTag")
.IsUnique();
b.ToTable("Tag", (string)null);
b.ToTable("Tag");
});
modelBuilder.Entity("API.Entities.Volume", b =>
@ -1015,7 +1015,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesId");
b.ToTable("Volume", (string)null);
b.ToTable("Volume");
});
modelBuilder.Entity("AppUserLibrary", b =>
@ -1030,7 +1030,7 @@ namespace API.Data.Migrations
b.HasIndex("LibrariesId");
b.ToTable("AppUserLibrary", (string)null);
b.ToTable("AppUserLibrary");
});
modelBuilder.Entity("ChapterGenre", b =>
@ -1045,7 +1045,7 @@ namespace API.Data.Migrations
b.HasIndex("GenresId");
b.ToTable("ChapterGenre", (string)null);
b.ToTable("ChapterGenre");
});
modelBuilder.Entity("ChapterPerson", b =>
@ -1060,7 +1060,7 @@ namespace API.Data.Migrations
b.HasIndex("PeopleId");
b.ToTable("ChapterPerson", (string)null);
b.ToTable("ChapterPerson");
});
modelBuilder.Entity("ChapterTag", b =>
@ -1075,7 +1075,7 @@ namespace API.Data.Migrations
b.HasIndex("TagsId");
b.ToTable("ChapterTag", (string)null);
b.ToTable("ChapterTag");
});
modelBuilder.Entity("CollectionTagSeriesMetadata", b =>
@ -1090,7 +1090,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesMetadatasId");
b.ToTable("CollectionTagSeriesMetadata", (string)null);
b.ToTable("CollectionTagSeriesMetadata");
});
modelBuilder.Entity("GenreSeriesMetadata", b =>
@ -1105,7 +1105,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesMetadatasId");
b.ToTable("GenreSeriesMetadata", (string)null);
b.ToTable("GenreSeriesMetadata");
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<int>", b =>
@ -1204,7 +1204,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesMetadatasId");
b.ToTable("PersonSeriesMetadata", (string)null);
b.ToTable("PersonSeriesMetadata");
});
modelBuilder.Entity("SeriesMetadataTag", b =>
@ -1219,7 +1219,7 @@ namespace API.Data.Migrations
b.HasIndex("TagsId");
b.ToTable("SeriesMetadataTag", (string)null);
b.ToTable("SeriesMetadataTag");
});
modelBuilder.Entity("API.Entities.AppUserBookmark", b =>
@ -1363,13 +1363,13 @@ namespace API.Data.Migrations
b.HasOne("API.Entities.Series", "Series")
.WithMany("Relations")
.HasForeignKey("SeriesId")
.OnDelete(DeleteBehavior.ClientCascade)
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.HasOne("API.Entities.Series", "TargetSeries")
.WithMany("RelationOf")
.HasForeignKey("TargetSeriesId")
.OnDelete(DeleteBehavior.ClientCascade)
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("Series");

View file

@ -1257,15 +1257,6 @@ public class SeriesRepository : ISeriesRepository
.Where(s => !ids.Contains(s.Id))
.ToListAsync();
// If the series to remove has Relation (related series), we must manually unlink due to the DB not being
// setup correctly (if this is not done, a foreign key constraint will be thrown)
foreach (var sr in seriesToRemove)
{
sr.Relations = new List<SeriesRelation>();
Update(sr);
}
_context.Series.RemoveRange(seriesToRemove);
return seriesToRemove;
@ -1387,14 +1378,26 @@ public class SeriesRepository : ISeriesRepository
AlternativeSettings = await GetRelatedSeriesQuery(seriesId, usersSeriesIds, RelationKind.AlternativeSetting, userRating),
AlternativeVersions = await GetRelatedSeriesQuery(seriesId, usersSeriesIds, RelationKind.AlternativeVersion, userRating),
Doujinshis = await GetRelatedSeriesQuery(seriesId, usersSeriesIds, RelationKind.Doujinshi, userRating),
Parent = await _context.Series
.SelectMany(s =>
s.RelationOf.Where(r => r.TargetSeriesId == seriesId
&& usersSeriesIds.Contains(r.TargetSeriesId)
&& r.RelationKind != RelationKind.Prequel
&& r.RelationKind != RelationKind.Sequel
&& r.RelationKind != RelationKind.Edition)
.Select(sr => sr.Series))
// Parent = await _context.Series
// .SelectMany(s =>
// s.TargetSeries.Where(r => r.TargetSeriesId == seriesId
// && usersSeriesIds.Contains(r.TargetSeriesId)
// && r.RelationKind != RelationKind.Prequel
// && r.RelationKind != RelationKind.Sequel
// && r.RelationKind != RelationKind.Edition)
// .Select(sr => sr.Series))
// .RestrictAgainstAgeRestriction(userRating)
// .AsSplitQuery()
// .AsNoTracking()
// .ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
// .ToListAsync(),
Parent = await _context.SeriesRelation
.Where(r => r.TargetSeriesId == seriesId
&& usersSeriesIds.Contains(r.TargetSeriesId)
&& r.RelationKind != RelationKind.Prequel
&& r.RelationKind != RelationKind.Sequel
&& r.RelationKind != RelationKind.Edition)
.Select(sr => sr.Series)
.RestrictAgainstAgeRestriction(userRating)
.AsSplitQuery()
.AsNoTracking()
@ -1477,13 +1480,14 @@ public class SeriesRepository : ISeriesRepository
public async Task<bool> IsSeriesInWantToRead(int userId, int seriesId)
{
// BUG: This is always returning true for any series
var libraryIds = GetLibraryIdsForUser(userId);
return await _context.AppUser
.Where(user => user.Id == userId)
.SelectMany(u => u.WantToRead)
.SelectMany(u => u.WantToRead.Where(s => s.Id == seriesId && libraryIds.Contains(s.LibraryId)))
.AsSplitQuery()
.AsNoTracking()
.AnyAsync(s => libraryIds.Contains(s.LibraryId) && s.Id == seriesId);
.AnyAsync();
}
public async Task<IDictionary<string, IList<SeriesModified>>> GetFolderPathMap(int libraryId)