Metadata Downloading (#3525)

This commit is contained in:
Joe Milazzo 2025-02-05 16:16:44 -06:00 committed by GitHub
parent eb66763078
commit f4fd7230ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
108 changed files with 6296 additions and 484 deletions

View file

@ -1,5 +1,7 @@
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using API.Entities;
@ -13,6 +15,7 @@ using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Identity.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.ChangeTracking;
using Microsoft.EntityFrameworkCore.Diagnostics;
namespace API.Data;
@ -70,7 +73,8 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
public DbSet<ChapterPeople> ChapterPeople { get; set; } = null!;
public DbSet<SeriesMetadataPeople> SeriesMetadataPeople { get; set; } = null!;
public DbSet<EmailHistory> EmailHistory { get; set; } = null!;
public DbSet<MetadataSettings> MetadataSettings { get; set; } = null!;
public DbSet<MetadataFieldMapping> MetadataFieldMapping { get; set; } = null!;
protected override void OnModelCreating(ModelBuilder builder)
{
@ -120,10 +124,19 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
.Property(b => b.Locale)
.IsRequired(true)
.HasDefaultValue("en");
builder.Entity<AppUserPreferences>()
.Property(b => b.AniListScrobblingEnabled)
.HasDefaultValue(true);
builder.Entity<AppUserPreferences>()
.Property(b => b.WantToReadSync)
.HasDefaultValue(true);
builder.Entity<Library>()
.Property(b => b.AllowScrobbling)
.HasDefaultValue(true);
builder.Entity<Library>()
.Property(b => b.AllowMetadataMatching)
.HasDefaultValue(true);
builder.Entity<Chapter>()
.Property(b => b.WebLinks)
@ -189,6 +202,31 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
.WithMany(p => p.SeriesMetadataPeople)
.HasForeignKey(smp => smp.PersonId)
.OnDelete(DeleteBehavior.Cascade);
builder.Entity<MetadataSettings>()
.Property(x => x.AgeRatingMappings)
.HasConversion(
v => JsonSerializer.Serialize(v, JsonSerializerOptions.Default),
v => JsonSerializer.Deserialize<Dictionary<string, AgeRating>>(v, JsonSerializerOptions.Default)
);
// Ensure blacklist is stored as a JSON array
builder.Entity<MetadataSettings>()
.Property(x => x.Blacklist)
.HasConversion(
v => JsonSerializer.Serialize(v, JsonSerializerOptions.Default),
v => JsonSerializer.Deserialize<List<string>>(v, JsonSerializerOptions.Default)
);
// Configure one-to-many relationship
builder.Entity<MetadataSettings>()
.HasMany(x => x.FieldMappings)
.WithOne(x => x.MetadataSettings)
.HasForeignKey(x => x.MetadataSettingsId)
.OnDelete(DeleteBehavior.Cascade);
builder.Entity<MetadataSettings>()
.Property(b => b.Enabled)
.HasDefaultValue(true);
}
#nullable enable

View file

@ -0,0 +1,52 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using API.Entities.History;
using API.Entities.Metadata;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.8.5 - Migrating Kavita+ Series that are Blacklisted but have valid ExternalSeries row
/// </summary>
public static class ManualMigrateInvalidBlacklistSeries
{
public static async Task Migrate(DataContext context, ILogger<Program> logger)
{
if (await context.ManualMigrationHistory.AnyAsync(m => m.Name == "ManualMigrateInvalidBlacklistSeries"))
{
return;
}
logger.LogCritical("Running ManualMigrateInvalidBlacklistSeries migration - Please be patient, this may take some time. This is not an error");
// Get all series in the Blacklist table and set their IsBlacklist = true
var blacklistedSeries = await context.Series
.Include(s => s.ExternalSeriesMetadata)
.Where(s => s.IsBlacklisted && s.ExternalSeriesMetadata.ValidUntilUtc > DateTime.MinValue)
.ToListAsync();
foreach (var series in blacklistedSeries)
{
series.IsBlacklisted = false;
context.Series.Entry(series).State = EntityState.Modified;
}
if (context.ChangeTracker.HasChanges())
{
await context.SaveChangesAsync();
}
await context.ManualMigrationHistory.AddAsync(new ManualMigrationHistory()
{
Name = "ManualMigrateInvalidBlacklistSeries",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await context.SaveChangesAsync();
logger.LogCritical("Running ManualMigrateInvalidBlacklistSeries migration - Completed. This is not an error");
}
}

File diff suppressed because it is too large Load diff

View file

@ -0,0 +1,112 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class KavitaPlusUserAndMetadataSettings : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "AllowMetadataMatching",
table: "Library",
type: "INTEGER",
nullable: false,
defaultValue: true);
migrationBuilder.AddColumn<bool>(
name: "AniListScrobblingEnabled",
table: "AppUserPreferences",
type: "INTEGER",
nullable: false,
defaultValue: true);
migrationBuilder.AddColumn<bool>(
name: "WantToReadSync",
table: "AppUserPreferences",
type: "INTEGER",
nullable: false,
defaultValue: true);
migrationBuilder.CreateTable(
name: "MetadataSettings",
columns: table => new
{
Id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
Enabled = table.Column<bool>(type: "INTEGER", nullable: false, defaultValue: true),
EnableSummary = table.Column<bool>(type: "INTEGER", nullable: false),
EnablePublicationStatus = table.Column<bool>(type: "INTEGER", nullable: false),
EnableRelationships = table.Column<bool>(type: "INTEGER", nullable: false),
EnablePeople = table.Column<bool>(type: "INTEGER", nullable: false),
EnableStartDate = table.Column<bool>(type: "INTEGER", nullable: false),
EnableLocalizedName = table.Column<bool>(type: "INTEGER", nullable: false),
EnableGenres = table.Column<bool>(type: "INTEGER", nullable: false),
EnableTags = table.Column<bool>(type: "INTEGER", nullable: false),
FirstLastPeopleNaming = table.Column<bool>(type: "INTEGER", nullable: false),
AgeRatingMappings = table.Column<string>(type: "TEXT", nullable: true),
Blacklist = table.Column<string>(type: "TEXT", nullable: true),
Whitelist = table.Column<string>(type: "TEXT", nullable: true),
PersonRoles = table.Column<string>(type: "TEXT", nullable: true)
},
constraints: table =>
{
table.PrimaryKey("PK_MetadataSettings", x => x.Id);
});
migrationBuilder.CreateTable(
name: "MetadataFieldMapping",
columns: table => new
{
Id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
SourceType = table.Column<int>(type: "INTEGER", nullable: false),
DestinationType = table.Column<int>(type: "INTEGER", nullable: false),
SourceValue = table.Column<string>(type: "TEXT", nullable: true),
DestinationValue = table.Column<string>(type: "TEXT", nullable: true),
ExcludeFromSource = table.Column<bool>(type: "INTEGER", nullable: false),
MetadataSettingsId = table.Column<int>(type: "INTEGER", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_MetadataFieldMapping", x => x.Id);
table.ForeignKey(
name: "FK_MetadataFieldMapping_MetadataSettings_MetadataSettingsId",
column: x => x.MetadataSettingsId,
principalTable: "MetadataSettings",
principalColumn: "Id",
onDelete: ReferentialAction.Cascade);
});
migrationBuilder.CreateIndex(
name: "IX_MetadataFieldMapping_MetadataSettingsId",
table: "MetadataFieldMapping",
column: "MetadataSettingsId");
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "MetadataFieldMapping");
migrationBuilder.DropTable(
name: "MetadataSettings");
migrationBuilder.DropColumn(
name: "AllowMetadataMatching",
table: "Library");
migrationBuilder.DropColumn(
name: "AniListScrobblingEnabled",
table: "AppUserPreferences");
migrationBuilder.DropColumn(
name: "WantToReadSync",
table: "AppUserPreferences");
}
}
}

View file

@ -15,7 +15,7 @@ namespace API.Data.Migrations
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "9.0.0");
modelBuilder.HasAnnotation("ProductVersion", "9.0.1");
modelBuilder.Entity("API.Entities.AppRole", b =>
{
@ -353,6 +353,11 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<bool>("AniListScrobblingEnabled")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(true);
b.Property<int>("AppUserId")
.HasColumnType("INTEGER");
@ -460,6 +465,11 @@ namespace API.Data.Migrations
b.Property<int?>("ThemeId")
.HasColumnType("INTEGER");
b.Property<bool>("WantToReadSync")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(true);
b.HasKey("Id");
b.HasIndex("AppUserId")
@ -1093,12 +1103,37 @@ namespace API.Data.Migrations
b.ToTable("Genre");
});
modelBuilder.Entity("API.Entities.History.ManualMigrationHistory", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("Name")
.HasColumnType("TEXT");
b.Property<string>("ProductVersion")
.HasColumnType("TEXT");
b.Property<DateTime>("RanAt")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("ManualMigrationHistory");
});
modelBuilder.Entity("API.Entities.Library", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<bool>("AllowMetadataMatching")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(true);
b.Property<bool>("AllowScrobbling")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
@ -1247,26 +1282,6 @@ namespace API.Data.Migrations
b.ToTable("MangaFile");
});
modelBuilder.Entity("API.Entities.ManualMigrationHistory", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("Name")
.HasColumnType("TEXT");
b.Property<string>("ProductVersion")
.HasColumnType("TEXT");
b.Property<DateTime>("RanAt")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("ManualMigrationHistory");
});
modelBuilder.Entity("API.Entities.MediaError", b =>
{
b.Property<int>("Id")
@ -1594,6 +1609,92 @@ namespace API.Data.Migrations
b.ToTable("SeriesRelation");
});
modelBuilder.Entity("API.Entities.MetadataFieldMapping", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<int>("DestinationType")
.HasColumnType("INTEGER");
b.Property<string>("DestinationValue")
.HasColumnType("TEXT");
b.Property<bool>("ExcludeFromSource")
.HasColumnType("INTEGER");
b.Property<int>("MetadataSettingsId")
.HasColumnType("INTEGER");
b.Property<int>("SourceType")
.HasColumnType("INTEGER");
b.Property<string>("SourceValue")
.HasColumnType("TEXT");
b.HasKey("Id");
b.HasIndex("MetadataSettingsId");
b.ToTable("MetadataFieldMapping");
});
modelBuilder.Entity("API.Entities.MetadataSettings", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("AgeRatingMappings")
.HasColumnType("TEXT");
b.Property<string>("Blacklist")
.HasColumnType("TEXT");
b.Property<bool>("EnableGenres")
.HasColumnType("INTEGER");
b.Property<bool>("EnableLocalizedName")
.HasColumnType("INTEGER");
b.Property<bool>("EnablePeople")
.HasColumnType("INTEGER");
b.Property<bool>("EnablePublicationStatus")
.HasColumnType("INTEGER");
b.Property<bool>("EnableRelationships")
.HasColumnType("INTEGER");
b.Property<bool>("EnableStartDate")
.HasColumnType("INTEGER");
b.Property<bool>("EnableSummary")
.HasColumnType("INTEGER");
b.Property<bool>("EnableTags")
.HasColumnType("INTEGER");
b.Property<bool>("Enabled")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER")
.HasDefaultValue(true);
b.Property<bool>("FirstLastPeopleNaming")
.HasColumnType("INTEGER");
b.PrimitiveCollection<string>("PersonRoles")
.HasColumnType("TEXT");
b.PrimitiveCollection<string>("Whitelist")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("MetadataSettings");
});
modelBuilder.Entity("API.Entities.Person", b =>
{
b.Property<int>("Id")
@ -2824,6 +2925,17 @@ namespace API.Data.Migrations
b.Navigation("TargetSeries");
});
modelBuilder.Entity("API.Entities.MetadataFieldMapping", b =>
{
b.HasOne("API.Entities.MetadataSettings", "MetadataSettings")
.WithMany("FieldMappings")
.HasForeignKey("MetadataSettingsId")
.OnDelete(DeleteBehavior.Cascade)
.IsRequired();
b.Navigation("MetadataSettings");
});
modelBuilder.Entity("API.Entities.ReadingList", b =>
{
b.HasOne("API.Entities.AppUser", "AppUser")
@ -3223,6 +3335,11 @@ namespace API.Data.Migrations
b.Navigation("People");
});
modelBuilder.Entity("API.Entities.MetadataSettings", b =>
{
b.Navigation("FieldMappings");
});
modelBuilder.Entity("API.Entities.Person", b =>
{
b.Navigation("ChapterPeople");

View file

@ -43,6 +43,7 @@ public interface IPersonRepository
Task<IEnumerable<SeriesDto>> GetSeriesKnownFor(int personId);
Task<IEnumerable<StandaloneChapterDto>> GetChaptersForPersonByRole(int personId, int userId, PersonRole role);
Task<IList<Person>> GetPeopleByNames(List<string> normalizedNames);
Task<Person?> GetPersonByAniListId(int aniListId);
}
public class PersonRepository : IPersonRepository
@ -263,6 +264,13 @@ public class PersonRepository : IPersonRepository
.ToListAsync();
}
public async Task<Person?> GetPersonByAniListId(int aniListId)
{
return await _context.Person
.Where(p => p.AniListId == aniListId)
.FirstOrDefaultAsync();
}
public async Task<IList<Person>> GetAllPeople()
{
return await _context.Person

View file

@ -79,6 +79,7 @@ public class ScrobbleRepository : IScrobbleRepository
.Include(s => s.Series)
.ThenInclude(s => s.Metadata)
.Include(s => s.AppUser)
.ThenInclude(u => u.UserPreferences)
.Where(s => s.ScrobbleEventType == type)
.Where(s => s.IsProcessed == isProcessed)
.AsSplitQuery()

View file

@ -146,7 +146,7 @@ public interface ISeriesRepository
Task<IEnumerable<Series>> GetAllSeriesByNameAsync(IList<string> normalizedNames,
int userId, SeriesIncludes includes = SeriesIncludes.None);
Task<Series?> GetFullSeriesByAnyName(string seriesName, string localizedName, int libraryId, MangaFormat format, bool withFullIncludes = true);
Task<Series?> GetSeriesByAnyName(string seriesName, string localizedName, IList<MangaFormat> formats, int userId);
Task<Series?> GetSeriesByAnyName(string seriesName, string localizedName, IList<MangaFormat> formats, int userId, int? aniListId = null, SeriesIncludes includes = SeriesIncludes.None);
public Task<IList<Series>> GetAllSeriesByAnyName(string seriesName, string localizedName, int libraryId,
MangaFormat format);
Task<IList<Series>> RemoveSeriesNotInList(IList<ParsedSeries> seenSeries, int libraryId);
@ -164,7 +164,7 @@ public interface ISeriesRepository
Task RemoveFromOnDeck(int seriesId, int userId);
Task ClearOnDeckRemoval(int seriesId, int userId);
Task<PagedList<SeriesDto>> GetSeriesDtoForLibraryIdV2Async(int userId, UserParams userParams, FilterV2Dto filterDto, QueryContext queryContext = QueryContext.None);
Task<PlusSeriesDto?> GetPlusSeriesDto(int seriesId);
Task<PlusSeriesRequestDto?> GetPlusSeriesDto(int seriesId);
Task<int> GetCountAsync();
Task<Series?> MatchSeries(ExternalSeriesDetailDto externalSeries);
}
@ -699,17 +699,16 @@ public class SeriesRepository : ISeriesRepository
var retSeries = query
.ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
//.AsSplitQuery()
.AsNoTracking();
return await PagedList<SeriesDto>.CreateAsync(retSeries, userParams.PageNumber, userParams.PageSize);
}
public async Task<PlusSeriesDto?> GetPlusSeriesDto(int seriesId)
public async Task<PlusSeriesRequestDto?> GetPlusSeriesDto(int seriesId)
{
return await _context.Series
.Where(s => s.Id == seriesId)
.Select(series => new PlusSeriesDto()
.Select(series => new PlusSeriesRequestDto()
{
MediaFormat = series.Library.Type.ConvertToPlusMediaFormat(series.Format),
SeriesName = series.Name,
@ -1725,24 +1724,36 @@ public class SeriesRepository : ISeriesRepository
#nullable enable
}
public async Task<Series?> GetSeriesByAnyName(string seriesName, string localizedName, IList<MangaFormat> formats, int userId)
public async Task<Series?> GetSeriesByAnyName(string seriesName, string localizedName, IList<MangaFormat> formats,
int userId, int? aniListId = null, SeriesIncludes includes = SeriesIncludes.None)
{
var libraryIds = GetLibraryIdsForUser(userId);
var normalizedSeries = seriesName.ToNormalized();
var normalizedLocalized = localizedName.ToNormalized();
return await _context.Series
var query = _context.Series
.Where(s => libraryIds.Contains(s.LibraryId))
.Where(s => formats.Contains(s.Format))
.Where(s =>
.Where(s => formats.Contains(s.Format));
if (aniListId.HasValue && aniListId.Value > 0)
{
// If AniList ID is provided, override name checks
query = query.Where(s => s.ExternalSeriesMetadata.AniListId == aniListId.Value);
}
else
{
// Otherwise, use name checks
query = query.Where(s =>
s.NormalizedName.Equals(normalizedSeries)
|| s.NormalizedName.Equals(normalizedLocalized)
|| s.NormalizedLocalizedName.Equals(normalizedSeries)
|| (!string.IsNullOrEmpty(normalizedLocalized) && s.NormalizedLocalizedName.Equals(normalizedLocalized))
|| (s.OriginalName != null && s.OriginalName.Equals(seriesName))
)
);
}
return await query
.Includes(includes)
.FirstOrDefaultAsync();
}

View file

@ -1,12 +1,14 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using API.DTOs.KavitaPlus.Metadata;
using API.DTOs.SeriesDetail;
using API.DTOs.Settings;
using API.Entities;
using API.Entities.Enums;
using API.Entities.Metadata;
using AutoMapper;
using AutoMapper.QueryableExtensions;
using Microsoft.EntityFrameworkCore;
namespace API.Data.Repositories;
@ -14,11 +16,15 @@ namespace API.Data.Repositories;
public interface ISettingsRepository
{
void Update(ServerSetting settings);
void Update(MetadataSettings settings);
void RemoveRange(List<MetadataFieldMapping> fieldMappings);
Task<ServerSettingDto> GetSettingsDtoAsync();
Task<ServerSetting> GetSettingAsync(ServerSettingKey key);
Task<IEnumerable<ServerSetting>> GetSettingsAsync();
void Remove(ServerSetting setting);
Task<ExternalSeriesMetadata?> GetExternalSeriesMetadata(int seriesId);
Task<MetadataSettings> GetMetadataSettings();
Task<MetadataSettingsDto> GetMetadataSettingDto();
}
public class SettingsRepository : ISettingsRepository
{
@ -36,6 +42,16 @@ public class SettingsRepository : ISettingsRepository
_context.Entry(settings).State = EntityState.Modified;
}
public void Update(MetadataSettings settings)
{
_context.Entry(settings).State = EntityState.Modified;
}
public void RemoveRange(List<MetadataFieldMapping> fieldMappings)
{
_context.MetadataFieldMapping.RemoveRange(fieldMappings);
}
public void Remove(ServerSetting setting)
{
_context.Remove(setting);
@ -48,6 +64,21 @@ public class SettingsRepository : ISettingsRepository
.FirstOrDefaultAsync();
}
public async Task<MetadataSettings> GetMetadataSettings()
{
return await _context.MetadataSettings
.Include(m => m.FieldMappings)
.FirstAsync();
}
public async Task<MetadataSettingsDto> GetMetadataSettingDto()
{
return await _context.MetadataSettings
.Include(m => m.FieldMappings)
.ProjectTo<MetadataSettingsDto>(_mapper.ConfigurationProvider)
.FirstAsync();
}
public async Task<ServerSettingDto> GetSettingsDtoAsync()
{
var settings = await _context.ServerSetting

View file

@ -262,12 +262,11 @@ public static class Seed
new() {Key = ServerSettingKey.EmailCustomizedTemplates, Value = "false"},
new() {Key = ServerSettingKey.FirstInstallVersion, Value = BuildInfo.Version.ToString()},
new() {Key = ServerSettingKey.FirstInstallDate, Value = DateTime.UtcNow.ToString()},
}.ToArray());
foreach (var defaultSetting in DefaultSettings)
{
var existing = context.ServerSetting.FirstOrDefault(s => s.Key == defaultSetting.Key);
var existing = await context.ServerSetting.FirstOrDefaultAsync(s => s.Key == defaultSetting.Key);
if (existing == null)
{
await context.ServerSetting.AddAsync(defaultSetting);
@ -291,6 +290,35 @@ public static class Seed
}
public static async Task SeedMetadataSettings(DataContext context)
{
await context.Database.EnsureCreatedAsync();
var existing = await context.MetadataSettings.FirstOrDefaultAsync();
if (existing == null)
{
existing = new MetadataSettings()
{
Enabled = true,
EnablePeople = true,
EnableRelationships = true,
EnableSummary = true,
EnablePublicationStatus = true,
EnableStartDate = true,
EnableTags = false,
EnableGenres = true,
EnableLocalizedName = false,
FirstLastPeopleNaming = false,
PersonRoles = [PersonRole.Writer, PersonRole.CoverArtist, PersonRole.Character]
};
await context.MetadataSettings.AddAsync(existing);
}
await context.SaveChangesAsync();
}
public static async Task SeedUserApiKeys(DataContext context)
{
await context.Database.EnsureCreatedAsync();