AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Comic Rework, New Scanner, Foundation Overahul (is this a full release?) (#2780)

Browse source
This commit is contained in:
Joe Milazzo 2024-03-17 12:58:32 -05:00 • committed by GitHub
parent d7e9e7c832
commit 7552c3f5fa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
182 changed files with 27630 additions and 3046 deletions
Download patch file Download diff file Expand all files Collapse all files

9
API/Data/DataContext.cs
View file

@ -156,10 +156,15 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
{
if (e.FromQuery || e.Entry.State != EntityState.Added || e.Entry.Entity is not IEntityDate entity) return;
entity.Created = DateTime.Now;
entity.LastModified = DateTime.Now;
entity.CreatedUtc = DateTime.UtcNow;
entity.LastModifiedUtc = DateTime.UtcNow;
// This allows for mocking
if (entity.Created == DateTime.MinValue)
{
entity.Created = DateTime.Now;
entity.CreatedUtc = DateTime.UtcNow;
}
}
private static void OnEntityStateChanged(object? sender, EntityStateChangedEventArgs e)

140
API/Data/ManualMigrations/ManualMigrateMixedSpecials.cs Normal file
View file

@ -0,0 +1,140 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using API.Helpers.Builders;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
public class UserProgressCsvRecord
{
public bool IsSpecial { get; set; }
public int AppUserId { get; set; }
public int PagesRead { get; set; }
public string Range { get; set; }
public string Number { get; set; }
public float MinNumber { get; set; }
public int SeriesId { get; set; }
public int VolumeId { get; set; }
}
/// <summary>
/// v0.8.0 migration to move Specials into their own volume and retain user progress.
/// </summary>
public static class MigrateMixedSpecials
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "ManualMigrateMixedSpecials"))
{
return;
}
logger.LogCritical(
"Running ManualMigrateMixedSpecials migration - Please be patient, this may take some time. This is not an error");
// First, group all the progresses into different series
// Get each series and move the specials from old volume to the new Volume()
// Create a new progress event from existing and store the Id of existing progress event to delete it
// Save per series
var progress = await dataContext.AppUserProgresses
.Join(dataContext.Chapter, p => p.ChapterId, c => c.Id, (p, c) => new UserProgressCsvRecord
{
IsSpecial = c.IsSpecial,
AppUserId = p.AppUserId,
PagesRead = p.PagesRead,
Range = c.Range,
Number = c.Number,
MinNumber = c.MinNumber,
SeriesId = p.SeriesId,
VolumeId = p.VolumeId
})
.Where(d => d.IsSpecial || d.Number == "0")
.Join(dataContext.Volume, d => d.VolumeId, v => v.Id, (d, v) => new
{
ProgressRecord = d,
Volume = v
})
.Where(d => d.Volume.Name == "0")
.ToListAsync();
// First, group all the progresses into different series
logger.LogCritical("Migrating {Count} progress events to new Volume structure - This may take over 10 minutes depending on size of DB. Please wait", progress.Count);
var progressesGroupedBySeries = progress.GroupBy(p => p.ProgressRecord.SeriesId);
foreach (var seriesGroup in progressesGroupedBySeries)
{
// Get each series and move the specials from the old volume to the new Volume
var seriesId = seriesGroup.Key;
var specialsInSeries = seriesGroup
.Where(p => p.ProgressRecord.IsSpecial)
.ToList();
// Get distinct Volumes by Id. For each one, create it then create the progress events
var distinctVolumes = specialsInSeries.DistinctBy(d => d.Volume.Id);
foreach (var distinctVolume in distinctVolumes)
{
// Create a new volume for each series with the appropriate number (-100000)
var chapters = await dataContext.Chapter
.Where(c => c.VolumeId == distinctVolume.Volume.Id && c.IsSpecial).ToListAsync();
var newVolume = new VolumeBuilder(Parser.SpecialVolume)
.WithSeriesId(seriesId)
.WithChapters(chapters)
.Build();
dataContext.Volume.Add(newVolume);
await dataContext.SaveChangesAsync(); // Save changes to generate the newVolumeId
// Migrate the progress event to the new volume
distinctVolume.ProgressRecord.VolumeId = newVolume.Id;
logger.LogInformation("Moving {Count} chapters from Volume Id {OldVolumeId} to New Volume {NewVolumeId}",
chapters.Count, distinctVolume.Volume.Id, newVolume.Id);
// Move the special chapters from the old volume to the new Volume
var specialChapters = await dataContext.Chapter
.Where(c => c.VolumeId == distinctVolume.ProgressRecord.VolumeId && c.IsSpecial)
.ToListAsync();
foreach (var specialChapter in specialChapters)
{
// Update the VolumeId on the existing progress event
specialChapter.VolumeId = newVolume.Id;
}
await dataContext.SaveChangesAsync();
}
}
// Save changes after processing all series
if (dataContext.ChangeTracker.HasChanges())
{
await dataContext.SaveChangesAsync();
}
// Update all Volumes with Name as "0" -> Special
logger.LogCritical("Updating all Volumes with Name 0 to SpecialNumber");
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "ManualMigrateMixedSpecials",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running ManualMigrateMixedSpecials migration - Completed. This is not an error");
}
}

89
API/Data/ManualMigrations/MigrateChapterFields.cs Normal file
View file

@ -0,0 +1,89 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.8.0, this migrates the existing Chapter and Volume 0 -> Parser defined, MangaFile.FileName
/// </summary>
public static class MigrateChapterFields
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateChapterFields"))
{
return;
}
logger.LogCritical(
"Running MigrateChapterFields migration - Please be patient, this may take some time. This is not an error");
// Update all volumes only have specials in them (rare)
var volumesWithJustSpecials = dataContext.Volume
.Include(v => v.Chapters)
.Where(v => v.Name == "0" && v.Chapters.All(c => c.IsSpecial))
.ToList();
logger.LogCritical(
"Running MigrateChapterFields migration - Updating {Count} volumes that only have specials in them", volumesWithJustSpecials.Count);
foreach (var volume in volumesWithJustSpecials)
{
volume.Name = $"{Parser.SpecialVolumeNumber}";
volume.MinNumber = Parser.SpecialVolumeNumber;
volume.MaxNumber = Parser.SpecialVolumeNumber;
}
// Update all volumes that only have loose leafs in them
var looseLeafVolumes = dataContext.Volume
.Include(v => v.Chapters)
.Where(v => v.Name == "0" && v.Chapters.All(c => !c.IsSpecial))
.ToList();
logger.LogCritical(
"Running MigrateChapterFields migration - Updating {Count} volumes that only have loose leaf chapters in them", looseLeafVolumes.Count);
foreach (var volume in looseLeafVolumes)
{
volume.Name = $"{Parser.DefaultChapterNumber}";
volume.MinNumber = Parser.DefaultChapterNumber;
volume.MaxNumber = Parser.DefaultChapterNumber;
}
// Update all MangaFile
logger.LogCritical(
"Running MigrateChapterFields migration - Updating all MangaFiles");
foreach (var mangaFile in dataContext.MangaFile)
{
mangaFile.FileName = Parser.RemoveExtensionIfSupported(mangaFile.FilePath);
}
var looseLeafChapters = await dataContext.Chapter.Where(c => c.Number == "0").ToListAsync();
logger.LogCritical(
"Running MigrateChapterFields migration - Updating {Count} loose leaf chapters", looseLeafChapters.Count);
foreach (var chapter in looseLeafChapters)
{
chapter.Number = Parser.DefaultChapter;
chapter.MinNumber = Parser.DefaultChapterNumber;
chapter.MaxNumber = Parser.DefaultChapterNumber;
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateChapterFields",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateChapterFields migration - Completed. This is not an error");
}
}

50
API/Data/ManualMigrations/MigrateChapterNumber.cs Normal file
View file

@ -0,0 +1,50 @@
using System;
using System.Threading.Tasks;
using API.Entities;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.8.0, this migrates the existing Chapter Range -> Chapter Min/Max Number
/// </summary>
public static class MigrateChapterNumber
{
public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateChapterNumber"))
{
return;
}
logger.LogCritical(
"Running MigrateChapterNumber migration - Please be patient, this may take some time. This is not an error");
// Get all volumes
foreach (var chapter in dataContext.Chapter)
{
if (chapter.IsSpecial)
{
chapter.MinNumber = Parser.DefaultChapterNumber;
chapter.MaxNumber = Parser.DefaultChapterNumber;
continue;
}
chapter.MinNumber = Parser.MinNumberFromRange(chapter.Range);
chapter.MaxNumber = Parser.MaxNumberFromRange(chapter.Range);
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateChapterNumber",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateChapterNumber migration - Completed. This is not an error");
}
}

55
API/Data/ManualMigrations/MigrateChapterRange.cs Normal file
View file

@ -0,0 +1,55 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using API.Helpers.Builders;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.8.0 changed the range to that it doesn't have filename by default
/// </summary>
public static class MigrateChapterRange
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateChapterRange"))
{
return;
}
logger.LogCritical(
"Running MigrateChapterRange migration - Please be patient, this may take some time. This is not an error");
var chapters = await dataContext.Chapter.ToListAsync();
foreach (var chapter in chapters)
{
if (Parser.MinNumberFromRange(chapter.Range) == 0.0f)
{
chapter.Range = chapter.GetNumberTitle();
}
}
// Save changes after processing all series
if (dataContext.ChangeTracker.HasChanges())
{
await dataContext.SaveChangesAsync();
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateChapterRange",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateChapterRange migration - Completed. This is not an error");
}
}

3
API/Data/ManualMigrations/MigrateLibrariesToHaveAllFileTypes.cs
View file

@ -15,9 +15,8 @@ public static class MigrateLibrariesToHaveAllFileTypes
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
if (await dataContext.Library.AnyAsync(l => l.LibraryFileTypes.Count == 0))
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateLibrariesToHaveAllFileTypes"))
{
logger.LogCritical("Running MigrateLibrariesToHaveAllFileTypes migration - Completed. This is not an error");
return;
}

2
API/Data/ManualMigrations/MigrateManualHistory.cs
View file

@ -16,8 +16,6 @@ public static class MigrateManualHistory
{
if (await dataContext.ManualMigrationHistory.AnyAsync())
{
logger.LogCritical(
"Running MigrateManualHistory migration - Completed. This is not an error");
return;
}

7
API/Data/ManualMigrations/MigrateSmartFilterEncoding.cs
View file

@ -4,6 +4,7 @@ using System.Text.RegularExpressions;
using System.Threading.Tasks;
using API.DTOs.Filtering.v2;
using API.Helpers;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
@ -21,8 +22,12 @@ public static class MigrateSmartFilterEncoding
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateSmartFilterEncoding migration - Please be patient, this may take some time. This is not an error");
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateSmartFilterEncoding"))
{
return;
}
logger.LogCritical("Running MigrateSmartFilterEncoding migration - Please be patient, this may take some time. This is not an error");
var smartFilters = dataContext.AppUserSmartFilter.ToList();
foreach (var filter in smartFilters)

4
API/Data/ManualMigrations/MigrateUserLibrarySideNavStream.cs
View file

@ -14,6 +14,10 @@ public static class MigrateUserLibrarySideNavStream
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateUserLibrarySideNavStream"))
{
return;
}
var usersWithLibraryStreams = await dataContext.AppUser
.Include(u => u.SideNavStreams)

41
API/Data/ManualMigrations/MigrateVolumeLookupName.cs Normal file
View file

@ -0,0 +1,41 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
public static class MigrateVolumeLookupName
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateVolumeLookupName"))
{
return;
}
logger.LogCritical(
"Running MigrateVolumeLookupName migration - Please be patient, this may take some time. This is not an error");
// Update all volumes to have LookupName as after this migration, name isn't used for lookup
var volumes = dataContext.Volume.ToList();
foreach (var volume in volumes)
{
volume.LookupName = volume.Name;
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateVolumeLookupName",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateVolumeLookupName migration - Completed. This is not an error");
}
}

7
API/Data/ManualMigrations/MigrateVolumeNumber.cs
View file

@ -13,8 +13,13 @@ namespace API.Data.ManualMigrations;
/// </summary>
public static class MigrateVolumeNumber
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateVolumeNumber"))
{
return;
}
if (await dataContext.Volume.AnyAsync(v => v.MaxNumber > 0))
{
logger.LogCritical(

5
API/Data/ManualMigrations/MigrateWantToReadExport.cs
View file

@ -20,6 +20,11 @@ public static class MigrateWantToReadExport
{
try
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateWantToReadExport"))
{
return;
}
var importFile = Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv");
if (File.Exists(importFile))
{

9
API/Data/ManualMigrations/MigrateWantToReadImport.cs
View file

@ -6,6 +6,7 @@ using API.Data.Repositories;
using API.Entities;
using API.Services;
using CsvHelper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
@ -15,8 +16,14 @@ namespace API.Data.ManualMigrations;
/// </summary>
public static class MigrateWantToReadImport
{
public static async Task Migrate(IUnitOfWork unitOfWork, IDirectoryService directoryService, ILogger<Program> logger)
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, IDirectoryService directoryService, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateWantToReadImport"))
{
return;
}
var importFile = Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv");
var outputFile = Path.Join(directoryService.ConfigDirectory, "imported-want-to-read-migration.csv");

14
API/Data/Metadata/ComicInfo.cs
View file

@ -127,7 +127,11 @@ public class ComicInfo
public string CoverArtist { get; set; } = string.Empty;
public string Editor { get; set; } = string.Empty;
public string Publisher { get; set; } = string.Empty;
public string Imprint { get; set; } = string.Empty;
public string Characters { get; set; } = string.Empty;
public string Teams { get; set; } = string.Empty;
public string Locations { get; set; } = string.Empty;
public static AgeRating ConvertAgeRatingToEnum(string value)
{
@ -151,9 +155,12 @@ public class ComicInfo
info.Letterer = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Letterer);
info.Penciller = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Penciller);
info.Publisher = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Publisher);
info.Imprint = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Imprint);
info.Characters = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Characters);
info.Translator = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Translator);
info.CoverArtist = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.CoverArtist);
info.Teams = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Teams);
info.Locations = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Locations);
// We need to convert GTIN to ISBN
if (!string.IsNullOrEmpty(info.GTIN))
@ -174,7 +181,12 @@ public class ComicInfo
if (!string.IsNullOrEmpty(info.Number))
{
info.Number = info.Number.Replace(",", "."); // Corrective measure for non English OSes
info.Number = info.Number.Trim().Replace(",", "."); // Corrective measure for non English OSes
}
if (!string.IsNullOrEmpty(info.Volume))
{
info.Volume = info.Volume.Trim();
}
}

2877
API/Data/Migrations/20240214232436_ChapterNumber.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

40
API/Data/Migrations/20240214232436_ChapterNumber.cs Normal file
View file

@ -0,0 +1,40 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class ChapterNumber : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<float>(
name: "MaxNumber",
table: "Chapter",
type: "REAL",
nullable: false,
defaultValue: 0f);
migrationBuilder.AddColumn<float>(
name: "MinNumber",
table: "Chapter",
type: "REAL",
nullable: false,
defaultValue: 0f);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "MaxNumber",
table: "Chapter");
migrationBuilder.DropColumn(
name: "MinNumber",
table: "Chapter");
}
}
}

2880
API/Data/Migrations/20240216000223_MangaFileNameTemp.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

28
API/Data/Migrations/20240216000223_MangaFileNameTemp.cs Normal file
View file

@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class MangaFileNameTemp : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "FileName",
table: "MangaFile",
type: "TEXT",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "FileName",
table: "MangaFile");
}
}
}

2883
API/Data/Migrations/20240222125420_ChapterIssueSort.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

29
API/Data/Migrations/20240222125420_ChapterIssueSort.cs Normal file
View file

@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class ChapterIssueSort : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<float>(
name: "SortOrder",
table: "Chapter",
type: "REAL",
nullable: false,
defaultValue: 0f);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "SortOrder",
table: "Chapter");
}
}
}

2886
API/Data/Migrations/20240225235816_VolumeLookupName.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

28
API/Data/Migrations/20240225235816_VolumeLookupName.cs Normal file
View file

@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class VolumeLookupName : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "LookupName",
table: "Volume",
type: "TEXT",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "LookupName",
table: "Volume");
}
}
}

2889
API/Data/Migrations/20240309140117_SeriesImprints.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

29
API/Data/Migrations/20240309140117_SeriesImprints.cs Normal file
View file

@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class SeriesImprints : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "ImprintLocked",
table: "SeriesMetadata",
type: "INTEGER",
nullable: false,
defaultValue: false);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ImprintLocked",
table: "SeriesMetadata");
}
}
}

2892
API/Data/Migrations/20240313112552_SeriesLowestFolderPath.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

28
API/Data/Migrations/20240313112552_SeriesLowestFolderPath.cs Normal file
View file

@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class SeriesLowestFolderPath : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "LowestFolderPath",
table: "Series",
type: "TEXT",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "LowestFolderPath",
table: "Series");
}
}
}

2898
API/Data/Migrations/20240314194402_TeamsAndLocations.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

40
API/Data/Migrations/20240314194402_TeamsAndLocations.cs Normal file
View file

@ -0,0 +1,40 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class TeamsAndLocations : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<bool>(
name: "LocationLocked",
table: "SeriesMetadata",
type: "INTEGER",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<bool>(
name: "TeamLocked",
table: "SeriesMetadata",
type: "INTEGER",
nullable: false,
defaultValue: false);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "LocationLocked",
table: "SeriesMetadata");
migrationBuilder.DropColumn(
name: "TeamLocked",
table: "SeriesMetadata");
}
}
}

29
API/Data/Migrations/DataContextModelSnapshot.cs
View file

@ -15,7 +15,7 @@ namespace API.Data.Migrations
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "8.0.1");
modelBuilder.HasAnnotation("ProductVersion", "8.0.3");
modelBuilder.Entity("API.Entities.AppRole", b =>
{
@ -679,9 +679,15 @@ namespace API.Data.Migrations
b.Property<int>("MaxHoursToRead")
.HasColumnType("INTEGER");
b.Property<float>("MaxNumber")
.HasColumnType("REAL");
b.Property<int>("MinHoursToRead")
.HasColumnType("INTEGER");
b.Property<float>("MinNumber")
.HasColumnType("REAL");
b.Property<string>("Number")
.HasColumnType("TEXT");
@ -697,6 +703,9 @@ namespace API.Data.Migrations
b.Property<string>("SeriesGroup")
.HasColumnType("TEXT");
b.Property<float>("SortOrder")
.HasColumnType("REAL");
b.Property<string>("StoryArc")
.HasColumnType("TEXT");
@ -973,6 +982,9 @@ namespace API.Data.Migrations
b.Property<string>("Extension")
.HasColumnType("TEXT");
b.Property<string>("FileName")
.HasColumnType("TEXT");
b.Property<string>("FilePath")
.HasColumnType("TEXT");
@ -1241,6 +1253,9 @@ namespace API.Data.Migrations
b.Property<bool>("GenresLocked")
.HasColumnType("INTEGER");
b.Property<bool>("ImprintLocked")
.HasColumnType("INTEGER");
b.Property<bool>("InkerLocked")
.HasColumnType("INTEGER");
@ -1253,6 +1268,9 @@ namespace API.Data.Migrations
b.Property<bool>("LettererLocked")
.HasColumnType("INTEGER");
b.Property<bool>("LocationLocked")
.HasColumnType("INTEGER");
b.Property<int>("MaxCount")
.HasColumnType("INTEGER");
@ -1290,6 +1308,9 @@ namespace API.Data.Migrations
b.Property<bool>("TagsLocked")
.HasColumnType("INTEGER");
b.Property<bool>("TeamLocked")
.HasColumnType("INTEGER");
b.Property<int>("TotalCount")
.HasColumnType("INTEGER");
@ -1665,6 +1686,9 @@ namespace API.Data.Migrations
b.Property<bool>("LocalizedNameLocked")
.HasColumnType("INTEGER");
b.Property<string>("LowestFolderPath")
.HasColumnType("TEXT");
b.Property<int>("MaxHoursToRead")
.HasColumnType("INTEGER");
@ -1839,6 +1863,9 @@ namespace API.Data.Migrations
b.Property<DateTime>("LastModifiedUtc")
.HasColumnType("TEXT");
b.Property<string>("LookupName")
.HasColumnType("TEXT");
b.Property<int>("MaxHoursToRead")
.HasColumnType("INTEGER");

6
API/Data/Repositories/AppUserProgressRepository.cs
View file

@ -167,9 +167,10 @@ public class AppUserProgressRepository : IAppUserProgressRepository
(appUserProgresses, chapter) => new {appUserProgresses, chapter})
.Where(p => p.appUserProgresses.SeriesId == seriesId && p.appUserProgresses.AppUserId == userId &&
p.appUserProgresses.PagesRead >= p.chapter.Pages)
.Select(p => p.chapter.Range)
.Where(p => p.chapter.MaxNumber != Parser.SpecialVolumeNumber)
.Select(p => p.chapter.MaxNumber)
.ToListAsync();
return list.Count == 0 ? 0 : list.DefaultIfEmpty().Where(d => d != null).Max(d => (int) Math.Floor(Parser.MaxNumberFromRange(d)));
return list.Count == 0 ? 0 : (int) list.DefaultIfEmpty().Max(d => d);
}
public async Task<float> GetHighestFullyReadVolumeForSeries(int seriesId, int userId)
@ -179,6 +180,7 @@ public class AppUserProgressRepository : IAppUserProgressRepository
(appUserProgresses, chapter) => new {appUserProgresses, chapter})
.Where(p => p.appUserProgresses.SeriesId == seriesId && p.appUserProgresses.AppUserId == userId &&
p.appUserProgresses.PagesRead >= p.chapter.Pages)
.Where(p => p.chapter.MaxNumber != Parser.SpecialVolumeNumber)
.Select(p => p.chapter.Volume.MaxNumber)
.ToListAsync();
return list.Count == 0 ? 0 : list.DefaultIfEmpty().Max();

14
API/Data/Repositories/ChapterRepository.cs
View file

@ -78,7 +78,7 @@ public class ChapterRepository : IChapterRepository
.Where(c => c.Id == chapterId)
.Join(_context.Volume, c => c.VolumeId, v => v.Id, (chapter, volume) => new
{
ChapterNumber = chapter.Range,
ChapterNumber = chapter.MinNumber,
VolumeNumber = volume.Name,
VolumeId = volume.Id,
chapter.IsSpecial,
@ -102,8 +102,8 @@ public class ChapterRepository : IChapterRepository
})
.Select(data => new ChapterInfoDto()
{
ChapterNumber = data.ChapterNumber,
VolumeNumber = data.VolumeNumber + string.Empty,
ChapterNumber = data.ChapterNumber + string.Empty, // TODO: Fix this
VolumeNumber = data.VolumeNumber + string.Empty, // TODO: Fix this
VolumeId = data.VolumeId,
IsSpecial = data.IsSpecial,
SeriesId = data.SeriesId,
@ -175,6 +175,7 @@ public class ChapterRepository : IChapterRepository
{
return await _context.Chapter
.Includes(includes)
.OrderBy(c => c.SortOrder)
.FirstOrDefaultAsync(c => c.Id == chapterId);
}
@ -187,6 +188,7 @@ public class ChapterRepository : IChapterRepository
{
return await _context.Chapter
.Where(c => c.VolumeId == volumeId)
.OrderBy(c => c.SortOrder)
.ToListAsync();
}
@ -267,10 +269,16 @@ public class ChapterRepository : IChapterRepository
return chapter;
}
/// <summary>
/// Includes Volumes
/// </summary>
/// <param name="seriesId"></param>
/// <returns></returns>
public IEnumerable<Chapter> GetChaptersForSeries(int seriesId)
{
return _context.Chapter
.Where(c => c.Volume.SeriesId == seriesId)
.OrderBy(c => c.SortOrder)
.Include(c => c.Volume)
.AsEnumerable();
}

8
API/Data/Repositories/ExternalSeriesMetadataRepository.cs
View file

@ -34,6 +34,7 @@ public interface IExternalSeriesMetadataRepository
Task<bool> ExternalSeriesMetadataNeedsRefresh(int seriesId);
Task<SeriesDetailPlusDto> GetSeriesDetailPlusDto(int seriesId);
Task LinkRecommendationsToSeries(Series series);
Task LinkRecommendationsToSeries(int seriesId);
Task<bool> IsBlacklistedSeries(int seriesId);
Task CreateBlacklistedSeries(int seriesId, bool saveChanges = true);
Task RemoveFromBlacklist(int seriesId);
@ -179,6 +180,13 @@ public class ExternalSeriesMetadataRepository : IExternalSeriesMetadataRepositor
return seriesDetailPlusDto;
}
public async Task LinkRecommendationsToSeries(int seriesId)
{
var series = await _context.Series.Where(s => s.Id == seriesId).AsNoTracking().SingleOrDefaultAsync();
if (series == null) return;
await LinkRecommendationsToSeries(series);
}
/// <summary>
/// Searches Recommendations without a SeriesId on record and attempts to link based on Series Name/Localized Name
/// </summary>

2
API/Data/Repositories/LibraryRepository.cs
View file

@ -318,7 +318,7 @@ public class LibraryRepository : ILibraryRepository
/// <returns></returns>
public async Task<bool> DoAnySeriesFoldersMatch(IEnumerable<string> folders)
{
var normalized = folders.Select(Services.Tasks.Scanner.Parser.Parser.NormalizePath);
var normalized = folders.Select(Parser.NormalizePath);
return await _context.Series.AnyAsync(s => normalized.Contains(s.FolderPath));
}

27
API/Data/Repositories/SeriesRepository.cs
View file

@ -498,6 +498,7 @@ public class SeriesRepository : ISeriesRepository
.Include(c => c.Files)
.Where(c => EF.Functions.Like(c.TitleName, $"%{searchQuery}%")
|| EF.Functions.Like(c.ISBN, $"%{searchQuery}%")
|| EF.Functions.Like(c.Range, $"%{searchQuery}%")
)
.Where(c => c.Files.All(f => fileIds.Contains(f.Id)))
.AsSplitQuery()
@ -1183,6 +1184,9 @@ public class SeriesRepository : ISeriesRepository
FilterField.Letterer => query.HasPeople(true, statement.Comparison, (IList<int>) value),
FilterField.Colorist => query.HasPeople(true, statement.Comparison, (IList<int>) value),
FilterField.Inker => query.HasPeople(true, statement.Comparison, (IList<int>) value),
FilterField.Imprint => query.HasPeople(true, statement.Comparison, (IList<int>) value),
FilterField.Team => query.HasPeople(true, statement.Comparison, (IList<int>) value),
FilterField.Location => query.HasPeople(true, statement.Comparison, (IList<int>) value),
FilterField.Penciller => query.HasPeople(true, statement.Comparison, (IList<int>) value),
FilterField.Writers => query.HasPeople(true, statement.Comparison, (IList<int>) value),
FilterField.Genres => query.HasGenre(true, statement.Comparison, (IList<int>) value),
@ -1817,19 +1821,7 @@ public class SeriesRepository : ISeriesRepository
AlternativeSettings = await GetRelatedSeriesQuery(seriesId, usersSeriesIds, RelationKind.AlternativeSetting, userRating),
AlternativeVersions = await GetRelatedSeriesQuery(seriesId, usersSeriesIds, RelationKind.AlternativeVersion, userRating),
Doujinshis = await GetRelatedSeriesQuery(seriesId, usersSeriesIds, RelationKind.Doujinshi, userRating),
// Parent = await _context.Series
// .SelectMany(s =>
// s.TargetSeries.Where(r => r.TargetSeriesId == seriesId
// && usersSeriesIds.Contains(r.TargetSeriesId)
// && r.RelationKind != RelationKind.Prequel
// && r.RelationKind != RelationKind.Sequel
// && r.RelationKind != RelationKind.Edition)
// .Select(sr => sr.Series))
// .RestrictAgainstAgeRestriction(userRating)
// .AsSplitQuery()
// .AsNoTracking()
// .ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
// .ToListAsync(),
Annuals = await GetRelatedSeriesQuery(seriesId, usersSeriesIds, RelationKind.Annual, userRating),
Parent = await _context.SeriesRelation
.Where(r => r.TargetSeriesId == seriesId
&& usersSeriesIds.Contains(r.TargetSeriesId)
@ -1891,8 +1883,8 @@ public class SeriesRepository : ISeriesRepository
VolumeId = c.VolumeId,
ChapterId = c.Id,
Format = c.Volume.Series.Format,
ChapterNumber = c.Number,
ChapterRange = c.Range,
ChapterNumber = c.MinNumber + string.Empty, // TODO: Refactor this
ChapterRange = c.Range, // TODO: Refactor this
IsSpecial = c.IsSpecial,
VolumeNumber = c.Volume.MinNumber,
ChapterTitle = c.Title,
@ -2063,7 +2055,7 @@ public class SeriesRepository : ISeriesRepository
foreach (var series in info)
{
if (series.FolderPath == null) continue;
if (!map.ContainsKey(series.FolderPath))
if (!map.TryGetValue(series.FolderPath, out var value))
{
map.Add(series.FolderPath, new List<SeriesModified>()
{
@ -2072,9 +2064,8 @@ public class SeriesRepository : ISeriesRepository
}
else
{
map[series.FolderPath].Add(series);
value.Add(series);
}
}
return map;

39
API/Data/Repositories/VolumeRepository.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using System.Threading.Tasks;
@ -6,6 +7,7 @@ using API.DTOs;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Extensions.QueryExtensions;
using API.Services;
using AutoMapper;
using AutoMapper.QueryableExtensions;
@ -14,6 +16,15 @@ using Microsoft.EntityFrameworkCore;
namespace API.Data.Repositories;
[Flags]
public enum VolumeIncludes
{
None = 1,
Chapters = 2,
People = 4,
Tags = 8,
}
public interface IVolumeRepository
{
void Add(Volume volume);
@ -22,7 +33,7 @@ public interface IVolumeRepository
Task<IList<MangaFile>> GetFilesForVolume(int volumeId);
Task<string?> GetVolumeCoverImageAsync(int volumeId);
Task<IList<int>> GetChapterIdsByVolumeIds(IReadOnlyList<int> volumeIds);
Task<IEnumerable<VolumeDto>> GetVolumesDtoAsync(int seriesId, int userId);
Task<IList<VolumeDto>> GetVolumesDtoAsync(int seriesId, int userId, VolumeIncludes includes = VolumeIncludes.Chapters);
Task<Volume?> GetVolumeAsync(int volumeId);
Task<VolumeDto?> GetVolumeDtoAsync(int volumeId, int userId);
Task<IEnumerable<Volume>> GetVolumesForSeriesAsync(IList<int> seriesIds, bool includeChapters = false);
@ -129,6 +140,7 @@ public class VolumeRepository : IVolumeRepository
.Include(vol => vol.Chapters)
.ThenInclude(c => c.Files)
.AsSplitQuery()
.OrderBy(v => v.MinNumber)
.ProjectTo<VolumeDto>(_mapper.ConfigurationProvider)
.SingleOrDefaultAsync(vol => vol.Id == volumeId);
@ -177,22 +189,22 @@ public class VolumeRepository : IVolumeRepository
/// <param name="seriesId"></param>
/// <param name="userId"></param>
/// <returns></returns>
public async Task<IEnumerable<VolumeDto>> GetVolumesDtoAsync(int seriesId, int userId)
public async Task<IList<VolumeDto>> GetVolumesDtoAsync(int seriesId, int userId, VolumeIncludes includes = VolumeIncludes.Chapters)
{
var volumes = await _context.Volume
.Where(vol => vol.SeriesId == seriesId)
.Include(vol => vol.Chapters)
.ThenInclude(c => c.People)
.Include(vol => vol.Chapters)
.ThenInclude(c => c.Tags)
.Includes(includes)
.OrderBy(volume => volume.MinNumber)
.ProjectTo<VolumeDto>(_mapper.ConfigurationProvider)
.AsNoTracking()
.AsSplitQuery()
.ToListAsync();
await AddVolumeModifiers(userId, volumes);
SortSpecialChapters(volumes);
foreach (var volume in volumes)
{
volume.Chapters = volume.Chapters.OrderBy(c => c.SortOrder).ToList();
}
return volumes;
}
@ -213,15 +225,6 @@ public class VolumeRepository : IVolumeRepository
}
private static void SortSpecialChapters(IEnumerable<VolumeDto> volumes)
{
foreach (var v in volumes.WhereLooseLeaf())
{
v.Chapters = v.Chapters.OrderByNatural(x => x.Range).ToList();
}
}
private async Task AddVolumeModifiers(int userId, IReadOnlyCollection<VolumeDto> volumes)
{
var volIds = volumes.Select(s => s.Id);