AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Comic Rework, New Scanner, Foundation Overahul (is this a full release?) (#2780)

Browse source
This commit is contained in:
Joe Milazzo 2024-03-17 12:58:32 -05:00 • committed by GitHub
parent d7e9e7c832
commit 7552c3f5fa
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
182 changed files with 27630 additions and 3046 deletions
Download patch file Download diff file Expand all files Collapse all files

140
API/Data/ManualMigrations/ManualMigrateMixedSpecials.cs Normal file
View file

@ -0,0 +1,140 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using API.Helpers.Builders;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
public class UserProgressCsvRecord
{
public bool IsSpecial { get; set; }
public int AppUserId { get; set; }
public int PagesRead { get; set; }
public string Range { get; set; }
public string Number { get; set; }
public float MinNumber { get; set; }
public int SeriesId { get; set; }
public int VolumeId { get; set; }
}
/// <summary>
/// v0.8.0 migration to move Specials into their own volume and retain user progress.
/// </summary>
public static class MigrateMixedSpecials
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "ManualMigrateMixedSpecials"))
{
return;
}
logger.LogCritical(
"Running ManualMigrateMixedSpecials migration - Please be patient, this may take some time. This is not an error");
// First, group all the progresses into different series
// Get each series and move the specials from old volume to the new Volume()
// Create a new progress event from existing and store the Id of existing progress event to delete it
// Save per series
var progress = await dataContext.AppUserProgresses
.Join(dataContext.Chapter, p => p.ChapterId, c => c.Id, (p, c) => new UserProgressCsvRecord
{
IsSpecial = c.IsSpecial,
AppUserId = p.AppUserId,
PagesRead = p.PagesRead,
Range = c.Range,
Number = c.Number,
MinNumber = c.MinNumber,
SeriesId = p.SeriesId,
VolumeId = p.VolumeId
})
.Where(d => d.IsSpecial || d.Number == "0")
.Join(dataContext.Volume, d => d.VolumeId, v => v.Id, (d, v) => new
{
ProgressRecord = d,
Volume = v
})
.Where(d => d.Volume.Name == "0")
.ToListAsync();
// First, group all the progresses into different series
logger.LogCritical("Migrating {Count} progress events to new Volume structure - This may take over 10 minutes depending on size of DB. Please wait", progress.Count);
var progressesGroupedBySeries = progress.GroupBy(p => p.ProgressRecord.SeriesId);
foreach (var seriesGroup in progressesGroupedBySeries)
{
// Get each series and move the specials from the old volume to the new Volume
var seriesId = seriesGroup.Key;
var specialsInSeries = seriesGroup
.Where(p => p.ProgressRecord.IsSpecial)
.ToList();
// Get distinct Volumes by Id. For each one, create it then create the progress events
var distinctVolumes = specialsInSeries.DistinctBy(d => d.Volume.Id);
foreach (var distinctVolume in distinctVolumes)
{
// Create a new volume for each series with the appropriate number (-100000)
var chapters = await dataContext.Chapter
.Where(c => c.VolumeId == distinctVolume.Volume.Id && c.IsSpecial).ToListAsync();
var newVolume = new VolumeBuilder(Parser.SpecialVolume)
.WithSeriesId(seriesId)
.WithChapters(chapters)
.Build();
dataContext.Volume.Add(newVolume);
await dataContext.SaveChangesAsync(); // Save changes to generate the newVolumeId
// Migrate the progress event to the new volume
distinctVolume.ProgressRecord.VolumeId = newVolume.Id;
logger.LogInformation("Moving {Count} chapters from Volume Id {OldVolumeId} to New Volume {NewVolumeId}",
chapters.Count, distinctVolume.Volume.Id, newVolume.Id);
// Move the special chapters from the old volume to the new Volume
var specialChapters = await dataContext.Chapter
.Where(c => c.VolumeId == distinctVolume.ProgressRecord.VolumeId && c.IsSpecial)
.ToListAsync();
foreach (var specialChapter in specialChapters)
{
// Update the VolumeId on the existing progress event
specialChapter.VolumeId = newVolume.Id;
}
await dataContext.SaveChangesAsync();
}
}
// Save changes after processing all series
if (dataContext.ChangeTracker.HasChanges())
{
await dataContext.SaveChangesAsync();
}
// Update all Volumes with Name as "0" -> Special
logger.LogCritical("Updating all Volumes with Name 0 to SpecialNumber");
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "ManualMigrateMixedSpecials",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running ManualMigrateMixedSpecials migration - Completed. This is not an error");
}
}

89
API/Data/ManualMigrations/MigrateChapterFields.cs Normal file
View file

@ -0,0 +1,89 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.8.0, this migrates the existing Chapter and Volume 0 -> Parser defined, MangaFile.FileName
/// </summary>
public static class MigrateChapterFields
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateChapterFields"))
{
return;
}
logger.LogCritical(
"Running MigrateChapterFields migration - Please be patient, this may take some time. This is not an error");
// Update all volumes only have specials in them (rare)
var volumesWithJustSpecials = dataContext.Volume
.Include(v => v.Chapters)
.Where(v => v.Name == "0" && v.Chapters.All(c => c.IsSpecial))
.ToList();
logger.LogCritical(
"Running MigrateChapterFields migration - Updating {Count} volumes that only have specials in them", volumesWithJustSpecials.Count);
foreach (var volume in volumesWithJustSpecials)
{
volume.Name = $"{Parser.SpecialVolumeNumber}";
volume.MinNumber = Parser.SpecialVolumeNumber;
volume.MaxNumber = Parser.SpecialVolumeNumber;
}
// Update all volumes that only have loose leafs in them
var looseLeafVolumes = dataContext.Volume
.Include(v => v.Chapters)
.Where(v => v.Name == "0" && v.Chapters.All(c => !c.IsSpecial))
.ToList();
logger.LogCritical(
"Running MigrateChapterFields migration - Updating {Count} volumes that only have loose leaf chapters in them", looseLeafVolumes.Count);
foreach (var volume in looseLeafVolumes)
{
volume.Name = $"{Parser.DefaultChapterNumber}";
volume.MinNumber = Parser.DefaultChapterNumber;
volume.MaxNumber = Parser.DefaultChapterNumber;
}
// Update all MangaFile
logger.LogCritical(
"Running MigrateChapterFields migration - Updating all MangaFiles");
foreach (var mangaFile in dataContext.MangaFile)
{
mangaFile.FileName = Parser.RemoveExtensionIfSupported(mangaFile.FilePath);
}
var looseLeafChapters = await dataContext.Chapter.Where(c => c.Number == "0").ToListAsync();
logger.LogCritical(
"Running MigrateChapterFields migration - Updating {Count} loose leaf chapters", looseLeafChapters.Count);
foreach (var chapter in looseLeafChapters)
{
chapter.Number = Parser.DefaultChapter;
chapter.MinNumber = Parser.DefaultChapterNumber;
chapter.MaxNumber = Parser.DefaultChapterNumber;
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateChapterFields",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateChapterFields migration - Completed. This is not an error");
}
}

50
API/Data/ManualMigrations/MigrateChapterNumber.cs Normal file
View file

@ -0,0 +1,50 @@
using System;
using System.Threading.Tasks;
using API.Entities;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.8.0, this migrates the existing Chapter Range -> Chapter Min/Max Number
/// </summary>
public static class MigrateChapterNumber
{
public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateChapterNumber"))
{
return;
}
logger.LogCritical(
"Running MigrateChapterNumber migration - Please be patient, this may take some time. This is not an error");
// Get all volumes
foreach (var chapter in dataContext.Chapter)
{
if (chapter.IsSpecial)
{
chapter.MinNumber = Parser.DefaultChapterNumber;
chapter.MaxNumber = Parser.DefaultChapterNumber;
continue;
}
chapter.MinNumber = Parser.MinNumberFromRange(chapter.Range);
chapter.MaxNumber = Parser.MaxNumberFromRange(chapter.Range);
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateChapterNumber",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateChapterNumber migration - Completed. This is not an error");
}
}

55
API/Data/ManualMigrations/MigrateChapterRange.cs Normal file
View file

@ -0,0 +1,55 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using API.Helpers.Builders;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.8.0 changed the range to that it doesn't have filename by default
/// </summary>
public static class MigrateChapterRange
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateChapterRange"))
{
return;
}
logger.LogCritical(
"Running MigrateChapterRange migration - Please be patient, this may take some time. This is not an error");
var chapters = await dataContext.Chapter.ToListAsync();
foreach (var chapter in chapters)
{
if (Parser.MinNumberFromRange(chapter.Range) == 0.0f)
{
chapter.Range = chapter.GetNumberTitle();
}
}
// Save changes after processing all series
if (dataContext.ChangeTracker.HasChanges())
{
await dataContext.SaveChangesAsync();
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateChapterRange",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateChapterRange migration - Completed. This is not an error");
}
}

3
API/Data/ManualMigrations/MigrateLibrariesToHaveAllFileTypes.cs
View file

@ -15,9 +15,8 @@ public static class MigrateLibrariesToHaveAllFileTypes
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
if (await dataContext.Library.AnyAsync(l => l.LibraryFileTypes.Count == 0))
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateLibrariesToHaveAllFileTypes"))
{
logger.LogCritical("Running MigrateLibrariesToHaveAllFileTypes migration - Completed. This is not an error");
return;
}

2
API/Data/ManualMigrations/MigrateManualHistory.cs
View file

@ -16,8 +16,6 @@ public static class MigrateManualHistory
{
if (await dataContext.ManualMigrationHistory.AnyAsync())
{
logger.LogCritical(
"Running MigrateManualHistory migration - Completed. This is not an error");
return;
}

7
API/Data/ManualMigrations/MigrateSmartFilterEncoding.cs
View file

@ -4,6 +4,7 @@ using System.Text.RegularExpressions;
using System.Threading.Tasks;
using API.DTOs.Filtering.v2;
using API.Helpers;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
@ -21,8 +22,12 @@ public static class MigrateSmartFilterEncoding
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateSmartFilterEncoding migration - Please be patient, this may take some time. This is not an error");
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateSmartFilterEncoding"))
{
return;
}
logger.LogCritical("Running MigrateSmartFilterEncoding migration - Please be patient, this may take some time. This is not an error");
var smartFilters = dataContext.AppUserSmartFilter.ToList();
foreach (var filter in smartFilters)

4
API/Data/ManualMigrations/MigrateUserLibrarySideNavStream.cs
View file

@ -14,6 +14,10 @@ public static class MigrateUserLibrarySideNavStream
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateUserLibrarySideNavStream"))
{
return;
}
var usersWithLibraryStreams = await dataContext.AppUser
.Include(u => u.SideNavStreams)

41
API/Data/ManualMigrations/MigrateVolumeLookupName.cs Normal file
View file

@ -0,0 +1,41 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
public static class MigrateVolumeLookupName
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateVolumeLookupName"))
{
return;
}
logger.LogCritical(
"Running MigrateVolumeLookupName migration - Please be patient, this may take some time. This is not an error");
// Update all volumes to have LookupName as after this migration, name isn't used for lookup
var volumes = dataContext.Volume.ToList();
foreach (var volume in volumes)
{
volume.LookupName = volume.Name;
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateVolumeLookupName",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateVolumeLookupName migration - Completed. This is not an error");
}
}

7
API/Data/ManualMigrations/MigrateVolumeNumber.cs
View file

@ -13,8 +13,13 @@ namespace API.Data.ManualMigrations;
/// </summary>
public static class MigrateVolumeNumber
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateVolumeNumber"))
{
return;
}
if (await dataContext.Volume.AnyAsync(v => v.MaxNumber > 0))
{
logger.LogCritical(

5
API/Data/ManualMigrations/MigrateWantToReadExport.cs
View file

@ -20,6 +20,11 @@ public static class MigrateWantToReadExport
{
try
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateWantToReadExport"))
{
return;
}
var importFile = Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv");
if (File.Exists(importFile))
{

9
API/Data/ManualMigrations/MigrateWantToReadImport.cs
View file

@ -6,6 +6,7 @@ using API.Data.Repositories;
using API.Entities;
using API.Services;
using CsvHelper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
@ -15,8 +16,14 @@ namespace API.Data.ManualMigrations;
/// </summary>
public static class MigrateWantToReadImport
{
public static async Task Migrate(IUnitOfWork unitOfWork, IDirectoryService directoryService, ILogger<Program> logger)
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, IDirectoryService directoryService, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateWantToReadImport"))
{
return;
}
var importFile = Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv");
var outputFile = Path.Join(directoryService.ConfigDirectory, "imported-want-to-read-migration.csv");