AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

AVIF Support & Much More! (#1992)

Browse source 
* Expand the list of potential favicon icons to grab.

* Added a url mapping functionality to use alternative urls for fetching icons

* Initial commit to streamline media encoding. No DB migration yet, No UI changes, no Task changes.

* Started refactoring code so that webp queries use encoding format instead.

* More refactoring to remove hardcoded webp references.

* Moved manual migrations to their own folder to keep things organized. Manually drop the obsolete webp keys.

* Removed old apis for converting media and now have one. Reworked where the conversion code was located and streamlined events and whatnot.

* Make favicon encode setting aware

* Cleaned up favicon conversion

* Updated format counter to now just use Extension from MangaFile now that it's been out a while.

* Tweaked jumpbar code to reduce a lookup to hashmap.

* Added AVIF (8-bit only) support.

* In UpdatePeopleList, use FirstOrDefault as Single adds extra checks that may not be needed.

* You can now remove weblinks from edit series page and you can leave empty cells, they will just be removed on backend.

* Forgot a file

* Don't prompt to write a review, just show the pencil. It's the same amount of clicks if you do, less if you dont.

* Fixed Refresh token using wrong Claim to look up the user.

* Refactored how we refresh authentication to perform it every 10 m ins to ensure we always stay authenticated.

* Changed Version update code to run more throughout the day. Updated some hangfire to newer method signatures.
This commit is contained in:
Joe Milazzo 2023-05-12 15:31:23 -05:00 • committed by GitHub
parent c1989e2819
commit 70690b747e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
73 changed files with 778 additions and 566 deletions
Download patch file Download diff file Expand all files Collapse all files

136
API/Data/ManualMigrations/MigrateBrokenGMT1Dates.cs Normal file
View file

@ -0,0 +1,136 @@
using System;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.7 introduced UTC dates and GMT+1 users would sometimes have dates stored as '0000-12-31 23:00:00'.
/// This Migration will update those dates.
/// </summary>
// ReSharper disable once InconsistentNaming
public static class MigrateBrokenGMT1Dates
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
// if current version is > 0.7, then we can exit and not perform
var settings = await unitOfWork.SettingsRepository.GetSettingsDtoAsync();
if (Version.Parse(settings.InstallVersion) > new Version(0, 7, 0, 2))
{
return;
}
logger.LogCritical("Running MigrateBrokenGMT1Dates migration. Please be patient, this may take some time depending on the size of your library. Do not abort, this can break your Database");
#region Series
logger.LogInformation("Updating Dates on Series...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Series SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE Series SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
UPDATE Series SET LastChapterAddedUtc = '0001-01-01 00:00:00' WHERE LastChapterAddedUtc = '0000-12-31 23:00:00';
UPDATE Series SET LastFolderScannedUtc = '0001-01-01 00:00:00' WHERE LastFolderScannedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on Series...Done");
#endregion
#region Library
logger.LogInformation("Updating Dates on Libraries...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Library SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE Library SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on Libraries...Done");
#endregion
#region Volume
try
{
logger.LogInformation("Updating Dates on Volumes...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Volume SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE Volume SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on Volumes...Done");
}
catch (Exception ex)
{
logger.LogCritical(ex, "Updating Dates on Volumes...Failed");
}
#endregion
#region Chapter
try
{
logger.LogInformation("Updating Dates on Chapters...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Chapter SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE Chapter SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on Chapters...Done");
}
catch (Exception ex)
{
logger.LogCritical(ex, "Updating Dates on Chapters...Failed");
}
#endregion
#region AppUserBookmark
logger.LogInformation("Updating Dates on Bookmarks...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE AppUserBookmark SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE AppUserBookmark SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on Bookmarks...Done");
#endregion
#region AppUserProgress
logger.LogInformation("Updating Dates on Progress...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE AppUserProgresses SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE AppUserProgresses SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on Progress...Done");
#endregion
#region Device
logger.LogInformation("Updating Dates on Device...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Device SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE Device SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
UPDATE Device SET LastUsedUtc = '0001-01-01 00:00:00' WHERE LastUsedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on Device...Done");
#endregion
#region MangaFile
logger.LogInformation("Updating Dates on MangaFile...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE MangaFile SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE MangaFile SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
UPDATE MangaFile SET LastFileAnalysisUtc = '0001-01-01 00:00:00' WHERE LastFileAnalysisUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on MangaFile...Done");
#endregion
#region ReadingList
logger.LogInformation("Updating Dates on ReadingList...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE ReadingList SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE ReadingList SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on ReadingList...Done");
#endregion
#region SiteTheme
logger.LogInformation("Updating Dates on SiteTheme...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE SiteTheme SET CreatedUtc = '0001-01-01 00:00:00' WHERE CreatedUtc = '0000-12-31 23:00:00';
UPDATE SiteTheme SET LastModifiedUtc = '0001-01-01 00:00:00' WHERE LastModifiedUtc = '0000-12-31 23:00:00';
");
logger.LogInformation("Updating Dates on SiteTheme...Done");
#endregion
logger.LogInformation("MigrateBrokenGMT1Dates migration finished");
}
}

30
API/Data/ManualMigrations/MigrateChangePasswordRoles.cs Normal file
View file

@ -0,0 +1,30 @@
using System.Threading.Tasks;
using API.Constants;
using API.Entities;
using Microsoft.AspNetCore.Identity;
namespace API.Data.ManualMigrations;
/// <summary>
/// New role introduced in v0.5.1. Adds the role to all users.
/// </summary>
public static class MigrateChangePasswordRoles
{
/// <summary>
/// Will not run if any users have the ChangePassword role already
/// </summary>
/// <param name="unitOfWork"></param>
/// <param name="userManager"></param>
public static async Task Migrate(IUnitOfWork unitOfWork, UserManager<AppUser> userManager)
{
var usersWithRole = await userManager.GetUsersInRoleAsync(PolicyConstants.ChangePasswordRole);
if (usersWithRole.Count != 0) return;
var allUsers = await unitOfWork.UserRepository.GetAllUsersAsync();
foreach (var user in allUsers)
{
await userManager.RemoveFromRoleAsync(user, "ChangePassword");
await userManager.AddToRoleAsync(user, PolicyConstants.ChangePasswordRole);
}
}
}

36
API/Data/ManualMigrations/MigrateChangeRestrictionRoles.cs Normal file
View file

@ -0,0 +1,36 @@
using System.Threading.Tasks;
using API.Constants;
using API.Entities;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// New role introduced in v0.6. Adds the role to all users.
/// </summary>
public static class MigrateChangeRestrictionRoles
{
/// <summary>
/// Will not run if any users have the <see cref="PolicyConstants.ChangeRestrictionRole"/> role already
/// </summary>
/// <param name="unitOfWork"></param>
/// <param name="userManager"></param>
/// <param name="logger"></param>
public static async Task Migrate(IUnitOfWork unitOfWork, UserManager<AppUser> userManager, ILogger<Program> logger)
{
var usersWithRole = await userManager.GetUsersInRoleAsync(PolicyConstants.ChangeRestrictionRole);
if (usersWithRole.Count != 0) return;
logger.LogCritical("Running MigrateChangeRestrictionRoles migration");
var allUsers = await unitOfWork.UserRepository.GetAllUsersAsync();
foreach (var user in allUsers)
{
await userManager.RemoveFromRoleAsync(user, PolicyConstants.ChangeRestrictionRole);
await userManager.AddToRoleAsync(user, PolicyConstants.ChangeRestrictionRole);
}
logger.LogInformation("MigrateChangeRestrictionRoles migration complete");
}
}

36
API/Data/ManualMigrations/MigrateLoginRole.cs Normal file
View file

@ -0,0 +1,36 @@
using System.Threading.Tasks;
using API.Constants;
using API.Entities;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Added in v0.7.1.18
/// </summary>
public static class MigrateLoginRoles
{
/// <summary>
/// Will not run if any users have the <see cref="PolicyConstants.LoginRole"/> role already
/// </summary>
/// <param name="unitOfWork"></param>
/// <param name="userManager"></param>
/// <param name="logger"></param>
public static async Task Migrate(IUnitOfWork unitOfWork, UserManager<AppUser> userManager, ILogger<Program> logger)
{
var usersWithRole = await userManager.GetUsersInRoleAsync(PolicyConstants.LoginRole);
if (usersWithRole.Count != 0) return;
logger.LogCritical("Running MigrateLoginRoles migration");
var allUsers = await unitOfWork.UserRepository.GetAllUsersAsync();
foreach (var user in allUsers)
{
await userManager.RemoveFromRoleAsync(user, PolicyConstants.LoginRole);
await userManager.AddToRoleAsync(user, PolicyConstants.LoginRole);
}
logger.LogInformation("MigrateLoginRoles migration complete");
}
}

118
API/Data/ManualMigrations/MigrateNormalizedEverything.cs Normal file
View file

@ -0,0 +1,118 @@
using System;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.6.0 introduced a change in how Normalization works and hence every normalized field needs to be re-calculated
/// </summary>
public static class MigrateNormalizedEverything
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
// if current version is > 0.5.6.5, then we can exit and not perform
var settings = await unitOfWork.SettingsRepository.GetSettingsDtoAsync();
if (Version.Parse(settings.InstallVersion) > new Version(0, 5, 6, 5))
{
return;
}
logger.LogCritical("Running MigrateNormalizedEverything migration. Please be patient, this may take some time depending on the size of your library. Do not abort, this can break your Database");
logger.LogInformation("Updating Normalization on Series...");
foreach (var series in await dataContext.Series.ToListAsync())
{
series.NormalizedLocalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(series.LocalizedName ?? string.Empty);
series.NormalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(series.Name ?? string.Empty);
logger.LogInformation("Updated Series: {SeriesName}", series.Name);
unitOfWork.SeriesRepository.Update(series);
}
if (unitOfWork.HasChanges())
{
await unitOfWork.CommitAsync();
}
logger.LogInformation("Updating Normalization on Series...Done");
// Genres
logger.LogInformation("Updating Normalization on Genres...");
foreach (var genre in await dataContext.Genre.ToListAsync())
{
genre.NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(genre.Title ?? string.Empty);
logger.LogInformation("Updated Genre: {Genre}", genre.Title);
unitOfWork.GenreRepository.Attach(genre);
}
if (unitOfWork.HasChanges())
{
await unitOfWork.CommitAsync();
}
logger.LogInformation("Updating Normalization on Genres...Done");
// Tags
logger.LogInformation("Updating Normalization on Tags...");
foreach (var tag in await dataContext.Tag.ToListAsync())
{
tag.NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(tag.Title ?? string.Empty);
logger.LogInformation("Updated Tag: {Tag}", tag.Title);
unitOfWork.TagRepository.Attach(tag);
}
if (unitOfWork.HasChanges())
{
await unitOfWork.CommitAsync();
}
logger.LogInformation("Updating Normalization on Tags...Done");
// People
logger.LogInformation("Updating Normalization on People...");
foreach (var person in await dataContext.Person.ToListAsync())
{
person.NormalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(person.Name ?? string.Empty);
logger.LogInformation("Updated Person: {Person}", person.Name);
unitOfWork.PersonRepository.Attach(person);
}
if (unitOfWork.HasChanges())
{
await unitOfWork.CommitAsync();
}
logger.LogInformation("Updating Normalization on People...Done");
// Collections
logger.LogInformation("Updating Normalization on Collections...");
foreach (var collection in await dataContext.CollectionTag.ToListAsync())
{
collection.NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(collection.Title ?? string.Empty);
logger.LogInformation("Updated Collection: {Collection}", collection.Title);
unitOfWork.CollectionTagRepository.Update(collection);
}
if (unitOfWork.HasChanges())
{
await unitOfWork.CommitAsync();
}
logger.LogInformation("Updating Normalization on Collections...Done");
// Reading Lists
logger.LogInformation("Updating Normalization on Reading Lists...");
foreach (var readingList in await dataContext.ReadingList.ToListAsync())
{
readingList.NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(readingList.Title ?? string.Empty);
logger.LogInformation("Updated Reading List: {ReadingList}", readingList.Title);
unitOfWork.ReadingListRepository.Update(readingList);
}
if (unitOfWork.HasChanges())
{
await unitOfWork.CommitAsync();
}
logger.LogInformation("Updating Normalization on Reading Lists...Done");
logger.LogInformation("MigrateNormalizedEverything migration finished");
}
}

38
API/Data/ManualMigrations/MigrateNormalizedLocalizedName.cs Normal file
View file

@ -0,0 +1,38 @@
using System.Linq;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.5.6 introduced Normalized Localized Name, which allows for faster lookups and less memory usage. This migration will calculate them once
/// </summary>
public static class MigrateNormalizedLocalizedName
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
if (!await dataContext.Series.Where(s => s.NormalizedLocalizedName == null).AnyAsync())
{
return;
}
logger.LogInformation("Running MigrateNormalizedLocalizedName migration. Please be patient, this may take some time");
foreach (var series in await dataContext.Series.ToListAsync())
{
series.NormalizedLocalizedName = Services.Tasks.Scanner.Parser.Parser.Normalize(series.LocalizedName ?? string.Empty);
logger.LogInformation("Updated {SeriesName} normalized localized name: {LocalizedName}", series.Name, series.NormalizedLocalizedName);
unitOfWork.SeriesRepository.Update(series);
}
if (unitOfWork.HasChanges())
{
await unitOfWork.CommitAsync();
}
logger.LogInformation("MigrateNormalizedLocalizedName migration finished");
}
}

40
API/Data/ManualMigrations/MigrateReadingListAgeRating.cs Normal file
View file

@ -0,0 +1,40 @@
using System;
using System.Threading.Tasks;
using API.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// New role introduced in v0.6. Calculates the Age Rating on all Reading Lists
/// </summary>
public static class MigrateReadingListAgeRating
{
/// <summary>
/// Will not run if any above v0.5.6.24 or v0.6.0
/// </summary>
/// <param name="unitOfWork"></param>
/// <param name="context"></param>
/// <param name="readingListService"></param>
/// <param name="logger"></param>
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext context, IReadingListService readingListService, ILogger<Program> logger)
{
var settings = await unitOfWork.SettingsRepository.GetSettingsDtoAsync();
if (Version.Parse(settings.InstallVersion) > new Version(0, 5, 6, 26))
{
return;
}
logger.LogInformation("MigrateReadingListAgeRating migration starting");
var readingLists = await context.ReadingList.Include(r => r.Items).ToListAsync();
foreach (var readingList in readingLists)
{
await readingListService.CalculateReadingListAgeRating(readingList);
context.ReadingList.Update(readingList);
}
await context.SaveChangesAsync();
logger.LogInformation("MigrateReadingListAgeRating migration complete");
}
}

55
API/Data/ManualMigrations/MigrateRemoveExtraThemes.cs Normal file
View file

@ -0,0 +1,55 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using API.Services.Tasks;
namespace API.Data.ManualMigrations;
/// <summary>
/// In v0.5.3, we removed Light and E-Ink themes. This migration will remove the themes from the DB and default anyone on
/// null, E-Ink, or Light to Dark.
/// </summary>
public static class MigrateRemoveExtraThemes
{
public static async Task Migrate(IUnitOfWork unitOfWork, IThemeService themeService)
{
var themes = (await unitOfWork.SiteThemeRepository.GetThemes()).ToList();
if (themes.FirstOrDefault(t => t.Name.Equals("Light")) == null)
{
return;
}
Console.WriteLine("Removing Dark and E-Ink themes");
var darkTheme = themes.Single(t => t.Name.Equals("Dark"));
var lightTheme = themes.Single(t => t.Name.Equals("Light"));
var eInkTheme = themes.Single(t => t.Name.Equals("E-Ink"));
// Update default theme if it's not Dark or a custom theme
await themeService.UpdateDefault(darkTheme.Id);
// Update all users to Dark theme if they are on Light/E-Ink
foreach (var pref in await unitOfWork.UserRepository.GetAllPreferencesByThemeAsync(lightTheme.Id))
{
pref.Theme = darkTheme;
}
foreach (var pref in await unitOfWork.UserRepository.GetAllPreferencesByThemeAsync(eInkTheme.Id))
{
pref.Theme = darkTheme;
}
// Remove Light/E-Ink themes
foreach (var siteTheme in themes.Where(t => t.Name.Equals("Light") || t.Name.Equals("E-Ink")))
{
unitOfWork.SiteThemeRepository.Remove(siteTheme);
}
// Commit and call it a day
await unitOfWork.CommitAsync();
Console.WriteLine("Completed removing Dark and E-Ink themes");
}
}

31
API/Data/ManualMigrations/MigrateRemoveWebPSettingRows.cs Normal file
View file

@ -0,0 +1,31 @@
using System.Threading.Tasks;
using API.Entities.Enums;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Added in v0.7.2.7/v0.7.3 in which the ConvertXToWebP Setting keys were removed. This migration will remove them.
/// </summary>
public static class MigrateRemoveWebPSettingRows
{
public static async Task Migrate(IUnitOfWork unitOfWork, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - Please be patient, this may take some time. This is not an error");
var key = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.ConvertBookmarkToWebP);
var key2 = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.ConvertCoverToWebP);
if (key == null && key2 == null)
{
logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - complete. Nothing to do");
return;
}
unitOfWork.SettingsRepository.Remove(key);
unitOfWork.SettingsRepository.Remove(key2);
await unitOfWork.CommitAsync();
logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - Completed. This is not an error");
}
}

103
API/Data/ManualMigrations/MigrateSeriesRelationsExport.cs Normal file
View file

@ -0,0 +1,103 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Entities.Enums;
using CsvHelper;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
internal sealed class SeriesRelationMigrationOutput
{
public required string SeriesName { get; set; }
public int SeriesId { get; set; }
public required string TargetSeriesName { get; set; }
public int TargetId { get; set; }
public RelationKind Relationship { get; set; }
}
/// <summary>
/// Introduced in v0.6.1.2 and v0.7, this exports to a temp file the existing series relationships. It is a 3 part migration.
/// This will run first, to export the data, then the DB migration will change the way the DB is constructed, then the last migration
/// will import said file and re-construct the relationships.
/// </summary>
public static class MigrateSeriesRelationsExport
{
private const string OutputFile = "config/relations.csv";
private const string CompleteOutputFile = "config/relations-imported.csv";
public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateSeriesRelationsExport migration - Please be patient, this may take some time. This is not an error");
if (BuildInfo.Version > new Version(0, 6, 1, 3)
|| new FileInfo(OutputFile).Exists
|| new FileInfo(CompleteOutputFile).Exists)
{
logger.LogCritical("Running MigrateSeriesRelationsExport migration - complete. Nothing to do");
return;
}
var seriesWithRelationships = await dataContext.Series
.Where(s => s.Relations.Any())
.Include(s => s.Relations)
.ThenInclude(r => r.TargetSeries)
.ToListAsync();
var records = new List<SeriesRelationMigrationOutput>();
var excludedRelationships = new List<RelationKind>()
{
RelationKind.Parent,
};
foreach (var series in seriesWithRelationships)
{
foreach (var relationship in series.Relations.Where(r => !excludedRelationships.Contains(r.RelationKind)))
{
records.Add(new SeriesRelationMigrationOutput()
{
SeriesId = series.Id,
SeriesName = series.Name,
Relationship = relationship.RelationKind,
TargetId = relationship.TargetSeriesId,
TargetSeriesName = relationship.TargetSeries.Name
});
}
}
await using var writer = new StreamWriter(OutputFile);
await using (var csv = new CsvWriter(writer, CultureInfo.InvariantCulture))
{
await csv.WriteRecordsAsync(records);
}
await writer.DisposeAsync();
logger.LogCritical("{OutputFile} has a backup of all data", OutputFile);
logger.LogCritical("Deleting all relationships in the DB. This is not an error");
var entities = await dataContext.SeriesRelation
.Include(s => s.Series)
.Include(s => s.TargetSeries)
.Select(s => s)
.ToListAsync();
foreach (var seriesWithRelationship in entities)
{
logger.LogCritical("Deleting {SeriesName} --{RelationshipKind}--> {TargetSeriesName}",
seriesWithRelationship.Series.Name, seriesWithRelationship.RelationKind, seriesWithRelationship.TargetSeries.Name);
dataContext.SeriesRelation.Remove(seriesWithRelationship);
await dataContext.SaveChangesAsync();
}
// In case of corrupted entities (where series were deleted but their Id still existed, we delete the rest of the table)
dataContext.SeriesRelation.RemoveRange(dataContext.SeriesRelation);
await dataContext.SaveChangesAsync();
logger.LogCritical("Running MigrateSeriesRelationsExport migration - Completed. This is not an error");
}
}

62
API/Data/ManualMigrations/MigrateSeriesRelationsImport.cs Normal file
View file

@ -0,0 +1,62 @@
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Entities.Metadata;
using CsvHelper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.6.1.2 and v0.7, this imports to a temp file the existing series relationships. It is a 3 part migration.
/// This will run last, to import the data and re-construct the relationships.
/// </summary>
public static class MigrateSeriesRelationsImport
{
private const string OutputFile = "config/relations.csv";
private const string CompleteOutputFile = "config/relations-imported.csv";
public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateSeriesRelationsImport migration - Please be patient, this may take some time. This is not an error");
if (!new FileInfo(OutputFile).Exists)
{
logger.LogCritical("Running MigrateSeriesRelationsImport migration - complete. Nothing to do");
return;
}
logger.LogCritical("Loading backed up relationships into the DB");
List<SeriesRelationMigrationOutput> records;
using var reader = new StreamReader(OutputFile);
using (var csv = new CsvReader(reader, CultureInfo.InvariantCulture))
{
records = csv.GetRecords<SeriesRelationMigrationOutput>().ToList();
}
foreach (var relation in records)
{
logger.LogCritical("Importing {SeriesName} --{RelationshipKind}--> {TargetSeriesName}",
relation.SeriesName, relation.Relationship, relation.TargetSeriesName);
// Filter out series that don't exist
if (!await dataContext.Series.AnyAsync(s => s.Id == relation.SeriesId) ||
!await dataContext.Series.AnyAsync(s => s.Id == relation.TargetId))
continue;
await dataContext.SeriesRelation.AddAsync(new SeriesRelation()
{
SeriesId = relation.SeriesId,
TargetSeriesId = relation.TargetId,
RelationKind = relation.Relationship
});
}
await dataContext.SaveChangesAsync();
File.Move(OutputFile, CompleteOutputFile);
logger.LogCritical("Running MigrateSeriesRelationsImport migration - Completed. This is not an error");
}
}

153
API/Data/ManualMigrations/MigrateToUtcDates.cs Normal file
View file

@ -0,0 +1,153 @@
using System;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.6.1.38 or v0.7.0,
/// </summary>
public static class MigrateToUtcDates
{
public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, ILogger<Program> logger)
{
// if current version is > 0.6.1.38, then we can exit and not perform
var settings = await unitOfWork.SettingsRepository.GetSettingsDtoAsync();
if (Version.Parse(settings.InstallVersion) > new Version(0, 6, 1, 38))
{
return;
}
logger.LogCritical("Running MigrateToUtcDates migration. Please be patient, this may take some time depending on the size of your library. Do not abort, this can break your Database");
#region Series
logger.LogInformation("Updating Dates on Series...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Series SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc'),
[LastChapterAddedUtc] = datetime([LastChapterAdded], 'utc'),
[LastFolderScannedUtc] = datetime([LastFolderScanned], 'utc')
;
");
logger.LogInformation("Updating Dates on Series...Done");
#endregion
#region Library
logger.LogInformation("Updating Dates on Libraries...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Library SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc')
;
");
logger.LogInformation("Updating Dates on Libraries...Done");
#endregion
#region Volume
try
{
logger.LogInformation("Updating Dates on Volumes...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Volume SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc');
");
logger.LogInformation("Updating Dates on Volumes...Done");
}
catch (Exception ex)
{
logger.LogCritical(ex, "Updating Dates on Volumes...Failed");
}
#endregion
#region Chapter
try
{
logger.LogInformation("Updating Dates on Chapters...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Chapter SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc')
;
");
logger.LogInformation("Updating Dates on Chapters...Done");
}
catch (Exception ex)
{
logger.LogCritical(ex, "Updating Dates on Chapters...Failed");
}
#endregion
#region AppUserBookmark
logger.LogInformation("Updating Dates on Bookmarks...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE AppUserBookmark SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc')
;
");
logger.LogInformation("Updating Dates on Bookmarks...Done");
#endregion
#region AppUserProgress
logger.LogInformation("Updating Dates on Progress...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE AppUserProgresses SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc')
;
");
logger.LogInformation("Updating Dates on Progress...Done");
#endregion
#region Device
logger.LogInformation("Updating Dates on Device...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE Device SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc'),
[LastUsedUtc] = datetime([LastUsed], 'utc')
;
");
logger.LogInformation("Updating Dates on Device...Done");
#endregion
#region MangaFile
logger.LogInformation("Updating Dates on MangaFile...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE MangaFile SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc'),
[LastFileAnalysisUtc] = datetime([LastFileAnalysis], 'utc')
;
");
logger.LogInformation("Updating Dates on MangaFile...Done");
#endregion
#region ReadingList
logger.LogInformation("Updating Dates on ReadingList...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE ReadingList SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc')
;
");
logger.LogInformation("Updating Dates on ReadingList...Done");
#endregion
#region SiteTheme
logger.LogInformation("Updating Dates on SiteTheme...");
await dataContext.Database.ExecuteSqlRawAsync(@"
UPDATE SiteTheme SET
[LastModifiedUtc] = datetime([LastModified], 'utc'),
[CreatedUtc] = datetime([Created], 'utc')
;
");
logger.LogInformation("Updating Dates on SiteTheme...Done");
#endregion
logger.LogInformation("MigrateToUtcDates migration finished");
}
}

34
API/Data/ManualMigrations/MigrateUserProgressLibraryId.cs Normal file
View file

@ -0,0 +1,34 @@
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.6.1.8 and v0.7, this adds library ids to all User Progress to allow for easier queries against progress
/// </summary>
public static class MigrateUserProgressLibraryId
{
public static async Task Migrate(IUnitOfWork unitOfWork, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateUserProgressLibraryId migration - Please be patient, this may take some time. This is not an error");
var progress = await unitOfWork.AppUserProgressRepository.GetAnyProgress();
if (progress == null || progress.LibraryId != 0)
{
logger.LogCritical("Running MigrateUserProgressLibraryId migration - complete. Nothing to do");
return;
}
var seriesIdsWithLibraryIds = await unitOfWork.SeriesRepository.GetLibraryIdsForSeriesAsync();
foreach (var prog in await unitOfWork.AppUserProgressRepository.GetAllProgress())
{
prog.LibraryId = seriesIdsWithLibraryIds[prog.SeriesId];
unitOfWork.AppUserProgressRepository.Update(prog);
}
await unitOfWork.CommitAsync();
logger.LogCritical("Running MigrateSeriesRelationsImport migration - Completed. This is not an error");
}
}