Metadata Downloading (#3525)

API/Data/ManualMigrations/v0.7.14/MigrateClearNightlyExternalSeriesRecords.cs

@@ -0,0 +1,43 @@
+using System;
+using System.Threading.Tasks;
+using API.Entities;
+using API.Entities.History;
+using Kavita.Common.EnvironmentInfo;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+/// <summary>
+/// For the v0.7.14 release, one of the nightlies had bad data that would cause issues. This drops those records
+/// </summary>
+public static class MigrateClearNightlyExternalSeriesRecords
+{
+    public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
+    {
+        if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateClearNightlyExternalSeriesRecords"))
+        {
+            return;
+        }
+
+        logger.LogCritical(
+            "Running MigrateClearNightlyExternalSeriesRecords migration - Please be patient, this may take some time. This is not an error");
+
+        dataContext.ExternalSeriesMetadata.RemoveRange(dataContext.ExternalSeriesMetadata);
+        dataContext.ExternalRating.RemoveRange(dataContext.ExternalRating);
+        dataContext.ExternalRecommendation.RemoveRange(dataContext.ExternalRecommendation);
+        dataContext.ExternalReview.RemoveRange(dataContext.ExternalReview);
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateClearNightlyExternalSeriesRecords",
+            ProductVersion = BuildInfo.Version.ToString(),
+            RanAt = DateTime.UtcNow
+        });
+
+        await dataContext.SaveChangesAsync();
+
+        logger.LogCritical(
+            "Running MigrateClearNightlyExternalSeriesRecords migration - Completed. This is not an error");
+    }
+}

API/Data/ManualMigrations/v0.7.14/MigrateEmailTemplates.cs

@@ -0,0 +1,59 @@
+using System;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using API.Services;
+using Flurl.Http;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+public static class MigrateEmailTemplates
+{
+    private const string EmailChange = "https://raw.githubusercontent.com/Kareadita/KavitaEmail/main/KavitaEmail/config/templates/EmailChange.html";
+    private const string EmailConfirm = "https://raw.githubusercontent.com/Kareadita/KavitaEmail/main/KavitaEmail/config/templates/EmailConfirm.html";
+    private const string EmailPasswordReset = "https://raw.githubusercontent.com/Kareadita/KavitaEmail/main/KavitaEmail/config/templates/EmailPasswordReset.html";
+    private const string SendToDevice = "https://raw.githubusercontent.com/Kareadita/KavitaEmail/main/KavitaEmail/config/templates/SendToDevice.html";
+    private const string EmailTest = "https://raw.githubusercontent.com/Kareadita/KavitaEmail/main/KavitaEmail/config/templates/EmailTest.html";
+
+    public static async Task Migrate(IDirectoryService directoryService, ILogger<Program> logger)
+    {
+        var files = directoryService.GetFiles(directoryService.CustomizedTemplateDirectory);
+        if (files.Any())
+        {
+            return;
+        }
+
+        logger.LogCritical("Running MigrateEmailTemplates migration - Please be patient, this may take some time. This is not an error");
+
+        // Write files to directory
+        await DownloadAndWriteToFile(EmailChange, Path.Join(directoryService.CustomizedTemplateDirectory, "EmailChange.html"), logger);
+        await DownloadAndWriteToFile(EmailConfirm, Path.Join(directoryService.CustomizedTemplateDirectory, "EmailConfirm.html"), logger);
+        await DownloadAndWriteToFile(EmailPasswordReset, Path.Join(directoryService.CustomizedTemplateDirectory, "EmailPasswordReset.html"), logger);
+        await DownloadAndWriteToFile(SendToDevice, Path.Join(directoryService.CustomizedTemplateDirectory, "SendToDevice.html"), logger);
+        await DownloadAndWriteToFile(EmailTest, Path.Join(directoryService.CustomizedTemplateDirectory, "EmailTest.html"), logger);
+
+
+        logger.LogCritical("Running MigrateEmailTemplates migration - Completed. This is not an error");
+    }
+
+    private static async Task DownloadAndWriteToFile(string url, string filePath, ILogger<Program> logger)
+    {
+        try
+        {
+            // Download the raw text using Flurl
+            var content = await url.GetStringAsync();
+
+            // Write the content to a file
+            await File.WriteAllTextAsync(filePath, content);
+
+            logger.LogInformation("{File} downloaded and written successfully", filePath);
+        }
+        catch (FlurlHttpException ex)
+        {
+            logger.LogError(ex, "Unable to download {Url} to {FilePath}. Please perform yourself!", url, filePath);
+        }
+    }
+
+
+}

API/Data/ManualMigrations/v0.7.14/MigrateManualHistory.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Threading.Tasks;
+using API.Entities;
+using API.Entities.History;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+/// <summary>
+/// Introduced in v0.7.14, will store history so that going forward, migrations can just check against the history
+/// and I don't need to remove old migrations
+/// </summary>
+public static class MigrateManualHistory
+{
+    public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
+    {
+        if (await dataContext.ManualMigrationHistory.AnyAsync())
+        {
+            return;
+        }
+
+        logger.LogCritical(
+            "Running MigrateManualHistory migration - Please be patient, this may take some time. This is not an error");
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateUserLibrarySideNavStream",
+            ProductVersion = "0.7.9.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateSmartFilterEncoding",
+            ProductVersion = "0.7.11.0",
+            RanAt = DateTime.UtcNow
+        });
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateLibrariesToHaveAllFileTypes",
+            ProductVersion = "0.7.11.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateEmailTemplates",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateVolumeNumber",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateWantToReadExport",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateWantToReadImport",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateManualHistory",
+            ProductVersion = "0.7.14.0",
+            RanAt = DateTime.UtcNow
+        });
+
+        await dataContext.SaveChangesAsync();
+
+        logger.LogCritical(
+            "Running MigrateManualHistory migration - Completed. This is not an error");
+    }
+}

API/Data/ManualMigrations/v0.7.14/MigrateVolumeLookupName.cs

@@ -0,0 +1,42 @@
+using System;
+using System.Linq;
+using System.Threading.Tasks;
+using API.Entities;
+using API.Entities.History;
+using Kavita.Common.EnvironmentInfo;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+public static class MigrateVolumeLookupName
+{
+    public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
+    {
+        if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateVolumeLookupName"))
+        {
+            return;
+        }
+
+        logger.LogCritical(
+            "Running MigrateVolumeLookupName migration - Please be patient, this may take some time. This is not an error");
+
+        // Update all volumes to have LookupName as after this migration, name isn't used for lookup
+        var volumes = dataContext.Volume.ToList();
+        foreach (var volume in volumes)
+        {
+            volume.LookupName = volume.Name;
+        }
+
+        dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
+        {
+            Name = "MigrateVolumeLookupName",
+            ProductVersion = BuildInfo.Version.ToString(),
+            RanAt = DateTime.UtcNow
+        });
+
+        await dataContext.SaveChangesAsync();
+        logger.LogCritical(
+            "Running MigrateVolumeLookupName migration - Completed. This is not an error");
+    }
+}

API/Data/ManualMigrations/v0.7.14/MigrateVolumeNumber.cs

@@ -0,0 +1,44 @@
+using System;
+using System.Threading.Tasks;
+using API.Entities.Enums;
+using API.Services.Tasks.Scanner.Parser;
+using Kavita.Common.EnvironmentInfo;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+/// <summary>
+/// Introduced in v0.7.14, this migrates the existing Volume Name -> Volume Min/Max Number
+/// </summary>
+public static class MigrateVolumeNumber
+{
+    public static async Task Migrate(DataContext dataContext, ILogger<Program> logger)
+    {
+        if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateVolumeNumber"))
+        {
+            return;
+        }
+
+        if (await dataContext.Volume.AnyAsync(v => v.MaxNumber > 0))
+        {
+            logger.LogCritical(
+                "Running MigrateVolumeNumber migration - Completed. This is not an error");
+            return;
+        }
+
+        logger.LogCritical(
+            "Running MigrateVolumeNumber migration - Please be patient, this may take some time. This is not an error");
+
+        // Get all volumes
+        foreach (var volume in dataContext.Volume)
+        {
+            volume.MinNumber = Parser.MinNumberFromRange(volume.Name);
+            volume.MaxNumber = Parser.MaxNumberFromRange(volume.Name);
+        }
+
+        await dataContext.SaveChangesAsync();
+        logger.LogCritical(
+            "Running MigrateVolumeNumber migration - Completed. This is not an error");
+    }
+}

API/Data/ManualMigrations/v0.7.14/MigrateWantToReadExport.cs

@@ -0,0 +1,85 @@
+using System;
+using System.Globalization;
+using System.IO;
+using System.Threading.Tasks;
+using API.Services;
+using CsvHelper;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+
+/// <summary>
+/// v0.7.13.12/v0.7.14 - Want to read is extracted and saved in a csv
+/// </summary>
+/// <remarks>This must run BEFORE any DB migrations</remarks>
+public static class MigrateWantToReadExport
+{
+    public static async Task Migrate(DataContext dataContext, IDirectoryService directoryService, ILogger<Program> logger)
+    {
+        try
+        {
+
+            if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateWantToReadExport"))
+            {
+                return;
+            }
+
+            var importFile = Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv");
+            if (File.Exists(importFile))
+            {
+                logger.LogCritical(
+                    "Running MigrateWantToReadExport migration - Completed. This is not an error");
+                return;
+            }
+
+            logger.LogCritical(
+                "Running MigrateWantToReadExport migration - Please be patient, this may take some time. This is not an error");
+
+            await using var command = dataContext.Database.GetDbConnection().CreateCommand();
+            command.CommandText = "Select AppUserId, Id from Series WHERE AppUserId IS NOT NULL ORDER BY AppUserId;";
+
+            await dataContext.Database.OpenConnectionAsync();
+            await using var result = await command.ExecuteReaderAsync();
+
+            await using var writer =
+                new StreamWriter(Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv"));
+            await using var csvWriter = new CsvWriter(writer, CultureInfo.InvariantCulture);
+
+            // Write header
+            csvWriter.WriteField("AppUserId");
+            csvWriter.WriteField("Id");
+            await csvWriter.NextRecordAsync();
+
+            // Write data
+            while (await result.ReadAsync())
+            {
+                var appUserId = result["AppUserId"].ToString();
+                var id = result["Id"].ToString();
+
+                csvWriter.WriteField(appUserId);
+                csvWriter.WriteField(id);
+                await csvWriter.NextRecordAsync();
+            }
+
+
+            try
+            {
+                await dataContext.Database.CloseConnectionAsync();
+                writer.Close();
+            }
+            catch (Exception)
+            {
+                /* Swallow */
+            }
+
+            logger.LogCritical(
+                "Running MigrateWantToReadExport migration - Completed. This is not an error");
+        }
+        catch (Exception ex)
+        {
+            // On new installs, the db isn't setup yet, so this has nothing to do
+        }
+    }
+}

API/Data/ManualMigrations/v0.7.14/MigrateWantToReadImport.cs

@@ -0,0 +1,67 @@
+using System.Globalization;
+using System.IO;
+using System.Linq;
+using System.Threading.Tasks;
+using API.Data.Repositories;
+using API.Entities;
+using API.Services;
+using CsvHelper;
+using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.Logging;
+
+namespace API.Data.ManualMigrations;
+
+/// <summary>
+/// v0.7.13.12/v0.7.14 - Want to read is imported from a csv
+/// </summary>
+public static class MigrateWantToReadImport
+{
+    public static async Task Migrate(IUnitOfWork unitOfWork, DataContext dataContext, IDirectoryService directoryService, ILogger<Program> logger)
+    {
+
+        if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateWantToReadImport"))
+        {
+            return;
+        }
+
+        var importFile = Path.Join(directoryService.ConfigDirectory, "want-to-read-migration.csv");
+        var outputFile = Path.Join(directoryService.ConfigDirectory, "imported-want-to-read-migration.csv");
+
+        if (!File.Exists(importFile) || File.Exists(outputFile))
+        {
+            logger.LogCritical(
+                "Running MigrateWantToReadImport migration - Completed. This is not an error");
+            return;
+        }
+
+        logger.LogCritical(
+            "Running MigrateWantToReadImport migration - Please be patient, this may take some time. This is not an error");
+
+        using var reader = new StreamReader(importFile);
+        using var csvReader = new CsvReader(reader, CultureInfo.InvariantCulture);
+        // Read the records from the CSV file
+        await csvReader.ReadAsync();
+        csvReader.ReadHeader(); // Skip the header row
+
+        while (await csvReader.ReadAsync())
+        {
+            // Read the values of AppUserId and Id columns
+            var appUserId = csvReader.GetField<int>("AppUserId");
+            var seriesId = csvReader.GetField<int>("Id");
+            var user = await unitOfWork.UserRepository.GetUserByIdAsync(appUserId, AppUserIncludes.WantToRead);
+            if (user == null || user.WantToRead.Any(w => w.SeriesId == seriesId)) continue;
+
+            user.WantToRead.Add(new AppUserWantToRead()
+            {
+                SeriesId = seriesId
+            });
+        }
+
+        await unitOfWork.CommitAsync();
+        reader.Close();
+
+        File.WriteAllLines(outputFile, await File.ReadAllLinesAsync(importFile));
+        logger.LogCritical(
+            "Running MigrateWantToReadImport migration - Completed. This is not an error");
+    }
+}