AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Foundational Rework (Round 2) (#2767)

Browse source
This commit is contained in:
Joe Milazzo 2024-03-07 07:13:36 -07:00 • committed by GitHub
parent 304fd8bc79
commit fc87dba0a7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
29 changed files with 249 additions and 84 deletions
Download patch file Download diff file Expand all files Collapse all files

9
API/DTOs/ChapterDto.cs
View file

@ -15,15 +15,22 @@ public class ChapterDto : IHasReadTimeEstimate
/// <summary>
/// Range of chapters. Chapter 2-4 -> "2-4". Chapter 2 -> "2". If special, will be special name.
/// </summary>
/// <remarks>This can be something like 19.HU or Alpha as some comics are like this</remarks>
public string Range { get; init; } = default!;
/// <summary>
/// Smallest number of the Range.
/// </summary>
[Obsolete("Use MinNumber and MaxNumber instead")]
public string Number { get; init; } = default!;
/// <summary>
/// This may be 0 under the circumstance that the Issue is "Alpha" or other non-standard numbers.
/// </summary>
public float MinNumber { get; init; }
public float MaxNumber { get; init; }
public float SortOrder { get; init; }
/// <summary>
/// The sorting order of the Chapter. Inherits from MinNumber, but can be overridden.
/// </summary>
public float SortOrder { get; set; }
/// <summary>
/// Total number of pages in all MangaFiles
/// </summary>

2
API/Data/ManualMigrations/MigrateChapterFields.cs
View file

@ -60,7 +60,7 @@ public static class MigrateChapterFields
"Running MigrateChapterFields migration - Updating all MangaFiles");
foreach (var mangaFile in dataContext.MangaFile)
{
mangaFile.FileName = Path.GetFileNameWithoutExtension(mangaFile.FilePath);
mangaFile.FileName = Parser.RemoveExtensionIfSupported(mangaFile.FilePath);
}
var looseLeafChapters = await dataContext.Chapter.Where(c => c.Number == "0").ToListAsync();

55
API/Data/ManualMigrations/MigrateChapterRange.cs Normal file
View file

@ -0,0 +1,55 @@
using System;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
using API.Helpers.Builders;
using API.Services.Tasks.Scanner.Parser;
using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.8.0 changed the range to that it doesn't have filename by default
/// </summary>
public static class MigrateChapterRange
{
public static async Task Migrate(DataContext dataContext, IUnitOfWork unitOfWork, ILogger<Program> logger)
{
if (await dataContext.ManualMigrationHistory.AnyAsync(m => m.Name == "MigrateChapterRange"))
{
return;
}
logger.LogCritical(
"Running MigrateChapterRange migration - Please be patient, this may take some time. This is not an error");
var chapters = await dataContext.Chapter.ToListAsync();
foreach (var chapter in chapters)
{
if (Parser.MinNumberFromRange(chapter.Range) == 0.0f)
{
chapter.Range = chapter.GetNumberTitle();
}
}
// Save changes after processing all series
if (dataContext.ChangeTracker.HasChanges())
{
await dataContext.SaveChangesAsync();
}
dataContext.ManualMigrationHistory.Add(new ManualMigrationHistory()
{
Name = "MigrateChapterRange",
ProductVersion = BuildInfo.Version.ToString(),
RanAt = DateTime.UtcNow
});
await dataContext.SaveChangesAsync();
logger.LogCritical(
"Running MigrateChapterRange migration - Completed. This is not an error");
}
}

1
API/Data/Repositories/SeriesRepository.cs
View file

@ -498,6 +498,7 @@ public class SeriesRepository : ISeriesRepository
.Include(c => c.Files)
.Where(c => EF.Functions.Like(c.TitleName, $"%{searchQuery}%")
|| EF.Functions.Like(c.ISBN, $"%{searchQuery}%")
|| EF.Functions.Like(c.Range, $"%{searchQuery}%")
)
.Where(c => c.Files.All(f => fileIds.Contains(f.Id)))
.AsSplitQuery()

14
API/Entities/Chapter.cs
View file

@ -149,10 +149,13 @@ public class Chapter : IEntityDate, IHasReadTimeEstimate
MinNumber = Parser.DefaultChapterNumber;
MaxNumber = Parser.DefaultChapterNumber;
}
// NOTE: This doesn't work well for all because Pdf usually should use into.Title or even filename
Title = (IsSpecial && info.Format == MangaFormat.Epub)
? info.Title
: Path.GetFileNameWithoutExtension(Range);
: Parser.RemoveExtensionIfSupported(Range);
var specialTreatment = info.IsSpecialInfo();
Range = specialTreatment ? info.Filename : info.Chapters;
}
/// <summary>
@ -165,13 +168,16 @@ public class Chapter : IEntityDate, IHasReadTimeEstimate
{
if (MinNumber.Is(Parser.DefaultChapterNumber) && IsSpecial)
{
return Title;
return Parser.RemoveExtensionIfSupported(Title);
}
else
if (MinNumber.Is(0) && !float.TryParse(Range, out _))
{
return $"{MinNumber}";
return $"{Range}";
}
return $"{MinNumber}";
}
return $"{MinNumber}-{MaxNumber}";
}

7
API/Extensions/ChapterListExtensions.cs
View file

@ -29,10 +29,11 @@ public static class ChapterListExtensions
/// <returns></returns>
public static Chapter? GetChapterByRange(this IEnumerable<Chapter> chapters, ParserInfo info)
{
var normalizedPath = Parser.NormalizePath(info.FullFilePath);
var specialTreatment = info.IsSpecialInfo();
return specialTreatment
? chapters.FirstOrDefault(c => c.Range == Path.GetFileNameWithoutExtension(info.Filename) || (c.Files.Select(f => f.FilePath).Contains(info.FullFilePath)))
: chapters.FirstOrDefault(c => c.Range == info.Chapters);
return specialTreatment
? chapters.FirstOrDefault(c => c.Range == Parser.RemoveExtensionIfSupported(info.Filename) || c.Files.Select(f => Parser.NormalizePath(f.FilePath)).Contains(normalizedPath))
: chapters.FirstOrDefault(c => c.Range == info.Chapters);
}
/// <summary>

7
API/Extensions/ParserInfoListExtensions.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.IO;
using System.Linq;
using API.Entities;
using API.Services.Tasks.Scanner.Parser;
@ -27,7 +28,9 @@ public static class ParserInfoListExtensions
/// <returns></returns>
public static bool HasInfo(this IList<ParserInfo> infos, Chapter chapter)
{
return chapter.IsSpecial ? infos.Any(v => v.Filename == chapter.Range)
: infos.Any(v => v.Chapters == chapter.Range);
var chapterFiles = chapter.Files.Select(x => Parser.NormalizePath(x.FilePath)).ToList();
var infoFiles = infos.Select(x => Parser.NormalizePath(x.FullFilePath)).ToList();
return infoFiles.Intersect(chapterFiles).Any();
}
}

20
API/Helpers/Builders/ChapterBuilder.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using API.Entities;
using API.Entities.Enums;
using API.Services.Tasks.Scanner.Parser;
@ -17,7 +18,7 @@ public class ChapterBuilder : IEntityBuilder<Chapter>
{
_chapter = new Chapter()
{
Range = string.IsNullOrEmpty(range) ? number : range,
Range = string.IsNullOrEmpty(range) ? number : Parser.RemoveExtensionIfSupported(range),
Title = string.IsNullOrEmpty(range) ? number : range,
Number = Parser.MinNumberFromRange(number).ToString(CultureInfo.InvariantCulture),
MinNumber = Parser.MinNumberFromRange(number),
@ -32,17 +33,14 @@ public class ChapterBuilder : IEntityBuilder<Chapter>
public static ChapterBuilder FromParserInfo(ParserInfo info)
{
var specialTreatment = info.IsSpecialInfo();
var specialTitle = specialTreatment ? info.Filename : info.Chapters;
var specialTitle = specialTreatment ? Parser.RemoveExtensionIfSupported(info.Filename) : info.Chapters;
var builder = new ChapterBuilder(Parser.DefaultChapter);
// TODO: Come back here and remove this side effect
return builder.WithNumber(specialTreatment ? Parser.DefaultChapter : Parser.MinNumberFromRange(info.Chapters) + string.Empty)
return builder.WithNumber(Parser.RemoveExtensionIfSupported(info.Chapters))
.WithRange(specialTreatment ? info.Filename : info.Chapters)
.WithTitle((specialTreatment && info.Format == MangaFormat.Epub)
? info.Title
: specialTitle)
// NEW
//.WithTitle(string.IsNullOrEmpty(info.Filename) ? specialTitle : info.Filename)
.WithTitle(info.Filename)
.WithIsSpecial(specialTreatment);
}
@ -53,7 +51,7 @@ public class ChapterBuilder : IEntityBuilder<Chapter>
}
public ChapterBuilder WithNumber(string number)
private ChapterBuilder WithNumber(string number)
{
_chapter.Number = number;
_chapter.MinNumber = Parser.MinNumberFromRange(number);
@ -79,11 +77,9 @@ public class ChapterBuilder : IEntityBuilder<Chapter>
return this;
}
private ChapterBuilder WithRange(string range)
public ChapterBuilder WithRange(string range)
{
_chapter.Range = range;
// TODO: HACK: Overriding range
_chapter.Range = _chapter.GetNumberTitle();
_chapter.Range = Parser.RemoveExtensionIfSupported(range);
return this;
}

3
API/Helpers/Builders/MangaFileBuilder.cs
View file

@ -2,6 +2,7 @@
using System.IO;
using API.Entities;
using API.Entities.Enums;
using API.Services.Tasks.Scanner.Parser;
namespace API.Helpers.Builders;
@ -19,7 +20,7 @@ public class MangaFileBuilder : IEntityBuilder<MangaFile>
Pages = pages,
LastModified = File.GetLastWriteTime(filePath),
LastModifiedUtc = File.GetLastWriteTimeUtc(filePath),
FileName = Path.GetFileNameWithoutExtension(filePath)
FileName = Parser.RemoveExtensionIfSupported(filePath)
};
}

1
API/Services/MetadataService.cs
View file

@ -82,6 +82,7 @@ public class MetadataService : IMetadataService
chapter.CoverImage = _readingItemService.GetCoverImage(firstFile.FilePath,
ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId), firstFile.Format, encodeFormat, coverImageSize);
_unitOfWork.ChapterRepository.Update(chapter);
_updateEvents.Add(MessageFactory.CoverUpdateEvent(chapter.Id, MessageFactoryEntityTypes.Chapter));
return Task.FromResult(true);
}

89
API/Services/ReadingItemService.cs
View file

@ -90,46 +90,6 @@ public class ReadingItemService : IReadingItemService
}
// This is first time ComicInfo is called
info.ComicInfo = GetComicInfo(path);
if (info.ComicInfo == null) return info;
if (!string.IsNullOrEmpty(info.ComicInfo.Volume))
{
info.Volumes = info.ComicInfo.Volume;
}
if (!string.IsNullOrEmpty(info.ComicInfo.Series))
{
info.Series = info.ComicInfo.Series.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
{
info.Chapters = info.ComicInfo.Number;
}
// Patch is SeriesSort from ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo.TitleSort))
{
info.SeriesSort = info.ComicInfo.TitleSort.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Format) && Parser.HasComicInfoSpecial(info.ComicInfo.Format))
{
info.IsSpecial = true;
info.Chapters = Parser.DefaultChapter;
info.Volumes = Parser.LooseLeafVolume;
}
if (!string.IsNullOrEmpty(info.ComicInfo.SeriesSort))
{
info.SeriesSort = info.ComicInfo.SeriesSort.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.LocalizedSeries))
{
info.LocalizedSeries = info.ComicInfo.LocalizedSeries.Trim();
}
return info;
}
@ -218,6 +178,53 @@ public class ReadingItemService : IReadingItemService
/// <returns></returns>
private ParserInfo? Parse(string path, string rootPath, LibraryType type)
{
return Parser.IsEpub(path) ? _bookService.ParseInfo(path) : _defaultParser.Parse(path, rootPath, type);
var info = Parser.IsEpub(path) ? _bookService.ParseInfo(path) : _defaultParser.Parse(path, rootPath, type);
if (info == null) return null;
info.ComicInfo = GetComicInfo(path);
if (info.ComicInfo == null) return info;
if (!string.IsNullOrEmpty(info.ComicInfo.Volume))
{
info.Volumes = info.ComicInfo.Volume;
}
if (!string.IsNullOrEmpty(info.ComicInfo.Series))
{
info.Series = info.ComicInfo.Series.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
{
info.Chapters = info.ComicInfo.Number;
if (info.IsSpecial && Parser.DefaultChapter != info.Chapters)
{
info.IsSpecial = false;
}
}
// Patch is SeriesSort from ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo.TitleSort))
{
info.SeriesSort = info.ComicInfo.TitleSort.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Format) && Parser.HasComicInfoSpecial(info.ComicInfo.Format))
{
info.IsSpecial = true;
info.Chapters = Parser.DefaultChapter;
info.Volumes = Parser.LooseLeafVolume;
}
if (!string.IsNullOrEmpty(info.ComicInfo.SeriesSort))
{
info.SeriesSort = info.ComicInfo.SeriesSort.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.LocalizedSeries))
{
info.LocalizedSeries = info.ComicInfo.LocalizedSeries.Trim();
}
return info;
}
}

2
API/Services/SeriesService.cs
View file

@ -510,7 +510,7 @@ public class SeriesService : ISeriesService
.SelectMany(v => v.Chapters
.Select(c =>
{
if (v.IsLooseLeaf()) return c;
if (v.IsLooseLeaf() || v.IsSpecial()) return c;
c.VolumeTitle = v.Name;
return c;
})

1
API/Services/TaskScheduler.cs
View file

@ -386,6 +386,7 @@ public class TaskScheduler : ITaskScheduler
}
if (RunningAnyTasksByMethod(ScanTasks, ScanQueue))
{
// BUG: This can end up triggering a ton of scan series calls
_logger.LogInformation("A Scan is already running, rescheduling ScanSeries in 10 minutes");
BackgroundJob.Schedule(() => ScanSeries(libraryId, seriesId, forceUpdate), TimeSpan.FromMinutes(10));
return;

8
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -336,7 +336,7 @@ public class ParseScannedFiles
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
if (files.Count == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty or is no longer in this location", folder);
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", folder);
return;
}
@ -416,6 +416,12 @@ public class ParseScannedFiles
}
else
{
// TODO: I think I need to bump by 0.1f as if the prevIssue matches counter
if (!string.IsNullOrEmpty(prevIssue) && prevIssue == counter + "")
{
// Bump by 0.1
counter += 0.1f;
}
chapter.IssueOrder = counter;
counter++;
prevIssue = chapter.Chapters;

11
API/Services/Tasks/Scanner/Parser/Parser.cs
View file

@ -1130,4 +1130,15 @@ public static class Parser
return null;
}
public static string RemoveExtensionIfSupported(string? filename)
{
if (string.IsNullOrEmpty(filename)) return filename;
if (Regex.IsMatch(filename, SupportedExtensions))
{
return Regex.Replace(filename, SupportedExtensions, string.Empty);
}
return filename;
}
}

3
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -246,6 +246,7 @@ public class ProcessSeries : IProcessSeries
catch (Exception ex)
{
_logger.LogError(ex, "[ScannerService] There was an exception updating series for {SeriesName}", series.Name);
return;
}
var settings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
@ -661,7 +662,7 @@ public class ProcessSeries : IProcessSeries
{
if (existingChapter.Files.Count == 0 || !parsedInfos.HasInfo(existingChapter))
{
_logger.LogDebug("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName}", existingChapter.GetNumberTitle(), volume.Name, parsedInfos[0].Series);
_logger.LogDebug("[ScannerService] Removed chapter {Chapter} for Volume {VolumeNumber} on {SeriesName}", existingChapter.Range, volume.Name, parsedInfos[0].Series);
volume.Chapters.Remove(existingChapter);
}
else

1
API/Startup.cs
View file

@ -257,6 +257,7 @@ public class Startup
await MigrateChapterNumber.Migrate(dataContext, logger);
await MigrateMixedSpecials.Migrate(dataContext, unitOfWork, logger);
await MigrateChapterFields.Migrate(dataContext, unitOfWork, logger);
await MigrateChapterRange.Migrate(dataContext, unitOfWork, logger);
// Update the version in the DB after all migrations are run
var installVersion = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallVersion);