AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Last Batch before Release (#2899)

Browse source
This commit is contained in:
Joe Milazzo 2024-04-21 10:53:40 -05:00 • committed by GitHub
parent 8d77b398b2
commit 32bedb4e06
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
32 changed files with 3302 additions and 124 deletions
Download patch file Download diff file Expand all files Collapse all files

13
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
@ -421,6 +422,7 @@ public class ParseScannedFiles
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
if (files.Count == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", folder);
@ -483,17 +485,17 @@ public class ParseScannedFiles
}
chapters = infos
.OrderByNatural(info => info.Chapters)
.OrderByNatural(info => info.Chapters, StringComparer.InvariantCulture)
.ToList();
counter = 0f;
var prevIssue = string.Empty;
foreach (var chapter in chapters)
{
if (float.TryParse(chapter.Chapters, out var parsedChapter))
if (float.TryParse(chapter.Chapters, CultureInfo.InvariantCulture, out var parsedChapter))
{
counter = parsedChapter;
if (!string.IsNullOrEmpty(prevIssue) && float.TryParse(prevIssue, out var prevIssueFloat) && parsedChapter.Is(prevIssueFloat))
if (!string.IsNullOrEmpty(prevIssue) && float.TryParse(prevIssue, CultureInfo.InvariantCulture, out var prevIssueFloat) && parsedChapter.Is(prevIssueFloat))
{
// Bump by 0.1
counter += 0.1f;
@ -565,7 +567,10 @@ public class ParseScannedFiles
// Normalize this as many of the cases is a capitalization difference
var nonLocalizedSeriesFound = infos
.Where(i => !i.IsSpecial)
.Select(i => i.Series).DistinctBy(Parser.Parser.Normalize).ToList();
.Select(i => i.Series)
.DistinctBy(Parser.Parser.Normalize)
.ToList();
if (nonLocalizedSeriesFound.Count == 1)
{
nonLocalizedSeries = nonLocalizedSeriesFound[0];

16
API/Services/Tasks/Scanner/Parser/BasicParser.cs
View file

@ -35,17 +35,15 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag
// This will be called if the epub is already parsed once then we call and merge the information, if the
if (Parser.IsEpub(filePath))
{
ret.Chapters = Parser.ParseChapter(fileName);
ret.Series = Parser.ParseSeries(fileName);
ret.Volumes = Parser.ParseVolume(fileName);
ret.Chapters = Parser.ParseChapter(fileName, type);
ret.Series = Parser.ParseSeries(fileName, type);
ret.Volumes = Parser.ParseVolume(fileName, type);
}
else
{
ret.Chapters = type == LibraryType.Comic
? Parser.ParseComicChapter(fileName)
: Parser.ParseChapter(fileName);
ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName);
ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName);
ret.Chapters = Parser.ParseChapter(fileName, type);
ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName, type);
ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName, type);
}
if (ret.Series == string.Empty || Parser.IsImage(filePath))
@ -61,7 +59,7 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag
ret.Edition = edition;
}
var isSpecial = type == LibraryType.Comic ? Parser.IsComicSpecial(fileName) : Parser.IsMangaSpecial(fileName);
var isSpecial = Parser.IsSpecial(fileName, type);
// We must ensure that we can only parse a special out. As some files will have v20 c171-180+Omake and that
// could cause a problem as Omake is a special term, but there is valid volume/chapter information.
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.LooseLeafVolume && isSpecial)

12
API/Services/Tasks/Scanner/Parser/BookParser.cs
View file

@ -13,25 +13,25 @@ public class BookParser(IDirectoryService directoryService, IBookService bookSer
info.ComicInfo = comicInfo;
// This catches when original library type is Manga/Comic and when parsing with non
if (Parser.ParseVolume(info.Series) != Parser.LooseLeafVolume) // Shouldn't this be info.Volume != DefaultVolume?
if (Parser.ParseVolume(info.Series, type) != Parser.LooseLeafVolume) // Shouldn't this be info.Volume != DefaultVolume?
{
var hasVolumeInTitle = !Parser.ParseVolume(info.Title)
var hasVolumeInTitle = !Parser.ParseVolume(info.Title, type)
.Equals(Parser.LooseLeafVolume);
var hasVolumeInSeries = !Parser.ParseVolume(info.Series)
var hasVolumeInSeries = !Parser.ParseVolume(info.Series, type)
.Equals(Parser.LooseLeafVolume);
if (string.IsNullOrEmpty(info.ComicInfo?.Volume) && hasVolumeInTitle && (hasVolumeInSeries || string.IsNullOrEmpty(info.Series)))
{
// NOTE: I'm not sure the comment is true. I've never seen this triggered
// This is likely a light novel for which we can set series from parsed title
info.Series = Parser.ParseSeries(info.Title);
info.Volumes = Parser.ParseVolume(info.Title);
info.Series = Parser.ParseSeries(info.Title, type);
info.Volumes = Parser.ParseVolume(info.Title, type);
}
else
{
var info2 = basicParser.Parse(filePath, rootPath, libraryRoot, LibraryType.Book, comicInfo);
info.Merge(info2);
if (hasVolumeInSeries && info2 != null && Parser.ParseVolume(info2.Series)
if (hasVolumeInSeries && info2 != null && Parser.ParseVolume(info2.Series, type)
.Equals(Parser.LooseLeafVolume))
{
// Override the Series name so it groups appropriately

6
API/Services/Tasks/Scanner/Parser/ComicVineParser.cs
View file

@ -37,8 +37,8 @@ public class ComicVineParser(IDirectoryService directoryService) : DefaultParser
FullFilePath = Parser.NormalizePath(filePath),
Series = string.Empty,
ComicInfo = comicInfo,
Chapters = Parser.ParseComicChapter(fileName),
Volumes = Parser.ParseComicVolume(fileName)
Chapters = Parser.ParseChapter(fileName, type),
Volumes = Parser.ParseVolume(fileName, type)
};
// See if we can formulate the name from the ComicInfo
@ -78,7 +78,7 @@ public class ComicVineParser(IDirectoryService directoryService) : DefaultParser
}
// Check if this is a Special/Annual
info.IsSpecial = Parser.IsComicSpecial(info.Filename) || Parser.IsComicSpecial(info.ComicInfo?.Format);
info.IsSpecial = Parser.IsSpecial(info.Filename, type) || Parser.IsSpecial(info.ComicInfo?.Format, type);
// Patch in other information from ComicInfo
UpdateFromComicInfo(info);

16
API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file

@ -39,13 +39,13 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
public void ParseFromFallbackFolders(string filePath, string rootPath, LibraryType type, ref ParserInfo ret)
{
var fallbackFolders = directoryService.GetFoldersTillRoot(rootPath, filePath)
.Where(f => !Parser.IsMangaSpecial(f))
.Where(f => !Parser.IsSpecial(f, type))
.ToList();
if (fallbackFolders.Count == 0)
{
var rootFolderName = directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
var series = Parser.ParseSeries(rootFolderName);
var series = Parser.ParseSeries(rootFolderName, type);
if (string.IsNullOrEmpty(series))
{
@ -64,16 +64,18 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
{
var folder = fallbackFolders[i];
var parsedVolume = type is LibraryType.Manga ? Parser.ParseVolume(folder) : Parser.ParseComicVolume(folder);
var parsedChapter = type is LibraryType.Manga ? Parser.ParseChapter(folder) : Parser.ParseComicChapter(folder);
var parsedVolume = Parser.ParseVolume(folder, type);
var parsedChapter = Parser.ParseChapter(folder, type);
if (!parsedVolume.Equals(Parser.LooseLeafVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
{
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.LooseLeafVolume)) && !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.LooseLeafVolume))
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.LooseLeafVolume))
&& !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.LooseLeafVolume))
{
ret.Volumes = parsedVolume;
}
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter))
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter))
&& !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}
@ -82,7 +84,7 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
// Generally users group in series folders. Let's try to parse series from the top folder
if (!folder.Equals(ret.Series) && i == fallbackFolders.Count - 1)
{
var series = Parser.ParseSeries(folder);
var series = Parser.ParseSeries(folder, type);
if (string.IsNullOrEmpty(series))
{

83
API/Services/Tasks/Scanner/Parser/Parser.cs
View file

@ -1,6 +1,6 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.RegularExpressions;
@ -722,20 +722,37 @@ public static class Parser
return int.Parse(match);
}
public static bool IsMangaSpecial(string filePath)
public static bool IsSpecial(string? filePath, LibraryType type)
{
filePath = ReplaceUnderscores(filePath);
return MangaSpecialRegex.IsMatch(filePath);
return type switch
{
LibraryType.Manga => IsMangaSpecial(filePath),
LibraryType.Comic => IsComicSpecial(filePath),
LibraryType.Book => IsMangaSpecial(filePath),
LibraryType.Image => IsMangaSpecial(filePath),
LibraryType.LightNovel => IsMangaSpecial(filePath),
LibraryType.ComicVine => IsComicSpecial(filePath),
_ => false
};
}
public static bool IsComicSpecial(string? filePath)
private static bool IsMangaSpecial(string? filePath)
{
if (string.IsNullOrEmpty(filePath)) return false;
filePath = ReplaceUnderscores(filePath);
return MangaSpecialRegex.IsMatch(filePath);
}
private static bool IsComicSpecial(string? filePath)
{
if (string.IsNullOrEmpty(filePath)) return false;
filePath = ReplaceUnderscores(filePath);
return ComicSpecialRegex.IsMatch(filePath);
}
public static string ParseSeries(string filename)
public static string ParseMangaSeries(string filename)
{
foreach (var regex in MangaSeriesRegex)
{
@ -762,7 +779,7 @@ public static class Parser
return string.Empty;
}
public static string ParseVolume(string filename)
public static string ParseMangaVolume(string filename)
{
foreach (var regex in MangaVolumeRegex)
{
@ -798,6 +815,7 @@ public static class Parser
return LooseLeafVolume;
}
private static string FormatValue(string value, bool hasPart)
{
if (!value.Contains('-'))
@ -807,6 +825,7 @@ public static class Parser
var tokens = value.Split("-");
var from = RemoveLeadingZeroes(tokens[0]);
if (tokens.Length != 2) return from;
// Occasionally users will use c01-c02 instead of c01-02, clean any leftover c
@ -818,7 +837,49 @@ public static class Parser
return $"{from}-{to}";
}
public static string ParseChapter(string filename)
public static string ParseSeries(string filename, LibraryType type)
{
return type switch
{
LibraryType.Manga => ParseMangaSeries(filename),
LibraryType.Comic => ParseComicSeries(filename),
LibraryType.Book => ParseMangaSeries(filename),
LibraryType.Image => ParseMangaSeries(filename),
LibraryType.LightNovel => ParseMangaSeries(filename),
LibraryType.ComicVine => ParseComicSeries(filename),
_ => string.Empty
};
}
public static string ParseVolume(string filename, LibraryType type)
{
return type switch
{
LibraryType.Manga => ParseMangaVolume(filename),
LibraryType.Comic => ParseComicVolume(filename),
LibraryType.Book => ParseMangaVolume(filename),
LibraryType.Image => ParseMangaVolume(filename),
LibraryType.LightNovel => ParseMangaVolume(filename),
LibraryType.ComicVine => ParseComicVolume(filename),
_ => LooseLeafVolume
};
}
public static string ParseChapter(string filename, LibraryType type)
{
return type switch
{
LibraryType.Manga => ParseMangaChapter(filename),
LibraryType.Comic => ParseComicChapter(filename),
LibraryType.Book => ParseMangaChapter(filename),
LibraryType.Image => ParseMangaChapter(filename),
LibraryType.LightNovel => ParseMangaChapter(filename),
LibraryType.ComicVine => ParseComicChapter(filename),
_ => DefaultChapter
};
}
private static string ParseMangaChapter(string filename)
{
foreach (var regex in MangaChapterRegex)
{
@ -847,7 +908,7 @@ public static class Parser
return $"{value}.5";
}
public static string ParseComicChapter(string filename)
private static string ParseComicChapter(string filename)
{
foreach (var regex in ComicChapterRegex)
{
@ -1003,7 +1064,7 @@ public static class Parser
return tokens.Min(t => t.AsFloat());
}
return float.Parse(range);
return range.AsFloat();
}
catch (Exception)
{
@ -1030,7 +1091,7 @@ public static class Parser
return tokens.Max(t => t.AsFloat());
}
return float.Parse(range);
return range.AsFloat();
}
catch (Exception)
{

10
API/Services/Tasks/Scanner/Parser/PdfParser.cs
View file

@ -17,9 +17,7 @@ public class PdfParser(IDirectoryService directoryService) : DefaultParser(direc
FullFilePath = Parser.NormalizePath(filePath),
Series = string.Empty,
ComicInfo = comicInfo,
Chapters = type == LibraryType.Comic
? Parser.ParseComicChapter(fileName)
: Parser.ParseChapter(fileName)
Chapters = Parser.ParseChapter(fileName, type)
};
if (type == LibraryType.Book)
@ -27,8 +25,8 @@ public class PdfParser(IDirectoryService directoryService) : DefaultParser(direc
ret.Chapters = Parser.DefaultChapter;
}
ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName);
ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName);
ret.Series = Parser.ParseSeries(fileName, type);
ret.Volumes = Parser.ParseVolume(fileName, type);
if (ret.Series == string.Empty)
{
@ -43,7 +41,7 @@ public class PdfParser(IDirectoryService directoryService) : DefaultParser(direc
ret.Edition = edition;
}
var isSpecial = type == LibraryType.Comic ? Parser.IsComicSpecial(fileName) : Parser.IsMangaSpecial(fileName);
var isSpecial = Parser.IsSpecial(fileName, type);
// We must ensure that we can only parse a special out. As some files will have v20 c171-180+Omake and that
// could cause a problem as Omake is a special term, but there is valid volume/chapter information.
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.LooseLeafVolume && isSpecial)

8
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -705,7 +705,10 @@ public class ProcessSeries : IProcessSeries
chapter.Number = Parser.Parser.MinNumberFromRange(info.Chapters).ToString(CultureInfo.InvariantCulture);
chapter.MinNumber = Parser.Parser.MinNumberFromRange(info.Chapters);
chapter.MaxNumber = Parser.Parser.MaxNumberFromRange(info.Chapters);
chapter.SortOrder = info.IssueOrder;
if (!chapter.SortOrderLocked)
{
chapter.SortOrder = info.IssueOrder;
}
chapter.Range = chapter.GetNumberTitle();
}
@ -725,7 +728,8 @@ public class ProcessSeries : IProcessSeries
// Ensure we remove any files that no longer exist AND order
existingChapter.Files = existingChapter.Files
.Where(f => parsedInfos.Any(p => Parser.Parser.NormalizePath(p.FullFilePath) == Parser.Parser.NormalizePath(f.FilePath)))
.OrderByNatural(f => f.FilePath).ToList();
.OrderByNatural(f => f.FilePath)
.ToList();
existingChapter.Pages = existingChapter.Files.Sum(f => f.Pages);
}
}