.NET 7 + Spring Cleaning (#1677)
* Updated to net7.0 * Updated GA to .net 7 * Updated System.IO.Abstractions to use New factory. * Converted Regex into SourceGenerator in Parser. * Updated more regex to source generators. * Enabled Nullability and more regex changes throughout codebase. * Parser is 100% GeneratedRegexified * Lots of nullability code * Enabled nullability for all repositories. * Fixed another unit test * Refactored some code around and took care of some todos. * Updating code for nullability and cleaning up methods that aren't used anymore. Refctored all uses of Parser.Normalize() to use new extension * More nullability exercises. 500 warnings to go. * Fixed a bug where custom file uploads for entities wouldn't save in webP. * Nullability is done for all DTOs * Fixed all unit tests and nullability for the project. Only OPDS is left which will be done with an upcoming OPDS enhancement. * Use localization in book service after validating * Code smells * Switched to preview build of swashbuckle for .net7 support * Fixed up merge issues * Disable emulate comic book when on single page reader * Fixed a regression where double page renderer wouldn't layout the images correctly * Updated to swashbuckle which support .net 7 * Fixed a bad GA action * Some code cleanup * More code smells * Took care of most of nullable issues * Fixed a broken test due to having more than one test run in parallel * I'm really not sure why the unit tests are failing or are so extremely slow on .net 7 * Updated all dependencies * Fixed up build and removed hardcoded framework from build scripts. (this merge removes Regex Source generators). Unit tests are completely busted. * Unit tests and code cleanup. Needs shakeout now. * Adjusted Series model since a few fields are not-nullable. Removed dead imports on the project. * Refactored to use Builder pattern for all unit tests. * Switched nullability down to warnings. It wasn't possible to switch due to constraint issues in DB Migration.
This commit is contained in:
Joe Milazzo
GitHub
parent
76fe3fd64a
commit
5d1dd7b3f0
283 changed files with 4221 additions and 4593 deletions
|
|
@ -7,7 +7,7 @@ namespace API.Services.Tasks.Scanner.Parser;
|
|||
|
||||
public interface IDefaultParser
|
||||
{
|
||||
ParserInfo Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga);
|
||||
ParserInfo? Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga);
|
||||
void ParseFromFallbackFolders(string filePath, string rootPath, LibraryType type, ref ParserInfo ret);
|
||||
}
|
||||
|
||||
|
|
@ -31,7 +31,7 @@ public class DefaultParser : IDefaultParser
|
|||
/// <param name="rootPath">Root folder</param>
|
||||
/// <param name="type">Defaults to Manga. Allows different Regex to be used for parsing.</param>
|
||||
/// <returns><see cref="ParserInfo"/> or null if Series was empty</returns>
|
||||
public ParserInfo Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga)
|
||||
public ParserInfo? Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga)
|
||||
{
|
||||
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
|
||||
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this.
|
||||
|
|
@ -134,7 +134,7 @@ public class DefaultParser : IDefaultParser
|
|||
|
||||
if (fallbackFolders.Count == 0)
|
||||
{
|
||||
var rootFolderName = _directoryService.FileSystem.DirectoryInfo.FromDirectoryName(rootPath).Name;
|
||||
var rootFolderName = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
|
||||
var series = Parser.ParseSeries(rootFolderName);
|
||||
|
||||
if (string.IsNullOrEmpty(series))
|
||||
|
|
|
|||
|
|
@ -11,11 +11,13 @@ public static class Parser
|
|||
{
|
||||
public const string DefaultChapter = "0";
|
||||
public const string DefaultVolume = "0";
|
||||
private const int RegexTimeoutMs = 5000000; // 500 ms
|
||||
public static readonly TimeSpan RegexTimeout = TimeSpan.FromMilliseconds(500);
|
||||
|
||||
public const string ImageFileExtensions = @"^(\.png|\.jpeg|\.jpg|\.webp|\.gif)";
|
||||
public const string ArchiveFileExtensions = @"\.cbz|\.zip|\.rar|\.cbr|\.tar.gz|\.7zip|\.7z|\.cb7|\.cbt";
|
||||
private const string BookFileExtensions = @"\.epub|\.pdf";
|
||||
private const string XmlRegexExtensions = @"\.xml";
|
||||
public const string MacOsMetadataFileStartsWith = @"._";
|
||||
|
||||
public const string SupportedExtensions =
|
||||
|
|
@ -24,6 +26,37 @@ public static class Parser
|
|||
private const RegexOptions MatchOptions =
|
||||
RegexOptions.IgnoreCase | RegexOptions.Compiled | RegexOptions.CultureInvariant;
|
||||
|
||||
private static readonly ImmutableArray<string> FormatTagSpecialKeywords = ImmutableArray.Create(
|
||||
"Special", "Reference", "Director's Cut", "Box Set", "Box-Set", "Annual", "Anthology", "Epilogue",
|
||||
"One Shot", "One-Shot", "Prologue", "TPB", "Trade Paper Back", "Omnibus", "Compendium", "Absolute", "Graphic Novel",
|
||||
"GN", "FCBD");
|
||||
|
||||
private static readonly char[] LeadingZeroesTrimChars = new[] { '0' };
|
||||
|
||||
private static readonly char[] SpacesAndSeparators = { '\0', '\t', '\r', ' ', '-', ','};
|
||||
|
||||
|
||||
private const string Number = @"\d+(\.\d)?";
|
||||
private const string NumberRange = Number + @"(-" + Number + @")?";
|
||||
|
||||
/// <summary>
|
||||
/// non greedy matching of a string where parenthesis are balanced
|
||||
/// </summary>
|
||||
public const string BalancedParen = @"(?:[^()]|(?<open>\()|(?<-open>\)))*?(?(open)(?!))";
|
||||
/// <summary>
|
||||
/// non greedy matching of a string where square brackets are balanced
|
||||
/// </summary>
|
||||
public const string BalancedBracket = @"(?:[^\[\]]|(?<open>\[)|(?<-open>\]))*?(?(open)(?!))";
|
||||
/// <summary>
|
||||
/// Matches [Complete], release tags like [kmts] but not [ Complete ] or [kmts ]
|
||||
/// </summary>
|
||||
private const string TagsInBrackets = $@"\[(?!\s){BalancedBracket}(?<!\s)\]";
|
||||
/// <summary>
|
||||
/// Common regex patterns present in both Comics and Mangas
|
||||
/// </summary>
|
||||
private const string CommonSpecial = @"Specials?|One[- ]?Shot|Extra(?:\sChapter)?(?=\s)|Art Collection|Side Stories|Bonus";
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Matches against font-family css syntax. Does not match if url import has data: starting, as that is binary data
|
||||
/// </summary>
|
||||
|
|
@ -44,7 +77,6 @@ public static class Parser
|
|||
MatchOptions, RegexTimeout);
|
||||
|
||||
|
||||
private const string XmlRegexExtensions = @"\.xml";
|
||||
private static readonly Regex ImageRegex = new Regex(ImageFileExtensions,
|
||||
MatchOptions, RegexTimeout);
|
||||
private static readonly Regex ArchiveFileRegex = new Regex(ArchiveFileExtensions,
|
||||
|
|
@ -67,14 +99,6 @@ public static class Parser
|
|||
private static readonly Regex SpecialTokenRegex = new Regex(@"SP\d+",
|
||||
MatchOptions, RegexTimeout);
|
||||
|
||||
private const string Number = @"\d+(\.\d)?";
|
||||
private const string NumberRange = Number + @"(-" + Number + @")?";
|
||||
|
||||
// Some generic reusage regex patterns:
|
||||
// - non greedy matching of a string where parenthesis are balanced
|
||||
public const string BalancedParen = @"(?:[^()]|(?<open>\()|(?<-open>\)))*?(?(open)(?!))";
|
||||
// - non greedy matching of a string where square brackets are balanced
|
||||
public const string BalancedBrack = @"(?:[^\[\]]|(?<open>\[)|(?<-open>\]))*?(?(open)(?!))";
|
||||
|
||||
private static readonly Regex[] MangaVolumeRegex = new[]
|
||||
{
|
||||
|
|
@ -86,7 +110,6 @@ public static class Parser
|
|||
new Regex(
|
||||
@"(?<Series>.*)(\b|_)(?!\[)(vol\.?)(?<Volume>\d+(-\d+)?)(?!\])",
|
||||
MatchOptions, RegexTimeout),
|
||||
// TODO: In .NET 7, update this to use raw literal strings and apply the NumberRange everywhere
|
||||
// Historys Strongest Disciple Kenichi_v11_c90-98.zip or Dance in the Vampire Bund v16-17
|
||||
new Regex(
|
||||
@"(?<Series>.*)(\b|_)(?!\[)v(?<Volume>" + NumberRange + @")(?!\])",
|
||||
|
|
@ -576,18 +599,12 @@ public static class Parser
|
|||
MatchOptions, RegexTimeout
|
||||
);
|
||||
|
||||
// Matches [Complete], release tags like [kmts] but not [ Complete ] or [kmts ]
|
||||
private const string TagsInBrackets = $@"\[(?!\s){BalancedBrack}(?<!\s)\]";
|
||||
|
||||
// Matches anything between balanced parenthesis, tags between brackets, {} and {Complete}
|
||||
private static readonly Regex CleanupRegex = new Regex(
|
||||
$@"(?:\({BalancedParen}\)|{TagsInBrackets}|\{{\}}|\{{Complete\}})",
|
||||
MatchOptions, RegexTimeout
|
||||
);
|
||||
|
||||
// Common regex patterns present in both Comics and Mangas
|
||||
private const string CommonSpecial = @"Specials?|One[- ]?Shot|Extra(?:\sChapter)?(?=\s)|Art Collection|Side Stories|Bonus";
|
||||
|
||||
private static readonly Regex MangaSpecialRegex = new Regex(
|
||||
// All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle.
|
||||
$@"\b(?:{CommonSpecial}|Omake)\b",
|
||||
|
|
@ -601,11 +618,12 @@ public static class Parser
|
|||
);
|
||||
|
||||
private static readonly Regex EuropeanComicRegex = new Regex(
|
||||
// All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle.
|
||||
// All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle.
|
||||
@"\b(?:Bd[-\s]Fr)\b",
|
||||
MatchOptions, RegexTimeout
|
||||
);
|
||||
|
||||
|
||||
// If SP\d+ is in the filename, we force treat it as a special regardless if volume or chapter might have been found.
|
||||
private static readonly Regex SpecialMarkerRegex = new Regex(
|
||||
@"SP\d+",
|
||||
|
|
@ -617,14 +635,7 @@ public static class Parser
|
|||
MatchOptions, RegexTimeout
|
||||
);
|
||||
|
||||
private static readonly ImmutableArray<string> FormatTagSpecialKeywords = ImmutableArray.Create(
|
||||
"Special", "Reference", "Director's Cut", "Box Set", "Box-Set", "Annual", "Anthology", "Epilogue",
|
||||
"One Shot", "One-Shot", "Prologue", "TPB", "Trade Paper Back", "Omnibus", "Compendium", "Absolute", "Graphic Novel",
|
||||
"GN", "FCBD");
|
||||
|
||||
private static readonly char[] LeadingZeroesTrimChars = new[] { '0' };
|
||||
|
||||
private static readonly char[] SpacesAndSeparators = { '\0', '\t', '\r', ' ', '-', ','};
|
||||
|
||||
public static MangaFormat ParseFormat(string filePath)
|
||||
{
|
||||
|
|
@ -669,11 +680,10 @@ public static class Parser
|
|||
foreach (var regex in MangaSeriesRegex)
|
||||
{
|
||||
var matches = regex.Matches(filename);
|
||||
foreach (var group in matches.Select(match => match.Groups["Series"])
|
||||
.Where(group => group.Success && group != Match.Empty))
|
||||
{
|
||||
return CleanTitle(group.Value);
|
||||
}
|
||||
var group = matches
|
||||
.Select(match => match.Groups["Series"])
|
||||
.FirstOrDefault(group => group.Success && group != Match.Empty);
|
||||
if (group != null) return CleanTitle(group.Value);
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
|
|
@ -683,11 +693,10 @@ public static class Parser
|
|||
foreach (var regex in ComicSeriesRegex)
|
||||
{
|
||||
var matches = regex.Matches(filename);
|
||||
foreach (var group in matches.Select(match => match.Groups["Series"])
|
||||
.Where(group => group.Success && group != Match.Empty))
|
||||
{
|
||||
return CleanTitle(group.Value, true);
|
||||
}
|
||||
var group = matches
|
||||
.Select(match => match.Groups["Series"])
|
||||
.FirstOrDefault(group => group.Success && group != Match.Empty);
|
||||
if (group != null) return CleanTitle(group.Value, true);
|
||||
}
|
||||
|
||||
return string.Empty;
|
||||
|
|
@ -1028,9 +1037,9 @@ public static class Parser
|
|||
/// <example>/manga/1\1 -> /manga/1/1</example>
|
||||
/// <param name="path"></param>
|
||||
/// <returns></returns>
|
||||
public static string NormalizePath(string path)
|
||||
public static string NormalizePath(string? path)
|
||||
{
|
||||
return path.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
|
||||
return string.IsNullOrEmpty(path) ? string.Empty : path.Replace(Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar)
|
||||
.Replace(@"//", Path.AltDirectorySeparatorChar + string.Empty);
|
||||
}
|
||||
|
||||
|
|
@ -1044,5 +1053,8 @@ public static class Parser
|
|||
return FormatTagSpecialKeywords.Contains(comicInfoFormat);
|
||||
}
|
||||
|
||||
private static string ReplaceUnderscores(string name) => name?.Replace("_", " ");
|
||||
private static string ReplaceUnderscores(string name)
|
||||
{
|
||||
return string.IsNullOrEmpty(name) ? string.Empty : name.Replace('_', ' ');
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -17,7 +17,7 @@ public class ParserInfo
|
|||
/// <summary>
|
||||
/// Represents the parsed series from the file or folder
|
||||
/// </summary>
|
||||
public string Series { get; set; } = string.Empty;
|
||||
public required string Series { get; set; } = string.Empty;
|
||||
/// <summary>
|
||||
/// This can be filled in from ComicInfo.xml/Epub during scanning. Will update the SortName field on <see cref="Entities.Series"/>
|
||||
/// </summary>
|
||||
|
|
@ -80,14 +80,14 @@ public class ParserInfo
|
|||
/// This will contain any EXTRA comicInfo information parsed from the epub or archive. If there is an archive with comicInfo.xml AND it contains
|
||||
/// series, volume information, that will override what we parsed.
|
||||
/// </summary>
|
||||
public ComicInfo ComicInfo { get; set; }
|
||||
public ComicInfo? ComicInfo { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Merges non empty/null properties from info2 into this entity.
|
||||
/// </summary>
|
||||
/// <remarks>This does not merge ComicInfo as they should always be the same</remarks>
|
||||
/// <param name="info2"></param>
|
||||
public void Merge(ParserInfo info2)
|
||||
public void Merge(ParserInfo? info2)
|
||||
{
|
||||
if (info2 == null) return;
|
||||
Chapters = string.IsNullOrEmpty(Chapters) || Chapters == "0" ? info2.Chapters: Chapters;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue