AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scan Loop Last Write Time Change (#1492)

Browse source 
* Refactored invite user flow to separate error handling on create user flow and email flow. This should help users that have unique situations.

* Switch to using files to check LastWriteTime. Debug code in for Robbie to test on rclone

* Updated Parser namespace. Changed the LastWriteTime to check all files and folders.
This commit is contained in:
Joseph Milazzo 2022-08-30 10:29:09 -05:00 • committed by GitHub
parent 521b2adc14
commit 85790dd71c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
59 changed files with 283 additions and 264 deletions
Download patch file Download diff file Expand all files Collapse all files

30
API/Services/ArchiveService.cs
View file

@ -60,7 +60,7 @@ namespace API.Services
/// <returns></returns>
public virtual ArchiveLibrary CanOpen(string archivePath)
{
if (string.IsNullOrEmpty(archivePath) || !(File.Exists(archivePath) && Parser.Parser.IsArchive(archivePath) || Parser.Parser.IsEpub(archivePath))) return ArchiveLibrary.NotSupported;
if (string.IsNullOrEmpty(archivePath) || !(File.Exists(archivePath) && Tasks.Scanner.Parser.Parser.IsArchive(archivePath) || Tasks.Scanner.Parser.Parser.IsEpub(archivePath))) return ArchiveLibrary.NotSupported;
var ext = _directoryService.FileSystem.Path.GetExtension(archivePath).ToUpper();
if (ext.Equals(".CBR") || ext.Equals(".RAR")) return ArchiveLibrary.SharpCompress;
@ -100,14 +100,14 @@ namespace API.Services
case ArchiveLibrary.Default:
{
using var archive = ZipFile.OpenRead(archivePath);
return archive.Entries.Count(e => !Parser.Parser.HasBlacklistedFolderInPath(e.FullName) && Parser.Parser.IsImage(e.FullName));
return archive.Entries.Count(e => !Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(e.FullName) && Tasks.Scanner.Parser.Parser.IsImage(e.FullName));
}
case ArchiveLibrary.SharpCompress:
{
using var archive = ArchiveFactory.Open(archivePath);
return archive.Entries.Count(entry => !entry.IsDirectory &&
!Parser.Parser.HasBlacklistedFolderInPath(Path.GetDirectoryName(entry.Key) ?? string.Empty)
&& Parser.Parser.IsImage(entry.Key));
!Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(Path.GetDirectoryName(entry.Key) ?? string.Empty)
&& Tasks.Scanner.Parser.Parser.IsImage(entry.Key));
}
case ArchiveLibrary.NotSupported:
_logger.LogWarning("[GetNumberOfPagesFromArchive] This archive cannot be read: {ArchivePath}. Defaulting to 0 pages", archivePath);
@ -132,9 +132,9 @@ namespace API.Services
public static string FindFolderEntry(IEnumerable<string> entryFullNames)
{
var result = entryFullNames
.Where(path => !(Path.EndsInDirectorySeparator(path) || Parser.Parser.HasBlacklistedFolderInPath(path) || path.StartsWith(Parser.Parser.MacOsMetadataFileStartsWith)))
.Where(path => !(Path.EndsInDirectorySeparator(path) || Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(path) || path.StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith)))
.OrderByNatural(Path.GetFileNameWithoutExtension)
.FirstOrDefault(Parser.Parser.IsCoverImage);
.FirstOrDefault(Tasks.Scanner.Parser.Parser.IsCoverImage);
return string.IsNullOrEmpty(result) ? null : result;
}
@ -150,7 +150,7 @@ namespace API.Services
// First check if there are any files that are not in a nested folder before just comparing by filename. This is needed
// because NaturalSortComparer does not work with paths and doesn't seem 001.jpg as before chapter 1/001.jpg.
var fullNames = entryFullNames
.Where(path => !(Path.EndsInDirectorySeparator(path) || Parser.Parser.HasBlacklistedFolderInPath(path) || path.StartsWith(Parser.Parser.MacOsMetadataFileStartsWith)) && Parser.Parser.IsImage(path))
.Where(path => !(Path.EndsInDirectorySeparator(path) || Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(path) || path.StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith)) && Tasks.Scanner.Parser.Parser.IsImage(path))
.OrderByNatural(c => c.GetFullPathWithoutExtension())
.ToList();
if (fullNames.Count == 0) return null;
@ -187,7 +187,7 @@ namespace API.Services
/// <summary>
/// Generates byte array of cover image.
/// Given a path to a compressed file <see cref="Parser.Parser.ArchiveFileExtensions"/>, will ensure the first image (respects directory structure) is returned unless
/// Given a path to a compressed file <see cref="Tasks.Scanner.Parser.Parser.ArchiveFileExtensions"/>, will ensure the first image (respects directory structure) is returned unless
/// a folder/cover.(image extension) exists in the the compressed file (if duplicate, the first is chosen)
///
/// This skips over any __MACOSX folder/file iteration.
@ -265,7 +265,7 @@ namespace API.Services
// Sometimes ZipArchive will list the directory and others it will just keep it in the FullName
return archive.Entries.Count > 0 &&
!Path.HasExtension(archive.Entries.ElementAt(0).FullName) ||
archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar) && !Parser.Parser.HasBlacklistedFolderInPath(e.FullName));
archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar) && !Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(e.FullName));
}
/// <summary>
@ -322,7 +322,7 @@ namespace API.Services
return false;
}
if (Parser.Parser.IsArchive(archivePath) || Parser.Parser.IsEpub(archivePath)) return true;
if (Tasks.Scanner.Parser.Parser.IsArchive(archivePath) || Tasks.Scanner.Parser.Parser.IsEpub(archivePath)) return true;
_logger.LogWarning("Archive {ArchivePath} is not a valid archive", archivePath);
return false;
@ -331,10 +331,10 @@ namespace API.Services
private static bool ValidComicInfoArchiveEntry(string fullName, string name)
{
var filenameWithoutExtension = Path.GetFileNameWithoutExtension(name).ToLower();
return !Parser.Parser.HasBlacklistedFolderInPath(fullName)
return !Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(fullName)
&& filenameWithoutExtension.Equals(ComicInfoFilename, StringComparison.InvariantCultureIgnoreCase)
&& !filenameWithoutExtension.StartsWith(Parser.Parser.MacOsMetadataFileStartsWith)
&& Parser.Parser.IsXml(name);
&& !filenameWithoutExtension.StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith)
&& Tasks.Scanner.Parser.Parser.IsXml(name);
}
/// <summary>
@ -467,8 +467,8 @@ namespace API.Services
{
using var archive = ArchiveFactory.Open(archivePath);
ExtractArchiveEntities(archive.Entries.Where(entry => !entry.IsDirectory
&& !Parser.Parser.HasBlacklistedFolderInPath(Path.GetDirectoryName(entry.Key) ?? string.Empty)
&& Parser.Parser.IsImage(entry.Key)), extractPath);
&& !Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(Path.GetDirectoryName(entry.Key) ?? string.Empty)
&& Tasks.Scanner.Parser.Parser.IsImage(entry.Key)), extractPath);
break;
}
case ArchiveLibrary.NotSupported:

28
API/Services/BookService.cs
View file

@ -167,7 +167,7 @@ namespace API.Services
// @Import statements will be handled by browser, so we must inline the css into the original file that request it, so they can be Scoped
var prepend = filename.Length > 0 ? filename.Replace(Path.GetFileName(filename), string.Empty) : string.Empty;
var importBuilder = new StringBuilder();
foreach (Match match in Parser.Parser.CssImportUrlRegex.Matches(stylesheetHtml))
foreach (Match match in Tasks.Scanner.Parser.Parser.CssImportUrlRegex.Matches(stylesheetHtml))
{
if (!match.Success) continue;
@ -218,7 +218,7 @@ namespace API.Services
private static void EscapeCssImportReferences(ref string stylesheetHtml, string apiBase, string prepend)
{
foreach (Match match in Parser.Parser.CssImportUrlRegex.Matches(stylesheetHtml))
foreach (Match match in Tasks.Scanner.Parser.Parser.CssImportUrlRegex.Matches(stylesheetHtml))
{
if (!match.Success) continue;
var importFile = match.Groups["Filename"].Value;
@ -228,7 +228,7 @@ namespace API.Services
private static void EscapeFontFamilyReferences(ref string stylesheetHtml, string apiBase, string prepend)
{
foreach (Match match in Parser.Parser.FontSrcUrlRegex.Matches(stylesheetHtml))
foreach (Match match in Tasks.Scanner.Parser.Parser.FontSrcUrlRegex.Matches(stylesheetHtml))
{
if (!match.Success) continue;
var importFile = match.Groups["Filename"].Value;
@ -238,7 +238,7 @@ namespace API.Services
private static void EscapeCssImageReferences(ref string stylesheetHtml, string apiBase, EpubBookRef book)
{
var matches = Parser.Parser.CssImageUrlRegex.Matches(stylesheetHtml);
var matches = Tasks.Scanner.Parser.Parser.CssImageUrlRegex.Matches(stylesheetHtml);
foreach (Match match in matches)
{
if (!match.Success) continue;
@ -394,7 +394,7 @@ namespace API.Services
public ComicInfo GetComicInfo(string filePath)
{
if (!IsValidFile(filePath) || Parser.Parser.IsPdf(filePath)) return null;
if (!IsValidFile(filePath) || Tasks.Scanner.Parser.Parser.IsPdf(filePath)) return null;
try
{
@ -425,7 +425,7 @@ namespace API.Services
var info = new ComicInfo()
{
Summary = epubBook.Schema.Package.Metadata.Description,
Writer = string.Join(",", epubBook.Schema.Package.Metadata.Creators.Select(c => Parser.Parser.CleanAuthor(c.Creator))),
Writer = string.Join(",", epubBook.Schema.Package.Metadata.Creators.Select(c => Tasks.Scanner.Parser.Parser.CleanAuthor(c.Creator))),
Publisher = string.Join(",", epubBook.Schema.Package.Metadata.Publishers),
Month = month,
Day = day,
@ -468,7 +468,7 @@ namespace API.Services
return false;
}
if (Parser.Parser.IsBook(filePath)) return true;
if (Tasks.Scanner.Parser.Parser.IsBook(filePath)) return true;
_logger.LogWarning("[BookService] Book {EpubFile} is not a valid EPUB/PDF", filePath);
return false;
@ -480,7 +480,7 @@ namespace API.Services
try
{
if (Parser.Parser.IsPdf(filePath))
if (Tasks.Scanner.Parser.Parser.IsPdf(filePath))
{
using var docReader = DocLib.Instance.GetDocReader(filePath, new PageDimensions(1080, 1920));
return docReader.GetPageCount();
@ -536,7 +536,7 @@ namespace API.Services
/// <returns></returns>
public ParserInfo ParseInfo(string filePath)
{
if (!Parser.Parser.IsEpub(filePath)) return null;
if (!Tasks.Scanner.Parser.Parser.IsEpub(filePath)) return null;
try
{
@ -601,7 +601,7 @@ namespace API.Services
}
var info = new ParserInfo()
{
Chapters = Parser.Parser.DefaultChapter,
Chapters = Tasks.Scanner.Parser.Parser.DefaultChapter,
Edition = string.Empty,
Format = MangaFormat.Epub,
Filename = Path.GetFileName(filePath),
@ -628,7 +628,7 @@ namespace API.Services
return new ParserInfo()
{
Chapters = Parser.Parser.DefaultChapter,
Chapters = Tasks.Scanner.Parser.Parser.DefaultChapter,
Edition = string.Empty,
Format = MangaFormat.Epub,
Filename = Path.GetFileName(filePath),
@ -636,7 +636,7 @@ namespace API.Services
FullFilePath = filePath,
IsSpecial = false,
Series = epubBook.Title.Trim(),
Volumes = Parser.Parser.DefaultVolume,
Volumes = Tasks.Scanner.Parser.Parser.DefaultVolume,
};
}
catch (Exception ex)
@ -876,7 +876,7 @@ namespace API.Services
{
if (!IsValidFile(fileFilePath)) return string.Empty;
if (Parser.Parser.IsPdf(fileFilePath))
if (Tasks.Scanner.Parser.Parser.IsPdf(fileFilePath))
{
return GetPdfCoverImage(fileFilePath, fileName, outputDirectory);
}
@ -887,7 +887,7 @@ namespace API.Services
{
// Try to get the cover image from OPF file, if not set, try to parse it from all the files, then result to the first one.
var coverImageContent = epubBook.Content.Cover
?? epubBook.Content.Images.Values.FirstOrDefault(file => Parser.Parser.IsCoverImage(file.FileName))
?? epubBook.Content.Images.Values.FirstOrDefault(file => Tasks.Scanner.Parser.Parser.IsCoverImage(file.FileName))
?? epubBook.Content.Images.Values.FirstOrDefault();
if (coverImageContent == null) return string.Empty;

4
API/Services/BookmarkService.cs
View file

@ -51,7 +51,7 @@ public class BookmarkService : IBookmarkService
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
var bookmarkFilesToDelete = bookmarks.Select(b => Parser.Parser.NormalizePath(
var bookmarkFilesToDelete = bookmarks.Select(b => Tasks.Scanner.Parser.Parser.NormalizePath(
_directoryService.FileSystem.Path.Join(bookmarkDirectory,
b.FileName))).ToList();
@ -165,7 +165,7 @@ public class BookmarkService : IBookmarkService
var bookmarks = await _unitOfWork.UserRepository.GetAllBookmarksByIds(bookmarkIds.ToList());
return bookmarks
.Select(b => Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(bookmarkDirectory,
.Select(b => Tasks.Scanner.Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(bookmarkDirectory,
b.FileName)));
}

4
API/Services/CacheService.cs
View file

@ -57,7 +57,7 @@ namespace API.Services
{
// Calculate what chapter the page belongs to
var path = GetBookmarkCachePath(seriesId);
var files = _directoryService.GetFilesWithExtension(path, Parser.Parser.ImageFileExtensions);
var files = _directoryService.GetFilesWithExtension(path, Tasks.Scanner.Parser.Parser.ImageFileExtensions);
files = files
.AsEnumerable()
.OrderByNatural(Path.GetFileNameWithoutExtension)
@ -214,7 +214,7 @@ namespace API.Services
// Calculate what chapter the page belongs to
var path = GetCachePath(chapter.Id);
// TODO: We can optimize this by extracting and renaming, so we don't need to scan for the files and can do a direct access
var files = _directoryService.GetFilesWithExtension(path, Parser.Parser.ImageFileExtensions)
var files = _directoryService.GetFilesWithExtension(path, Tasks.Scanner.Parser.Parser.ImageFileExtensions)
.OrderByNatural(Path.GetFileNameWithoutExtension)
.ToArray();

37
API/Services/DirectoryService.cs
View file

@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.IO;
using System.IO.Abstractions;
using System.Linq;
@ -116,7 +117,7 @@ namespace API.Services
/// <summary>
/// Given a set of regex search criteria, get files in the given path.
/// </summary>
/// <remarks>This will always exclude <see cref="Parser.Parser.MacOsMetadataFileStartsWith"/> patterns</remarks>
/// <remarks>This will always exclude <see cref="Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith"/> patterns</remarks>
/// <param name="path">Directory to search</param>
/// <param name="searchPatternExpression">Regex version of search pattern (ie \.mp3|\.mp4). Defaults to * meaning all files.</param>
/// <param name="searchOption">SearchOption to use, defaults to TopDirectoryOnly</param>
@ -130,7 +131,7 @@ namespace API.Services
return FileSystem.Directory.EnumerateFiles(path, "*", searchOption)
.Where(file =>
reSearchPattern.IsMatch(FileSystem.Path.GetExtension(file)) && !FileSystem.Path.GetFileName(file).StartsWith(Parser.Parser.MacOsMetadataFileStartsWith));
reSearchPattern.IsMatch(FileSystem.Path.GetExtension(file)) && !FileSystem.Path.GetFileName(file).StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith));
}
@ -207,12 +208,12 @@ namespace API.Services
{
var fileName = FileSystem.Path.GetFileName(file);
return reSearchPattern.IsMatch(fileName) &&
!fileName.StartsWith(Parser.Parser.MacOsMetadataFileStartsWith);
!fileName.StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith);
});
}
return FileSystem.Directory.EnumerateFiles(path, "*", searchOption).Where(file =>
!FileSystem.Path.GetFileName(file).StartsWith(Parser.Parser.MacOsMetadataFileStartsWith));
!FileSystem.Path.GetFileName(file).StartsWith(Tasks.Scanner.Parser.Parser.MacOsMetadataFileStartsWith));
}
/// <summary>
@ -496,10 +497,10 @@ namespace API.Services
{
var stopLookingForDirectories = false;
var dirs = new Dictionary<string, string>();
foreach (var folder in libraryFolders.Select(Parser.Parser.NormalizePath))
foreach (var folder in libraryFolders.Select(Tasks.Scanner.Parser.Parser.NormalizePath))
{
if (stopLookingForDirectories) break;
foreach (var file in filePaths.Select(Parser.Parser.NormalizePath))
foreach (var file in filePaths.Select(Tasks.Scanner.Parser.Parser.NormalizePath))
{
if (!file.Contains(folder)) continue;
@ -512,7 +513,7 @@ namespace API.Services
break;
}
var fullPath = Parser.Parser.NormalizePath(Path.Join(folder, parts.Last()));
var fullPath = Tasks.Scanner.Parser.Parser.NormalizePath(Path.Join(folder, parts.Last()));
if (!dirs.ContainsKey(fullPath))
{
dirs.Add(fullPath, string.Empty);
@ -579,7 +580,7 @@ namespace API.Services
{
try
{
return Parser.Parser.NormalizePath(Directory.GetParent(fileOrFolder)?.FullName);
return Tasks.Scanner.Parser.Parser.NormalizePath(Directory.GetParent(fileOrFolder)?.FullName);
}
catch (Exception)
{
@ -621,12 +622,12 @@ namespace API.Services
// Get the matcher from either ignore or global (default setup)
if (matcher == null)
{
files.AddRange(GetFilesWithCertainExtensions(folderPath, Parser.Parser.SupportedExtensions));
files.AddRange(GetFilesWithCertainExtensions(folderPath, Tasks.Scanner.Parser.Parser.SupportedExtensions));
}
else
{
var foundFiles = GetFilesWithCertainExtensions(folderPath,
Parser.Parser.SupportedExtensions)
Tasks.Scanner.Parser.Parser.SupportedExtensions)
.Where(file => !matcher.ExcludeMatches(FileSystem.FileInfo.FromFileName(file).Name));
files.AddRange(foundFiles);
}
@ -635,18 +636,14 @@ namespace API.Services
}
/// <summary>
/// Recursively scans a folder and returns the max last write time on any folders
/// Recursively scans a folder and returns the max last write time on any folders and files
/// </summary>
/// <param name="folderPath"></param>
/// <returns>Max Last Write Time</returns>
public DateTime GetLastWriteTime(string folderPath)
{
if (!FileSystem.Directory.Exists(folderPath)) throw new IOException($"{folderPath} does not exist");
var directories = GetAllDirectories(folderPath).ToList();
if (directories.Count == 0) return FileSystem.Directory.GetLastWriteTime(folderPath);
return directories.Max(d => FileSystem.Directory.GetLastWriteTime(d));
return Directory.GetFileSystemEntries(folderPath, "*.*", SearchOption.AllDirectories).Max(path => FileSystem.File.GetLastWriteTime(path));
}
/// <summary>
@ -848,7 +845,7 @@ namespace API.Services
/// <param name="directoryName">Fully qualified directory</param>
public void RemoveNonImages(string directoryName)
{
DeleteFiles(GetFiles(directoryName, searchOption:SearchOption.AllDirectories).Where(file => !Parser.Parser.IsImage(file)));
DeleteFiles(GetFiles(directoryName, searchOption:SearchOption.AllDirectories).Where(file => !Tasks.Scanner.Parser.Parser.IsImage(file)));
}
@ -921,9 +918,9 @@ namespace API.Services
foreach (var file in directory.EnumerateFiles().OrderByNatural(file => file.FullName))
{
if (file.Directory == null) continue;
var paddedIndex = Parser.Parser.PadZeros(directoryIndex + "");
var paddedIndex = Tasks.Scanner.Parser.Parser.PadZeros(directoryIndex + "");
// We need to rename the files so that after flattening, they are in the order we found them
var newName = $"{paddedIndex}_{Parser.Parser.PadZeros(fileIndex + "")}{file.Extension}";
var newName = $"{paddedIndex}_{Tasks.Scanner.Parser.Parser.PadZeros(fileIndex + "")}{file.Extension}";
var newPath = Path.Join(root.FullName, newName);
if (!File.Exists(newPath)) file.MoveTo(newPath);
fileIndex++;
@ -935,7 +932,7 @@ namespace API.Services
foreach (var subDirectory in directory.EnumerateDirectories().OrderByNatural(d => d.FullName))
{
// We need to check if the directory is not a blacklisted (ie __MACOSX)
if (Parser.Parser.HasBlacklistedFolderInPath(subDirectory.FullName)) continue;
if (Tasks.Scanner.Parser.Parser.HasBlacklistedFolderInPath(subDirectory.FullName)) continue;
FlattenDirectory(root, subDirectory, ref directoryIndex);
}

11
API/Services/EmailService.cs
View file

@ -82,8 +82,15 @@ public class EmailService : IEmailService
public async Task<bool> CheckIfAccessible(string host)
{
// This is the only exception for using the default because we need an external service to check if the server is accessible for emails
if (IsLocalIpAddress(host)) return false;
return await SendEmailWithGet(DefaultApiUrl + "/api/email/reachable?host=" + host);
try
{
if (IsLocalIpAddress(host)) return false;
return await SendEmailWithGet(DefaultApiUrl + "/api/email/reachable?host=" + host);
}
catch (Exception)
{
return false;
}
}
public async Task<bool> SendMigrationEmail(EmailMigrationDto data)

2
API/Services/ImageService.cs
View file

@ -63,7 +63,7 @@ public class ImageService : IImageService
else
{
_directoryService.CopyDirectoryToDirectory(Path.GetDirectoryName(fileFilePath), targetDirectory,
Parser.Parser.ImageFileExtensions);
Tasks.Scanner.Parser.Parser.ImageFileExtensions);
}
}

4
API/Services/ReaderService.cs
View file

@ -59,7 +59,7 @@ public class ReaderService : IReaderService
public static string FormatBookmarkFolderPath(string baseDirectory, int userId, int seriesId, int chapterId)
{
return Parser.Parser.NormalizePath(Path.Join(baseDirectory, $"{userId}", $"{seriesId}", $"{chapterId}"));
return Tasks.Scanner.Parser.Parser.NormalizePath(Path.Join(baseDirectory, $"{userId}", $"{seriesId}", $"{chapterId}"));
}
/// <summary>
@ -496,7 +496,7 @@ public class ReaderService : IReaderService
{
var chapters = volume.Chapters
.OrderBy(c => float.Parse(c.Number))
.Where(c => !c.IsSpecial && Parser.Parser.MaxNumberFromRange(c.Range) <= chapterNumber);
.Where(c => !c.IsSpecial && Tasks.Scanner.Parser.Parser.MaxNumberFromRange(c.Range) <= chapterNumber);
await MarkChaptersAsRead(user, volume.SeriesId, chapters);
}
}

14
API/Services/ReadingItemService.cs
View file

@ -40,12 +40,12 @@ public class ReadingItemService : IReadingItemService
/// <returns></returns>
public ComicInfo? GetComicInfo(string filePath)
{
if (Parser.Parser.IsEpub(filePath))
if (Tasks.Scanner.Parser.Parser.IsEpub(filePath))
{
return _bookService.GetComicInfo(filePath);
}
if (Parser.Parser.IsComicInfoExtension(filePath))
if (Tasks.Scanner.Parser.Parser.IsComicInfoExtension(filePath))
{
return _archiveService.GetComicInfo(filePath);
}
@ -69,7 +69,7 @@ public class ReadingItemService : IReadingItemService
// This catches when original library type is Manga/Comic and when parsing with non
if (Parser.Parser.IsEpub(path) && Parser.Parser.ParseVolume(info.Series) != Parser.Parser.DefaultVolume) // Shouldn't this be info.Volume != DefaultVolume?
if (Tasks.Scanner.Parser.Parser.IsEpub(path) && Tasks.Scanner.Parser.Parser.ParseVolume(info.Series) != Tasks.Scanner.Parser.Parser.DefaultVolume) // Shouldn't this be info.Volume != DefaultVolume?
{
info = _defaultParser.Parse(path, rootPath, LibraryType.Book);
var info2 = Parse(path, rootPath, type);
@ -98,11 +98,11 @@ public class ReadingItemService : IReadingItemService
info.SeriesSort = info.ComicInfo.TitleSort.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Format) && Parser.Parser.HasComicInfoSpecial(info.ComicInfo.Format))
if (!string.IsNullOrEmpty(info.ComicInfo.Format) && Tasks.Scanner.Parser.Parser.HasComicInfoSpecial(info.ComicInfo.Format))
{
info.IsSpecial = true;
info.Chapters = Parser.Parser.DefaultChapter;
info.Volumes = Parser.Parser.DefaultVolume;
info.Chapters = Tasks.Scanner.Parser.Parser.DefaultChapter;
info.Volumes = Tasks.Scanner.Parser.Parser.DefaultVolume;
}
if (!string.IsNullOrEmpty(info.ComicInfo.SeriesSort))
@ -200,6 +200,6 @@ public class ReadingItemService : IReadingItemService
/// <returns></returns>
public ParserInfo Parse(string path, string rootPath, LibraryType type)
{
return Parser.Parser.IsEpub(path) ? _bookService.ParseInfo(path) : _defaultParser.Parse(path, rootPath, type);
return Tasks.Scanner.Parser.Parser.IsEpub(path) ? _bookService.ParseInfo(path) : _defaultParser.Parse(path, rootPath, type);
}
}

2
API/Services/ReadingListService.cs
View file

@ -166,7 +166,7 @@ public class ReadingListService : IReadingListService
var existingChapterExists = readingList.Items.Select(rli => rli.ChapterId).ToHashSet();
var chaptersForSeries = (await _unitOfWork.ChapterRepository.GetChaptersByIdsAsync(chapterIds))
.OrderBy(c => Parser.Parser.MinNumberFromRange(c.Volume.Name))
.OrderBy(c => Tasks.Scanner.Parser.Parser.MinNumberFromRange(c.Volume.Name))
.ThenBy(x => double.Parse(x.Number), _chapterSortComparerForInChapterSorting);
var index = lastOrder + 1;

12
API/Services/SeriesService.cs
View file

@ -254,7 +254,7 @@ public class SeriesService : ISeriesService
// At this point, all tags that aren't in dto have been removed.
foreach (var tagTitle in tags.Select(t => t.Title))
{
var normalizedTitle = Parser.Parser.Normalize(tagTitle);
var normalizedTitle = Tasks.Scanner.Parser.Parser.Normalize(tagTitle);
var existingTag = allTags.SingleOrDefault(t => t.NormalizedTitle == normalizedTitle);
if (existingTag != null)
{
@ -295,7 +295,7 @@ public class SeriesService : ISeriesService
// At this point, all tags that aren't in dto have been removed.
foreach (var tagTitle in tags.Select(t => t.Title))
{
var normalizedTitle = Parser.Parser.Normalize(tagTitle);
var normalizedTitle = Tasks.Scanner.Parser.Parser.Normalize(tagTitle);
var existingTag = allTags.SingleOrDefault(t => t.NormalizedTitle.Equals(normalizedTitle));
if (existingTag != null)
{
@ -465,7 +465,7 @@ public class SeriesService : ISeriesService
var libraryType = await _unitOfWork.LibraryRepository.GetLibraryTypeAsync(series.LibraryId);
var volumes = (await _unitOfWork.VolumeRepository.GetVolumesDtoAsync(seriesId, userId))
.OrderBy(v => Parser.Parser.MinNumberFromRange(v.Name))
.OrderBy(v => Tasks.Scanner.Parser.Parser.MinNumberFromRange(v.Name))
.ToList();
// For books, the Name of the Volume is remapped to the actual name of the book, rather than Volume number.
@ -542,7 +542,7 @@ public class SeriesService : ISeriesService
/// <returns></returns>
private static bool ShouldIncludeChapter(ChapterDto chapter)
{
return !chapter.IsSpecial && !chapter.Number.Equals(Parser.Parser.DefaultChapter);
return !chapter.IsSpecial && !chapter.Number.Equals(Tasks.Scanner.Parser.Parser.DefaultChapter);
}
public static void RenameVolumeName(ChapterDto firstChapter, VolumeDto volume, LibraryType libraryType)
@ -551,7 +551,7 @@ public class SeriesService : ISeriesService
{
if (string.IsNullOrEmpty(firstChapter.TitleName))
{
if (firstChapter.Range.Equals(Parser.Parser.DefaultVolume)) return;
if (firstChapter.Range.Equals(Tasks.Scanner.Parser.Parser.DefaultVolume)) return;
var title = Path.GetFileNameWithoutExtension(firstChapter.Range);
if (string.IsNullOrEmpty(title)) return;
volume.Name += $" - {title}";
@ -572,7 +572,7 @@ public class SeriesService : ISeriesService
{
if (isSpecial)
{
return Parser.Parser.CleanSpecialTitle(chapterTitle);
return Tasks.Scanner.Parser.Parser.CleanSpecialTitle(chapterTitle);
}
var hashSpot = withHash ? "#" : string.Empty;

2
API/Services/TaskScheduler.cs
View file

@ -168,7 +168,7 @@ public class TaskScheduler : ITaskScheduler
public void ScanFolder(string folderPath)
{
_scannerService.ScanFolder(Parser.Parser.NormalizePath(folderPath));
_scannerService.ScanFolder(Tasks.Scanner.Parser.Parser.NormalizePath(folderPath));
}
#endregion

62
API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file

@ -36,15 +36,15 @@ public class DefaultParser : IDefaultParser
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
ParserInfo ret;
if (Parser.IsEpub(filePath))
if (Services.Tasks.Scanner.Parser.Parser.IsEpub(filePath))
{
ret = new ParserInfo()
{
Chapters = Parser.ParseChapter(fileName) ?? Parser.ParseComicChapter(fileName),
Series = Parser.ParseSeries(fileName) ?? Parser.ParseComicSeries(fileName),
Volumes = Parser.ParseVolume(fileName) ?? Parser.ParseComicVolume(fileName),
Chapters = Services.Tasks.Scanner.Parser.Parser.ParseChapter(fileName) ?? Services.Tasks.Scanner.Parser.Parser.ParseComicChapter(fileName),
Series = Services.Tasks.Scanner.Parser.Parser.ParseSeries(fileName) ?? Services.Tasks.Scanner.Parser.Parser.ParseComicSeries(fileName),
Volumes = Services.Tasks.Scanner.Parser.Parser.ParseVolume(fileName) ?? Services.Tasks.Scanner.Parser.Parser.ParseComicVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Format = Services.Tasks.Scanner.Parser.Parser.ParseFormat(filePath),
FullFilePath = filePath
};
}
@ -52,65 +52,65 @@ public class DefaultParser : IDefaultParser
{
ret = new ParserInfo()
{
Chapters = type == LibraryType.Comic ? Parser.ParseComicChapter(fileName) : Parser.ParseChapter(fileName),
Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName),
Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName),
Chapters = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicChapter(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseChapter(fileName),
Series = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicSeries(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseSeries(fileName),
Volumes = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicVolume(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Format = Services.Tasks.Scanner.Parser.Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath
};
}
if (Parser.IsImage(filePath) && Parser.IsCoverImage(filePath)) return null;
if (Services.Tasks.Scanner.Parser.Parser.IsImage(filePath) && Services.Tasks.Scanner.Parser.Parser.IsCoverImage(filePath)) return null;
if (Parser.IsImage(filePath))
if (Services.Tasks.Scanner.Parser.Parser.IsImage(filePath))
{
// Reset Chapters, Volumes, and Series as images are not good to parse information out of. Better to use folders.
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
ret.Volumes = Services.Tasks.Scanner.Parser.Parser.DefaultVolume;
ret.Chapters = Services.Tasks.Scanner.Parser.Parser.DefaultChapter;
ret.Series = string.Empty;
}
if (ret.Series == string.Empty || Parser.IsImage(filePath))
if (ret.Series == string.Empty || Services.Tasks.Scanner.Parser.Parser.IsImage(filePath))
{
// Try to parse information out of each folder all the way to rootPath
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
var edition = Parser.ParseEdition(fileName);
var edition = Services.Tasks.Scanner.Parser.Parser.ParseEdition(fileName);
if (!string.IsNullOrEmpty(edition))
{
ret.Series = Parser.CleanTitle(ret.Series.Replace(edition, ""), type is LibraryType.Comic);
ret.Series = Services.Tasks.Scanner.Parser.Parser.CleanTitle(ret.Series.Replace(edition, ""), type is LibraryType.Comic);
ret.Edition = edition;
}
var isSpecial = type == LibraryType.Comic ? Parser.ParseComicSpecial(fileName) : Parser.ParseMangaSpecial(fileName);
var isSpecial = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicSpecial(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseMangaSpecial(fileName);
// We must ensure that we can only parse a special out. As some files will have v20 c171-180+Omake and that
// could cause a problem as Omake is a special term, but there is valid volume/chapter information.
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.DefaultVolume && !string.IsNullOrEmpty(isSpecial))
if (ret.Chapters == Services.Tasks.Scanner.Parser.Parser.DefaultChapter && ret.Volumes == Services.Tasks.Scanner.Parser.Parser.DefaultVolume && !string.IsNullOrEmpty(isSpecial))
{
ret.IsSpecial = true;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret); // NOTE: This can cause some complications, we should try to be a bit less aggressive to fallback to folder
}
// If we are a special with marker, we need to ensure we use the correct series name. we can do this by falling back to Folder name
if (Parser.HasSpecialMarker(fileName))
if (Services.Tasks.Scanner.Parser.Parser.HasSpecialMarker(fileName))
{
ret.IsSpecial = true;
ret.Chapters = Parser.DefaultChapter;
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Services.Tasks.Scanner.Parser.Parser.DefaultChapter;
ret.Volumes = Services.Tasks.Scanner.Parser.Parser.DefaultVolume;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
if (string.IsNullOrEmpty(ret.Series))
{
ret.Series = Parser.CleanTitle(fileName, type is LibraryType.Comic);
ret.Series = Services.Tasks.Scanner.Parser.Parser.CleanTitle(fileName, type is LibraryType.Comic);
}
// Pdfs may have .pdf in the series name, remove that
if (Parser.IsPdf(filePath) && ret.Series.ToLower().EndsWith(".pdf"))
if (Services.Tasks.Scanner.Parser.Parser.IsPdf(filePath) && ret.Series.ToLower().EndsWith(".pdf"))
{
ret.Series = ret.Series.Substring(0, ret.Series.Length - ".pdf".Length);
}
@ -131,18 +131,18 @@ public class DefaultParser : IDefaultParser
for (var i = 0; i < fallbackFolders.Count; i++)
{
var folder = fallbackFolders[i];
if (!string.IsNullOrEmpty(Parser.ParseMangaSpecial(folder))) continue;
if (!string.IsNullOrEmpty(Services.Tasks.Scanner.Parser.Parser.ParseMangaSpecial(folder))) continue;
var parsedVolume = type is LibraryType.Manga ? Parser.ParseVolume(folder) : Parser.ParseComicVolume(folder);
var parsedChapter = type is LibraryType.Manga ? Parser.ParseChapter(folder) : Parser.ParseComicChapter(folder);
var parsedVolume = type is LibraryType.Manga ? Services.Tasks.Scanner.Parser.Parser.ParseVolume(folder) : Services.Tasks.Scanner.Parser.Parser.ParseComicVolume(folder);
var parsedChapter = type is LibraryType.Manga ? Services.Tasks.Scanner.Parser.Parser.ParseChapter(folder) : Services.Tasks.Scanner.Parser.Parser.ParseComicChapter(folder);
if (!parsedVolume.Equals(Parser.DefaultVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
if (!parsedVolume.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultVolume) || !parsedChapter.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultChapter))
{
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !parsedVolume.Equals(Parser.DefaultVolume))
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultVolume)) && !parsedVolume.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultVolume))
{
ret.Volumes = parsedVolume;
}
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !parsedChapter.Equals(Parser.DefaultChapter))
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultChapter)) && !parsedChapter.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}
@ -151,11 +151,11 @@ public class DefaultParser : IDefaultParser
// Generally users group in series folders. Let's try to parse series from the top folder
if (!folder.Equals(ret.Series) && i == fallbackFolders.Count - 1)
{
var series = Parser.ParseSeries(folder);
var series = Services.Tasks.Scanner.Parser.Parser.ParseSeries(folder);
if (string.IsNullOrEmpty(series))
{
ret.Series = Parser.CleanTitle(folder, type is LibraryType.Comic);
ret.Series = Services.Tasks.Scanner.Parser.Parser.CleanTitle(folder, type is LibraryType.Comic);
break;
}

2
API/Services/Tasks/Scanner/Parser/Parser.cs
View file

@ -5,7 +5,7 @@ using System.Linq;
using System.Text.RegularExpressions;
using API.Entities.Enums;
namespace API.Parser
namespace API.Services.Tasks.Scanner.Parser
{
public static class Parser
{

1
API/Services/Tasks/Scanner/Parser/ParserInfo.cs
View file

@ -1,5 +1,6 @@
using API.Data.Metadata;
using API.Entities.Enums;
using API.Services.Tasks.Scanner.Parser;
namespace API.Parser
{

10
API/Services/Tasks/ScannerService.cs
View file

@ -111,11 +111,11 @@ public class ScannerService : IScannerService
var libraries = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).ToList();
var libraryFolders = libraries.SelectMany(l => l.Folders);
var libraryFolder = libraryFolders.Select(Parser.Parser.NormalizePath).SingleOrDefault(f => f.Contains(parentDirectory));
var libraryFolder = libraryFolders.Select(Scanner.Parser.Parser.NormalizePath).SingleOrDefault(f => f.Contains(parentDirectory));
if (string.IsNullOrEmpty(libraryFolder)) return;
var library = libraries.FirstOrDefault(l => l.Folders.Select(Parser.Parser.NormalizePath).Contains(libraryFolder));
var library = libraries.FirstOrDefault(l => l.Folders.Select(Scanner.Parser.Parser.NormalizePath).Contains(libraryFolder));
if (library != null)
{
BackgroundJob.Enqueue(() => ScanLibrary(library.Id, false));
@ -188,7 +188,7 @@ public class ScannerService : IScannerService
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles.First().Series,
NormalizedName = Parser.Parser.Normalize(parsedFiles.First().Series),
NormalizedName = Scanner.Parser.Parser.Normalize(parsedFiles.First().Series),
Format = parsedFiles.First().Format
};
@ -457,7 +457,7 @@ public class ScannerService : IScannerService
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles.First().Series,
NormalizedName = Parser.Parser.Normalize(parsedFiles.First().Series),
NormalizedName = Scanner.Parser.Parser.Normalize(parsedFiles.First().Series),
Format = parsedFiles.First().Format
};
@ -466,7 +466,7 @@ public class ScannerService : IScannerService
seenSeries.AddRange(parsedFiles.Select(pf => new ParsedSeries()
{
Name = pf.Series,
NormalizedName = Parser.Parser.Normalize(pf.Series),
NormalizedName = Scanner.Parser.Parser.Normalize(pf.Series),
Format = pf.Format
}));
return;

8
API/Services/Tasks/SiteThemeService.cs
View file

@ -55,8 +55,8 @@ public class ThemeService : IThemeService
_directoryService.ExistOrCreate(_directoryService.SiteThemeDirectory);
var reservedNames = Seed.DefaultThemes.Select(t => t.NormalizedName).ToList();
var themeFiles = _directoryService
.GetFilesWithExtension(Parser.Parser.NormalizePath(_directoryService.SiteThemeDirectory), @"\.css")
.Where(name => !reservedNames.Contains(Parser.Parser.Normalize(name))).ToList();
.GetFilesWithExtension(Scanner.Parser.Parser.NormalizePath(_directoryService.SiteThemeDirectory), @"\.css")
.Where(name => !reservedNames.Contains(Scanner.Parser.Parser.Normalize(name))).ToList();
var allThemes = (await _unitOfWork.SiteThemeRepository.GetThemes()).ToList();
@ -64,7 +64,7 @@ public class ThemeService : IThemeService
var userThemes = allThemes.Where(t => t.Provider == ThemeProvider.User).ToList();
foreach (var userTheme in userThemes)
{
var filepath = Parser.Parser.NormalizePath(
var filepath = Scanner.Parser.Parser.NormalizePath(
_directoryService.FileSystem.Path.Join(_directoryService.SiteThemeDirectory, userTheme.FileName));
if (_directoryService.FileSystem.File.Exists(filepath)) continue;
@ -78,7 +78,7 @@ public class ThemeService : IThemeService
foreach (var themeFile in themeFiles)
{
var themeName =
Parser.Parser.Normalize(_directoryService.FileSystem.Path.GetFileNameWithoutExtension(themeFile));
Scanner.Parser.Parser.Normalize(_directoryService.FileSystem.Path.GetFileNameWithoutExtension(themeFile));
if (allThemeNames.Contains(themeName)) continue;
_unitOfWork.SiteThemeRepository.Add(new SiteTheme()