AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scan Loop Last Write Time Change (#1492)

Browse source 
* Refactored invite user flow to separate error handling on create user flow and email flow. This should help users that have unique situations.

* Switch to using files to check LastWriteTime. Debug code in for Robbie to test on rclone

* Updated Parser namespace. Changed the LastWriteTime to check all files and folders.
This commit is contained in:
Joseph Milazzo 2022-08-30 10:29:09 -05:00 • committed by GitHub
parent 521b2adc14
commit 85790dd71c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
59 changed files with 283 additions and 264 deletions
Download patch file Download diff file Expand all files Collapse all files

62
API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file

@ -36,15 +36,15 @@ public class DefaultParser : IDefaultParser
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
ParserInfo ret;
if (Parser.IsEpub(filePath))
if (Services.Tasks.Scanner.Parser.Parser.IsEpub(filePath))
{
ret = new ParserInfo()
{
Chapters = Parser.ParseChapter(fileName) ?? Parser.ParseComicChapter(fileName),
Series = Parser.ParseSeries(fileName) ?? Parser.ParseComicSeries(fileName),
Volumes = Parser.ParseVolume(fileName) ?? Parser.ParseComicVolume(fileName),
Chapters = Services.Tasks.Scanner.Parser.Parser.ParseChapter(fileName) ?? Services.Tasks.Scanner.Parser.Parser.ParseComicChapter(fileName),
Series = Services.Tasks.Scanner.Parser.Parser.ParseSeries(fileName) ?? Services.Tasks.Scanner.Parser.Parser.ParseComicSeries(fileName),
Volumes = Services.Tasks.Scanner.Parser.Parser.ParseVolume(fileName) ?? Services.Tasks.Scanner.Parser.Parser.ParseComicVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Format = Services.Tasks.Scanner.Parser.Parser.ParseFormat(filePath),
FullFilePath = filePath
};
}
@ -52,65 +52,65 @@ public class DefaultParser : IDefaultParser
{
ret = new ParserInfo()
{
Chapters = type == LibraryType.Comic ? Parser.ParseComicChapter(fileName) : Parser.ParseChapter(fileName),
Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName),
Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName),
Chapters = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicChapter(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseChapter(fileName),
Series = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicSeries(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseSeries(fileName),
Volumes = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicVolume(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Format = Services.Tasks.Scanner.Parser.Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath
};
}
if (Parser.IsImage(filePath) && Parser.IsCoverImage(filePath)) return null;
if (Services.Tasks.Scanner.Parser.Parser.IsImage(filePath) && Services.Tasks.Scanner.Parser.Parser.IsCoverImage(filePath)) return null;
if (Parser.IsImage(filePath))
if (Services.Tasks.Scanner.Parser.Parser.IsImage(filePath))
{
// Reset Chapters, Volumes, and Series as images are not good to parse information out of. Better to use folders.
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
ret.Volumes = Services.Tasks.Scanner.Parser.Parser.DefaultVolume;
ret.Chapters = Services.Tasks.Scanner.Parser.Parser.DefaultChapter;
ret.Series = string.Empty;
}
if (ret.Series == string.Empty || Parser.IsImage(filePath))
if (ret.Series == string.Empty || Services.Tasks.Scanner.Parser.Parser.IsImage(filePath))
{
// Try to parse information out of each folder all the way to rootPath
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
var edition = Parser.ParseEdition(fileName);
var edition = Services.Tasks.Scanner.Parser.Parser.ParseEdition(fileName);
if (!string.IsNullOrEmpty(edition))
{
ret.Series = Parser.CleanTitle(ret.Series.Replace(edition, ""), type is LibraryType.Comic);
ret.Series = Services.Tasks.Scanner.Parser.Parser.CleanTitle(ret.Series.Replace(edition, ""), type is LibraryType.Comic);
ret.Edition = edition;
}
var isSpecial = type == LibraryType.Comic ? Parser.ParseComicSpecial(fileName) : Parser.ParseMangaSpecial(fileName);
var isSpecial = type == LibraryType.Comic ? Services.Tasks.Scanner.Parser.Parser.ParseComicSpecial(fileName) : Services.Tasks.Scanner.Parser.Parser.ParseMangaSpecial(fileName);
// We must ensure that we can only parse a special out. As some files will have v20 c171-180+Omake and that
// could cause a problem as Omake is a special term, but there is valid volume/chapter information.
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.DefaultVolume && !string.IsNullOrEmpty(isSpecial))
if (ret.Chapters == Services.Tasks.Scanner.Parser.Parser.DefaultChapter && ret.Volumes == Services.Tasks.Scanner.Parser.Parser.DefaultVolume && !string.IsNullOrEmpty(isSpecial))
{
ret.IsSpecial = true;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret); // NOTE: This can cause some complications, we should try to be a bit less aggressive to fallback to folder
}
// If we are a special with marker, we need to ensure we use the correct series name. we can do this by falling back to Folder name
if (Parser.HasSpecialMarker(fileName))
if (Services.Tasks.Scanner.Parser.Parser.HasSpecialMarker(fileName))
{
ret.IsSpecial = true;
ret.Chapters = Parser.DefaultChapter;
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Services.Tasks.Scanner.Parser.Parser.DefaultChapter;
ret.Volumes = Services.Tasks.Scanner.Parser.Parser.DefaultVolume;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
if (string.IsNullOrEmpty(ret.Series))
{
ret.Series = Parser.CleanTitle(fileName, type is LibraryType.Comic);
ret.Series = Services.Tasks.Scanner.Parser.Parser.CleanTitle(fileName, type is LibraryType.Comic);
}
// Pdfs may have .pdf in the series name, remove that
if (Parser.IsPdf(filePath) && ret.Series.ToLower().EndsWith(".pdf"))
if (Services.Tasks.Scanner.Parser.Parser.IsPdf(filePath) && ret.Series.ToLower().EndsWith(".pdf"))
{
ret.Series = ret.Series.Substring(0, ret.Series.Length - ".pdf".Length);
}
@ -131,18 +131,18 @@ public class DefaultParser : IDefaultParser
for (var i = 0; i < fallbackFolders.Count; i++)
{
var folder = fallbackFolders[i];
if (!string.IsNullOrEmpty(Parser.ParseMangaSpecial(folder))) continue;
if (!string.IsNullOrEmpty(Services.Tasks.Scanner.Parser.Parser.ParseMangaSpecial(folder))) continue;
var parsedVolume = type is LibraryType.Manga ? Parser.ParseVolume(folder) : Parser.ParseComicVolume(folder);
var parsedChapter = type is LibraryType.Manga ? Parser.ParseChapter(folder) : Parser.ParseComicChapter(folder);
var parsedVolume = type is LibraryType.Manga ? Services.Tasks.Scanner.Parser.Parser.ParseVolume(folder) : Services.Tasks.Scanner.Parser.Parser.ParseComicVolume(folder);
var parsedChapter = type is LibraryType.Manga ? Services.Tasks.Scanner.Parser.Parser.ParseChapter(folder) : Services.Tasks.Scanner.Parser.Parser.ParseComicChapter(folder);
if (!parsedVolume.Equals(Parser.DefaultVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
if (!parsedVolume.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultVolume) || !parsedChapter.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultChapter))
{
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !parsedVolume.Equals(Parser.DefaultVolume))
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultVolume)) && !parsedVolume.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultVolume))
{
ret.Volumes = parsedVolume;
}
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !parsedChapter.Equals(Parser.DefaultChapter))
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultChapter)) && !parsedChapter.Equals(Services.Tasks.Scanner.Parser.Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}
@ -151,11 +151,11 @@ public class DefaultParser : IDefaultParser
// Generally users group in series folders. Let's try to parse series from the top folder
if (!folder.Equals(ret.Series) && i == fallbackFolders.Count - 1)
{
var series = Parser.ParseSeries(folder);
var series = Services.Tasks.Scanner.Parser.Parser.ParseSeries(folder);
if (string.IsNullOrEmpty(series))
{
ret.Series = Parser.CleanTitle(folder, type is LibraryType.Comic);
ret.Series = Services.Tasks.Scanner.Parser.Parser.CleanTitle(folder, type is LibraryType.Comic);
break;
}

2
API/Services/Tasks/Scanner/Parser/Parser.cs
View file

@ -5,7 +5,7 @@ using System.Linq;
using System.Text.RegularExpressions;
using API.Entities.Enums;
namespace API.Parser
namespace API.Services.Tasks.Scanner.Parser
{
public static class Parser
{

1
API/Services/Tasks/Scanner/Parser/ParserInfo.cs
View file

@ -1,5 +1,6 @@
using API.Data.Metadata;
using API.Entities.Enums;
using API.Services.Tasks.Scanner.Parser;
namespace API.Parser
{

10
API/Services/Tasks/ScannerService.cs
View file

@ -111,11 +111,11 @@ public class ScannerService : IScannerService
var libraries = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).ToList();
var libraryFolders = libraries.SelectMany(l => l.Folders);
var libraryFolder = libraryFolders.Select(Parser.Parser.NormalizePath).SingleOrDefault(f => f.Contains(parentDirectory));
var libraryFolder = libraryFolders.Select(Scanner.Parser.Parser.NormalizePath).SingleOrDefault(f => f.Contains(parentDirectory));
if (string.IsNullOrEmpty(libraryFolder)) return;
var library = libraries.FirstOrDefault(l => l.Folders.Select(Parser.Parser.NormalizePath).Contains(libraryFolder));
var library = libraries.FirstOrDefault(l => l.Folders.Select(Scanner.Parser.Parser.NormalizePath).Contains(libraryFolder));
if (library != null)
{
BackgroundJob.Enqueue(() => ScanLibrary(library.Id, false));
@ -188,7 +188,7 @@ public class ScannerService : IScannerService
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles.First().Series,
NormalizedName = Parser.Parser.Normalize(parsedFiles.First().Series),
NormalizedName = Scanner.Parser.Parser.Normalize(parsedFiles.First().Series),
Format = parsedFiles.First().Format
};
@ -457,7 +457,7 @@ public class ScannerService : IScannerService
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles.First().Series,
NormalizedName = Parser.Parser.Normalize(parsedFiles.First().Series),
NormalizedName = Scanner.Parser.Parser.Normalize(parsedFiles.First().Series),
Format = parsedFiles.First().Format
};
@ -466,7 +466,7 @@ public class ScannerService : IScannerService
seenSeries.AddRange(parsedFiles.Select(pf => new ParsedSeries()
{
Name = pf.Series,
NormalizedName = Parser.Parser.Normalize(pf.Series),
NormalizedName = Scanner.Parser.Parser.Normalize(pf.Series),
Format = pf.Format
}));
return;

8
API/Services/Tasks/SiteThemeService.cs
View file

@ -55,8 +55,8 @@ public class ThemeService : IThemeService
_directoryService.ExistOrCreate(_directoryService.SiteThemeDirectory);
var reservedNames = Seed.DefaultThemes.Select(t => t.NormalizedName).ToList();
var themeFiles = _directoryService
.GetFilesWithExtension(Parser.Parser.NormalizePath(_directoryService.SiteThemeDirectory), @"\.css")
.Where(name => !reservedNames.Contains(Parser.Parser.Normalize(name))).ToList();
.GetFilesWithExtension(Scanner.Parser.Parser.NormalizePath(_directoryService.SiteThemeDirectory), @"\.css")
.Where(name => !reservedNames.Contains(Scanner.Parser.Parser.Normalize(name))).ToList();
var allThemes = (await _unitOfWork.SiteThemeRepository.GetThemes()).ToList();
@ -64,7 +64,7 @@ public class ThemeService : IThemeService
var userThemes = allThemes.Where(t => t.Provider == ThemeProvider.User).ToList();
foreach (var userTheme in userThemes)
{
var filepath = Parser.Parser.NormalizePath(
var filepath = Scanner.Parser.Parser.NormalizePath(
_directoryService.FileSystem.Path.Join(_directoryService.SiteThemeDirectory, userTheme.FileName));
if (_directoryService.FileSystem.File.Exists(filepath)) continue;
@ -78,7 +78,7 @@ public class ThemeService : IThemeService
foreach (var themeFile in themeFiles)
{
var themeName =
Parser.Parser.Normalize(_directoryService.FileSystem.Path.GetFileNameWithoutExtension(themeFile));
Scanner.Parser.Parser.Normalize(_directoryService.FileSystem.Path.GetFileNameWithoutExtension(themeFile));
if (allThemeNames.Contains(themeName)) continue;
_unitOfWork.SiteThemeRepository.Add(new SiteTheme()