Logging Enhancements (#1521)
* Recreated Kavita Logging with Serilog instead of Default. This needs to be move out of the appsettings now, to allow auto updater to patch. * Refactored the code to be completely configured via Code rather than appsettings.json. This is a required step for Auto Updating. * Added in the ability to send logs directly to the UI only for users on the log route. Stopping implementation as Alerts page will handle the rest of the implementation. * Fixed up the backup service to not rely on Config from appsettings.json * Tweaked the Logging levels available * Moved everything over to File-scoped namespaces * Moved everything over to File-scoped namespaces * Code cleanup, removed an old migration and changed so debug logging doesn't print sensitive db data * Removed dead code
This commit is contained in:
Joseph Milazzo
GitHub
parent
9f715cc35f
commit
d1a14f7e68
212 changed files with 16599 additions and 16834 deletions
|
|
@ -9,353 +9,352 @@ using API.Parser;
|
|||
using API.SignalR;
|
||||
using Microsoft.Extensions.Logging;
|
||||
|
||||
namespace API.Services.Tasks.Scanner
|
||||
namespace API.Services.Tasks.Scanner;
|
||||
|
||||
public class ParsedSeries
|
||||
{
|
||||
public class ParsedSeries
|
||||
{
|
||||
/// <summary>
|
||||
/// Name of the Series
|
||||
/// </summary>
|
||||
public string Name { get; init; }
|
||||
/// <summary>
|
||||
/// Normalized Name of the Series
|
||||
/// </summary>
|
||||
public string NormalizedName { get; init; }
|
||||
/// <summary>
|
||||
/// Format of the Series
|
||||
/// </summary>
|
||||
public MangaFormat Format { get; init; }
|
||||
}
|
||||
/// <summary>
|
||||
/// Name of the Series
|
||||
/// </summary>
|
||||
public string Name { get; init; }
|
||||
/// <summary>
|
||||
/// Normalized Name of the Series
|
||||
/// </summary>
|
||||
public string NormalizedName { get; init; }
|
||||
/// <summary>
|
||||
/// Format of the Series
|
||||
/// </summary>
|
||||
public MangaFormat Format { get; init; }
|
||||
}
|
||||
|
||||
public enum Modified
|
||||
{
|
||||
Modified = 1,
|
||||
NotModified = 2
|
||||
}
|
||||
public enum Modified
|
||||
{
|
||||
Modified = 1,
|
||||
NotModified = 2
|
||||
}
|
||||
|
||||
public class SeriesModified
|
||||
public class SeriesModified
|
||||
{
|
||||
public string FolderPath { get; set; }
|
||||
public string SeriesName { get; set; }
|
||||
public DateTime LastScanned { get; set; }
|
||||
public MangaFormat Format { get; set; }
|
||||
}
|
||||
|
||||
|
||||
public class ParseScannedFiles
|
||||
{
|
||||
private readonly ILogger _logger;
|
||||
private readonly IDirectoryService _directoryService;
|
||||
private readonly IReadingItemService _readingItemService;
|
||||
private readonly IEventHub _eventHub;
|
||||
|
||||
/// <summary>
|
||||
/// An instance of a pipeline for processing files and returning a Map of Series -> ParserInfos.
|
||||
/// Each instance is separate from other threads, allowing for no cross over.
|
||||
/// </summary>
|
||||
/// <param name="logger">Logger of the parent class that invokes this</param>
|
||||
/// <param name="directoryService">Directory Service</param>
|
||||
/// <param name="readingItemService">ReadingItemService Service for extracting information on a number of formats</param>
|
||||
/// <param name="eventHub">For firing off SignalR events</param>
|
||||
public ParseScannedFiles(ILogger logger, IDirectoryService directoryService,
|
||||
IReadingItemService readingItemService, IEventHub eventHub)
|
||||
{
|
||||
public string FolderPath { get; set; }
|
||||
public string SeriesName { get; set; }
|
||||
public DateTime LastScanned { get; set; }
|
||||
public MangaFormat Format { get; set; }
|
||||
_logger = logger;
|
||||
_directoryService = directoryService;
|
||||
_readingItemService = readingItemService;
|
||||
_eventHub = eventHub;
|
||||
}
|
||||
|
||||
|
||||
public class ParseScannedFiles
|
||||
/// <summary>
|
||||
/// This will Scan all files in a folder path. For each folder within the folderPath, FolderAction will be invoked for all files contained
|
||||
/// </summary>
|
||||
/// <param name="scanDirectoryByDirectory">Scan directory by directory and for each, call folderAction</param>
|
||||
/// <param name="folderPath">A library folder or series folder</param>
|
||||
/// <param name="folderAction">A callback async Task to be called once all files for each folder path are found</param>
|
||||
/// <param name="forceCheck">If we should bypass any folder last write time checks on the scan and force I/O</param>
|
||||
public async Task ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
|
||||
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<IList<string>, string,Task> folderAction, bool forceCheck = false)
|
||||
{
|
||||
private readonly ILogger _logger;
|
||||
private readonly IDirectoryService _directoryService;
|
||||
private readonly IReadingItemService _readingItemService;
|
||||
private readonly IEventHub _eventHub;
|
||||
|
||||
/// <summary>
|
||||
/// An instance of a pipeline for processing files and returning a Map of Series -> ParserInfos.
|
||||
/// Each instance is separate from other threads, allowing for no cross over.
|
||||
/// </summary>
|
||||
/// <param name="logger">Logger of the parent class that invokes this</param>
|
||||
/// <param name="directoryService">Directory Service</param>
|
||||
/// <param name="readingItemService">ReadingItemService Service for extracting information on a number of formats</param>
|
||||
/// <param name="eventHub">For firing off SignalR events</param>
|
||||
public ParseScannedFiles(ILogger logger, IDirectoryService directoryService,
|
||||
IReadingItemService readingItemService, IEventHub eventHub)
|
||||
string normalizedPath;
|
||||
if (scanDirectoryByDirectory)
|
||||
{
|
||||
_logger = logger;
|
||||
_directoryService = directoryService;
|
||||
_readingItemService = readingItemService;
|
||||
_eventHub = eventHub;
|
||||
// This is used in library scan, so we should check first for a ignore file and use that here as well
|
||||
var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(folderPath, DirectoryService.KavitaIgnoreFile);
|
||||
var directories = _directoryService.GetDirectories(folderPath, _directoryService.CreateMatcherFromFile(potentialIgnoreFile)).ToList();
|
||||
|
||||
foreach (var directory in directories)
|
||||
{
|
||||
normalizedPath = Parser.Parser.NormalizePath(directory);
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
|
||||
{
|
||||
await folderAction(new List<string>(), directory);
|
||||
}
|
||||
else
|
||||
{
|
||||
// For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication
|
||||
await folderAction(_directoryService.ScanFiles(directory), directory);
|
||||
}
|
||||
}
|
||||
|
||||
return;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// This will Scan all files in a folder path. For each folder within the folderPath, FolderAction will be invoked for all files contained
|
||||
/// </summary>
|
||||
/// <param name="scanDirectoryByDirectory">Scan directory by directory and for each, call folderAction</param>
|
||||
/// <param name="folderPath">A library folder or series folder</param>
|
||||
/// <param name="folderAction">A callback async Task to be called once all files for each folder path are found</param>
|
||||
/// <param name="forceCheck">If we should bypass any folder last write time checks on the scan and force I/O</param>
|
||||
public async Task ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
|
||||
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<IList<string>, string,Task> folderAction, bool forceCheck = false)
|
||||
normalizedPath = Parser.Parser.NormalizePath(folderPath);
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
|
||||
{
|
||||
string normalizedPath;
|
||||
if (scanDirectoryByDirectory)
|
||||
{
|
||||
// This is used in library scan, so we should check first for a ignore file and use that here as well
|
||||
var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(folderPath, DirectoryService.KavitaIgnoreFile);
|
||||
var directories = _directoryService.GetDirectories(folderPath, _directoryService.CreateMatcherFromFile(potentialIgnoreFile)).ToList();
|
||||
await folderAction(new List<string>(), folderPath);
|
||||
return;
|
||||
}
|
||||
await folderAction(_directoryService.ScanFiles(folderPath), folderPath);
|
||||
}
|
||||
|
||||
foreach (var directory in directories)
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to either add a new instance of a show mapping to the _scannedSeries bag or adds to an existing.
|
||||
/// This will check if the name matches an existing series name (multiple fields) <see cref="MergeName"/>
|
||||
/// </summary>
|
||||
/// <param name="scannedSeries">A localized list of a series' parsed infos</param>
|
||||
/// <param name="info"></param>
|
||||
private void TrackSeries(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, ParserInfo info)
|
||||
{
|
||||
if (info.Series == string.Empty) return;
|
||||
|
||||
// Check if normalized info.Series already exists and if so, update info to use that name instead
|
||||
info.Series = MergeName(scannedSeries, info);
|
||||
|
||||
var normalizedSeries = Parser.Parser.Normalize(info.Series);
|
||||
var normalizedSortSeries = Parser.Parser.Normalize(info.SeriesSort);
|
||||
var normalizedLocalizedSeries = Parser.Parser.Normalize(info.LocalizedSeries);
|
||||
|
||||
try
|
||||
{
|
||||
var existingKey = scannedSeries.Keys.SingleOrDefault(ps =>
|
||||
ps.Format == info.Format && (ps.NormalizedName.Equals(normalizedSeries)
|
||||
|| ps.NormalizedName.Equals(normalizedLocalizedSeries)
|
||||
|| ps.NormalizedName.Equals(normalizedSortSeries)));
|
||||
existingKey ??= new ParsedSeries()
|
||||
{
|
||||
Format = info.Format,
|
||||
Name = info.Series,
|
||||
NormalizedName = normalizedSeries
|
||||
};
|
||||
|
||||
scannedSeries.AddOrUpdate(existingKey, new List<ParserInfo>() {info}, (_, oldValue) =>
|
||||
{
|
||||
oldValue ??= new List<ParserInfo>();
|
||||
if (!oldValue.Contains(info))
|
||||
{
|
||||
normalizedPath = Parser.Parser.NormalizePath(directory);
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
|
||||
{
|
||||
await folderAction(new List<string>(), directory);
|
||||
}
|
||||
else
|
||||
{
|
||||
// For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication
|
||||
await folderAction(_directoryService.ScanFiles(directory), directory);
|
||||
}
|
||||
oldValue.Add(info);
|
||||
}
|
||||
|
||||
return;
|
||||
return oldValue;
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogCritical(ex, "{SeriesName} matches against multiple series in the parsed series. This indicates a critical kavita issue. Key will be skipped", info.Series);
|
||||
foreach (var seriesKey in scannedSeries.Keys.Where(ps =>
|
||||
ps.Format == info.Format && (ps.NormalizedName.Equals(normalizedSeries)
|
||||
|| ps.NormalizedName.Equals(normalizedLocalizedSeries)
|
||||
|| ps.NormalizedName.Equals(normalizedSortSeries))))
|
||||
{
|
||||
_logger.LogCritical("Matches: {SeriesName} matches on {SeriesKey}", info.Series, seriesKey.Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Using a normalized name from the passed ParserInfo, this checks against all found series so far and if an existing one exists with
|
||||
/// same normalized name, it merges into the existing one. This is important as some manga may have a slight difference with punctuation or capitalization.
|
||||
/// </summary>
|
||||
/// <param name="info"></param>
|
||||
/// <returns>Series Name to group this info into</returns>
|
||||
private string MergeName(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, ParserInfo info)
|
||||
{
|
||||
var normalizedSeries = Parser.Parser.Normalize(info.Series);
|
||||
var normalizedLocalSeries = Parser.Parser.Normalize(info.LocalizedSeries);
|
||||
|
||||
try
|
||||
{
|
||||
var existingName =
|
||||
scannedSeries.SingleOrDefault(p =>
|
||||
(Parser.Parser.Normalize(p.Key.NormalizedName).Equals(normalizedSeries) ||
|
||||
Parser.Parser.Normalize(p.Key.NormalizedName).Equals(normalizedLocalSeries)) &&
|
||||
p.Key.Format == info.Format)
|
||||
.Key;
|
||||
|
||||
if (existingName != null && !string.IsNullOrEmpty(existingName.Name))
|
||||
{
|
||||
return existingName.Name;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogCritical(ex, "Multiple series detected for {SeriesName} ({File})! This is critical to fix! There should only be 1", info.Series, info.FullFilePath);
|
||||
var values = scannedSeries.Where(p =>
|
||||
(Parser.Parser.Normalize(p.Key.NormalizedName) == normalizedSeries ||
|
||||
Parser.Parser.Normalize(p.Key.NormalizedName) == normalizedLocalSeries) &&
|
||||
p.Key.Format == info.Format);
|
||||
foreach (var pair in values)
|
||||
{
|
||||
_logger.LogCritical("Duplicate Series in DB matches with {SeriesName}: {DuplicateName}", info.Series, pair.Key.Name);
|
||||
}
|
||||
|
||||
normalizedPath = Parser.Parser.NormalizePath(folderPath);
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
|
||||
{
|
||||
await folderAction(new List<string>(), folderPath);
|
||||
return;
|
||||
}
|
||||
await folderAction(_directoryService.ScanFiles(folderPath), folderPath);
|
||||
}
|
||||
|
||||
return info.Series;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Attempts to either add a new instance of a show mapping to the _scannedSeries bag or adds to an existing.
|
||||
/// This will check if the name matches an existing series name (multiple fields) <see cref="MergeName"/>
|
||||
/// </summary>
|
||||
/// <param name="scannedSeries">A localized list of a series' parsed infos</param>
|
||||
/// <param name="info"></param>
|
||||
private void TrackSeries(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, ParserInfo info)
|
||||
|
||||
/// <summary>
|
||||
/// This will process series by folder groups.
|
||||
/// </summary>
|
||||
/// <param name="libraryType"></param>
|
||||
/// <param name="folders"></param>
|
||||
/// <param name="libraryName"></param>
|
||||
/// <returns></returns>
|
||||
public async Task ScanLibrariesForSeries(LibraryType libraryType,
|
||||
IEnumerable<string> folders, string libraryName, bool isLibraryScan,
|
||||
IDictionary<string, IList<SeriesModified>> seriesPaths, Action<Tuple<bool, IList<ParserInfo>>> processSeriesInfos, bool forceCheck = false)
|
||||
{
|
||||
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", libraryName, ProgressEventType.Started));
|
||||
|
||||
foreach (var folderPath in folders)
|
||||
{
|
||||
if (info.Series == string.Empty) return;
|
||||
|
||||
// Check if normalized info.Series already exists and if so, update info to use that name instead
|
||||
info.Series = MergeName(scannedSeries, info);
|
||||
|
||||
var normalizedSeries = Parser.Parser.Normalize(info.Series);
|
||||
var normalizedSortSeries = Parser.Parser.Normalize(info.SeriesSort);
|
||||
var normalizedLocalizedSeries = Parser.Parser.Normalize(info.LocalizedSeries);
|
||||
|
||||
try
|
||||
{
|
||||
var existingKey = scannedSeries.Keys.SingleOrDefault(ps =>
|
||||
ps.Format == info.Format && (ps.NormalizedName.Equals(normalizedSeries)
|
||||
|| ps.NormalizedName.Equals(normalizedLocalizedSeries)
|
||||
|| ps.NormalizedName.Equals(normalizedSortSeries)));
|
||||
existingKey ??= new ParsedSeries()
|
||||
await ProcessFiles(folderPath, isLibraryScan, seriesPaths, async (files, folder) =>
|
||||
{
|
||||
Format = info.Format,
|
||||
Name = info.Series,
|
||||
NormalizedName = normalizedSeries
|
||||
};
|
||||
|
||||
scannedSeries.AddOrUpdate(existingKey, new List<ParserInfo>() {info}, (_, oldValue) =>
|
||||
{
|
||||
oldValue ??= new List<ParserInfo>();
|
||||
if (!oldValue.Contains(info))
|
||||
var normalizedFolder = Parser.Parser.NormalizePath(folder);
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedFolder, forceCheck))
|
||||
{
|
||||
oldValue.Add(info);
|
||||
var parsedInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
|
||||
{
|
||||
Series = fp.SeriesName,
|
||||
Format = fp.Format,
|
||||
}).ToList();
|
||||
processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
|
||||
_logger.LogDebug("Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
|
||||
return;
|
||||
}
|
||||
_logger.LogDebug("Found {Count} files for {Folder}", files.Count, folder);
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent(folderPath, libraryName, ProgressEventType.Updated));
|
||||
if (files.Count == 0)
|
||||
{
|
||||
_logger.LogInformation("[ScannerService] {Folder} is empty", folder);
|
||||
return;
|
||||
}
|
||||
var scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
|
||||
var infos = files
|
||||
.Select(file => _readingItemService.ParseFile(file, folderPath, libraryType))
|
||||
.Where(info => info != null)
|
||||
.ToList();
|
||||
|
||||
|
||||
MergeLocalizedSeriesWithSeries(infos);
|
||||
|
||||
foreach (var info in infos)
|
||||
{
|
||||
try
|
||||
{
|
||||
TrackSeries(scannedSeries, info);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "There was an exception that occurred during tracking {FilePath}. Skipping this file", info.FullFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
return oldValue;
|
||||
});
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogCritical(ex, "{SeriesName} matches against multiple series in the parsed series. This indicates a critical kavita issue. Key will be skipped", info.Series);
|
||||
foreach (var seriesKey in scannedSeries.Keys.Where(ps =>
|
||||
ps.Format == info.Format && (ps.NormalizedName.Equals(normalizedSeries)
|
||||
|| ps.NormalizedName.Equals(normalizedLocalizedSeries)
|
||||
|| ps.NormalizedName.Equals(normalizedSortSeries))))
|
||||
{
|
||||
_logger.LogCritical("Matches: {SeriesName} matches on {SeriesKey}", info.Series, seriesKey.Name);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Using a normalized name from the passed ParserInfo, this checks against all found series so far and if an existing one exists with
|
||||
/// same normalized name, it merges into the existing one. This is important as some manga may have a slight difference with punctuation or capitalization.
|
||||
/// </summary>
|
||||
/// <param name="info"></param>
|
||||
/// <returns>Series Name to group this info into</returns>
|
||||
private string MergeName(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, ParserInfo info)
|
||||
{
|
||||
var normalizedSeries = Parser.Parser.Normalize(info.Series);
|
||||
var normalizedLocalSeries = Parser.Parser.Normalize(info.LocalizedSeries);
|
||||
|
||||
try
|
||||
{
|
||||
var existingName =
|
||||
scannedSeries.SingleOrDefault(p =>
|
||||
(Parser.Parser.Normalize(p.Key.NormalizedName).Equals(normalizedSeries) ||
|
||||
Parser.Parser.Normalize(p.Key.NormalizedName).Equals(normalizedLocalSeries)) &&
|
||||
p.Key.Format == info.Format)
|
||||
.Key;
|
||||
|
||||
if (existingName != null && !string.IsNullOrEmpty(existingName.Name))
|
||||
{
|
||||
return existingName.Name;
|
||||
}
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogCritical(ex, "Multiple series detected for {SeriesName} ({File})! This is critical to fix! There should only be 1", info.Series, info.FullFilePath);
|
||||
var values = scannedSeries.Where(p =>
|
||||
(Parser.Parser.Normalize(p.Key.NormalizedName) == normalizedSeries ||
|
||||
Parser.Parser.Normalize(p.Key.NormalizedName) == normalizedLocalSeries) &&
|
||||
p.Key.Format == info.Format);
|
||||
foreach (var pair in values)
|
||||
{
|
||||
_logger.LogCritical("Duplicate Series in DB matches with {SeriesName}: {DuplicateName}", info.Series, pair.Key.Name);
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
return info.Series;
|
||||
}
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// This will process series by folder groups.
|
||||
/// </summary>
|
||||
/// <param name="libraryType"></param>
|
||||
/// <param name="folders"></param>
|
||||
/// <param name="libraryName"></param>
|
||||
/// <returns></returns>
|
||||
public async Task ScanLibrariesForSeries(LibraryType libraryType,
|
||||
IEnumerable<string> folders, string libraryName, bool isLibraryScan,
|
||||
IDictionary<string, IList<SeriesModified>> seriesPaths, Action<Tuple<bool, IList<ParserInfo>>> processSeriesInfos, bool forceCheck = false)
|
||||
{
|
||||
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", libraryName, ProgressEventType.Started));
|
||||
|
||||
foreach (var folderPath in folders)
|
||||
{
|
||||
try
|
||||
{
|
||||
await ProcessFiles(folderPath, isLibraryScan, seriesPaths, async (files, folder) =>
|
||||
// It would be really cool if we can emit an event when a folder hasn't been changed so we don't parse everything, but the first item to ensure we don't delete it
|
||||
// Otherwise, we can do a last step in the DB where we validate all files on disk exist and if not, delete them. (easy but slow)
|
||||
foreach (var series in scannedSeries.Keys)
|
||||
{
|
||||
var normalizedFolder = Parser.Parser.NormalizePath(folder);
|
||||
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedFolder, forceCheck))
|
||||
if (scannedSeries[series].Count > 0 && processSeriesInfos != null)
|
||||
{
|
||||
var parsedInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
|
||||
{
|
||||
Series = fp.SeriesName,
|
||||
Format = fp.Format,
|
||||
}).ToList();
|
||||
processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
|
||||
_logger.LogDebug("Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
|
||||
return;
|
||||
processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
|
||||
}
|
||||
_logger.LogDebug("Found {Count} files for {Folder}", files.Count, folder);
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent(folderPath, libraryName, ProgressEventType.Updated));
|
||||
if (files.Count == 0)
|
||||
{
|
||||
_logger.LogInformation("[ScannerService] {Folder} is empty", folder);
|
||||
return;
|
||||
}
|
||||
var scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
|
||||
var infos = files
|
||||
.Select(file => _readingItemService.ParseFile(file, folderPath, libraryType))
|
||||
.Where(info => info != null)
|
||||
.ToList();
|
||||
|
||||
|
||||
MergeLocalizedSeriesWithSeries(infos);
|
||||
|
||||
foreach (var info in infos)
|
||||
{
|
||||
try
|
||||
{
|
||||
TrackSeries(scannedSeries, info);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "There was an exception that occurred during tracking {FilePath}. Skipping this file", info.FullFilePath);
|
||||
}
|
||||
}
|
||||
|
||||
// It would be really cool if we can emit an event when a folder hasn't been changed so we don't parse everything, but the first item to ensure we don't delete it
|
||||
// Otherwise, we can do a last step in the DB where we validate all files on disk exist and if not, delete them. (easy but slow)
|
||||
foreach (var series in scannedSeries.Keys)
|
||||
{
|
||||
if (scannedSeries[series].Count > 0 && processSeriesInfos != null)
|
||||
{
|
||||
processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
|
||||
}
|
||||
}
|
||||
}, forceCheck);
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
_logger.LogError(ex, "The directory '{FolderPath}' does not exist", folderPath);
|
||||
}
|
||||
}
|
||||
}, forceCheck);
|
||||
}
|
||||
catch (ArgumentException ex)
|
||||
{
|
||||
_logger.LogError(ex, "The directory '{FolderPath}' does not exist", folderPath);
|
||||
}
|
||||
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Done", libraryName, ProgressEventType.Ended));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks against all folder paths on file if the last scanned is >= the directory's last write down to the second
|
||||
/// </summary>
|
||||
/// <param name="seriesPaths"></param>
|
||||
/// <param name="normalizedFolder"></param>
|
||||
/// <param name="forceCheck"></param>
|
||||
/// <returns></returns>
|
||||
private bool HasSeriesFolderNotChangedSinceLastScan(IDictionary<string, IList<SeriesModified>> seriesPaths, string normalizedFolder, bool forceCheck = false)
|
||||
{
|
||||
if (forceCheck) return false;
|
||||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Done", libraryName, ProgressEventType.Ended));
|
||||
}
|
||||
|
||||
return seriesPaths.ContainsKey(normalizedFolder) && seriesPaths[normalizedFolder].All(f => f.LastScanned.Truncate(TimeSpan.TicksPerSecond) >=
|
||||
_directoryService.GetLastWriteTime(normalizedFolder).Truncate(TimeSpan.TicksPerSecond));
|
||||
/// <summary>
|
||||
/// Checks against all folder paths on file if the last scanned is >= the directory's last write down to the second
|
||||
/// </summary>
|
||||
/// <param name="seriesPaths"></param>
|
||||
/// <param name="normalizedFolder"></param>
|
||||
/// <param name="forceCheck"></param>
|
||||
/// <returns></returns>
|
||||
private bool HasSeriesFolderNotChangedSinceLastScan(IDictionary<string, IList<SeriesModified>> seriesPaths, string normalizedFolder, bool forceCheck = false)
|
||||
{
|
||||
if (forceCheck) return false;
|
||||
|
||||
return seriesPaths.ContainsKey(normalizedFolder) && seriesPaths[normalizedFolder].All(f => f.LastScanned.Truncate(TimeSpan.TicksPerSecond) >=
|
||||
_directoryService.GetLastWriteTime(normalizedFolder).Truncate(TimeSpan.TicksPerSecond));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if there are any ParserInfos that have a Series that matches the LocalizedSeries field in any other info. If so,
|
||||
/// rewrites the infos with series name instead of the localized name, so they stack.
|
||||
/// </summary>
|
||||
/// <example>
|
||||
/// Accel World v01.cbz has Series "Accel World" and Localized Series "World of Acceleration"
|
||||
/// World of Acceleration v02.cbz has Series "World of Acceleration"
|
||||
/// After running this code, we'd have:
|
||||
/// World of Acceleration v02.cbz having Series "Accel World" and Localized Series of "World of Acceleration"
|
||||
/// </example>
|
||||
/// <param name="infos">A collection of ParserInfos</param>
|
||||
private void MergeLocalizedSeriesWithSeries(IReadOnlyCollection<ParserInfo> infos)
|
||||
{
|
||||
var hasLocalizedSeries = infos.Any(i => !string.IsNullOrEmpty(i.LocalizedSeries));
|
||||
if (!hasLocalizedSeries) return;
|
||||
|
||||
var localizedSeries = infos
|
||||
.Where(i => !i.IsSpecial)
|
||||
.Select(i => i.LocalizedSeries)
|
||||
.Distinct()
|
||||
.FirstOrDefault(i => !string.IsNullOrEmpty(i));
|
||||
if (string.IsNullOrEmpty(localizedSeries)) return;
|
||||
|
||||
// NOTE: If we have multiple series in a folder with a localized title, then this will fail. It will group into one series. User needs to fix this themselves.
|
||||
string nonLocalizedSeries;
|
||||
// Normalize this as many of the cases is a capitalization difference
|
||||
var nonLocalizedSeriesFound = infos
|
||||
.Where(i => !i.IsSpecial)
|
||||
.Select(i => i.Series).DistinctBy(Parser.Parser.Normalize).ToList();
|
||||
if (nonLocalizedSeriesFound.Count == 1)
|
||||
{
|
||||
nonLocalizedSeries = nonLocalizedSeriesFound.First();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There can be a case where there are multiple series in a folder that causes merging.
|
||||
if (nonLocalizedSeriesFound.Count > 2)
|
||||
{
|
||||
_logger.LogError("[ScannerService] There are multiple series within one folder that contain localized series. This will cause them to group incorrectly. Please separate series into their own dedicated folder or ensure there is only 2 potential series (localized and series): {LocalizedSeries}", string.Join(", ", nonLocalizedSeriesFound));
|
||||
}
|
||||
nonLocalizedSeries = nonLocalizedSeriesFound.FirstOrDefault(s => !s.Equals(localizedSeries));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Checks if there are any ParserInfos that have a Series that matches the LocalizedSeries field in any other info. If so,
|
||||
/// rewrites the infos with series name instead of the localized name, so they stack.
|
||||
/// </summary>
|
||||
/// <example>
|
||||
/// Accel World v01.cbz has Series "Accel World" and Localized Series "World of Acceleration"
|
||||
/// World of Acceleration v02.cbz has Series "World of Acceleration"
|
||||
/// After running this code, we'd have:
|
||||
/// World of Acceleration v02.cbz having Series "Accel World" and Localized Series of "World of Acceleration"
|
||||
/// </example>
|
||||
/// <param name="infos">A collection of ParserInfos</param>
|
||||
private void MergeLocalizedSeriesWithSeries(IReadOnlyCollection<ParserInfo> infos)
|
||||
if (string.IsNullOrEmpty(nonLocalizedSeries)) return;
|
||||
|
||||
var normalizedNonLocalizedSeries = Parser.Parser.Normalize(nonLocalizedSeries);
|
||||
foreach (var infoNeedingMapping in infos.Where(i =>
|
||||
!Parser.Parser.Normalize(i.Series).Equals(normalizedNonLocalizedSeries)))
|
||||
{
|
||||
var hasLocalizedSeries = infos.Any(i => !string.IsNullOrEmpty(i.LocalizedSeries));
|
||||
if (!hasLocalizedSeries) return;
|
||||
|
||||
var localizedSeries = infos
|
||||
.Where(i => !i.IsSpecial)
|
||||
.Select(i => i.LocalizedSeries)
|
||||
.Distinct()
|
||||
.FirstOrDefault(i => !string.IsNullOrEmpty(i));
|
||||
if (string.IsNullOrEmpty(localizedSeries)) return;
|
||||
|
||||
// NOTE: If we have multiple series in a folder with a localized title, then this will fail. It will group into one series. User needs to fix this themselves.
|
||||
string nonLocalizedSeries;
|
||||
// Normalize this as many of the cases is a capitalization difference
|
||||
var nonLocalizedSeriesFound = infos
|
||||
.Where(i => !i.IsSpecial)
|
||||
.Select(i => i.Series).DistinctBy(Parser.Parser.Normalize).ToList();
|
||||
if (nonLocalizedSeriesFound.Count == 1)
|
||||
{
|
||||
nonLocalizedSeries = nonLocalizedSeriesFound.First();
|
||||
}
|
||||
else
|
||||
{
|
||||
// There can be a case where there are multiple series in a folder that causes merging.
|
||||
if (nonLocalizedSeriesFound.Count > 2)
|
||||
{
|
||||
_logger.LogError("[ScannerService] There are multiple series within one folder that contain localized series. This will cause them to group incorrectly. Please separate series into their own dedicated folder or ensure there is only 2 potential series (localized and series): {LocalizedSeries}", string.Join(", ", nonLocalizedSeriesFound));
|
||||
}
|
||||
nonLocalizedSeries = nonLocalizedSeriesFound.FirstOrDefault(s => !s.Equals(localizedSeries));
|
||||
}
|
||||
|
||||
if (string.IsNullOrEmpty(nonLocalizedSeries)) return;
|
||||
|
||||
var normalizedNonLocalizedSeries = Parser.Parser.Normalize(nonLocalizedSeries);
|
||||
foreach (var infoNeedingMapping in infos.Where(i =>
|
||||
!Parser.Parser.Normalize(i.Series).Equals(normalizedNonLocalizedSeries)))
|
||||
{
|
||||
infoNeedingMapping.Series = nonLocalizedSeries;
|
||||
infoNeedingMapping.LocalizedSeries = localizedSeries;
|
||||
}
|
||||
infoNeedingMapping.Series = nonLocalizedSeries;
|
||||
infoNeedingMapping.LocalizedSeries = localizedSeries;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
|
|||
File diff suppressed because it is too large
Load diff
|
|
@ -2,100 +2,99 @@
|
|||
using API.Entities.Enums;
|
||||
using API.Services.Tasks.Scanner.Parser;
|
||||
|
||||
namespace API.Parser
|
||||
namespace API.Parser;
|
||||
|
||||
/// <summary>
|
||||
/// This represents all parsed information from a single file
|
||||
/// </summary>
|
||||
public class ParserInfo
|
||||
{
|
||||
/// <summary>
|
||||
/// This represents all parsed information from a single file
|
||||
/// Represents the parsed chapters from a file. By default, will be 0 which means nothing could be parsed.
|
||||
/// <remarks>The chapters can only be a single float or a range of float ie) 1-2. Mainly floats should be multiples of 0.5 representing specials</remarks>
|
||||
/// </summary>
|
||||
public class ParserInfo
|
||||
public string Chapters { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Represents the parsed series from the file or folder
|
||||
/// </summary>
|
||||
public string Series { get; set; } = string.Empty;
|
||||
/// <summary>
|
||||
/// This can be filled in from ComicInfo.xml/Epub during scanning. Will update the SortName field on <see cref="Entities.Series"/>
|
||||
/// </summary>
|
||||
public string SeriesSort { get; set; } = string.Empty;
|
||||
/// <summary>
|
||||
/// This can be filled in from ComicInfo.xml/Epub during scanning. Will update the LocalizedName field on <see cref="Entities.Series"/>
|
||||
/// </summary>
|
||||
public string LocalizedSeries { get; set; } = string.Empty;
|
||||
/// <summary>
|
||||
/// Represents the parsed volumes from a file. By default, will be 0 which means that nothing could be parsed.
|
||||
/// If Volumes is 0 and Chapters is 0, the file is a special. If Chapters is non-zero, then no volume could be parsed.
|
||||
/// <example>Beastars Vol 3-4 will map to "3-4"</example>
|
||||
/// <remarks>The volumes can only be a single int or a range of ints ie) 1-2. Float based volumes are not supported.</remarks>
|
||||
/// </summary>
|
||||
public string Volumes { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Filename of the underlying file
|
||||
/// <example>Beastars v01 (digital).cbz</example>
|
||||
/// </summary>
|
||||
public string Filename { get; init; } = "";
|
||||
/// <summary>
|
||||
/// Full filepath of the underlying file
|
||||
/// <example>C:/Manga/Beastars v01 (digital).cbz</example>
|
||||
/// </summary>
|
||||
public string FullFilePath { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="MangaFormat"/> that represents the type of the file
|
||||
/// <remarks>Mainly used to show in the UI and so caching service knows how to cache for reading.</remarks>
|
||||
/// </summary>
|
||||
public MangaFormat Format { get; set; } = MangaFormat.Unknown;
|
||||
|
||||
/// <summary>
|
||||
/// This can potentially story things like "Omnibus, Color, Full Contact Edition, Extra, Final, etc"
|
||||
/// </summary>
|
||||
/// <remarks>Not Used in Database</remarks>
|
||||
public string Edition { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// If the file contains no volume/chapter information or contains Special Keywords <see cref="Parser.MangaSpecialRegex"/>
|
||||
/// </summary>
|
||||
public bool IsSpecial { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Used for specials or books, stores what the UI should show.
|
||||
/// <remarks>Manga does not use this field</remarks>
|
||||
/// </summary>
|
||||
public string Title { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// If the ParserInfo has the IsSpecial tag or both volumes and chapters are default aka 0
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public bool IsSpecialInfo()
|
||||
{
|
||||
/// <summary>
|
||||
/// Represents the parsed chapters from a file. By default, will be 0 which means nothing could be parsed.
|
||||
/// <remarks>The chapters can only be a single float or a range of float ie) 1-2. Mainly floats should be multiples of 0.5 representing specials</remarks>
|
||||
/// </summary>
|
||||
public string Chapters { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Represents the parsed series from the file or folder
|
||||
/// </summary>
|
||||
public string Series { get; set; } = string.Empty;
|
||||
/// <summary>
|
||||
/// This can be filled in from ComicInfo.xml/Epub during scanning. Will update the SortName field on <see cref="Entities.Series"/>
|
||||
/// </summary>
|
||||
public string SeriesSort { get; set; } = string.Empty;
|
||||
/// <summary>
|
||||
/// This can be filled in from ComicInfo.xml/Epub during scanning. Will update the LocalizedName field on <see cref="Entities.Series"/>
|
||||
/// </summary>
|
||||
public string LocalizedSeries { get; set; } = string.Empty;
|
||||
/// <summary>
|
||||
/// Represents the parsed volumes from a file. By default, will be 0 which means that nothing could be parsed.
|
||||
/// If Volumes is 0 and Chapters is 0, the file is a special. If Chapters is non-zero, then no volume could be parsed.
|
||||
/// <example>Beastars Vol 3-4 will map to "3-4"</example>
|
||||
/// <remarks>The volumes can only be a single int or a range of ints ie) 1-2. Float based volumes are not supported.</remarks>
|
||||
/// </summary>
|
||||
public string Volumes { get; set; } = "";
|
||||
/// <summary>
|
||||
/// Filename of the underlying file
|
||||
/// <example>Beastars v01 (digital).cbz</example>
|
||||
/// </summary>
|
||||
public string Filename { get; init; } = "";
|
||||
/// <summary>
|
||||
/// Full filepath of the underlying file
|
||||
/// <example>C:/Manga/Beastars v01 (digital).cbz</example>
|
||||
/// </summary>
|
||||
public string FullFilePath { get; set; } = "";
|
||||
return (IsSpecial || (Volumes == "0" && Chapters == "0"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// <see cref="MangaFormat"/> that represents the type of the file
|
||||
/// <remarks>Mainly used to show in the UI and so caching service knows how to cache for reading.</remarks>
|
||||
/// </summary>
|
||||
public MangaFormat Format { get; set; } = MangaFormat.Unknown;
|
||||
/// <summary>
|
||||
/// This will contain any EXTRA comicInfo information parsed from the epub or archive. If there is an archive with comicInfo.xml AND it contains
|
||||
/// series, volume information, that will override what we parsed.
|
||||
/// </summary>
|
||||
public ComicInfo ComicInfo { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// This can potentially story things like "Omnibus, Color, Full Contact Edition, Extra, Final, etc"
|
||||
/// </summary>
|
||||
/// <remarks>Not Used in Database</remarks>
|
||||
public string Edition { get; set; } = "";
|
||||
|
||||
/// <summary>
|
||||
/// If the file contains no volume/chapter information or contains Special Keywords <see cref="Parser.MangaSpecialRegex"/>
|
||||
/// </summary>
|
||||
public bool IsSpecial { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Used for specials or books, stores what the UI should show.
|
||||
/// <remarks>Manga does not use this field</remarks>
|
||||
/// </summary>
|
||||
public string Title { get; set; } = string.Empty;
|
||||
|
||||
/// <summary>
|
||||
/// If the ParserInfo has the IsSpecial tag or both volumes and chapters are default aka 0
|
||||
/// </summary>
|
||||
/// <returns></returns>
|
||||
public bool IsSpecialInfo()
|
||||
{
|
||||
return (IsSpecial || (Volumes == "0" && Chapters == "0"));
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// This will contain any EXTRA comicInfo information parsed from the epub or archive. If there is an archive with comicInfo.xml AND it contains
|
||||
/// series, volume information, that will override what we parsed.
|
||||
/// </summary>
|
||||
public ComicInfo ComicInfo { get; set; }
|
||||
|
||||
/// <summary>
|
||||
/// Merges non empty/null properties from info2 into this entity.
|
||||
/// </summary>
|
||||
/// <remarks>This does not merge ComicInfo as they should always be the same</remarks>
|
||||
/// <param name="info2"></param>
|
||||
public void Merge(ParserInfo info2)
|
||||
{
|
||||
if (info2 == null) return;
|
||||
Chapters = string.IsNullOrEmpty(Chapters) || Chapters == "0" ? info2.Chapters: Chapters;
|
||||
Volumes = string.IsNullOrEmpty(Volumes) || Volumes == "0" ? info2.Volumes : Volumes;
|
||||
Edition = string.IsNullOrEmpty(Edition) ? info2.Edition : Edition;
|
||||
Title = string.IsNullOrEmpty(Title) ? info2.Title : Title;
|
||||
Series = string.IsNullOrEmpty(Series) ? info2.Series : Series;
|
||||
IsSpecial = IsSpecial || info2.IsSpecial;
|
||||
}
|
||||
/// <summary>
|
||||
/// Merges non empty/null properties from info2 into this entity.
|
||||
/// </summary>
|
||||
/// <remarks>This does not merge ComicInfo as they should always be the same</remarks>
|
||||
/// <param name="info2"></param>
|
||||
public void Merge(ParserInfo info2)
|
||||
{
|
||||
if (info2 == null) return;
|
||||
Chapters = string.IsNullOrEmpty(Chapters) || Chapters == "0" ? info2.Chapters: Chapters;
|
||||
Volumes = string.IsNullOrEmpty(Volumes) || Volumes == "0" ? info2.Volumes : Volumes;
|
||||
Edition = string.IsNullOrEmpty(Edition) ? info2.Edition : Edition;
|
||||
Title = string.IsNullOrEmpty(Title) ? info2.Title : Title;
|
||||
Series = string.IsNullOrEmpty(Series) ? info2.Series : Series;
|
||||
IsSpecial = IsSpecial || info2.IsSpecial;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue