AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

A hefty refactor of the ScanLibrary code. There were significant fallouts due to duplicate entities getting created and SingleOrDefaults failing.

Browse source
This commit is contained in:
Joseph Milazzo 2021-02-08 12:03:52 -06:00
parent 39fa750d96
commit 9461b89725
15 changed files with 1075 additions and 153 deletions
Download patch file Download diff file Expand all files Collapse all files

316
API/Services/ScannerService.cs
View file

@ -10,6 +10,7 @@ using API.Entities;
using API.Entities.Enums;
using API.Interfaces;
using API.Parser;
using Hangfire;
using Microsoft.Extensions.Logging;
namespace API.Services
@ -20,6 +21,7 @@ namespace API.Services
private readonly ILogger<ScannerService> _logger;
private readonly IArchiveService _archiveService;
private ConcurrentDictionary<string, List<ParserInfo>> _scannedSeries;
private bool _forceUpdate;
public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger, IArchiveService archiveService)
{
@ -28,6 +30,7 @@ namespace API.Services
_archiveService = archiveService;
}
[DisableConcurrentExecution(timeoutInSeconds: 120)]
public void ScanLibraries()
{
var libraries = Task.Run(() => _unitOfWork.LibraryRepository.GetLibrariesAsync()).Result.ToList();
@ -37,9 +40,31 @@ namespace API.Services
}
}
private bool ShouldSkipFolderScan(FolderPath folder, ref int skippedFolders)
{
// NOTE: This solution isn't the best, but it has potential. We need to handle a few other cases so it works great.
return false;
// if (/*_environment.IsProduction() && */!_forceUpdate && Directory.GetLastWriteTime(folder.Path) < folder.LastScanned)
// {
// _logger.LogDebug($"{folder.Path} hasn't been updated since last scan. Skipping.");
// skippedFolders += 1;
// return true;
// }
//
// return false;
}
private void Cleanup()
{
_scannedSeries = null;
_forceUpdate = false;
}
[DisableConcurrentExecution(timeoutInSeconds: 120)]
public void ScanLibrary(int libraryId, bool forceUpdate)
{
{
_forceUpdate = forceUpdate;
var sw = Stopwatch.StartNew();
Library library;
try
@ -60,14 +85,8 @@ namespace API.Services
var skippedFolders = 0;
foreach (var folderPath in library.Folders)
{
// if (!forceUpdate && Directory.GetLastWriteTime(folderPath.Path) <= folderPath.LastScanned)
// {
// // NOTE: This solution isn't the best, but it has potential. We need to handle a few other cases so it works great.
// _logger.LogDebug($"{folderPath.Path} hasn't been updated since last scan. Skipping.");
// skippedFolders += 1;
// continue;
// }
if (ShouldSkipFolderScan(folderPath, ref skippedFolders)) continue;
try {
totalFiles += DirectoryService.TraverseTreeParallelForEach(folderPath.Path, (f) =>
{
@ -77,91 +96,110 @@ namespace API.Services
}
catch (FileNotFoundException exception)
{
_logger.LogError(exception, "The file could not be found");
_logger.LogError(exception, $"The file {f} could not be found");
}
});
}
catch (ArgumentException ex) {
_logger.LogError(ex, $"The directory '{folderPath}' does not exist");
_logger.LogError(ex, $"The directory '{folderPath.Path}' does not exist");
}
folderPath.LastScanned = DateTime.Now;
}
var scanElapsedTime = sw.ElapsedMilliseconds;
_logger.LogInformation("Folders Scanned {0} files in {1} milliseconds", totalFiles, scanElapsedTime);
sw.Restart();
if (skippedFolders == library.Folders.Count)
{
_logger.LogInformation("All Folders were skipped due to no modifications to the directories.");
_unitOfWork.LibraryRepository.Update(library);
_scannedSeries = null;
_logger.LogInformation("Processed {0} files in {1} milliseconds for {2}", totalFiles, sw.ElapsedMilliseconds, library.Name);
Cleanup();
return;
}
// Remove any series where there were no parsed infos
var filtered = _scannedSeries.Where(kvp => kvp.Value.Count != 0);
var series = filtered.ToImmutableDictionary(v => v.Key, v => v.Value);
// Perform DB activities
var allSeries = UpsertSeries(libraryId, forceUpdate, series, library);
// Remove series that are no longer on disk
RemoveSeriesNotOnDisk(allSeries, series, library);
//foreach (var folder in library.Folders) folder.LastScanned = DateTime.Now;
UpdateLibrary(libraryId, series, library);
_unitOfWork.LibraryRepository.Update(library);
if (Task.Run(() => _unitOfWork.Complete()).Result)
{
_logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series.");
_logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series in {sw.ElapsedMilliseconds} ms.");
}
else
{
_logger.LogError("There was a critical error that resulted in a failed scan. Please rescan.");
_logger.LogError("There was a critical error that resulted in a failed scan. Please check logs and rescan.");
}
_scannedSeries = null;
_logger.LogInformation("Processed {0} files in {1} milliseconds for {2}", totalFiles, sw.ElapsedMilliseconds, library.Name);
_logger.LogInformation("Processed {0} files in {1} milliseconds for {2}", totalFiles, sw.ElapsedMilliseconds + scanElapsedTime, library.Name);
Cleanup();
}
private List<Series> UpsertSeries(int libraryId, bool forceUpdate, ImmutableDictionary<string, List<ParserInfo>> series, Library library)
private void UpdateLibrary(int libraryId, ImmutableDictionary<string, List<ParserInfo>> parsedSeries, Library library)
{
var allSeries = Task.Run(() => _unitOfWork.SeriesRepository.GetSeriesForLibraryIdAsync(libraryId)).Result.ToList();
foreach (var seriesKey in series.Keys)
_logger.LogInformation($"Updating Library {library.Name}");
// Perform DB activities
UpsertSeries(library, parsedSeries, allSeries);
// Remove series that are no longer on disk
RemoveSeriesNotOnDisk(allSeries, parsedSeries, library);
foreach (var folder in library.Folders) folder.LastScanned = DateTime.Now;
}
private void UpsertSeries(Library library, ImmutableDictionary<string, List<ParserInfo>> parsedSeries,
IList<Series> allSeries)
{
// NOTE: This is a great point to break the parsing into threads and join back. Each thread can take X series.
foreach (var seriesKey in parsedSeries.Keys)
{
var mangaSeries = allSeries.SingleOrDefault(s => s.Name == seriesKey) ?? new Series
var mangaSeries = ExistingOrDefault(library, allSeries, seriesKey) ?? new Series
{
Name = seriesKey,
OriginalName = seriesKey,
NormalizedName = Parser.Parser.Normalize(seriesKey),
SortName = seriesKey,
Summary = ""
};
mangaSeries.NormalizedName = Parser.Parser.Normalize(seriesKey);
try
{
mangaSeries = UpdateSeries(mangaSeries, series[seriesKey].ToArray(), forceUpdate);
_logger.LogInformation($"Created/Updated series {mangaSeries.Name} for {library.Name} library");
library.Series ??= new List<Series>();
library.Series.Add(mangaSeries);
UpdateSeries(ref mangaSeries, parsedSeries[seriesKey].ToArray());
if (!library.Series.Any(s => s.NormalizedName == mangaSeries.NormalizedName))
{
_logger.LogInformation($"Added series {mangaSeries.Name}");
library.Series.Add(mangaSeries);
}
}
catch (Exception ex)
{
_logger.LogError(ex, $"There was an error during scanning of library. {seriesKey} will be skipped.");
}
}
return allSeries;
}
private void RemoveSeriesNotOnDisk(List<Series> allSeries, ImmutableDictionary<string, List<ParserInfo>> series, Library library)
private void RemoveSeriesNotOnDisk(IEnumerable<Series> allSeries, ImmutableDictionary<string, List<ParserInfo>> series, Library library)
{
_logger.LogInformation("Removing any series that are no longer on disk.");
var count = 0;
foreach (var existingSeries in allSeries)
var foundSeries = series.Select(s => Parser.Parser.Normalize(s.Key)).ToList();
var missingSeries = allSeries.Where(existingSeries =>
!foundSeries.Contains(existingSeries.NormalizedName) || !series.ContainsKey(existingSeries.Name) ||
!series.ContainsKey(existingSeries.OriginalName));
foreach (var existingSeries in missingSeries)
{
if (!series.ContainsKey(existingSeries.Name) || !series.ContainsKey(existingSeries.OriginalName))
{
// Delete series, there is no file to backup any longer.
library.Series?.Remove(existingSeries);
count++;
}
// Delete series, there is no file to backup any longer.
library.Series?.Remove(existingSeries);
count++;
}
_logger.LogInformation($"Removed {count} series that are no longer on disk");
}
@ -206,33 +244,33 @@ namespace API.Services
TrackSeries(info);
}
private Series UpdateSeries(Series series, ParserInfo[] infos, bool forceUpdate)
private void UpdateSeries(ref Series series, ParserInfo[] infos)
{
var volumes = UpdateVolumesWithChapters(series, infos, forceUpdate);
series.Volumes = volumes;
series.Pages = volumes.Sum(v => v.Pages);
if (ShouldFindCoverImage(forceUpdate, series.CoverImage))
_logger.LogInformation($"Updating entries for {series.Name}. {infos.Length} related files.");
UpdateVolumes(series, infos);
series.Pages = series.Volumes.Sum(v => v.Pages);
if (ShouldFindCoverImage(series.CoverImage))
{
var firstCover = volumes.OrderBy(x => x.Number).FirstOrDefault(x => x.Number != 0);
if (firstCover == null && volumes.Any())
var firstCover = series.Volumes.OrderBy(x => x.Number).FirstOrDefault(x => x.Number != 0);
if (firstCover == null && series.Volumes.Any())
{
firstCover = volumes.FirstOrDefault(x => x.Number == 0);
firstCover = series.Volumes.FirstOrDefault(x => x.Number == 0);
}
series.CoverImage = firstCover?.CoverImage;
}
if (string.IsNullOrEmpty(series.Summary) || forceUpdate)
if (string.IsNullOrEmpty(series.Summary) || _forceUpdate)
{
series.Summary = "";
}
return series;
_logger.LogDebug($"Created {series.Volumes.Count} volumes on {series.Name}");
}
private MangaFile CreateMangaFile(ParserInfo info)
{
_logger.LogDebug($"Creating File Entry for {info.FullFilePath}");
return new MangaFile()
{
FilePath = info.FullFilePath,
@ -241,99 +279,138 @@ namespace API.Services
};
}
private bool ShouldFindCoverImage(bool forceUpdate, byte[] coverImage)
private bool ShouldFindCoverImage(byte[] coverImage)
{
return forceUpdate || coverImage == null || !coverImage.Any();
return _forceUpdate || coverImage == null || !coverImage.Any();
}
/// <summary>
///
/// </summary>
/// <param name="volume"></param>
/// <param name="infos"></param>
/// <param name="forceUpdate"></param>
/// <returns></returns>
private ICollection<Chapter> UpdateChapters(Volume volume, IEnumerable<ParserInfo> infos, bool forceUpdate)
{
var chapters = new List<Chapter>();
private void UpdateChapters(Volume volume, IEnumerable<ParserInfo> infos) // ICollection<Chapter>
{
volume.Chapters ??= new List<Chapter>();
foreach (var info in infos)
{
volume.Chapters ??= new List<Chapter>();
var chapter = volume.Chapters.SingleOrDefault(c => c.Range == info.Chapters) ??
chapters.SingleOrDefault(v => v.Range == info.Chapters) ??
new Chapter()
{
Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + "",
Range = info.Chapters,
};
chapter.Files ??= new List<MangaFile>();
var existingFile = chapter.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
if (existingFile != null)
try
{
existingFile.Format = info.Format;
existingFile.NumberOfPages = _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath);
var chapter = volume.Chapters.SingleOrDefault(c => c.Range == info.Chapters) ??
new Chapter()
{
Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + "",
Range = info.Chapters,
};
AddOrUpdateFileForChapter(chapter, info);
chapter.Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + "";
chapter.Range = info.Chapters;
if (volume.Chapters.All(c => c.Range != info.Chapters))
{
volume.Chapters.Add(chapter);
}
}
else
catch (Exception ex)
{
if (info.Format == MangaFormat.Archive)
{
chapter.Files.Add(CreateMangaFile(info));
}
else
{
_logger.LogDebug($"Ignoring {info.Filename} as it is not an archive.");
}
_logger.LogWarning(ex, $"There was an exception parsing {info.Series} - Volume {volume.Number}'s chapters. Skipping Chapter.");
}
chapter.Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + "";
chapter.Range = info.Chapters;
chapters.Add(chapter);
}
foreach (var chapter in chapters)
foreach (var chapter in volume.Chapters)
{
chapter.Pages = chapter.Files.Sum(f => f.NumberOfPages);
if (ShouldFindCoverImage(forceUpdate, chapter.CoverImage))
if (ShouldFindCoverImage(chapter.CoverImage))
{
chapter.Files ??= new List<MangaFile>();
var firstFile = chapter.Files.OrderBy(x => x.Chapter).FirstOrDefault();
if (firstFile != null) chapter.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true);
}
}
return chapters;
}
private void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info)
{
chapter.Files ??= new List<MangaFile>();
var existingFile = chapter.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
if (existingFile != null)
{
existingFile.Format = info.Format;
existingFile.NumberOfPages = _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath);
}
else
{
if (info.Format == MangaFormat.Archive)
{
chapter.Files.Add(CreateMangaFile(info));
}
else
{
_logger.LogDebug($"Ignoring {info.Filename}. Non-archives are not supported yet.");
}
}
}
public static Volume ExistingOrDefault(IList<Volume> existingVolumes, ICollection<Volume> volumes, string volumeName)
{
return volumes.SingleOrDefault(v => v.Name == volumeName) ?? existingVolumes.SingleOrDefault(v => v.Name == volumeName);
}
public static Series ExistingOrDefault(Library library, IEnumerable<Series> allSeries, string seriesName)
{
var name = Parser.Parser.Normalize(seriesName);
library.Series ??= new List<Series>();
return library.Series.SingleOrDefault(s => Parser.Parser.Normalize(s.Name) == name) ??
allSeries.SingleOrDefault(s => Parser.Parser.Normalize(s.Name) == name);
}
private ICollection<Volume> UpdateVolumesWithChapters(Series series, ParserInfo[] infos, bool forceUpdate)
private void UpdateVolumes(Series series, ParserInfo[] infos)
{
ICollection<Volume> volumes = new List<Volume>();
series.Volumes ??= new List<Volume>();
_logger.LogDebug($"Updating Volumes for {series.Name}. {infos.Length} related files.");
IList<Volume> existingVolumes = _unitOfWork.SeriesRepository.GetVolumes(series.Id).ToList();
foreach (var info in infos)
{
var volume = (existingVolumes.SingleOrDefault(v => v.Name == info.Volumes) ??
volumes.SingleOrDefault(v => v.Name == info.Volumes)) ?? new Volume
try
{
Name = info.Volumes,
Number = Parser.Parser.MinimumNumberFromRange(info.Volumes),
};
var volume = ExistingOrDefault(existingVolumes, series.Volumes, info.Volumes) ?? new Volume
{
Name = info.Volumes,
Number = (int) Parser.Parser.MinimumNumberFromRange(info.Volumes),
IsSpecial = false,
Chapters = new List<Chapter>()
};
if (series.Volumes.Any(v => v.Name == volume.Name)) continue;
series.Volumes.Add(volume);
}
catch (Exception ex)
{
_logger.LogError(ex, $"There was an exception when creating volume {info.Volumes}. Skipping volume.");
}
}
var chapters = UpdateChapters(volume, infos.Where(pi => pi.Volumes == volume.Name).ToArray(), forceUpdate);
volume.Chapters = chapters;
volume.Pages = chapters.Sum(c => c.Pages);
volumes.Add(volume);
foreach (var volume in series.Volumes)
{
try
{
var justVolumeInfos = infos.Where(pi => pi.Volumes == volume.Name).ToArray();
UpdateChapters(volume, justVolumeInfos);
volume.Pages = volume.Chapters.Sum(c => c.Pages);
_logger.LogDebug($"Created {volume.Chapters.Count} chapters on {series.Name} - Volume {volume.Name}");
} catch (Exception ex)
{
_logger.LogError(ex, $"There was an exception when creating volume {volume.Name}. Skipping volume.");
}
}
foreach (var volume in volumes)
foreach (var volume in series.Volumes)
{
if (ShouldFindCoverImage(forceUpdate, volume.CoverImage))
if (ShouldFindCoverImage(volume.CoverImage))
{
// TODO: Create a custom sorter for Chapters so it's consistent across the application
var firstChapter = volume.Chapters.OrderBy(x => Double.Parse(x.Number)).FirstOrDefault();
@ -341,17 +418,6 @@ namespace API.Services
if (firstFile != null) volume.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true);
}
}
return volumes;
}
public void ScanSeries(int libraryId, int seriesId)
{
throw new NotImplementedException();
}
}
}