AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Refactored DB to be Series->Volumes->Chapters instead. All functionality that previously worked still works. Cleanup still needed.

Browse source
This commit is contained in:
Joseph Milazzo 2021-02-01 09:29:11 -06:00
parent a42e54a078
commit 53e85317f9
33 changed files with 2084 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

2
API/Services/ArchiveService.cs
View file

@ -161,7 +161,7 @@ namespace API.Services
var needsFlattening = ArchiveNeedsFlattening(archive);
if (!archive.HasFiles() && !needsFlattening) return;
archive.ExtractToDirectory(extractPath);
archive.ExtractToDirectory(extractPath, true);
_logger.LogDebug($"Extracted archive to {extractPath} in {sw.ElapsedMilliseconds} milliseconds.");
if (needsFlattening)

68
API/Services/CacheService.cs
View file

@ -18,7 +18,7 @@ namespace API.Services
private readonly IArchiveService _archiveService;
private readonly IDirectoryService _directoryService;
private readonly NumericComparer _numericComparer;
public static readonly string CacheDirectory = Path.GetFullPath(Path.Join(Directory.GetCurrentDirectory(), "../cache/"));
public static readonly string CacheDirectory = Path.GetFullPath(Path.Join(Directory.GetCurrentDirectory(), "cache/"));
public CacheService(ILogger<CacheService> logger, IUnitOfWork unitOfWork, IArchiveService archiveService, IDirectoryService directoryService)
{
@ -29,40 +29,36 @@ namespace API.Services
_numericComparer = new NumericComparer();
}
public bool CacheDirectoryIsAccessible()
public void EnsureCacheDirectory()
{
_logger.LogDebug($"Checking if valid Cache directory: {CacheDirectory}");
var di = new DirectoryInfo(CacheDirectory);
return di.Exists;
if (!di.Exists)
{
_logger.LogError($"Cache directory {CacheDirectory} is not accessible or does not exist. Creating...");
Directory.CreateDirectory(CacheDirectory);
}
}
public async Task<Volume> Ensure(int volumeId)
public async Task<Chapter> Ensure(int chapterId)
{
if (!CacheDirectoryIsAccessible())
{
return null;
}
Volume volume = await _unitOfWork.SeriesRepository.GetVolumeAsync(volumeId);
EnsureCacheDirectory();
Chapter chapter = await _unitOfWork.VolumeRepository.GetChapterAsync(chapterId);
foreach (var file in volume.Files)
foreach (var file in chapter.Files)
{
var extractPath = GetVolumeCachePath(volumeId, file);
var extractPath = GetCachePath(chapterId, file);
_archiveService.ExtractArchive(file.FilePath, extractPath);
}
return volume;
return chapter;
}
public void Cleanup()
{
_logger.LogInformation("Performing cleanup of Cache directory");
if (!CacheDirectoryIsAccessible())
{
_logger.LogError($"Cache directory {CacheDirectory} is not accessible or does not exist.");
return;
}
EnsureCacheDirectory();
DirectoryInfo di = new DirectoryInfo(CacheDirectory);
try
@ -79,6 +75,7 @@ namespace API.Services
public void CleanupVolumes(int[] volumeIds)
{
// TODO: Fix this code to work with chapters
_logger.LogInformation($"Running Cache cleanup on Volumes");
foreach (var volume in volumeIds)
@ -96,13 +93,19 @@ namespace API.Services
public string GetVolumeCachePath(int volumeId, MangaFile file)
/// <summary>
/// Returns the cache path for a given Chapter. Should be cacheDirectory/{chapterId}/
/// </summary>
/// <param name="chapterId"></param>
/// <param name="file"></param>
/// <returns></returns>
public string GetCachePath(int chapterId, MangaFile file)
{
var extractPath = Path.GetFullPath(Path.Join(Directory.GetCurrentDirectory(), $"../cache/{volumeId}/"));
if (file.Chapter > 0)
{
extractPath = Path.Join(extractPath, file.Chapter + "");
}
var extractPath = Path.GetFullPath(Path.Join(CacheDirectory, $"{chapterId}/"));
// if (file.Chapter != null)
// {
// extractPath = Path.Join(extractPath, chapterId + "");
// }
return extractPath;
}
@ -110,28 +113,29 @@ namespace API.Services
{
// BUG: This causes a problem because total pages on a volume assumes "specials" to be there
//return files.OrderBy(f => f.Chapter).Where(f => f.Chapter > 0 || f.Volume.Number != 0);
return files.OrderBy(f => f.Chapter, new ChapterSortComparer());
return files;
//return files.OrderBy(f => f.Chapter, new ChapterSortComparer());
}
public (string path, MangaFile file) GetCachedPagePath(Volume volume, int page)
public async Task<(string path, MangaFile file)> GetCachedPagePath(Chapter chapter, int page)
{
// Calculate what chapter the page belongs to
var pagesSoFar = 0;
// Do not allow chapters with 0, as those are specials and break ordering for reading.
var orderedChapters = GetOrderedChapters(volume.Files);
foreach (var mangaFile in orderedChapters)
var chapterFiles = chapter.Files ?? await _unitOfWork.VolumeRepository.GetFilesForChapter(chapter.Id);
foreach (var mangaFile in chapterFiles)
{
if (page + 1 < (mangaFile.NumberOfPages + pagesSoFar))
{
var path = GetVolumeCachePath(volume.Id, mangaFile);
var path = GetCachePath(chapter.Id, mangaFile);
var files = _directoryService.GetFiles(path);
Array.Sort(files, _numericComparer);
return (files.ElementAt(page - pagesSoFar), mangaFile);
}
pagesSoFar += mangaFile.NumberOfPages;
}
return ("", null);
}
}

12
API/Services/DirectoryService.cs
View file

@ -8,12 +8,19 @@ using System.Threading;
using System.Threading.Tasks;
using API.DTOs;
using API.Interfaces;
using Microsoft.Extensions.Logging;
using NetVips;
namespace API.Services
{
public class DirectoryService : IDirectoryService
{
private readonly ILogger<DirectoryService> _logger;
public DirectoryService(ILogger<DirectoryService> logger)
{
_logger = logger;
}
/// <summary>
/// Given a set of regex search criteria, get files in the given path.
@ -52,6 +59,11 @@ namespace API.Services
public async Task<ImageDto> ReadImageAsync(string imagePath)
{
if (!File.Exists(imagePath))
{
_logger.LogError("Image does not exist on disk.");
return null;
}
using var image = Image.NewFromFile(imagePath);
return new ImageDto

160
API/Services/ScannerService.cs
View file

@ -61,6 +61,12 @@ namespace API.Services
var totalFiles = 0;
foreach (var folderPath in library.Folders)
{
// if (!forceUpdate && Directory.GetLastWriteTime(folderPath.Path) <= folderPath.LastScanned)
// {
// _logger.LogDebug($"{folderPath.Path} hasn't been updated since last scan. Skipping.");
// continue;
// }
try {
totalFiles += DirectoryService.TraverseTreeParallelForEach(folderPath.Path, (f) =>
{
@ -88,10 +94,12 @@ namespace API.Services
// Remove series that are no longer on disk
RemoveSeriesNotOnDisk(allSeries, series, library);
foreach (var folder in library.Folders) folder.LastScanned = DateTime.Now;
_unitOfWork.LibraryRepository.Update(library);
if (Task.Run(() => _unitOfWork.Complete()).Result)
{
_logger.LogInformation($"Scan completed on {library.Name}. Parsed {series.Keys.Count()} series.");
}
else
@ -188,10 +196,10 @@ namespace API.Services
private Series UpdateSeries(Series series, ParserInfo[] infos, bool forceUpdate)
{
var volumes = UpdateVolumes(series, infos, forceUpdate);
var volumes = UpdateVolumesWithChapters(series, infos, forceUpdate);
series.Volumes = volumes;
series.Pages = volumes.Sum(v => v.Pages);
if (series.CoverImage == null || forceUpdate)
if (ShouldFindCoverImage(forceUpdate, series.CoverImage))
{
var firstCover = volumes.OrderBy(x => x.Number).FirstOrDefault(x => x.Number != 0);
if (firstCover == null && volumes.Any())
@ -213,16 +221,20 @@ namespace API.Services
{
_logger.LogDebug($"Creating File Entry for {info.FullFilePath}");
int.TryParse(info.Chapters, out var chapter);
_logger.LogDebug($"Found Chapter: {chapter}");
return new MangaFile()
{
FilePath = info.FullFilePath,
Chapter = chapter,
Format = info.Format,
NumberOfPages = info.Format == MangaFormat.Archive ? _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath): 1
};
}
private bool ShouldFindCoverImage(bool forceUpdate, byte[] coverImage)
{
return forceUpdate || coverImage == null || !coverImage.Any();
}
@ -237,16 +249,20 @@ namespace API.Services
{
ICollection<Volume> volumes = new List<Volume>();
IList<Volume> existingVolumes = _unitOfWork.SeriesRepository.GetVolumes(series.Id).ToList();
//var justVolumes = infos.Select(pi => pi.Chapters == "0");
foreach (var info in infos)
{
var existingVolume = existingVolumes.SingleOrDefault(v => v.Name == info.Volumes);
if (existingVolume != null)
{
var existingFile = existingVolume.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
//var existingFile = existingVolume.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
var existingFile = new MangaFile();
if (existingFile != null)
{
existingFile.Chapter = Parser.Parser.MinimumNumberFromRange(info.Chapters);
//existingFile.Chapter = Parser.Parser.MinimumNumberFromRange(info.Chapters);
existingFile.Format = info.Format;
existingFile.NumberOfPages = _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath);
}
@ -254,7 +270,7 @@ namespace API.Services
{
if (info.Format == MangaFormat.Archive)
{
existingVolume.Files.Add(CreateMangaFile(info));
// existingVolume.Files.Add(CreateMangaFile(info));
}
else
{
@ -271,7 +287,7 @@ namespace API.Services
existingVolume = volumes.SingleOrDefault(v => v.Name == info.Volumes);
if (existingVolume != null)
{
existingVolume.Files.Add(CreateMangaFile(info));
//existingVolume.Files.Add(CreateMangaFile(info));
}
else
{
@ -279,10 +295,10 @@ namespace API.Services
{
Name = info.Volumes,
Number = Parser.Parser.MinimumNumberFromRange(info.Volumes),
Files = new List<MangaFile>()
{
CreateMangaFile(info)
}
// Files = new List<MangaFile>()
// {
// CreateMangaFile(info)
// }
};
volumes.Add(vol);
}
@ -290,23 +306,121 @@ namespace API.Services
_logger.LogInformation($"Adding volume {volumes.Last().Number} with File: {info.Filename}");
}
foreach (var volume in volumes)
{
if (forceUpdate || volume.CoverImage == null || !volume.Files.Any())
{
var firstFile = volume.Files.OrderBy(x => x.Chapter).FirstOrDefault();
if (firstFile != null) volume.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true);
}
volume.Pages = volume.Files.Sum(x => x.NumberOfPages);
// if (forceUpdate || volume.CoverImage == null || !volume.Files.Any())
// {
// var firstFile = volume.Files.OrderBy(x => x.Chapter).FirstOrDefault();
// if (firstFile != null) volume.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true);
// }
//volume.Pages = volume.Files.Sum(x => x.NumberOfPages);
}
return volumes;
}
/// <summary>
///
/// </summary>
/// <param name="volume"></param>
/// <param name="infos"></param>
/// <param name="forceUpdate"></param>
/// <returns></returns>
private ICollection<Chapter> UpdateChapters(Volume volume, IEnumerable<ParserInfo> infos, bool forceUpdate)
{
var chapters = new List<Chapter>();
foreach (var info in infos)
{
volume.Chapters ??= new List<Chapter>();
var chapter = volume.Chapters.SingleOrDefault(c => c.Range == info.Chapters) ??
chapters.SingleOrDefault(v => v.Range == info.Chapters) ??
new Chapter()
{
Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + "",
Range = info.Chapters,
};
chapter.Files ??= new List<MangaFile>();
var existingFile = chapter?.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
if (existingFile != null)
{
existingFile.Format = info.Format;
existingFile.NumberOfPages = _archiveService.GetNumberOfPagesFromArchive(info.FullFilePath);
}
else
{
if (info.Format == MangaFormat.Archive)
{
chapter.Files.Add(CreateMangaFile(info));
}
else
{
_logger.LogDebug($"Ignoring {info.Filename} as it is not an archive.");
}
}
chapter.Number = Parser.Parser.MinimumNumberFromRange(info.Chapters) + "";
chapter.Range = info.Chapters;
chapters.Add(chapter);
}
foreach (var chapter in chapters)
{
chapter.Pages = chapter.Files.Sum(f => f.NumberOfPages);
if (ShouldFindCoverImage(forceUpdate, chapter.CoverImage))
{
var firstFile = chapter?.Files.OrderBy(x => x.Chapter).FirstOrDefault();
if (firstFile != null) chapter.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true);
}
}
return chapters;
}
private ICollection<Volume> UpdateVolumesWithChapters(Series series, ParserInfo[] infos, bool forceUpdate)
{
ICollection<Volume> volumes = new List<Volume>();
IList<Volume> existingVolumes = _unitOfWork.SeriesRepository.GetVolumes(series.Id).ToList();
foreach (var info in infos)
{
var volume = (existingVolumes.SingleOrDefault(v => v.Name == info.Volumes) ??
volumes.SingleOrDefault(v => v.Name == info.Volumes)) ?? new Volume
{
Name = info.Volumes,
Number = Parser.Parser.MinimumNumberFromRange(info.Volumes),
};
var chapters = UpdateChapters(volume, infos.Where(pi => pi.Volumes == volume.Name).ToArray(), forceUpdate);
volume.Chapters = chapters;
volume.Pages = chapters.Sum(c => c.Pages);
volumes.Add(volume);
}
foreach (var volume in volumes)
{
if (ShouldFindCoverImage(forceUpdate, volume.CoverImage))
{
// TODO: Create a custom sorter for Chapters so it's consistent across the application
var firstChapter = volume.Chapters.OrderBy(x => Double.Parse(x.Number)).FirstOrDefault();
var firstFile = firstChapter?.Files.OrderBy(x => x.Chapter).FirstOrDefault();
if (firstFile != null) volume.CoverImage = _archiveService.GetCoverImage(firstFile.FilePath, true);
}
}
return volumes;
}
public void ScanSeries(int libraryId, int seriesId)
{