AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scanner Fixes (#3619)

Browse source 
Co-authored-by: Fesaa <77553571+Fesaa@users.noreply.github.com>
This commit is contained in:
Joe Milazzo 2025-03-12 17:25:15 -05:00 • committed by GitHub
parent b4061e3711
commit ab540c0ea6
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
8 changed files with 340 additions and 19 deletions
Download patch file Download diff file Expand all files Collapse all files

8
API.Tests/Helpers/ScannerHelper.cs
View file

@ -63,10 +63,10 @@ public class ScannerHelper
return library;
}
public ScannerService CreateServices()
public ScannerService CreateServices(DirectoryService ds = null, IFileSystem fs = null)
{
var fs = new FileSystem();
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fs);
fs ??= new FileSystem();
ds ??= new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fs);
var archiveService = new ArchiveService(Substitute.For<ILogger<ArchiveService>>(), ds,
Substitute.For<IImageService>(), Substitute.For<IMediaErrorService>());
var readingItemService = new ReadingItemService(archiveService, Substitute.For<IBookService>(),
@ -133,7 +133,7 @@ public class ScannerHelper
_testOutputHelper.WriteLine($"Test Directory Path: {testDirectory}");
return testDirectory;
return Path.GetFullPath(testDirectory);
}

106
API.Tests/Services/ParseScannedFilesTests.cs
View file

@ -1,8 +1,11 @@
using System;
using System.Collections.Generic;
using System.Data.Common;
using System.IO;
using System.IO.Abstractions;
using System.IO.Abstractions.TestingHelpers;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.Data.Metadata;
@ -15,13 +18,16 @@ using API.Services;
using API.Services.Tasks.Scanner;
using API.Services.Tasks.Scanner.Parser;
using API.SignalR;
using API.Tests.Helpers;
using AutoMapper;
using Hangfire;
using Microsoft.Data.Sqlite;
using Microsoft.EntityFrameworkCore;
using Microsoft.EntityFrameworkCore.Infrastructure;
using Microsoft.Extensions.Logging;
using NSubstitute;
using Xunit;
using Xunit.Abstractions;
namespace API.Tests.Services;
@ -97,11 +103,13 @@ public class MockReadingItemService : IReadingItemService
public class ParseScannedFilesTests : AbstractDbTest
{
private readonly ILogger<ParseScannedFiles> _logger = Substitute.For<ILogger<ParseScannedFiles>>();
private readonly ScannerHelper _scannerHelper;
public ParseScannedFilesTests()
public ParseScannedFilesTests(ITestOutputHelper testOutputHelper)
{
// Since ProcessFile relies on _readingItemService, we can implement our own versions of _readingItemService so we have control over how the calls work
GlobalConfiguration.Configuration.UseInMemoryStorage();
_scannerHelper = new ScannerHelper(_unitOfWork, testOutputHelper);
}
protected override async Task ResetDb()
@ -349,4 +357,98 @@ public class ParseScannedFilesTests : AbstractDbTest
#endregion
[Fact]
public async Task HasSeriesFolderNotChangedSinceLastScan_AllSeriesFoldersHaveChanges()
{
const string testcase = "Subfolders always scanning all series changes - Manga.json";
var infos = new Dictionary<string, ComicInfo>();
var library = await _scannerHelper.GenerateScannerData(testcase, infos);
var testDirectoryPath = library.Folders.First().Path;
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
var fs = new FileSystem();
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fs);
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(ds, Substitute.For<IBookService>()), Substitute.For<IEventHub>());
var scanner = _scannerHelper.CreateServices(ds, fs);
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Equal(4, postLib.Series.Count);
var spiceAndWolf = postLib.Series.First(x => x.Name == "Spice and Wolf");
Assert.Equal(2, spiceAndWolf.Volumes.Count);
var frieren = postLib.Series.First(x => x.Name == "Frieren - Beyond Journey's End");
Assert.Single(frieren.Volumes);
var executionerAndHerWayOfLife = postLib.Series.First(x => x.Name == "The Executioner and Her Way of Life");
Assert.Equal(2, executionerAndHerWayOfLife.Volumes.Count);
Thread.Sleep(1100); // Ensure at least one second has passed since library scan
// Add a new chapter to a volume of the series, and scan. Validate that only, and all directories of this
// series are marked as HasChanged
var executionerCopyDir = Path.Join(Path.Join(testDirectoryPath, "The Executioner and Her Way of Life"),
"The Executioner and Her Way of Life Vol. 1");
File.Copy(Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1 Ch. 0001.cbz"),
Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1 Ch. 0002.cbz"));
// 4 series, of which 2 have volumes as directories
var folderMap = await _unitOfWork.SeriesRepository.GetFolderPathMap(postLib.Id);
Assert.Equal(6, folderMap.Count);
var res = await psf.ScanFiles(testDirectoryPath, true, folderMap, postLib);
var changes = res.Where(sc => sc.HasChanged).ToList();
Assert.Equal(2, changes.Count);
// Only volumes of The Executioner and Her Way of Life should be marked as HasChanged (Spice and Wolf also has 2 volumes dirs)
Assert.Equal(2, changes.Count(sc => sc.Folder.Contains("The Executioner and Her Way of Life")));
}
[Fact]
public async Task HasSeriesFolderNotChangedSinceLastScan_PublisherLayout()
{
const string testcase = "Subfolder always scanning fix publisher layout - Comic.json";
var infos = new Dictionary<string, ComicInfo>();
var library = await _scannerHelper.GenerateScannerData(testcase, infos);
var testDirectoryPath = library.Folders.First().Path;
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
var fs = new FileSystem();
var ds = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), fs);
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(ds, Substitute.For<IBookService>()), Substitute.For<IEventHub>());
var scanner = _scannerHelper.CreateServices(ds, fs);
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Equal(4, postLib.Series.Count);
var spiceAndWolf = postLib.Series.First(x => x.Name == "Spice and Wolf");
Assert.Equal(2, spiceAndWolf.Volumes.Count);
var frieren = postLib.Series.First(x => x.Name == "Frieren - Beyond Journey's End");
Assert.Equal(2, frieren.Volumes.Count);
Thread.Sleep(1100); // Ensure at least one second has passed since library scan
// Add a volume to a series, and scan. Ensure only this series is marked as HasChanged
var executionerCopyDir = Path.Join(Path.Join(testDirectoryPath, "YenPress"), "The Executioner and Her Way of Life");
File.Copy(Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1.cbz"),
Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 2.cbz"));
var res = await psf.ScanFiles(testDirectoryPath, true,
await _unitOfWork.SeriesRepository.GetFolderPathMap(postLib.Id), postLib);
var changes = res.Count(sc => sc.HasChanged);
Assert.Equal(1, changes);
}
}

153
API.Tests/Services/ScannerServiceTests.cs
View file

@ -6,6 +6,7 @@ using System.IO.Compression;
using System.Linq;
using System.Text;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using System.Xml;
using System.Xml.Serialization;
@ -519,7 +520,7 @@ public class ScannerServiceTests : AbstractDbTest
}
[Fact]
public async Task ScanLibrary_MultipleRoots_MultipleScans_DataPersists()
public async Task ScanLibrary_MultipleRoots_MultipleScans_DataPersists_Forced()
{
const string testcase = "Multiple Roots - Manga.json";
@ -552,22 +553,83 @@ public class ScannerServiceTests : AbstractDbTest
var s2 = postLib.Series.First(s => s.Name == "Accel");
Assert.Single(s2.Volumes);
// Make a change (copy a file into only 1 root)
var root1PlushFolder = Path.Join(testDirectoryPath, "Root 1/Antarctic Press/Plush");
File.Copy(Path.Join(root1PlushFolder, "Plush v02.cbz"), Path.Join(root1PlushFolder, "Plush v03.cbz"));
// Rescan to ensure nothing changes yet again
await scanner.ScanLibrary(library.Id, true);
postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.Equal(2, postLib.Series.Count);
s = postLib.Series.First(s => s.Name == "Plush");
Assert.Equal(2, s.Volumes.Count);
Assert.Equal(3, s.Volumes.Count);
s2 = postLib.Series.First(s => s.Name == "Accel");
Assert.Single(s2.Volumes);
}
//[Fact]
/// <summary>
/// Regression bug appeared where multi-root and one root gets a new file, on next scan of library,
/// the series in the other root are deleted. (This is actually failing because the file in Root 1 isn't being detected)
/// </summary>
[Fact]
public async Task ScanLibrary_MultipleRoots_MultipleScans_DataPersists_NonForced()
{
const string testcase = "Multiple Roots - Manga.json";
// Get the first file and generate a ComicInfo
var infos = new Dictionary<string, ComicInfo>();
var library = await _scannerHelper.GenerateScannerData(testcase, infos);
var testDirectoryPath =
Path.Join(
Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ScannerService/ScanTests"),
testcase.Replace(".json", string.Empty));
library.Folders =
[
new FolderPath() {Path = Path.Join(testDirectoryPath, "Root 1")},
new FolderPath() {Path = Path.Join(testDirectoryPath, "Root 2")}
];
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
var scanner = _scannerHelper.CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Equal(2, postLib.Series.Count);
var s = postLib.Series.First(s => s.Name == "Plush");
Assert.Equal(2, s.Volumes.Count);
var s2 = postLib.Series.First(s => s.Name == "Accel");
Assert.Single(s2.Volumes);
// Make a change (copy a file into only 1 root)
var root1PlushFolder = Path.Join(testDirectoryPath, "Root 1/Antarctic Press/Plush");
File.Copy(Path.Join(root1PlushFolder, "Plush v02.cbz"), Path.Join(root1PlushFolder, "Plush v03.cbz"));
// Emulate time passage by updating lastFolderScan to be a min in the past
s.LastFolderScanned = DateTime.Now.Subtract(TimeSpan.FromMinutes(1));
_context.Series.Update(s);
await _context.SaveChangesAsync();
// Rescan to ensure nothing changes yet again
await scanner.ScanLibrary(library.Id, false);
postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.Equal(2, postLib.Series.Count);
s = postLib.Series.First(s => s.Name == "Plush");
Assert.Equal(3, s.Volumes.Count);
s2 = postLib.Series.First(s => s.Name == "Accel");
Assert.Single(s2.Volumes);
}
[Fact]
public async Task ScanLibrary_AlternatingRemoval_IssueReplication()
{
// https://github.com/Kareadita/Kavita/issues/3476#issuecomment-2661635558
// TODO: Come back to this, it's complicated
const string testcase = "Alternating Removal - Manga.json";
// Setup: Generate test library
@ -602,6 +664,14 @@ public class ScannerServiceTests : AbstractDbTest
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
// Emulate time passage by updating lastFolderScan to be a min in the past
foreach (var s in postLib.Series)
{
s.LastFolderScanned = DateTime.Now.Subtract(TimeSpan.FromMinutes(1));
_context.Series.Update(s);
}
await _context.SaveChangesAsync();
await scanner.ScanLibrary(library.Id);
postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
@ -617,12 +687,28 @@ public class ScannerServiceTests : AbstractDbTest
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
// Emulate time passage by updating lastFolderScan to be a min in the past
foreach (var s in postLib.Series)
{
s.LastFolderScanned = DateTime.Now.Subtract(TimeSpan.FromMinutes(1));
_context.Series.Update(s);
}
await _context.SaveChangesAsync();
await scanner.ScanLibrary(library.Id);
postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.Contains(postLib.Series, s => s.Name == "Accel"); // Accel should be back
Assert.Contains(postLib.Series, s => s.Name == "Plush");
// Emulate time passage by updating lastFolderScan to be a min in the past
foreach (var s in postLib.Series)
{
s.LastFolderScanned = DateTime.Now.Subtract(TimeSpan.FromMinutes(1));
_context.Series.Update(s);
}
await _context.SaveChangesAsync();
// Fourth Scan: Run again to check stability (should not remove Accel)
await scanner.ScanLibrary(library.Id);
postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
@ -677,4 +763,63 @@ public class ScannerServiceTests : AbstractDbTest
Assert.Contains(postLib.Series, s => s.Name == "Accel"); // Ensure Accel is gone
Assert.Contains(postLib.Series, s => s.Name == "Plush");
}
[Fact]
public async Task SubFolders_NoRemovals_ChangesFound()
{
const string testcase = "Subfolders always scanning all series changes - Manga.json";
var infos = new Dictionary<string, ComicInfo>();
var library = await _scannerHelper.GenerateScannerData(testcase, infos);
var testDirectoryPath = library.Folders.First().Path;
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
var scanner = _scannerHelper.CreateServices();
await scanner.ScanLibrary(library.Id);
var postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Equal(4, postLib.Series.Count);
var spiceAndWolf = postLib.Series.First(x => x.Name == "Spice and Wolf");
Assert.Equal(2, spiceAndWolf.Volumes.Count);
Assert.Equal(3, spiceAndWolf.Volumes.Sum(v => v.Chapters.Count));
var frieren = postLib.Series.First(x => x.Name == "Frieren - Beyond Journey's End");
Assert.Single(frieren.Volumes);
Assert.Equal(2, frieren.Volumes.Sum(v => v.Chapters.Count));
var executionerAndHerWayOfLife = postLib.Series.First(x => x.Name == "The Executioner and Her Way of Life");
Assert.Equal(2, executionerAndHerWayOfLife.Volumes.Count);
Assert.Equal(2, executionerAndHerWayOfLife.Volumes.Sum(v => v.Chapters.Count));
Thread.Sleep(1100); // Ensure at least one second has passed since library scan
// Add a new chapter to a volume of the series, and scan. Validate that no chapters were lost, and the new
// chapter was added
var executionerCopyDir = Path.Join(Path.Join(testDirectoryPath, "The Executioner and Her Way of Life"),
"The Executioner and Her Way of Life Vol. 1");
File.Copy(Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1 Ch. 0001.cbz"),
Path.Join(executionerCopyDir, "The Executioner and Her Way of Life Vol. 1 Ch. 0002.cbz"));
await scanner.ScanLibrary(library.Id);
await _unitOfWork.CommitAsync();
postLib = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(library.Id, LibraryIncludes.Series);
Assert.NotNull(postLib);
Assert.Equal(4, postLib.Series.Count);
spiceAndWolf = postLib.Series.First(x => x.Name == "Spice and Wolf");
Assert.Equal(2, spiceAndWolf.Volumes.Count);
Assert.Equal(3, spiceAndWolf.Volumes.Sum(v => v.Chapters.Count));
frieren = postLib.Series.First(x => x.Name == "Frieren - Beyond Journey's End");
Assert.Single(frieren.Volumes);
Assert.Equal(2, frieren.Volumes.Sum(v => v.Chapters.Count));
executionerAndHerWayOfLife = postLib.Series.First(x => x.Name == "The Executioner and Her Way of Life");
Assert.Equal(2, executionerAndHerWayOfLife.Volumes.Count);
Assert.Equal(3, executionerAndHerWayOfLife.Volumes.Sum(v => v.Chapters.Count)); // Incremented by 1
}
}

8
API.Tests/Services/Test Data/ScannerService/TestCases/Subfolder always scanning fix publisher layout - Comic.json Normal file
View file

@ -0,0 +1,8 @@
[
"VizMedia/Frieren - Beyond Journey's End/Frieren - Beyond Journey's End Vol. 1.cbz",
"VizMedia/Frieren - Beyond Journey's End/Frieren - Beyond Journey's End Vol. 2.cbz",
"VizMedia/Seraph of the End/Seraph of the End Vol. 1.cbz",
"YenPress/Spice and Wolf/Spice and Wolf Vol. 1.cbz",
"YenPress/Spice and Wolf/Spice and Wolf Vol. 2.cbz",
"YenPress/The Executioner and Her Way of Life/The Executioner and Her Way of Life Vol. 1.cbz"
]

11
API.Tests/Services/Test Data/ScannerService/TestCases/Subfolders always scanning all series changes - Manga.json Normal file
View file

@ -0,0 +1,11 @@
[
"Frieren - Beyond Journey's End/Frieren - Beyond Journey's End Vol. 1/Frieren - Beyond Journey's End Ch. 0001.cbz",
"Frieren - Beyond Journey's End/Frieren - Beyond Journey's End Vol. 1/Frieren - Beyond Journey's End Ch. 0002.cbz",
"Seraph of the End/Seraph of the End Vol. 1/Seraph of the End Ch. 0001.cbz",
"Spice and Wolf/Spice and Wolf Vol. 1/Spice and Wolf Vol. 1 Ch. 0001.cbz",
"Spice and Wolf/Spice and Wolf Vol. 1/Spice and Wolf Vol. 1 Ch. 0002.cbz",
"Spice and Wolf/Spice and Wolf Vol. 2/Spice and Wolf Vol. 2 Ch. 0003.cbz",
"The Executioner and Her Way of Life/The Executioner and Her Way of Life Vol. 1/The Executioner and Her Way of Life Vol. 1 Ch. 0001.cbz",
"The Executioner and Her Way of Life/The Executioner and Her Way of Life Vol. 2/The Executioner and Her Way of Life Vol. 2 Ch. 0003.cbz"
]