AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scanner Tech Debt (#2777)

Browse source
This commit is contained in:
Joe Milazzo 2024-03-12 15:10:43 -06:00 • committed by GitHub
parent f5a31b9a02
commit 08cc7c7cbd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 839 additions and 676 deletions
Download patch file Download diff file Expand all files Collapse all files

36
API.Tests/Helpers/GenreHelperTests.cs
View file

@ -1,6 +1,7 @@
using System.Collections.Generic;
using API.Data;
using API.Entities;
using API.Extensions;
using API.Helpers;
using API.Helpers.Builders;
using Xunit;
@ -12,42 +13,51 @@ public class GenreHelperTests
[Fact]
public void UpdateGenre_ShouldAddNewGenre()
{
var allGenres = new List<Genre>
var allGenres = new Dictionary<string, Genre>
{
new GenreBuilder("Action").Build(),
new GenreBuilder("action").Build(),
new GenreBuilder("Sci-fi").Build(),
{"Action".ToNormalized(), new GenreBuilder("Action").Build()},
{"Sci-fi".ToNormalized(), new GenreBuilder("Sci-fi").Build()}
};
var genreAdded = new List<Genre>();
var addedCount = 0;
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Adventure"}, genre =>
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Adventure"}, (genre, isNew) =>
{
if (isNew)
{
addedCount++;
}
genreAdded.Add(genre);
});
Assert.Equal(2, genreAdded.Count);
Assert.Equal(4, allGenres.Count);
Assert.Equal(1, addedCount);
Assert.Equal(3, allGenres.Count);
}
[Fact]
public void UpdateGenre_ShouldNotAddDuplicateGenre()
{
var allGenres = new List<Genre>
var allGenres = new Dictionary<string, Genre>
{
new GenreBuilder("Action").Build(),
new GenreBuilder("action").Build(),
new GenreBuilder("Sci-fi").Build(),
{"Action".ToNormalized(), new GenreBuilder("Action").Build()},
{"Sci-fi".ToNormalized(), new GenreBuilder("Sci-fi").Build()}
};
var genreAdded = new List<Genre>();
var addedCount = 0;
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Scifi"}, genre =>
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Scifi"}, (genre, isNew) =>
{
if (isNew)
{
addedCount++;
}
genreAdded.Add(genre);
});
Assert.Equal(3, allGenres.Count);
Assert.Equal(0, addedCount);
Assert.Equal(2, genreAdded.Count);
Assert.Equal(2, allGenres.Count);
}
[Fact]

40
API.Tests/Helpers/TagHelperTests.cs
View file

@ -1,6 +1,8 @@
using System.Collections.Generic;
using System.Linq;
using API.Data;
using API.Entities;
using API.Extensions;
using API.Helpers;
using API.Helpers.Builders;
using Xunit;
@ -12,50 +14,50 @@ public class TagHelperTests
[Fact]
public void UpdateTag_ShouldAddNewTag()
{
var allTags = new List<Tag>
var allTags = new Dictionary<string, Tag>
{
new TagBuilder("Action").Build(),
new TagBuilder("action").Build(),
new TagBuilder("Sci-fi").Build(),
{"Action".ToNormalized(), new TagBuilder("Action").Build()},
{"Sci-fi".ToNormalized(), new TagBuilder("Sci-fi").Build()}
};
var tagAdded = new List<Tag>();
var tagCalled = new List<Tag>();
var addedCount = 0;
TagHelper.UpdateTag(allTags, new[] {"Action", "Adventure"}, (tag, added) =>
{
if (added)
{
tagAdded.Add(tag);
addedCount++;
}
tagCalled.Add(tag);
});
Assert.Single(tagAdded);
Assert.Equal(4, allTags.Count);
Assert.Equal(1, addedCount);
Assert.Equal(2, tagCalled.Count());
Assert.Equal(3, allTags.Count);
}
[Fact]
public void UpdateTag_ShouldNotAddDuplicateTag()
{
var allTags = new List<Tag>
var allTags = new Dictionary<string, Tag>
{
new TagBuilder("Action").Build(),
new TagBuilder("action").Build(),
new TagBuilder("Sci-fi").Build(),
{"Action".ToNormalized(), new TagBuilder("Action").Build()},
{"Sci-fi".ToNormalized(), new TagBuilder("Sci-fi").Build()}
};
var tagAdded = new List<Tag>();
var tagCalled = new List<Tag>();
var addedCount = 0;
TagHelper.UpdateTag(allTags, new[] {"Action", "Scifi"}, (tag, added) =>
{
if (added)
{
tagAdded.Add(tag);
addedCount++;
}
TagHelper.AddTagIfNotExists(allTags, tag);
tagCalled.Add(tag);
});
Assert.Equal(3, allTags.Count);
Assert.Empty(tagAdded);
Assert.Equal(2, allTags.Count);
Assert.Equal(0, addedCount);
}
[Fact]

189
API.Tests/Services/ParseScannedFilesTests.cs
View file

@ -65,88 +65,23 @@ internal class MockReadingItemService : IReadingItemService
}
}
public class ParseScannedFilesTests
public class ParseScannedFilesTests : AbstractDbTest
{
private readonly ILogger<ParseScannedFiles> _logger = Substitute.For<ILogger<ParseScannedFiles>>();
private readonly IUnitOfWork _unitOfWork;
private readonly DbConnection _connection;
private readonly DataContext _context;
private const string CacheDirectory = "C:/kavita/config/cache/";
private const string CoverImageDirectory = "C:/kavita/config/covers/";
private const string BackupDirectory = "C:/kavita/config/backups/";
private const string DataDirectory = "C:/data/";
public ParseScannedFilesTests()
{
var contextOptions = new DbContextOptionsBuilder()
.UseSqlite(CreateInMemoryDatabase())
.Options;
_connection = RelationalOptionsExtension.Extract(contextOptions).Connection;
_context = new DataContext(contextOptions);
Task.Run(SeedDb).GetAwaiter().GetResult();
_unitOfWork = new UnitOfWork(_context, Substitute.For<IMapper>(), null);
// Since ProcessFile relies on _readingItemService, we can implement our own versions of _readingItemService so we have control over how the calls work
}
#region Setup
private static DbConnection CreateInMemoryDatabase()
{
var connection = new SqliteConnection("Filename=:memory:");
connection.Open();
return connection;
}
private async Task<bool> SeedDb()
{
await _context.Database.MigrateAsync();
var filesystem = CreateFileSystem();
await Seed.SeedSettings(_context, new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), filesystem));
var setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.CacheDirectory).SingleAsync();
setting.Value = CacheDirectory;
setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.BackupDirectory).SingleAsync();
setting.Value = BackupDirectory;
_context.ServerSetting.Update(setting);
_context.Library.Add(new LibraryBuilder("Manga")
.WithFolderPath(new FolderPathBuilder(DataDirectory).Build())
.Build());
return await _context.SaveChangesAsync() > 0;
}
private async Task ResetDB()
protected override async Task ResetDb()
{
_context.Series.RemoveRange(_context.Series.ToList());
await _context.SaveChangesAsync();
}
private static MockFileSystem CreateFileSystem()
{
var fileSystem = new MockFileSystem();
fileSystem.Directory.SetCurrentDirectory("C:/kavita/");
fileSystem.AddDirectory("C:/kavita/config/");
fileSystem.AddDirectory(CacheDirectory);
fileSystem.AddDirectory(CoverImageDirectory);
fileSystem.AddDirectory(BackupDirectory);
fileSystem.AddDirectory(DataDirectory);
return fileSystem;
}
#endregion
#region MergeName
// NOTE: I don't think I can test MergeName as it relies on Tracking Files, which is more complicated than I need
@ -219,6 +154,15 @@ public class ParseScannedFilesTests
#region ScanLibrariesForSeries
/// <summary>
/// Test that when a folder has 2 series with a localizedSeries, they combine into one final series
/// </summary>
// [Fact]
// public async Task ScanLibrariesForSeries_ShouldCombineSeries()
// {
// // TODO: Implement these unit tests
// }
[Fact]
public async Task ScanLibrariesForSeries_ShouldFindFiles()
{
@ -233,34 +177,40 @@ public class ParseScannedFilesTests
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return Task.CompletedTask;
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles.First().Series,
NormalizedName = parsedFiles.First().Series.ToNormalized(),
Format = parsedFiles.First().Format
};
parsedSeries.Add(foundParsedSeries, parsedFiles);
return Task.CompletedTask;
}
// var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
//
// Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
// {
// var skippedScan = parsedInfo.Item1;
// var parsedFiles = parsedInfo.Item2;
// if (parsedFiles.Count == 0) return Task.CompletedTask;
//
// var foundParsedSeries = new ParsedSeries()
// {
// Name = parsedFiles.First().Series,
// NormalizedName = parsedFiles.First().Series.ToNormalized(),
// Format = parsedFiles.First().Format
// };
//
// parsedSeries.Add(foundParsedSeries, parsedFiles);
// return Task.CompletedTask;
// }
var library =
await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
Assert.NotNull(library);
library.Type = LibraryType.Manga;
await psf.ScanLibrariesForSeries(library, new List<string>() {"C:/Data/"}, false, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), TrackFiles);
var parsedSeries = await psf.ScanLibrariesForSeries(library, new List<string>() {"C:/Data/"}, false,
await _unitOfWork.SeriesRepository.GetFolderPathMap(1));
Assert.Equal(3, parsedSeries.Values.Count);
Assert.NotEmpty(parsedSeries.Keys.Where(p => p.Format == MangaFormat.Archive && p.Name.Equals("Accel World")));
// Assert.Equal(3, parsedSeries.Values.Count);
// Assert.NotEmpty(parsedSeries.Keys.Where(p => p.Format == MangaFormat.Archive && p.Name.Equals("Accel World")));
Assert.Equal(3, parsedSeries.Count);
Assert.NotEmpty(parsedSeries.Select(p => p.ParsedSeries).Where(p => p.Format == MangaFormat.Archive && p.Name.Equals("Accel World")));
}
#endregion
@ -292,15 +242,13 @@ public class ParseScannedFilesTests
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var directoriesSeen = new HashSet<string>();
var library =
await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
await psf.ProcessFiles("C:/Data/", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1),
(files, directoryPath, libraryFolder) =>
var scanResults = psf.ProcessFiles("C:/Data/", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
foreach (var scanResult in scanResults)
{
directoriesSeen.Add(directoryPath);
return Task.CompletedTask;
}, library);
directoriesSeen.Add(scanResult.Folder);
}
Assert.Equal(2, directoriesSeen.Count);
}
@ -313,14 +261,18 @@ public class ParseScannedFilesTests
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
Assert.NotNull(library);
var directoriesSeen = new HashSet<string>();
await psf.ProcessFiles("C:/Data/", false, await _unitOfWork.SeriesRepository.GetFolderPathMap(1),
(files, directoryPath, libraryFolder) =>
var scanResults = psf.ProcessFiles("C:/Data/", false,
await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
foreach (var scanResult in scanResults)
{
directoriesSeen.Add(directoryPath);
return Task.CompletedTask;
}, await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes));
directoriesSeen.Add(scanResult.Folder);
}
Assert.Single(directoriesSeen);
directoriesSeen.TryGetValue("C:/Data/", out var actual);
@ -344,17 +296,12 @@ public class ParseScannedFilesTests
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var callCount = 0;
await psf.ProcessFiles("C:/Data", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1),
(files, folderPath, libraryFolder) =>
{
callCount++;
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
Assert.NotNull(library);
var scanResults = psf.ProcessFiles("C:/Data", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
return Task.CompletedTask;
}, await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes));
Assert.Equal(2, callCount);
Assert.Equal(2, scanResults.Count);
}
@ -378,17 +325,17 @@ public class ParseScannedFilesTests
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var callCount = 0;
await psf.ProcessFiles("C:/Data", false, await _unitOfWork.SeriesRepository.GetFolderPathMap(1),
(files, folderPath, libraryFolder) =>
{
callCount++;
return Task.CompletedTask;
}, await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes));
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
Assert.NotNull(library);
var scanResults = psf.ProcessFiles("C:/Data", false,
await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
Assert.Equal(1, callCount);
Assert.Single(scanResults);
}
#endregion
}

6
API/Controllers/LicenseController.cs
View file

@ -32,7 +32,11 @@ public class LicenseController(
public async Task<ActionResult<bool>> HasValidLicense(bool forceCheck = false)
{
var result = await licenseService.HasActiveLicense(forceCheck);
await taskScheduler.ScheduleKavitaPlusTasks();
if (result)
{
await taskScheduler.ScheduleKavitaPlusTasks();
}
return Ok(result);
}

7
API/Data/Metadata/ComicInfo.cs
View file

@ -177,7 +177,12 @@ public class ComicInfo
if (!string.IsNullOrEmpty(info.Number))
{
info.Number = info.Number.Replace(",", "."); // Corrective measure for non English OSes
info.Number = info.Number.Trim().Replace(",", "."); // Corrective measure for non English OSes
}
if (!string.IsNullOrEmpty(info.Volume))
{
info.Volume = info.Volume.Trim();
}
}

8
API/Data/Repositories/ExternalSeriesMetadataRepository.cs
View file

@ -34,6 +34,7 @@ public interface IExternalSeriesMetadataRepository
Task<bool> ExternalSeriesMetadataNeedsRefresh(int seriesId);
Task<SeriesDetailPlusDto> GetSeriesDetailPlusDto(int seriesId);
Task LinkRecommendationsToSeries(Series series);
Task LinkRecommendationsToSeries(int seriesId);
Task<bool> IsBlacklistedSeries(int seriesId);
Task CreateBlacklistedSeries(int seriesId, bool saveChanges = true);
Task RemoveFromBlacklist(int seriesId);
@ -179,6 +180,13 @@ public class ExternalSeriesMetadataRepository : IExternalSeriesMetadataRepositor
return seriesDetailPlusDto;
}
public async Task LinkRecommendationsToSeries(int seriesId)
{
var series = await _context.Series.Where(s => s.Id == seriesId).AsNoTracking().SingleOrDefaultAsync();
if (series == null) return;
await LinkRecommendationsToSeries(series);
}
/// <summary>
/// Searches Recommendations without a SeriesId on record and attempts to link based on Series Name/Localized Name
/// </summary>

1
API/Entities/Series.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using API.Entities.Enums;
using API.Entities.Interfaces;
using API.Entities.Metadata;
using API.Services.Tasks.Scanner.Parser;
namespace API.Entities;

1
API/Extensions/ApplicationServiceExtensions.cs
View file

@ -60,6 +60,7 @@ public static class ApplicationServiceExtensions
services.AddScoped<ILibraryWatcher, LibraryWatcher>();
services.AddScoped<ITachiyomiService, TachiyomiService>();
services.AddScoped<ICollectionTagService, CollectionTagService>();
services.AddScoped<ITagManagerService, TagManagerService>();
services.AddScoped<IFileSystem, FileSystem>();
services.AddScoped<IDirectoryService, DirectoryService>();

22
API/Helpers/GenreHelper.cs
View file

@ -12,25 +12,28 @@ namespace API.Helpers;
public static class GenreHelper
{
public static void UpdateGenre(ICollection<Genre> allGenres, IEnumerable<string> names, Action<Genre> action)
public static void UpdateGenre(Dictionary<string, Genre> allGenres,
IEnumerable<string> names, Action<Genre, bool> action)
{
foreach (var name in names)
{
if (string.IsNullOrEmpty(name.Trim())) continue;
var normalizedName = name.ToNormalized();
var genre = allGenres.FirstOrDefault(p => p.NormalizedTitle != null && p.NormalizedTitle.Equals(normalizedName));
if (genre == null)
if (string.IsNullOrEmpty(normalizedName)) continue;
if (allGenres.TryGetValue(normalizedName, out var genre))
{
action(genre, false);
}
else
{
genre = new GenreBuilder(name).Build();
allGenres.Add(genre);
allGenres.Add(normalizedName, genre);
action(genre, true);
}
action(genre);
}
}
public static void KeepOnlySameGenreBetweenLists(ICollection<Genre> existingGenres, ICollection<Genre> removeAllExcept, Action<Genre>? action = null)
{
var existing = existingGenres.ToList();
@ -64,6 +67,7 @@ public static class GenreHelper
public static void UpdateGenreList(ICollection<GenreTagDto>? tags, Series series,
IReadOnlyCollection<Genre> allTags, Action<Genre> handleAdd, Action onModified)
{
// TODO: Write some unit tests
if (tags == null) return;
var isModified = false;
// I want a union of these 2 lists. Return only elements that are in both lists, but the list types are different

2
API/Helpers/PersonHelper.cs
View file

@ -12,6 +12,7 @@ namespace API.Helpers;
public static class PersonHelper
{
/// <summary>
/// Given a list of all existing people, this will check the new names and roles and if it doesn't exist in allPeople, will create and
/// add an entry. For each person in name, the callback will be executed.
@ -24,7 +25,6 @@ public static class PersonHelper
/// <param name="action"></param>
public static void UpdatePeople(ICollection<Person> allPeople, IEnumerable<string> names, PersonRole role, Action<Person> action)
{
// TODO: Validate if we need this, not used
var allPeopleTypeRole = allPeople.Where(p => p.Role == role).ToList();
foreach (var name in names)

40
API/Helpers/TagHelper.cs
View file

@ -1,43 +1,37 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq;
using API.Data;
using API.DTOs.Metadata;
using API.Entities;
using API.Extensions;
using API.Helpers.Builders;
using API.Services.Tasks.Scanner.Parser;
namespace API.Helpers;
#nullable enable
public static class TagHelper
{
/// <summary>
///
/// </summary>
/// <param name="allTags"></param>
/// <param name="names"></param>
/// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param>
public static void UpdateTag(ICollection<Tag> allTags, IEnumerable<string> names, Action<Tag, bool> action)
public static void UpdateTag(Dictionary<string, Tag> allTags, IEnumerable<string> names, Action<Tag, bool> action)
{
foreach (var name in names)
{
if (string.IsNullOrEmpty(name.Trim())) continue;
var added = false;
var normalizedName = name.ToNormalized();
allTags.TryGetValue(normalizedName, out var tag);
var genre = allTags.FirstOrDefault(p =>
p.NormalizedTitle.Equals(normalizedName));
if (genre == null)
var added = tag == null;
if (tag == null)
{
added = true;
genre = new TagBuilder(name).Build();
allTags.Add(genre);
tag = new TagBuilder(name).Build();
allTags.Add(normalizedName, tag);
}
action(genre, added);
action(tag, added);
}
}
@ -79,6 +73,22 @@ public static class TagHelper
}
}
public static IList<string> GetTagValues(string comicInfoTagSeparatedByComma)
{
// TODO: Unit tests needed
if (string.IsNullOrEmpty(comicInfoTagSeparatedByComma))
{
return ImmutableList<string>.Empty;
}
return comicInfoTagSeparatedByComma.Split(",")
.Select(s => s.Trim())
.DistinctBy(Parser.Normalize)
.ToList();
}
/// <summary>
/// Remove tags on a list
/// </summary>

3
API/Services/Plus/ExternalMetadataService.cs
View file

@ -70,7 +70,7 @@ public class ExternalMetadataService : IExternalMetadataService
private readonly IMapper _mapper;
private readonly ILicenseService _licenseService;
private readonly TimeSpan _externalSeriesMetadataCache = TimeSpan.FromDays(30);
public static readonly ImmutableArray<LibraryType> NonEligibleLibraryTypes = ImmutableArray.Create<LibraryType>(LibraryType.Comic, LibraryType.Book);
public static readonly ImmutableArray<LibraryType> NonEligibleLibraryTypes = ImmutableArray.Create<LibraryType>(LibraryType.Comic, LibraryType.Book, LibraryType.Image, LibraryType.ComicVine);
private readonly SeriesDetailPlusDto _defaultReturn = new()
{
Recommendations = null,
@ -155,6 +155,7 @@ public class ExternalMetadataService : IExternalMetadataService
public async Task GetNewSeriesData(int seriesId, LibraryType libraryType)
{
if (!IsPlusEligible(libraryType)) return;
if (!await _licenseService.HasActiveLicense()) return;
// Generate key based on seriesId and libraryType or any unique identifier for the request
// Check if the request is allowed based on the rate limit

16
API/Services/ReadingListService.cs
View file

@ -46,6 +46,8 @@ public interface IReadingListService
/// <param name="library"></param>
/// <returns></returns>
Task CreateReadingListsFromSeries(Series series, Library library);
Task CreateReadingListsFromSeries(int libraryId, int seriesId);
}
/// <summary>
@ -407,6 +409,20 @@ public class ReadingListService : IReadingListService
return index > lastOrder + 1;
}
/// <summary>
/// Create Reading lists from a Series
/// </summary>
/// <remarks>Execute this from Hangfire</remarks>
/// <param name="libraryId"></param>
/// <param name="seriesId"></param>
public async Task CreateReadingListsFromSeries(int libraryId, int seriesId)
{
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId);
if (series == null || library == null) return;
await CreateReadingListsFromSeries(series, library);
}
public async Task CreateReadingListsFromSeries(Series series, Library library)
{
if (!library.ManageReadingLists) return;

16
API/Services/TaskScheduler.cs
View file

@ -328,13 +328,13 @@ public class TaskScheduler : ITaskScheduler
}
if (RunningAnyTasksByMethod(ScanTasks, ScanQueue))
{
_logger.LogInformation("A Library Scan is already running, rescheduling ScanLibrary in 3 hours");
_logger.LogInformation("A Scan is already running, rescheduling ScanLibrary in 3 hours");
BackgroundJob.Schedule(() => ScanLibrary(libraryId, force), TimeSpan.FromHours(3));
return;
}
_logger.LogInformation("Enqueuing library scan for: {LibraryId}", libraryId);
BackgroundJob.Enqueue(() => _scannerService.ScanLibrary(libraryId, force));
BackgroundJob.Enqueue(() => _scannerService.ScanLibrary(libraryId, force, true));
// When we do a scan, force cache to re-unpack in case page numbers change
BackgroundJob.Enqueue(() => _cleanupService.CleanupCacheAndTempDirectories());
}
@ -386,7 +386,7 @@ public class TaskScheduler : ITaskScheduler
}
if (RunningAnyTasksByMethod(ScanTasks, ScanQueue))
{
// BUG: This can end up triggering a ton of scan series calls
// BUG: This can end up triggering a ton of scan series calls (but i haven't seen in practice)
_logger.LogInformation("A Scan is already running, rescheduling ScanSeries in 10 minutes");
BackgroundJob.Schedule(() => ScanSeries(libraryId, seriesId, forceUpdate), TimeSpan.FromMinutes(10));
return;
@ -428,8 +428,14 @@ public class TaskScheduler : ITaskScheduler
public static bool HasScanTaskRunningForLibrary(int libraryId, bool checkRunningJobs = true)
{
return
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, true}, ScanQueue, checkRunningJobs) ||
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, false}, ScanQueue, checkRunningJobs);
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, true, true}, ScanQueue,
checkRunningJobs) ||
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, false, true}, ScanQueue,
checkRunningJobs) ||
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, true, false}, ScanQueue,
checkRunningJobs) ||
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, false, false}, ScanQueue,
checkRunningJobs);
}
/// <summary>

311
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -31,6 +31,49 @@ public class ParsedSeries
public required MangaFormat Format { get; init; }
}
public class ScanResult
{
/// <summary>
/// A list of files in the Folder. Empty if HasChanged = false
/// </summary>
public IList<string> Files { get; set; }
/// <summary>
/// A nested folder from Library Root (at any level)
/// </summary>
public string Folder { get; set; }
/// <summary>
/// The library root
/// </summary>
public string LibraryRoot { get; set; }
/// <summary>
/// Was the Folder scanned or not. If not modified since last scan, this will be false and Files empty
/// </summary>
public bool HasChanged { get; set; }
/// <summary>
/// Set in Stage 2: Parsed Info from the Files
/// </summary>
public IList<ParserInfo> ParserInfos { get; set; }
}
/// <summary>
/// The final product of ParseScannedFiles. This has all the processed parserInfo and is ready for tracking/processing into entities
/// </summary>
public class ScannedSeriesResult
{
/// <summary>
/// Was the Folder scanned or not. If not modified since last scan, this will be false and indicates that upstream should count this as skipped
/// </summary>
public bool HasChanged { get; set; }
/// <summary>
/// The Parsed Series information used for tracking
/// </summary>
public ParsedSeries ParsedSeries { get; set; }
/// <summary>
/// Parsed files
/// </summary>
public IList<ParserInfo> ParsedInfos { get; set; }
}
public class SeriesModified
{
public required string FolderPath { get; set; }
@ -75,112 +118,79 @@ public class ParseScannedFiles
/// <param name="scanDirectoryByDirectory">Scan directory by directory and for each, call folderAction</param>
/// <param name="seriesPaths">A dictionary mapping a normalized path to a list of <see cref="SeriesModified"/> to help scanner skip I/O</param>
/// <param name="folderPath">A library folder or series folder</param>
/// <param name="folderAction">A callback async Task to be called once all files for each folder path are found</param>
/// <param name="forceCheck">If we should bypass any folder last write time checks on the scan and force I/O</param>
public async Task ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<IList<string>, string, string, Task> folderAction, Library library, bool forceCheck = false)
public IList<ScanResult> ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
IDictionary<string, IList<SeriesModified>> seriesPaths, Library library, bool forceCheck = false)
{
string normalizedPath;
var result = new List<ScanResult>();
var fileExtensions = string.Join("|", library.LibraryFileTypes.Select(l => l.FileTypeGroup.GetRegex()));
if (scanDirectoryByDirectory)
{
// This is used in library scan, so we should check first for a ignore file and use that here as well
var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(folderPath, DirectoryService.KavitaIgnoreFile);
var matcher = _directoryService.CreateMatcherFromFile(potentialIgnoreFile);
if (matcher != null)
var matcher = new GlobMatcher();
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
{
_logger.LogWarning(".kavitaignore found! Ignore files is deprecated in favor of Library Settings. Please update and remove file at {Path}", potentialIgnoreFile);
matcher.AddExclude(pattern.Pattern);
}
if (library.LibraryExcludePatterns.Count != 0)
{
matcher ??= new GlobMatcher();
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
{
matcher.AddExclude(pattern.Pattern);
}
}
var directories = _directoryService.GetDirectories(folderPath, matcher).ToList();
foreach (var directory in directories)
{
// Since this is a loop, we need a list return
normalizedPath = Parser.Parser.NormalizePath(directory);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
{
await folderAction(new List<string>(), directory, folderPath);
result.Add(new ScanResult()
{
Files = ArraySegment<string>.Empty,
Folder = directory,
LibraryRoot = folderPath,
HasChanged = false
});
}
else
{
// For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication
await folderAction(_directoryService.ScanFiles(directory, fileExtensions, matcher), directory, folderPath);
result.Add(new ScanResult()
{
Files = _directoryService.ScanFiles(directory, fileExtensions, matcher),
Folder = directory,
LibraryRoot = folderPath,
HasChanged = true
});
}
}
return;
return result;
}
normalizedPath = Parser.Parser.NormalizePath(folderPath);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
{
await folderAction(new List<string>(), folderPath, folderPath);
return;
}
// We need to calculate all folders till library root and see if any kavitaignores
var seriesMatcher = BuildIgnoreFromLibraryRoot(folderPath, seriesPaths);
await folderAction(_directoryService.ScanFiles(folderPath, fileExtensions, seriesMatcher), folderPath, folderPath);
}
/// <summary>
/// Used in ScanSeries, which enters at a lower level folder and hence needs a .kavitaignore from higher (up to root) to be built before
/// the scan takes place.
/// </summary>
/// <param name="folderPath"></param>
/// <param name="seriesPaths"></param>
/// <returns>A GlobMatter. Empty if not applicable</returns>
private GlobMatcher BuildIgnoreFromLibraryRoot(string folderPath, IDictionary<string, IList<SeriesModified>> seriesPaths)
{
var seriesMatcher = new GlobMatcher();
try
{
var roots = seriesPaths[folderPath][0].LibraryRoots.Select(Parser.Parser.NormalizePath).ToList();
var libraryFolder = roots.SingleOrDefault(folderPath.Contains);
if (string.IsNullOrEmpty(libraryFolder) || !Directory.Exists(folderPath))
result.Add(new ScanResult()
{
return seriesMatcher;
}
var allParents = _directoryService.GetFoldersTillRoot(libraryFolder, folderPath);
var path = libraryFolder;
// Apply the library root level kavitaignore
var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(path, DirectoryService.KavitaIgnoreFile);
seriesMatcher.Merge(_directoryService.CreateMatcherFromFile(potentialIgnoreFile));
// Then apply kavitaignores for each folder down to where the series folder is
foreach (var folderPart in allParents.Reverse())
{
path = Parser.Parser.NormalizePath(Path.Join(libraryFolder, folderPart));
potentialIgnoreFile = _directoryService.FileSystem.Path.Join(path, DirectoryService.KavitaIgnoreFile);
seriesMatcher.Merge(_directoryService.CreateMatcherFromFile(potentialIgnoreFile));
}
Files = ArraySegment<string>.Empty,
Folder = folderPath,
LibraryRoot = folderPath,
HasChanged = false
});
}
catch (Exception ex)
result.Add(new ScanResult()
{
_logger.LogError(ex,
"[ScannerService] There was an error trying to find and apply .kavitaignores above the Series Folder. Scanning without them present");
}
Files = _directoryService.ScanFiles(folderPath, fileExtensions),
Folder = folderPath,
LibraryRoot = folderPath,
HasChanged = true
});
return seriesMatcher;
return result;
}
/// <summary>
/// Attempts to either add a new instance of a show mapping to the _scannedSeries bag or adds to an existing.
/// Attempts to either add a new instance of a series mapping to the _scannedSeries bag or adds to an existing.
/// This will check if the name matches an existing series name (multiple fields) <see cref="MergeName"/>
/// </summary>
/// <param name="scannedSeries">A localized list of a series' parsed infos</param>
@ -290,20 +300,62 @@ public class ParseScannedFiles
/// <param name="folders"></param>
/// <param name="isLibraryScan">If true, does a directory scan first (resulting in folders being tackled in parallel), else does an immediate scan files</param>
/// <param name="seriesPaths">A map of Series names -> existing folder paths to handle skipping folders</param>
/// <param name="processSeriesInfos">Action which returns if the folder was skipped and the infos from said folder</param>
/// <param name="forceCheck">Defaults to false</param>
/// <returns></returns>
public async Task ScanLibrariesForSeries(Library library,
public async Task<IList<ScannedSeriesResult>> ScanLibrariesForSeries(Library library,
IEnumerable<string> folders, bool isLibraryScan,
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<Tuple<bool, IList<ParserInfo>>, Task>? processSeriesInfos, bool forceCheck = false)
IDictionary<string, IList<SeriesModified>> seriesPaths, bool forceCheck = false)
{
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", library.Name, ProgressEventType.Started));
var processedScannedSeries = new List<ScannedSeriesResult>();
foreach (var folderPath in folders)
{
try
{
await ProcessFiles(folderPath, isLibraryScan, seriesPaths, ProcessFolder, library, forceCheck);
var scanResults = ProcessFiles(folderPath, isLibraryScan, seriesPaths, library, forceCheck);
foreach (var scanResult in scanResults)
{
// scanResult is updated with the parsed infos
await ProcessScanResult(scanResult, seriesPaths, library);
// We now have all the parsed infos from the scan result, perform any merging that is necessary and post processing steps
var scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
// Merge any series together (like Nagatoro/nagator.cbz, japanesename.cbz) -> Nagator series
MergeLocalizedSeriesWithSeries(scanResult.ParserInfos);
// Combine everything into scannedSeries
foreach (var info in scanResult.ParserInfos)
{
try
{
TrackSeries(scannedSeries, info);
}
catch (Exception ex)
{
_logger.LogError(ex,
"[ScannerService] There was an exception that occurred during tracking {FilePath}. Skipping this file",
info?.FullFilePath);
}
}
foreach (var series in scannedSeries.Keys)
{
if (scannedSeries[series].Count <= 0) continue;
UpdateSortOrder(scannedSeries, series);
processedScannedSeries.Add(new ScannedSeriesResult()
{
HasChanged = scanResult.HasChanged,
ParsedSeries = series,
ParsedInfos = scannedSeries[series]
});
}
}
}
catch (ArgumentException ex)
{
@ -313,66 +365,61 @@ public class ParseScannedFiles
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Done", library.Name, ProgressEventType.Ended));
async Task ProcessFolder(IList<string> files, string folder, string libraryRoot)
{
var normalizedFolder = Parser.Parser.NormalizePath(folder);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedFolder, forceCheck))
{
var parsedInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
{
Series = fp.SeriesName,
Format = fp.Format,
}).ToList();
if (processSeriesInfos != null)
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent("Skipped " + normalizedFolder, library.Name, ProgressEventType.Updated));
return;
}
return processedScannedSeries;
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
if (files.Count == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", folder);
return;
}
var scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
var infos = files
.Select(file => _readingItemService.ParseFile(file, folder, libraryRoot, library.Type))
.Where(info => info != null)
.ToList();
MergeLocalizedSeriesWithSeries(infos);
foreach (var info in infos)
{
try
{
TrackSeries(scannedSeries, info);
}
catch (Exception ex)
{
_logger.LogError(ex,
"[ScannerService] There was an exception that occurred during tracking {FilePath}. Skipping this file",
info?.FullFilePath);
}
}
foreach (var series in scannedSeries.Keys)
{
if (scannedSeries[series].Count <= 0 || processSeriesInfos == null) continue;
UpdateSortOrder(scannedSeries, series);
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
}
}
}
/// <summary>
/// For a given ScanResult, sets the ParserInfos on the result
/// </summary>
/// <param name="result"></param>
/// <param name="seriesPaths"></param>
/// <param name="library"></param>
private async Task ProcessScanResult(ScanResult result, IDictionary<string, IList<SeriesModified>> seriesPaths, Library library)
{
// If the folder hasn't changed, generate fake ParserInfos for the Series that were in that folder.
if (!result.HasChanged)
{
var normalizedFolder = Parser.Parser.NormalizePath(result.Folder);
result.ParserInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
{
Series = fp.SeriesName,
Format = fp.Format,
}).ToList();
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", normalizedFolder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent("Skipped " + normalizedFolder, library.Name, ProgressEventType.Updated));
return;
}
var files = result.Files;
var folder = result.Folder;
var libraryRoot = result.LibraryRoot;
// When processing files for a folder and we do enter, we need to parse the information and combine parser infos
// NOTE: We might want to move the merge step later in the process, like return and combine.
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
if (files.Count == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", folder);
result.ParserInfos = ArraySegment<ParserInfo>.Empty;
return;
}
// Multiple Series can exist within a folder. We should instead put these infos on the result and perform merging above
IList<ParserInfo> infos = files
.Select(file => _readingItemService.ParseFile(file, folder, libraryRoot, library.Type))
.Where(info => info != null)
.ToList()!;
result.ParserInfos = infos;
return;
}
private void UpdateSortOrder(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, ParsedSeries series)
{
try
@ -461,7 +508,7 @@ public class ParseScannedFiles
/// World of Acceleration v02.cbz having Series "Accel World" and Localized Series of "World of Acceleration"
/// </example>
/// <param name="infos">A collection of ParserInfos</param>
private void MergeLocalizedSeriesWithSeries(IReadOnlyCollection<ParserInfo?> infos)
private void MergeLocalizedSeriesWithSeries(IList<ParserInfo> infos)
{
var hasLocalizedSeries = infos.Any(i => !string.IsNullOrEmpty(i.LocalizedSeries));
if (!hasLocalizedSeries) return;

3
API/Services/Tasks/Scanner/Parser/BasicParser.cs
View file

@ -92,6 +92,9 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag
ret.Series = ret.Series.Substring(0, ret.Series.Length - ".pdf".Length);
}
// Patch in other information from ComicInfo
UpdateFromComicInfo(ret);
// v0.8.x: Introducing a change where Specials will go in a separate Volume with a reserved number
if (ret.IsSpecial)
{

29
API/Services/Tasks/Scanner/Parser/ComicVineParser.cs
View file

@ -98,33 +98,4 @@ public class ComicVineParser(IDirectoryService directoryService) : DefaultParser
{
return type == LibraryType.ComicVine;
}
private void UpdateFromComicInfo(ParserInfo info)
{
if (info.ComicInfo == null) return;
if (!string.IsNullOrEmpty(info.ComicInfo.Volume))
{
info.Volumes = info.ComicInfo.Volume;
}
if (string.IsNullOrEmpty(info.Series) && !string.IsNullOrEmpty(info.ComicInfo.Series))
{
info.Series = info.ComicInfo.Series.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
{
info.Chapters = info.ComicInfo.Number;
if (info.IsSpecial && Parser.DefaultChapter != info.Chapters)
{
info.IsSpecial = false;
info.Volumes = $"{Parser.SpecialVolumeNumber}";
}
}
// Patch is SeriesSort from ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo.TitleSort))
{
info.SeriesSort = info.ComicInfo.TitleSort.Trim();
}
}
}

33
API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file

@ -99,6 +99,39 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
}
}
protected void UpdateFromComicInfo(ParserInfo info)
{
if (info.ComicInfo == null) return;
if (!string.IsNullOrEmpty(info.ComicInfo.Volume))
{
info.Volumes = info.ComicInfo.Volume;
}
if (string.IsNullOrEmpty(info.Series) && !string.IsNullOrEmpty(info.ComicInfo.Series))
{
info.Series = info.ComicInfo.Series.Trim();
}
if (string.IsNullOrEmpty(info.LocalizedSeries) && !string.IsNullOrEmpty(info.ComicInfo.LocalizedSeries))
{
info.LocalizedSeries = info.ComicInfo.LocalizedSeries.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
{
info.Chapters = info.ComicInfo.Number;
if (info.IsSpecial && Parser.DefaultChapter != info.Chapters)
{
info.IsSpecial = false;
info.Volumes = $"{Parser.SpecialVolumeNumber}";
}
}
// Patch is SeriesSort from ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo.TitleSort))
{
info.SeriesSort = info.ComicInfo.TitleSort.Trim();
}
}
public abstract bool IsApplicable(string filePath, LibraryType type);
protected static bool IsEmptyOrDefault(string volumes, string chapters)

310
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -1,14 +1,11 @@
using System;
using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
using API.Data.Metadata;
using API.Data.Repositories;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
@ -32,15 +29,9 @@ public interface IProcessSeries
/// </summary>
/// <returns></returns>
Task Prime();
Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false);
void EnqueuePostSeriesProcessTasks(int libraryId, int seriesId, bool forceUpdate = false);
// These exists only for Unit testing
void UpdateSeriesMetadata(Series series, Library library);
void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false);
void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos, bool forceUpdate = false);
void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info, bool forceUpdate = false);
void UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? comicInfo, bool forceUpdate = false);
void Reset();
Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false);
}
/// <summary>
@ -60,16 +51,14 @@ public class ProcessSeries : IProcessSeries
private readonly ICollectionTagService _collectionTagService;
private readonly IReadingListService _readingListService;
private readonly IExternalMetadataService _externalMetadataService;
private readonly ITagManagerService _tagManagerService;
private Dictionary<string, Genre> _genres;
private IList<Person> _people;
private Dictionary<string, Tag> _tags;
private Dictionary<string, CollectionTag> _collectionTags;
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
IFileService fileService, IMetadataService metadataService, IWordCountAnalyzerService wordCountAnalyzerService,
ICollectionTagService collectionTagService, IReadingListService readingListService, IExternalMetadataService externalMetadataService)
ICollectionTagService collectionTagService, IReadingListService readingListService,
IExternalMetadataService externalMetadataService, ITagManagerService tagManagerService)
{
_unitOfWork = unitOfWork;
_logger = logger;
@ -83,12 +72,7 @@ public class ProcessSeries : IProcessSeries
_collectionTagService = collectionTagService;
_readingListService = readingListService;
_externalMetadataService = externalMetadataService;
_genres = new Dictionary<string, Genre>();
_people = new List<Person>();
_tags = new Dictionary<string, Tag>();
_collectionTags = new Dictionary<string, CollectionTag>();
_tagManagerService = tagManagerService;
}
/// <summary>
@ -96,12 +80,15 @@ public class ProcessSeries : IProcessSeries
/// </summary>
public async Task Prime()
{
_genres = (await _unitOfWork.GenreRepository.GetAllGenresAsync()).ToDictionary(t => t.NormalizedTitle);
_people = await _unitOfWork.PersonRepository.GetAllPeople();
_tags = (await _unitOfWork.TagRepository.GetAllTagsAsync()).ToDictionary(t => t.NormalizedTitle);
_collectionTags = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync(CollectionTagIncludes.SeriesMetadata))
.ToDictionary(t => t.NormalizedTitle);
await _tagManagerService.Prime();
}
/// <summary>
/// Frees up memory
/// </summary>
public void Reset()
{
_tagManagerService.Reset();
}
public async Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false)
@ -128,29 +115,7 @@ public class ProcessSeries : IProcessSeries
}
catch (Exception ex)
{
var seriesCollisions = await _unitOfWork.SeriesRepository.GetAllSeriesByAnyName(firstInfo.LocalizedSeries, string.Empty, library.Id, firstInfo.Format);
seriesCollisions = seriesCollisions.Where(collision =>
collision.Name != firstInfo.Series || collision.LocalizedName != firstInfo.LocalizedSeries).ToList();
if (seriesCollisions.Count > 1)
{
var firstCollision = seriesCollisions[0];
var secondCollision = seriesCollisions[1];
var tableRows = $"<tr><td>Name: {firstCollision.Name}</td><td>Name: {secondCollision.Name}</td></tr>" +
$"<tr><td>Localized: {firstCollision.LocalizedName}</td><td>Localized: {secondCollision.LocalizedName}</td></tr>" +
$"<tr><td>Filename: {Parser.Parser.NormalizePath(firstCollision.FolderPath)}</td><td>Filename: {Parser.Parser.NormalizePath(secondCollision.FolderPath)}</td></tr>";
var htmlTable = $"<table class='table table-striped'><thead><tr><th>Series 1</th><th>Series 2</th></tr></thead><tbody>{string.Join(string.Empty, tableRows)}</tbody></table>";
_logger.LogError(ex, "Scanner found a Series {SeriesName} which matched another Series {LocalizedName} in a different folder parallel to Library {LibraryName} root folder. This is not allowed. Please correct",
firstInfo.Series, firstInfo.LocalizedSeries, library.Name);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"Library {library.Name} Series collision on {firstInfo.Series}",
htmlTable));
}
await ReportDuplicateSeriesLookup(library, firstInfo, ex);
return;
}
@ -172,7 +137,7 @@ public class ProcessSeries : IProcessSeries
// parsedInfos[0] is not the first volume or chapter. We need to find it using a ComicInfo check (as it uses firstParsedInfo for series sort)
var firstParsedInfo = parsedInfos.FirstOrDefault(p => p.ComicInfo != null, firstInfo);
UpdateVolumes(series, parsedInfos, forceUpdate);
await UpdateVolumes(series, parsedInfos, forceUpdate);
series.Pages = series.Volumes.Sum(v => v.Pages);
series.NormalizedName = series.Name.ToNormalized();
@ -203,7 +168,7 @@ public class ProcessSeries : IProcessSeries
series.NormalizedLocalizedName = series.LocalizedName.ToNormalized();
}
UpdateSeriesMetadata(series, library);
await UpdateSeriesMetadata(series, library);
// Update series FolderPath here
await UpdateSeriesFolderPath(parsedInfos, library, series);
@ -229,18 +194,25 @@ public class ProcessSeries : IProcessSeries
return;
}
// Process reading list after commit as we need to commit per list
await _readingListService.CreateReadingListsFromSeries(series, library);
BackgroundJob.Enqueue(() => _readingListService.CreateReadingListsFromSeries(library.Id, series.Id));
if (seriesAdded)
{
// See if any recommendations can link up to the series and pre-fetch external metadata for the series
_logger.LogInformation("Linking up External Recommendations new series (if applicable)");
await _externalMetadataService.GetNewSeriesData(series.Id, series.Library.Type);
await _unitOfWork.ExternalSeriesMetadataRepository.LinkRecommendationsToSeries(series);
BackgroundJob.Enqueue(() =>
_externalMetadataService.GetNewSeriesData(series.Id, series.Library.Type));
await _eventHub.SendMessageAsync(MessageFactory.SeriesAdded,
MessageFactory.SeriesAddedEvent(series.Id, series.Name, series.LibraryId), false);
}
else
{
await _unitOfWork.ExternalSeriesMetadataRepository.LinkRecommendationsToSeries(series);
}
_logger.LogInformation("[ScannerService] Finished series update on {SeriesName} in {Milliseconds} ms", seriesName, scanWatch.ElapsedMilliseconds);
}
@ -253,7 +225,34 @@ public class ProcessSeries : IProcessSeries
var settings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
await _metadataService.GenerateCoversForSeries(series, settings.EncodeMediaAs, settings.CoverImageSize);
EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id);
BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(series.LibraryId, series.Id, forceUpdate));
}
private async Task ReportDuplicateSeriesLookup(Library library, ParserInfo firstInfo, Exception ex)
{
var seriesCollisions = await _unitOfWork.SeriesRepository.GetAllSeriesByAnyName(firstInfo.LocalizedSeries, string.Empty, library.Id, firstInfo.Format);
seriesCollisions = seriesCollisions.Where(collision =>
collision.Name != firstInfo.Series || collision.LocalizedName != firstInfo.LocalizedSeries).ToList();
if (seriesCollisions.Count > 1)
{
var firstCollision = seriesCollisions[0];
var secondCollision = seriesCollisions[1];
var tableRows = $"<tr><td>Name: {firstCollision.Name}</td><td>Name: {secondCollision.Name}</td></tr>" +
$"<tr><td>Localized: {firstCollision.LocalizedName}</td><td>Localized: {secondCollision.LocalizedName}</td></tr>" +
$"<tr><td>Filename: {Parser.Parser.NormalizePath(firstCollision.FolderPath)}</td><td>Filename: {Parser.Parser.NormalizePath(secondCollision.FolderPath)}</td></tr>";
var htmlTable = $"<table class='table table-striped'><thead><tr><th>Series 1</th><th>Series 2</th></tr></thead><tbody>{string.Join(string.Empty, tableRows)}</tbody></table>";
_logger.LogError(ex, "Scanner found a Series {SeriesName} which matched another Series {LocalizedName} in a different folder parallel to Library {LibraryName} root folder. This is not allowed. Please correct",
firstInfo.Series, firstInfo.LocalizedSeries, library.Name);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"Library {library.Name} Series collision on {firstInfo.Series}",
htmlTable));
}
}
@ -280,12 +279,8 @@ public class ProcessSeries : IProcessSeries
}
}
public void EnqueuePostSeriesProcessTasks(int libraryId, int seriesId, bool forceUpdate = false)
{
BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(libraryId, seriesId, forceUpdate));
}
public void UpdateSeriesMetadata(Series series, Library library)
private async Task UpdateSeriesMetadata(Series series, Library library)
{
series.Metadata ??= new SeriesMetadataBuilder().Build();
var firstChapter = SeriesService.GetFirstChapterForMetadata(series);
@ -359,14 +354,9 @@ public class ProcessSeries : IProcessSeries
_logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
var normalizedName = Parser.Parser.Normalize(collection);
if (!_collectionTags.TryGetValue(normalizedName, out var tag))
{
tag = _collectionTagService.CreateTag(collection);
_collectionTags.Add(normalizedName, tag);
}
_collectionTagService.AddTagToSeriesMetadata(tag, series.Metadata);
var t = await _tagManagerService.GetCollectionTag(collection);
if (t == null) continue;
_collectionTagService.AddTagToSeriesMetadata(t, series.Metadata);
}
}
@ -541,7 +531,7 @@ public class ProcessSeries : IProcessSeries
}
public void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
private async Task UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
{
// Add new volumes and update chapters per volume
var distinctVolumes = parsedInfos.DistinctVolumes();
@ -586,7 +576,7 @@ public class ProcessSeries : IProcessSeries
try
{
var firstChapterInfo = infos.SingleOrDefault(i => i.FullFilePath.Equals(firstFile.FilePath));
UpdateChapterFromComicInfo(chapter, firstChapterInfo?.ComicInfo, forceUpdate);
await UpdateChapterFromComicInfo(chapter, firstChapterInfo?.ComicInfo, forceUpdate);
}
catch (Exception ex)
{
@ -622,7 +612,7 @@ public class ProcessSeries : IProcessSeries
}
}
public void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
private void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
{
// Add new chapters
foreach (var info in parsedInfos)
@ -689,7 +679,7 @@ public class ProcessSeries : IProcessSeries
}
}
public void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info, bool forceUpdate = false)
private void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info, bool forceUpdate = false)
{
chapter.Files ??= new List<MangaFile>();
var existingFile = chapter.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
@ -700,7 +690,7 @@ public class ProcessSeries : IProcessSeries
if (!forceUpdate && !_fileService.HasFileBeenModifiedSince(existingFile.FilePath, existingFile.LastModified) && existingFile.Pages != 0) return;
existingFile.Pages = _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format);
existingFile.Extension = fileInfo.Extension.ToLowerInvariant();
existingFile.FileName = Path.GetFileNameWithoutExtension(existingFile.FilePath);
existingFile.FileName = Parser.Parser.RemoveExtensionIfSupported(existingFile.FilePath);
existingFile.Bytes = fileInfo.Length;
// We skip updating DB here with last modified time so that metadata refresh can do it
}
@ -715,7 +705,7 @@ public class ProcessSeries : IProcessSeries
}
}
public void UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? comicInfo, bool forceUpdate = false)
private async Task UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? comicInfo, bool forceUpdate = false)
{
if (comicInfo == null) return;
var firstFile = chapter.Files.MinBy(x => x.Chapter);
@ -774,9 +764,7 @@ public class ProcessSeries : IProcessSeries
if (!string.IsNullOrEmpty(comicInfo.Web))
{
chapter.WebLinks = string.Join(",", comicInfo.Web
.Split(",")
.Where(s => !string.IsNullOrEmpty(s))
.Select(s => s.Trim())
.Split(",", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
);
// For each weblink, try to parse out some MetadataIds and store in the Chapter directly for matching (CBL)
@ -795,21 +783,6 @@ public class ProcessSeries : IProcessSeries
// This needs to check against both Number and Volume to calculate Count
chapter.Count = comicInfo.CalculatedCount();
void AddPerson(Person person)
{
PersonHelper.AddPersonIfNotExists(chapter.People, person);
}
void AddGenre(Genre genre, bool newTag)
{
chapter.Genres.Add(genre);
}
void AddTag(Tag tag, bool added)
{
chapter.Tags.Add(tag);
}
if (comicInfo.Year > 0)
{
@ -818,152 +791,79 @@ public class ProcessSeries : IProcessSeries
chapter.ReleaseDate = new DateTime(comicInfo.Year, month, day);
}
var people = GetTagValues(comicInfo.Colorist);
var people = TagHelper.GetTagValues(comicInfo.Colorist);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Colorist);
UpdatePeople(people, PersonRole.Colorist, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Colorist);
people = GetTagValues(comicInfo.Characters);
people = TagHelper.GetTagValues(comicInfo.Characters);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Character);
UpdatePeople(people, PersonRole.Character, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Character);
people = GetTagValues(comicInfo.Translator);
people = TagHelper.GetTagValues(comicInfo.Translator);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Translator);
UpdatePeople(people, PersonRole.Translator, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Translator);
people = GetTagValues(comicInfo.Writer);
people = TagHelper.GetTagValues(comicInfo.Writer);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Writer);
UpdatePeople(people, PersonRole.Writer, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Writer);
people = GetTagValues(comicInfo.Editor);
people = TagHelper.GetTagValues(comicInfo.Editor);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Editor);
UpdatePeople(people, PersonRole.Editor, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Editor);
people = GetTagValues(comicInfo.Inker);
people = TagHelper.GetTagValues(comicInfo.Inker);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Inker);
UpdatePeople(people, PersonRole.Inker, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Inker);
people = GetTagValues(comicInfo.Letterer);
people = TagHelper.GetTagValues(comicInfo.Letterer);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Letterer);
UpdatePeople(people, PersonRole.Letterer, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Letterer);
people = GetTagValues(comicInfo.Penciller);
people = TagHelper.GetTagValues(comicInfo.Penciller);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Penciller);
UpdatePeople(people, PersonRole.Penciller, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Penciller);
people = GetTagValues(comicInfo.CoverArtist);
people = TagHelper.GetTagValues(comicInfo.CoverArtist);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.CoverArtist);
UpdatePeople(people, PersonRole.CoverArtist, AddPerson);
await UpdatePeople(chapter, people, PersonRole.CoverArtist);
people = GetTagValues(comicInfo.Publisher);
people = TagHelper.GetTagValues(comicInfo.Publisher);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Publisher);
UpdatePeople(people, PersonRole.Publisher, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Publisher);
people = GetTagValues(comicInfo.Imprint);
people = TagHelper.GetTagValues(comicInfo.Imprint);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Imprint);
UpdatePeople(people, PersonRole.Imprint, AddPerson);
await UpdatePeople(chapter, people, PersonRole.Imprint);
var genres = GetTagValues(comicInfo.Genre);
var genres = TagHelper.GetTagValues(comicInfo.Genre);
GenreHelper.KeepOnlySameGenreBetweenLists(chapter.Genres,
genres.Select(g => new GenreBuilder(g).Build()).ToList());
UpdateGenre(genres, AddGenre);
foreach (var genre in genres)
{
var g = await _tagManagerService.GetGenre(genre);
if (g == null) continue;
chapter.Genres.Add(g);
}
var tags = GetTagValues(comicInfo.Tags);
var tags = TagHelper.GetTagValues(comicInfo.Tags);
TagHelper.KeepOnlySameTagBetweenLists(chapter.Tags, tags.Select(t => new TagBuilder(t).Build()).ToList());
UpdateTag(tags, AddTag);
}
private static IList<string> GetTagValues(string comicInfoTagSeparatedByComma)
{
// TODO: Move this to an extension and test it
if (string.IsNullOrEmpty(comicInfoTagSeparatedByComma))
foreach (var tag in tags)
{
return ImmutableList<string>.Empty;
}
return comicInfoTagSeparatedByComma.Split(",")
.Select(s => s.Trim())
.DistinctBy(Parser.Parser.Normalize)
.ToList();
}
/// <summary>
/// Given a list of all existing people, this will check the new names and roles and if it doesn't exist in allPeople, will create and
/// add an entry. For each person in name, the callback will be executed.
/// </summary>
/// <remarks>This does not remove people if an empty list is passed into names</remarks>
/// <remarks>This is used to add new people to a list without worrying about duplicating rows in the DB</remarks>
/// <param name="names"></param>
/// <param name="role"></param>
/// <param name="action"></param>
private void UpdatePeople(IEnumerable<string> names, PersonRole role, Action<Person> action)
{
var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
foreach (var name in names)
{
var normalizedName = name.ToNormalized();
var person = allPeopleTypeRole.Find(p =>
p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
if (person == null)
{
person = new PersonBuilder(name, role).Build();
_people.Add(person);
}
action(person);
var t = await _tagManagerService.GetTag(tag);
if (t == null) continue;
chapter.Tags.Add(t);
}
}
/// <summary>
///
/// </summary>
/// <param name="names"></param>
/// <param name="action">Executes for each tag</param>
private void UpdateGenre(IEnumerable<string> names, Action<Genre, bool> action)
private async Task UpdatePeople(Chapter chapter, IList<string> people, PersonRole role)
{
foreach (var name in names)
foreach (var person in people)
{
var normalizedName = name.ToNormalized();
if (string.IsNullOrEmpty(normalizedName)) continue;
_genres.TryGetValue(normalizedName, out var genre);
var newTag = genre == null;
if (newTag)
{
genre = new GenreBuilder(name).Build();
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}
action(genre!, newTag);
var p = await _tagManagerService.GetPerson(person, role);
if (p == null) continue;
chapter.People.Add(p);
}
}
/// <summary>
///
/// </summary>
/// <param name="names"></param>
/// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param>
private void UpdateTag(IEnumerable<string> names, Action<Tag, bool> action)
{
foreach (var name in names)
{
if (string.IsNullOrEmpty(name.Trim())) continue;
var normalizedName = name.ToNormalized();
_tags.TryGetValue(normalizedName, out var tag);
var added = tag == null;
if (tag == null)
{
tag = new TagBuilder(name).Build();
_tags.Add(normalizedName, tag);
}
action(tag, added);
}
}
}

207
API/Services/Tasks/Scanner/TagManagerService.cs Normal file
View file

@ -0,0 +1,207 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers.Builders;
namespace API.Services.Tasks.Scanner;
#nullable enable
public interface ITagManagerService
{
/// <summary>
/// Should be called once before any usage
/// </summary>
/// <returns></returns>
Task Prime();
/// <summary>
/// Should be called after all work is done, will free up memory
/// </summary>
/// <returns></returns>
void Reset();
Task<Genre?> GetGenre(string genre);
Task<Tag?> GetTag(string tag);
Task<Person?> GetPerson(string name, PersonRole role);
Task<CollectionTag?> GetCollectionTag(string name);
}
/// <summary>
/// This is responsible for handling existing and new tags during the scan. When a new tag doesn't exist, it will create it.
/// This is Thread Safe.
/// </summary>
public class TagManagerService : ITagManagerService
{
private readonly IUnitOfWork _unitOfWork;
private Dictionary<string, Genre> _genres;
private Dictionary<string, Tag> _tags;
private Dictionary<string, Person> _people;
private Dictionary<string, CollectionTag> _collectionTags;
private readonly SemaphoreSlim _genreSemaphore = new SemaphoreSlim(1, 1);
private readonly SemaphoreSlim _tagSemaphore = new SemaphoreSlim(1, 1);
private readonly SemaphoreSlim _personSemaphore = new SemaphoreSlim(1, 1);
private readonly SemaphoreSlim _collectionTagSemaphore = new SemaphoreSlim(1, 1);
public TagManagerService(IUnitOfWork unitOfWork)
{
_unitOfWork = unitOfWork;
Reset();
}
public void Reset()
{
_genres = new Dictionary<string, Genre>();
_tags = new Dictionary<string, Tag>();
_people = new Dictionary<string, Person>();
_collectionTags = new Dictionary<string, CollectionTag>();
}
public async Task Prime()
{
_genres = (await _unitOfWork.GenreRepository.GetAllGenresAsync()).ToDictionary(t => t.NormalizedTitle);
_tags = (await _unitOfWork.TagRepository.GetAllTagsAsync()).ToDictionary(t => t.NormalizedTitle);
_people = (await _unitOfWork.PersonRepository.GetAllPeople()).ToDictionary(GetPersonKey);
_collectionTags = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync(CollectionTagIncludes.SeriesMetadata))
.ToDictionary(t => t.NormalizedTitle);
}
/// <summary>
/// Gets the Genre entity for the given string. If one doesn't exist, one will be created and committed.
/// </summary>
/// <param name="genre"></param>
/// <returns></returns>
public async Task<Genre?> GetGenre(string genre)
{
if (string.IsNullOrEmpty(genre)) return null;
await _genreSemaphore.WaitAsync();
try
{
if (_genres.TryGetValue(genre.ToNormalized(), out var result))
{
return result;
}
// We need to create a new Genre
result = new GenreBuilder(genre).Build();
_unitOfWork.GenreRepository.Attach(result);
await _unitOfWork.CommitAsync();
_genres.Add(result.NormalizedTitle, result);
return result;
}
finally
{
_genreSemaphore.Release();
}
}
/// <summary>
/// Gets the Tag entity for the given string. If one doesn't exist, one will be created and committed.
/// </summary>
/// <param name="tag"></param>
/// <returns></returns>
public async Task<Tag?> GetTag(string tag)
{
if (string.IsNullOrEmpty(tag)) return null;
await _tagSemaphore.WaitAsync();
try
{
if (_tags.TryGetValue(tag.ToNormalized(), out var result))
{
return result;
}
// We need to create a new Genre
result = new TagBuilder(tag).Build();
_unitOfWork.TagRepository.Attach(result);
await _unitOfWork.CommitAsync();
_tags.Add(result.NormalizedTitle, result);
return result;
}
finally
{
_tagSemaphore.Release();
}
}
/// <summary>
/// Gets the Person entity for the given string and role. If one doesn't exist, one will be created and committed.
/// </summary>
/// <param name="name">Person Name</param>
/// <param name="role"></param>
/// <returns></returns>
public async Task<Person?> GetPerson(string name, PersonRole role)
{
if (string.IsNullOrEmpty(name)) return null;
await _personSemaphore.WaitAsync();
try
{
var key = GetPersonKey(name.ToNormalized(), role);
if (_people.TryGetValue(key, out var result))
{
return result;
}
// We need to create a new Genre
result = new PersonBuilder(name, role).Build();
_unitOfWork.PersonRepository.Attach(result);
await _unitOfWork.CommitAsync();
_people.Add(key, result);
return result;
}
finally
{
_personSemaphore.Release();
}
}
private static string GetPersonKey(string normalizedName, PersonRole role)
{
return normalizedName + "_" + role;
}
private static string GetPersonKey(Person p)
{
return GetPersonKey(p.NormalizedName, p.Role);
}
/// <summary>
/// Gets the CollectionTag entity for the given string. If one doesn't exist, one will be created and committed.
/// </summary>
/// <param name="tag"></param>
/// <returns></returns>
public async Task<CollectionTag?> GetCollectionTag(string tag)
{
if (string.IsNullOrEmpty(tag)) return null;
await _collectionTagSemaphore.WaitAsync();
try
{
if (_collectionTags.TryGetValue(tag.ToNormalized(), out var result))
{
return result;
}
// We need to create a new Genre
result = new CollectionTagBuilder(tag).Build();
_unitOfWork.CollectionTagRepository.Add(result);
await _unitOfWork.CommitAsync();
_collectionTags.Add(result.NormalizedTitle, result);
return result;
}
finally
{
_collectionTagSemaphore.Release();
}
}
}

231
API/Services/Tasks/ScannerService.cs
View file

@ -33,7 +33,7 @@ public interface IScannerService
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task ScanLibrary(int libraryId, bool forceUpdate = false);
Task ScanLibrary(int libraryId, bool forceUpdate = false, bool isSingleScan = true);
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
@ -86,8 +86,6 @@ public class ScannerService : IScannerService
private readonly IProcessSeries _processSeries;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly SemaphoreSlim _seriesProcessingSemaphore = new SemaphoreSlim(1, 1);
public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger,
IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub,
IDirectoryService directoryService, IReadingItemService readingItemService,
@ -171,7 +169,7 @@ public class ScannerService : IScannerService
var libraries = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).ToList();
var libraryFolders = libraries.SelectMany(l => l.Folders);
var libraryFolder = libraryFolders.Select(Scanner.Parser.Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory));
var libraryFolder = libraryFolders.Select(Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory));
if (string.IsNullOrEmpty(libraryFolder)) return;
var library = libraries.Find(l => l.Folders.Select(Parser.NormalizePath).Contains(libraryFolder));
@ -183,7 +181,7 @@ public class ScannerService : IScannerService
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
return;
}
BackgroundJob.Schedule(() => ScanLibrary(library.Id, false), TimeSpan.FromMinutes(1));
BackgroundJob.Schedule(() => ScanLibrary(library.Id, false, true), TimeSpan.FromMinutes(1));
}
}
@ -196,12 +194,14 @@ public class ScannerService : IScannerService
public async Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true)
{
var sw = Stopwatch.StartNew();
var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId);
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update
var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var existingChapterIdsToClean = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
if (library == null) return;
var libraryPaths = library.Folders.Select(f => f.Path).ToList();
if (await ShouldScanSeries(seriesId, library, libraryPaths, series, true) != ScanCancelReason.NoCancel)
{
@ -214,6 +214,7 @@ public class ScannerService : IScannerService
if (string.IsNullOrEmpty(folderPath) || !_directoryService.Exists(folderPath))
{
// We don't care if it's multiple due to new scan loop enforcing all in one root directory
var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId);
var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(libraryPaths, files.Select(f => f.FilePath).ToList());
if (seriesDirs.Keys.Count == 0)
{
@ -243,17 +244,19 @@ public class ScannerService : IScannerService
// If the series path doesn't exist anymore, it was either moved or renamed. We need to essentially delete it
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
await _processSeries.Prime();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
_logger.LogInformation("Beginning file scan on {SeriesName}", series.Name);
var scanElapsedTime = await ScanFiles(library, new []{ folderPath }, false, TrackFiles, true);
var (scanElapsedTime, processedSeries) = await ScanFiles(library, new []{ folderPath },
false, true);
// Transform seen series into the parsedSeries (I think we can actually just have processedSeries be used instead
TrackFoundSeriesAndFiles(parsedSeries, processedSeries);
_logger.LogInformation("ScanFiles for {Series} took {Time}", series.Name, scanElapsedTime);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
// We now technically have all scannedSeries, we could invoke each Series to be scanned
// Remove any parsedSeries keys that don't belong to our series. This can occur when users store 2 series in the same folder
RemoveParsedInfosNotForSeries(parsedSeries, series);
@ -261,36 +264,57 @@ public class ScannerService : IScannerService
// If nothing was found, first validate any of the files still exist. If they don't then we have a deletion and can skip the rest of the logic flow
if (parsedSeries.Count == 0)
{
var seriesFiles = (await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id));
if (!string.IsNullOrEmpty(series.FolderPath) && !seriesFiles.Where(f => f.FilePath.Contains(series.FolderPath)).Any(m => File.Exists(m.FilePath)))
{
try
var seriesFiles = (await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id));
if (!string.IsNullOrEmpty(series.FolderPath) && !seriesFiles.Where(f => f.FilePath.Contains(series.FolderPath)).Any(m => File.Exists(m.FilePath)))
{
_unitOfWork.SeriesRepository.Remove(series);
await CommitAndSend(1, sw, scanElapsedTime, series);
await _eventHub.SendMessageAsync(MessageFactory.SeriesRemoved,
MessageFactory.SeriesRemovedEvent(seriesId, string.Empty, series.LibraryId), false);
try
{
_unitOfWork.SeriesRepository.Remove(series);
await CommitAndSend(1, sw, scanElapsedTime, series);
await _eventHub.SendMessageAsync(MessageFactory.SeriesRemoved,
MessageFactory.SeriesRemovedEvent(seriesId, string.Empty, series.LibraryId), false);
}
catch (Exception ex)
{
_logger.LogCritical(ex, "There was an error during ScanSeries to delete the series as no files could be found. Aborting scan");
await _unitOfWork.RollbackAsync();
return;
}
}
catch (Exception ex)
else
{
_logger.LogCritical(ex, "There was an error during ScanSeries to delete the series as no files could be found. Aborting scan");
// I think we should just fail and tell user to fix their setup. This is extremely expensive for an edge case
_logger.LogCritical("We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan");
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"Error scanning {series.Name}", "We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan"));
await _unitOfWork.RollbackAsync();
return;
}
}
else
{
// I think we should just fail and tell user to fix their setup. This is extremely expensive for an edge case
_logger.LogCritical("We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan");
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"Error scanning {series.Name}", "We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan"));
await _unitOfWork.RollbackAsync();
return;
}
// At this point, parsedSeries will have at least one key and we can perform the update. If it still doesn't, just return and don't do anything
if (parsedSeries.Count == 0) return;
}
// At this point, parsedSeries will have at least one key and we can perform the update. If it still doesn't, just return and don't do anything
if (parsedSeries.Count == 0) return;
// Don't allow any processing on files that aren't part of this series
var toProcess = parsedSeries.Keys.Where(key =>
key.NormalizedName.Equals(series.NormalizedName) ||
key.NormalizedName.Equals(series.OriginalName?.ToNormalized()))
.ToList();
if (toProcess.Count > 0)
{
await _processSeries.Prime();
}
foreach (var pSeries in toProcess)
{
// Process Series
await _processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, bypassFolderOptimizationChecks);
}
_processSeries.Reset();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
// Tell UI that this series is done
@ -298,32 +322,17 @@ public class ScannerService : IScannerService
MessageFactory.ScanSeriesEvent(library.Id, seriesId, series.Name));
await _metadataService.RemoveAbandonedMetadataKeys();
//BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(series.LibraryId, seriesId, false));
//BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(library.Id, seriesId, false));
BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(chapterIds));
BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(existingChapterIdsToClean));
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return;
}
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
private void TrackFoundSeriesAndFiles(Dictionary<ParsedSeries, IList<ParserInfo>> parsedSeries, IList<ScannedSeriesResult> seenSeries)
{
foreach (var series in seenSeries.Where(s => s.ParsedInfos.Count > 0))
{
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return;
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles[0].Series,
NormalizedName = parsedFiles[0].Series.ToNormalized(),
Format = parsedFiles[0].Format
};
// For Scan Series, we need to filter out anything that isn't our Series
if (!foundParsedSeries.NormalizedName.Equals(series.NormalizedName) && !foundParsedSeries.NormalizedName.Equals(series.OriginalName?.ToNormalized()))
{
return;
}
await _processSeries.ProcessSeriesAsync(parsedFiles, library, bypassFolderOptimizationChecks);
parsedSeries.Add(foundParsedSeries, parsedFiles);
var parsedFiles = series.ParsedInfos;
parsedSeries.Add(series.ParsedSeries, parsedFiles);
}
}
@ -451,11 +460,12 @@ public class ScannerService : IScannerService
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibraries(bool forceUpdate = false)
{
_logger.LogInformation("Starting Scan of All Libraries");
_logger.LogInformation("Starting Scan of All Libraries, Forced: {Forced}", forceUpdate);
foreach (var lib in await _unitOfWork.LibraryRepository.GetLibrariesAsync())
{
await ScanLibrary(lib.Id, forceUpdate);
await ScanLibrary(lib.Id, forceUpdate, true);
}
_processSeries.Reset();
_logger.LogInformation("Scan of All Libraries Finished");
}
@ -467,10 +477,11 @@ public class ScannerService : IScannerService
/// </summary>
/// <param name="libraryId"></param>
/// <param name="forceUpdate">Defaults to false</param>
/// <param name="isSingleScan">Defaults to true. Is this a standalone invocation or is it in a loop?</param>
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibrary(int libraryId, bool forceUpdate = false)
public async Task ScanLibrary(int libraryId, bool forceUpdate = false, bool isSingleScan = true)
{
var sw = Stopwatch.StartNew();
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
@ -490,19 +501,33 @@ public class ScannerService : IScannerService
var totalFiles = 0;
var seenSeries = new List<ParsedSeries>();
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
var (scanElapsedTime, processedSeries) = await ScanFiles(library, libraryFolderPaths,
shouldUseLibraryScan, forceUpdate);
await _processSeries.Prime();
//var processTasks = new List<Func<Task>>();
TrackFoundSeriesAndFiles(parsedSeries, processedSeries);
var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate);
// We need to remove any keys where there is no actual parser info
var toProcess = parsedSeries.Keys
.Where(k => parsedSeries[k].Any() && !string.IsNullOrEmpty(parsedSeries[k][0].Filename))
.ToList();
if (toProcess.Count > 0)
{
// This grabs all the shared entities, like tags, genre, people. To be solved later in this refactor on how to not have blocking access.
await _processSeries.Prime();
}
var tasks = new List<Task>();
foreach (var pSeries in toProcess)
{
totalFiles += parsedSeries[pSeries].Count;
tasks.Add(_processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, forceUpdate));
}
await Task.WhenAll(tasks);
// NOTE: This runs sync after every file is scanned
// foreach (var task in processTasks)
// {
// await task();
// }
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(string.Empty, library.Name, ProgressEventType.Ended));
@ -521,17 +546,22 @@ public class ScannerService : IScannerService
_unitOfWork.LibraryRepository.Update(library);
if (await _unitOfWork.CommitAsync())
{
if (isSingleScan)
{
_processSeries.Reset();
}
if (totalFiles == 0)
{
_logger.LogInformation(
"[ScannerService] Finished library scan of {ParsedSeriesCount} series in {ElapsedScanTime} milliseconds for {LibraryName}. There were no changes",
seenSeries.Count, sw.ElapsedMilliseconds, library.Name);
parsedSeries.Count, sw.ElapsedMilliseconds, library.Name);
}
else
{
_logger.LogInformation(
"[ScannerService] Finished library scan of {TotalFiles} files and {ParsedSeriesCount} series in {ElapsedScanTime} milliseconds for {LibraryName}",
totalFiles, seenSeries.Count, sw.ElapsedMilliseconds, library.Name);
totalFiles, parsedSeries.Count, sw.ElapsedMilliseconds, library.Name);
}
try
@ -539,7 +569,7 @@ public class ScannerService : IScannerService
// Could I delete anything in a Library's Series where the LastScan date is before scanStart?
// NOTE: This implementation is expensive
_logger.LogDebug("[ScannerService] Removing Series that were not found during the scan");
var removedSeries = await _unitOfWork.SeriesRepository.RemoveSeriesNotInList(seenSeries, library.Id);
var removedSeries = await _unitOfWork.SeriesRepository.RemoveSeriesNotInList(parsedSeries.Keys.ToList(), library.Id);
_logger.LogDebug("[ScannerService] Found {Count} series that needs to be removed: {SeriesList}",
removedSeries.Count, removedSeries.Select(s => s.Name));
_logger.LogDebug("[ScannerService] Removing Series that were not found during the scan - complete");
@ -567,63 +597,20 @@ public class ScannerService : IScannerService
await _metadataService.RemoveAbandonedMetadataKeys();
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return;
// Responsible for transforming parsedInfo into an actual ParsedSeries then calling the actual processing of the series
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return;
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles[0].Series,
NormalizedName = Parser.Normalize(parsedFiles[0].Series),
Format = parsedFiles[0].Format,
};
if (skippedScan)
{
seenSeries.AddRange(parsedFiles.Select(pf => new ParsedSeries()
{
Name = pf.Series,
NormalizedName = Parser.Normalize(pf.Series),
Format = pf.Format
}));
return;
}
totalFiles += parsedFiles.Count;
seenSeries.Add(foundParsedSeries);
// TODO: This is extremely expensive to lock the thread on this. We should instead move this onto Hangfire
// or in a queue to be processed.
await _seriesProcessingSemaphore.WaitAsync();
try
{
await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate);
}
finally
{
_seriesProcessingSemaphore.Release();
}
}
}
private async Task<long> ScanFiles(Library library, IEnumerable<string> dirs,
bool isLibraryScan, Func<Tuple<bool, IList<ParserInfo>>, Task>? processSeriesInfos = null, bool forceChecks = false)
private async Task<Tuple<long, IList<ScannedSeriesResult>>> ScanFiles(Library library, IEnumerable<string> dirs,
bool isLibraryScan, bool forceChecks = false)
{
var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService, _eventHub);
var scanWatch = Stopwatch.StartNew();
await scanner.ScanLibrariesForSeries(library, dirs,
isLibraryScan, await _unitOfWork.SeriesRepository.GetFolderPathMap(library.Id), processSeriesInfos, forceChecks);
var processedSeries = await scanner.ScanLibrariesForSeries(library, dirs,
isLibraryScan, await _unitOfWork.SeriesRepository.GetFolderPathMap(library.Id), forceChecks);
var scanElapsedTime = scanWatch.ElapsedMilliseconds;
return scanElapsedTime;
return Tuple.Create(scanElapsedTime, processedSeries);
}
public static IEnumerable<Series> FindSeriesNotOnDisk(IEnumerable<Series> existingSeries, Dictionary<ParsedSeries, IList<ParserInfo>> parsedSeries)

2
UI/Web/src/assets/langs/en.json
View file

@ -1527,7 +1527,7 @@
"general-tab": "General",
"cover-image-tab": "Cover Image",
"close": "{{common.close}}",
"save": "{common.save}}",
"save": "{{common.save}}",
"year-validation": "Must be greater than 1000, 0 or blank",
"month-validation": "Must be between 1 and 12 or blank",
"name-unique-validation": "Name must be unique",

2
openapi.json
View file

@ -7,7 +7,7 @@
"name": "GPL-3.0",
"url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE"
},
"version": "0.7.14.7"
"version": "0.7.14.8"
},
"servers": [
{