AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scanner Tech Debt (#2777)

Browse source
This commit is contained in:
Joe Milazzo 2024-03-12 15:10:43 -06:00 • committed by GitHub
parent f5a31b9a02
commit 08cc7c7cbd
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
23 changed files with 839 additions and 676 deletions
Download patch file Download diff file Expand all files Collapse all files

36
API.Tests/Helpers/GenreHelperTests.cs
View file

@ -1,6 +1,7 @@
using System.Collections.Generic; using System.Collections.Generic;
using API.Data; using API.Data;
using API.Entities; using API.Entities;
using API.Extensions;
using API.Helpers; using API.Helpers;
using API.Helpers.Builders; using API.Helpers.Builders;
using Xunit; using Xunit;
@ -12,42 +13,51 @@ public class GenreHelperTests
[Fact] [Fact]
public void UpdateGenre_ShouldAddNewGenre() public void UpdateGenre_ShouldAddNewGenre()
{ {
var allGenres = new List<Genre> var allGenres = new Dictionary<string, Genre>
{ {
new GenreBuilder("Action").Build(), {"Action".ToNormalized(), new GenreBuilder("Action").Build()},
new GenreBuilder("action").Build(), {"Sci-fi".ToNormalized(), new GenreBuilder("Sci-fi").Build()}
new GenreBuilder("Sci-fi").Build(),
}; };
var genreAdded = new List<Genre>(); var genreAdded = new List<Genre>();
var addedCount = 0;
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Adventure"}, genre => GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Adventure"}, (genre, isNew) =>
{ {
if (isNew)
{
addedCount++;
}
genreAdded.Add(genre); genreAdded.Add(genre);
}); });
Assert.Equal(2, genreAdded.Count); Assert.Equal(2, genreAdded.Count);
Assert.Equal(4, allGenres.Count); Assert.Equal(1, addedCount);
Assert.Equal(3, allGenres.Count);
} }
[Fact] [Fact]
public void UpdateGenre_ShouldNotAddDuplicateGenre() public void UpdateGenre_ShouldNotAddDuplicateGenre()
{ {
var allGenres = new List<Genre> var allGenres = new Dictionary<string, Genre>
{ {
new GenreBuilder("Action").Build(), {"Action".ToNormalized(), new GenreBuilder("Action").Build()},
new GenreBuilder("action").Build(), {"Sci-fi".ToNormalized(), new GenreBuilder("Sci-fi").Build()}
new GenreBuilder("Sci-fi").Build(),
}; };
var genreAdded = new List<Genre>(); var genreAdded = new List<Genre>();
var addedCount = 0;
GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Scifi"}, genre => GenreHelper.UpdateGenre(allGenres, new[] {"Action", "Scifi"}, (genre, isNew) =>
{ {
if (isNew)
{
addedCount++;
}
genreAdded.Add(genre); genreAdded.Add(genre);
}); });
Assert.Equal(3, allGenres.Count); Assert.Equal(0, addedCount);
Assert.Equal(2, genreAdded.Count); Assert.Equal(2, genreAdded.Count);
Assert.Equal(2, allGenres.Count);
} }
[Fact] [Fact]

40
API.Tests/Helpers/TagHelperTests.cs
View file

@ -1,6 +1,8 @@
using System.Collections.Generic; using System.Collections.Generic;
using System.Linq;
using API.Data; using API.Data;
using API.Entities; using API.Entities;
using API.Extensions;
using API.Helpers; using API.Helpers;
using API.Helpers.Builders; using API.Helpers.Builders;
using Xunit; using Xunit;
@ -12,50 +14,50 @@ public class TagHelperTests
[Fact] [Fact]
public void UpdateTag_ShouldAddNewTag() public void UpdateTag_ShouldAddNewTag()
{ {
var allTags = new List<Tag> var allTags = new Dictionary<string, Tag>
{ {
new TagBuilder("Action").Build(), {"Action".ToNormalized(), new TagBuilder("Action").Build()},
new TagBuilder("action").Build(), {"Sci-fi".ToNormalized(), new TagBuilder("Sci-fi").Build()}
new TagBuilder("Sci-fi").Build(),
}; };
var tagAdded = new List<Tag>(); var tagCalled = new List<Tag>();
var addedCount = 0;
TagHelper.UpdateTag(allTags, new[] {"Action", "Adventure"}, (tag, added) => TagHelper.UpdateTag(allTags, new[] {"Action", "Adventure"}, (tag, added) =>
{ {
if (added) if (added)
{ {
tagAdded.Add(tag); addedCount++;
} }
tagCalled.Add(tag);
}); });
Assert.Single(tagAdded); Assert.Equal(1, addedCount);
Assert.Equal(4, allTags.Count); Assert.Equal(2, tagCalled.Count());
Assert.Equal(3, allTags.Count);
} }
[Fact] [Fact]
public void UpdateTag_ShouldNotAddDuplicateTag() public void UpdateTag_ShouldNotAddDuplicateTag()
{ {
var allTags = new List<Tag> var allTags = new Dictionary<string, Tag>
{ {
new TagBuilder("Action").Build(), {"Action".ToNormalized(), new TagBuilder("Action").Build()},
new TagBuilder("action").Build(), {"Sci-fi".ToNormalized(), new TagBuilder("Sci-fi").Build()}
new TagBuilder("Sci-fi").Build(),
}; };
var tagAdded = new List<Tag>(); var tagCalled = new List<Tag>();
var addedCount = 0;
TagHelper.UpdateTag(allTags, new[] {"Action", "Scifi"}, (tag, added) => TagHelper.UpdateTag(allTags, new[] {"Action", "Scifi"}, (tag, added) =>
{ {
if (added) if (added)
{ {
tagAdded.Add(tag); addedCount++;
} }
TagHelper.AddTagIfNotExists(allTags, tag); tagCalled.Add(tag);
}); });
Assert.Equal(3, allTags.Count); Assert.Equal(2, allTags.Count);
Assert.Empty(tagAdded); Assert.Equal(0, addedCount);
} }
[Fact] [Fact]

189
API.Tests/Services/ParseScannedFilesTests.cs
View file

@ -65,88 +65,23 @@ internal class MockReadingItemService : IReadingItemService
} }
} }
public class ParseScannedFilesTests public class ParseScannedFilesTests : AbstractDbTest
{ {
private readonly ILogger<ParseScannedFiles> _logger = Substitute.For<ILogger<ParseScannedFiles>>(); private readonly ILogger<ParseScannedFiles> _logger = Substitute.For<ILogger<ParseScannedFiles>>();
private readonly IUnitOfWork _unitOfWork;
private readonly DbConnection _connection;
private readonly DataContext _context;
private const string CacheDirectory = "C:/kavita/config/cache/";
private const string CoverImageDirectory = "C:/kavita/config/covers/";
private const string BackupDirectory = "C:/kavita/config/backups/";
private const string DataDirectory = "C:/data/";
public ParseScannedFilesTests() public ParseScannedFilesTests()
{ {
var contextOptions = new DbContextOptionsBuilder()
.UseSqlite(CreateInMemoryDatabase())
.Options;
_connection = RelationalOptionsExtension.Extract(contextOptions).Connection;
_context = new DataContext(contextOptions);
Task.Run(SeedDb).GetAwaiter().GetResult();
_unitOfWork = new UnitOfWork(_context, Substitute.For<IMapper>(), null);
// Since ProcessFile relies on _readingItemService, we can implement our own versions of _readingItemService so we have control over how the calls work // Since ProcessFile relies on _readingItemService, we can implement our own versions of _readingItemService so we have control over how the calls work
} }
#region Setup protected override async Task ResetDb()
private static DbConnection CreateInMemoryDatabase()
{
var connection = new SqliteConnection("Filename=:memory:");
connection.Open();
return connection;
}
private async Task<bool> SeedDb()
{
await _context.Database.MigrateAsync();
var filesystem = CreateFileSystem();
await Seed.SeedSettings(_context, new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), filesystem));
var setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.CacheDirectory).SingleAsync();
setting.Value = CacheDirectory;
setting = await _context.ServerSetting.Where(s => s.Key == ServerSettingKey.BackupDirectory).SingleAsync();
setting.Value = BackupDirectory;
_context.ServerSetting.Update(setting);
_context.Library.Add(new LibraryBuilder("Manga")
.WithFolderPath(new FolderPathBuilder(DataDirectory).Build())
.Build());
return await _context.SaveChangesAsync() > 0;
}
private async Task ResetDB()
{ {
_context.Series.RemoveRange(_context.Series.ToList()); _context.Series.RemoveRange(_context.Series.ToList());
await _context.SaveChangesAsync(); await _context.SaveChangesAsync();
} }
private static MockFileSystem CreateFileSystem()
{
var fileSystem = new MockFileSystem();
fileSystem.Directory.SetCurrentDirectory("C:/kavita/");
fileSystem.AddDirectory("C:/kavita/config/");
fileSystem.AddDirectory(CacheDirectory);
fileSystem.AddDirectory(CoverImageDirectory);
fileSystem.AddDirectory(BackupDirectory);
fileSystem.AddDirectory(DataDirectory);
return fileSystem;
}
#endregion
#region MergeName #region MergeName
// NOTE: I don't think I can test MergeName as it relies on Tracking Files, which is more complicated than I need // NOTE: I don't think I can test MergeName as it relies on Tracking Files, which is more complicated than I need
@ -219,6 +154,15 @@ public class ParseScannedFilesTests
#region ScanLibrariesForSeries #region ScanLibrariesForSeries
/// <summary>
/// Test that when a folder has 2 series with a localizedSeries, they combine into one final series
/// </summary>
// [Fact]
// public async Task ScanLibrariesForSeries_ShouldCombineSeries()
// {
// // TODO: Implement these unit tests
// }
[Fact] [Fact]
public async Task ScanLibrariesForSeries_ShouldFindFiles() public async Task ScanLibrariesForSeries_ShouldFindFiles()
{ {
@ -233,34 +177,40 @@ public class ParseScannedFilesTests
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds, var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>()); new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>(); // var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
//
Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo) // Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{ // {
var skippedScan = parsedInfo.Item1; // var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2; // var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return Task.CompletedTask; // if (parsedFiles.Count == 0) return Task.CompletedTask;
//
var foundParsedSeries = new ParsedSeries() // var foundParsedSeries = new ParsedSeries()
{ // {
Name = parsedFiles.First().Series, // Name = parsedFiles.First().Series,
NormalizedName = parsedFiles.First().Series.ToNormalized(), // NormalizedName = parsedFiles.First().Series.ToNormalized(),
Format = parsedFiles.First().Format // Format = parsedFiles.First().Format
}; // };
//
parsedSeries.Add(foundParsedSeries, parsedFiles); // parsedSeries.Add(foundParsedSeries, parsedFiles);
return Task.CompletedTask; // return Task.CompletedTask;
} // }
var library = var library =
await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1, await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes); LibraryIncludes.Folders | LibraryIncludes.FileTypes);
Assert.NotNull(library);
library.Type = LibraryType.Manga; library.Type = LibraryType.Manga;
await psf.ScanLibrariesForSeries(library, new List<string>() {"C:/Data/"}, false, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), TrackFiles); var parsedSeries = await psf.ScanLibrariesForSeries(library, new List<string>() {"C:/Data/"}, false,
await _unitOfWork.SeriesRepository.GetFolderPathMap(1));
Assert.Equal(3, parsedSeries.Values.Count); // Assert.Equal(3, parsedSeries.Values.Count);
Assert.NotEmpty(parsedSeries.Keys.Where(p => p.Format == MangaFormat.Archive && p.Name.Equals("Accel World"))); // Assert.NotEmpty(parsedSeries.Keys.Where(p => p.Format == MangaFormat.Archive && p.Name.Equals("Accel World")));
Assert.Equal(3, parsedSeries.Count);
Assert.NotEmpty(parsedSeries.Select(p => p.ParsedSeries).Where(p => p.Format == MangaFormat.Archive && p.Name.Equals("Accel World")));
} }
#endregion #endregion
@ -292,15 +242,13 @@ public class ParseScannedFilesTests
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>()); new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var directoriesSeen = new HashSet<string>(); var directoriesSeen = new HashSet<string>();
var library = var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes); LibraryIncludes.Folders | LibraryIncludes.FileTypes);
await psf.ProcessFiles("C:/Data/", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), var scanResults = psf.ProcessFiles("C:/Data/", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
(files, directoryPath, libraryFolder) => foreach (var scanResult in scanResults)
{ {
directoriesSeen.Add(directoryPath); directoriesSeen.Add(scanResult.Folder);
return Task.CompletedTask; }
}, library);
Assert.Equal(2, directoriesSeen.Count); Assert.Equal(2, directoriesSeen.Count);
} }
@ -313,14 +261,18 @@ public class ParseScannedFilesTests
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds, var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>()); new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes);
Assert.NotNull(library);
var directoriesSeen = new HashSet<string>(); var directoriesSeen = new HashSet<string>();
await psf.ProcessFiles("C:/Data/", false, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), var scanResults = psf.ProcessFiles("C:/Data/", false,
(files, directoryPath, libraryFolder) => await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
foreach (var scanResult in scanResults)
{ {
directoriesSeen.Add(directoryPath); directoriesSeen.Add(scanResult.Folder);
return Task.CompletedTask; }
}, await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes));
Assert.Single(directoriesSeen); Assert.Single(directoriesSeen);
directoriesSeen.TryGetValue("C:/Data/", out var actual); directoriesSeen.TryGetValue("C:/Data/", out var actual);
@ -344,17 +296,12 @@ public class ParseScannedFilesTests
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds, var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>()); new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var callCount = 0; var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
await psf.ProcessFiles("C:/Data", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), LibraryIncludes.Folders | LibraryIncludes.FileTypes);
(files, folderPath, libraryFolder) => Assert.NotNull(library);
{ var scanResults = psf.ProcessFiles("C:/Data", true, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
callCount++;
return Task.CompletedTask; Assert.Equal(2, scanResults.Count);
}, await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes));
Assert.Equal(2, callCount);
} }
@ -378,17 +325,17 @@ public class ParseScannedFilesTests
var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds, var psf = new ParseScannedFiles(Substitute.For<ILogger<ParseScannedFiles>>(), ds,
new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>()); new MockReadingItemService(new BasicParser(ds, new ImageParser(ds))), Substitute.For<IEventHub>());
var callCount = 0; var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
await psf.ProcessFiles("C:/Data", false, await _unitOfWork.SeriesRepository.GetFolderPathMap(1), LibraryIncludes.Folders | LibraryIncludes.FileTypes);
(files, folderPath, libraryFolder) => Assert.NotNull(library);
{ var scanResults = psf.ProcessFiles("C:/Data", false,
callCount++; await _unitOfWork.SeriesRepository.GetFolderPathMap(1), library);
return Task.CompletedTask;
}, await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(1,
LibraryIncludes.Folders | LibraryIncludes.FileTypes));
Assert.Equal(1, callCount); Assert.Single(scanResults);
} }
#endregion #endregion
} }

6
API/Controllers/LicenseController.cs
View file

@ -32,7 +32,11 @@ public class LicenseController(
public async Task<ActionResult<bool>> HasValidLicense(bool forceCheck = false) public async Task<ActionResult<bool>> HasValidLicense(bool forceCheck = false)
{ {
var result = await licenseService.HasActiveLicense(forceCheck); var result = await licenseService.HasActiveLicense(forceCheck);
await taskScheduler.ScheduleKavitaPlusTasks(); if (result)
{
await taskScheduler.ScheduleKavitaPlusTasks();
}
return Ok(result); return Ok(result);
} }

7
API/Data/Metadata/ComicInfo.cs
View file

@ -177,7 +177,12 @@ public class ComicInfo
if (!string.IsNullOrEmpty(info.Number)) if (!string.IsNullOrEmpty(info.Number))
{ {
info.Number = info.Number.Replace(",", "."); // Corrective measure for non English OSes info.Number = info.Number.Trim().Replace(",", "."); // Corrective measure for non English OSes
}
if (!string.IsNullOrEmpty(info.Volume))
{
info.Volume = info.Volume.Trim();
} }
} }

8
API/Data/Repositories/ExternalSeriesMetadataRepository.cs
View file

@ -34,6 +34,7 @@ public interface IExternalSeriesMetadataRepository
Task<bool> ExternalSeriesMetadataNeedsRefresh(int seriesId); Task<bool> ExternalSeriesMetadataNeedsRefresh(int seriesId);
Task<SeriesDetailPlusDto> GetSeriesDetailPlusDto(int seriesId); Task<SeriesDetailPlusDto> GetSeriesDetailPlusDto(int seriesId);
Task LinkRecommendationsToSeries(Series series); Task LinkRecommendationsToSeries(Series series);
Task LinkRecommendationsToSeries(int seriesId);
Task<bool> IsBlacklistedSeries(int seriesId); Task<bool> IsBlacklistedSeries(int seriesId);
Task CreateBlacklistedSeries(int seriesId, bool saveChanges = true); Task CreateBlacklistedSeries(int seriesId, bool saveChanges = true);
Task RemoveFromBlacklist(int seriesId); Task RemoveFromBlacklist(int seriesId);
@ -179,6 +180,13 @@ public class ExternalSeriesMetadataRepository : IExternalSeriesMetadataRepositor
return seriesDetailPlusDto; return seriesDetailPlusDto;
} }
public async Task LinkRecommendationsToSeries(int seriesId)
{
var series = await _context.Series.Where(s => s.Id == seriesId).AsNoTracking().SingleOrDefaultAsync();
if (series == null) return;
await LinkRecommendationsToSeries(series);
}
/// <summary> /// <summary>
/// Searches Recommendations without a SeriesId on record and attempts to link based on Series Name/Localized Name /// Searches Recommendations without a SeriesId on record and attempts to link based on Series Name/Localized Name
/// </summary> /// </summary>

1
API/Entities/Series.cs
View file

@ -3,6 +3,7 @@ using System.Collections.Generic;
using API.Entities.Enums; using API.Entities.Enums;
using API.Entities.Interfaces; using API.Entities.Interfaces;
using API.Entities.Metadata; using API.Entities.Metadata;
using API.Services.Tasks.Scanner.Parser;
namespace API.Entities; namespace API.Entities;

1
API/Extensions/ApplicationServiceExtensions.cs
View file

@ -60,6 +60,7 @@ public static class ApplicationServiceExtensions
services.AddScoped<ILibraryWatcher, LibraryWatcher>(); services.AddScoped<ILibraryWatcher, LibraryWatcher>();
services.AddScoped<ITachiyomiService, TachiyomiService>(); services.AddScoped<ITachiyomiService, TachiyomiService>();
services.AddScoped<ICollectionTagService, CollectionTagService>(); services.AddScoped<ICollectionTagService, CollectionTagService>();
services.AddScoped<ITagManagerService, TagManagerService>();
services.AddScoped<IFileSystem, FileSystem>(); services.AddScoped<IFileSystem, FileSystem>();
services.AddScoped<IDirectoryService, DirectoryService>(); services.AddScoped<IDirectoryService, DirectoryService>();

22
API/Helpers/GenreHelper.cs
View file

@ -12,25 +12,28 @@ namespace API.Helpers;
public static class GenreHelper public static class GenreHelper
{ {
public static void UpdateGenre(ICollection<Genre> allGenres, IEnumerable<string> names, Action<Genre> action)
public static void UpdateGenre(Dictionary<string, Genre> allGenres,
IEnumerable<string> names, Action<Genre, bool> action)
{ {
foreach (var name in names) foreach (var name in names)
{ {
if (string.IsNullOrEmpty(name.Trim())) continue;
var normalizedName = name.ToNormalized(); var normalizedName = name.ToNormalized();
var genre = allGenres.FirstOrDefault(p => p.NormalizedTitle != null && p.NormalizedTitle.Equals(normalizedName)); if (string.IsNullOrEmpty(normalizedName)) continue;
if (genre == null)
if (allGenres.TryGetValue(normalizedName, out var genre))
{
action(genre, false);
}
else
{ {
genre = new GenreBuilder(name).Build(); genre = new GenreBuilder(name).Build();
allGenres.Add(genre); allGenres.Add(normalizedName, genre);
action(genre, true);
} }
action(genre);
} }
} }
public static void KeepOnlySameGenreBetweenLists(ICollection<Genre> existingGenres, ICollection<Genre> removeAllExcept, Action<Genre>? action = null) public static void KeepOnlySameGenreBetweenLists(ICollection<Genre> existingGenres, ICollection<Genre> removeAllExcept, Action<Genre>? action = null)
{ {
var existing = existingGenres.ToList(); var existing = existingGenres.ToList();
@ -64,6 +67,7 @@ public static class GenreHelper
public static void UpdateGenreList(ICollection<GenreTagDto>? tags, Series series, public static void UpdateGenreList(ICollection<GenreTagDto>? tags, Series series,
IReadOnlyCollection<Genre> allTags, Action<Genre> handleAdd, Action onModified) IReadOnlyCollection<Genre> allTags, Action<Genre> handleAdd, Action onModified)
{ {
// TODO: Write some unit tests
if (tags == null) return; if (tags == null) return;
var isModified = false; var isModified = false;
// I want a union of these 2 lists. Return only elements that are in both lists, but the list types are different // I want a union of these 2 lists. Return only elements that are in both lists, but the list types are different

2
API/Helpers/PersonHelper.cs
View file

@ -12,6 +12,7 @@ namespace API.Helpers;
public static class PersonHelper public static class PersonHelper
{ {
/// <summary> /// <summary>
/// Given a list of all existing people, this will check the new names and roles and if it doesn't exist in allPeople, will create and /// Given a list of all existing people, this will check the new names and roles and if it doesn't exist in allPeople, will create and
/// add an entry. For each person in name, the callback will be executed. /// add an entry. For each person in name, the callback will be executed.
@ -24,7 +25,6 @@ public static class PersonHelper
/// <param name="action"></param> /// <param name="action"></param>
public static void UpdatePeople(ICollection<Person> allPeople, IEnumerable<string> names, PersonRole role, Action<Person> action) public static void UpdatePeople(ICollection<Person> allPeople, IEnumerable<string> names, PersonRole role, Action<Person> action)
{ {
// TODO: Validate if we need this, not used
var allPeopleTypeRole = allPeople.Where(p => p.Role == role).ToList(); var allPeopleTypeRole = allPeople.Where(p => p.Role == role).ToList();
foreach (var name in names) foreach (var name in names)

40
API/Helpers/TagHelper.cs
View file

@ -1,43 +1,37 @@
using System; using System;
using System.Collections.Concurrent; using System.Collections.Concurrent;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Immutable;
using System.Linq; using System.Linq;
using API.Data; using API.Data;
using API.DTOs.Metadata; using API.DTOs.Metadata;
using API.Entities; using API.Entities;
using API.Extensions; using API.Extensions;
using API.Helpers.Builders; using API.Helpers.Builders;
using API.Services.Tasks.Scanner.Parser;
namespace API.Helpers; namespace API.Helpers;
#nullable enable #nullable enable
public static class TagHelper public static class TagHelper
{ {
/// <summary> public static void UpdateTag(Dictionary<string, Tag> allTags, IEnumerable<string> names, Action<Tag, bool> action)
///
/// </summary>
/// <param name="allTags"></param>
/// <param name="names"></param>
/// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param>
public static void UpdateTag(ICollection<Tag> allTags, IEnumerable<string> names, Action<Tag, bool> action)
{ {
foreach (var name in names) foreach (var name in names)
{ {
if (string.IsNullOrEmpty(name.Trim())) continue; if (string.IsNullOrEmpty(name.Trim())) continue;
var added = false;
var normalizedName = name.ToNormalized(); var normalizedName = name.ToNormalized();
allTags.TryGetValue(normalizedName, out var tag);
var genre = allTags.FirstOrDefault(p => var added = tag == null;
p.NormalizedTitle.Equals(normalizedName)); if (tag == null)
if (genre == null)
{ {
added = true; tag = new TagBuilder(name).Build();
genre = new TagBuilder(name).Build(); allTags.Add(normalizedName, tag);
allTags.Add(genre);
} }
action(genre, added); action(tag, added);
} }
} }
@ -79,6 +73,22 @@ public static class TagHelper
} }
} }
public static IList<string> GetTagValues(string comicInfoTagSeparatedByComma)
{
// TODO: Unit tests needed
if (string.IsNullOrEmpty(comicInfoTagSeparatedByComma))
{
return ImmutableList<string>.Empty;
}
return comicInfoTagSeparatedByComma.Split(",")
.Select(s => s.Trim())
.DistinctBy(Parser.Normalize)
.ToList();
}
/// <summary> /// <summary>
/// Remove tags on a list /// Remove tags on a list
/// </summary> /// </summary>

3
API/Services/Plus/ExternalMetadataService.cs
View file

@ -70,7 +70,7 @@ public class ExternalMetadataService : IExternalMetadataService
private readonly IMapper _mapper; private readonly IMapper _mapper;
private readonly ILicenseService _licenseService; private readonly ILicenseService _licenseService;
private readonly TimeSpan _externalSeriesMetadataCache = TimeSpan.FromDays(30); private readonly TimeSpan _externalSeriesMetadataCache = TimeSpan.FromDays(30);
public static readonly ImmutableArray<LibraryType> NonEligibleLibraryTypes = ImmutableArray.Create<LibraryType>(LibraryType.Comic, LibraryType.Book); public static readonly ImmutableArray<LibraryType> NonEligibleLibraryTypes = ImmutableArray.Create<LibraryType>(LibraryType.Comic, LibraryType.Book, LibraryType.Image, LibraryType.ComicVine);
private readonly SeriesDetailPlusDto _defaultReturn = new() private readonly SeriesDetailPlusDto _defaultReturn = new()
{ {
Recommendations = null, Recommendations = null,
@ -155,6 +155,7 @@ public class ExternalMetadataService : IExternalMetadataService
public async Task GetNewSeriesData(int seriesId, LibraryType libraryType) public async Task GetNewSeriesData(int seriesId, LibraryType libraryType)
{ {
if (!IsPlusEligible(libraryType)) return; if (!IsPlusEligible(libraryType)) return;
if (!await _licenseService.HasActiveLicense()) return;
// Generate key based on seriesId and libraryType or any unique identifier for the request // Generate key based on seriesId and libraryType or any unique identifier for the request
// Check if the request is allowed based on the rate limit // Check if the request is allowed based on the rate limit

16
API/Services/ReadingListService.cs
View file

@ -46,6 +46,8 @@ public interface IReadingListService
/// <param name="library"></param> /// <param name="library"></param>
/// <returns></returns> /// <returns></returns>
Task CreateReadingListsFromSeries(Series series, Library library); Task CreateReadingListsFromSeries(Series series, Library library);
Task CreateReadingListsFromSeries(int libraryId, int seriesId);
} }
/// <summary> /// <summary>
@ -407,6 +409,20 @@ public class ReadingListService : IReadingListService
return index > lastOrder + 1; return index > lastOrder + 1;
} }
/// <summary>
/// Create Reading lists from a Series
/// </summary>
/// <remarks>Execute this from Hangfire</remarks>
/// <param name="libraryId"></param>
/// <param name="seriesId"></param>
public async Task CreateReadingListsFromSeries(int libraryId, int seriesId)
{
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId);
if (series == null || library == null) return;
await CreateReadingListsFromSeries(series, library);
}
public async Task CreateReadingListsFromSeries(Series series, Library library) public async Task CreateReadingListsFromSeries(Series series, Library library)
{ {
if (!library.ManageReadingLists) return; if (!library.ManageReadingLists) return;

16
API/Services/TaskScheduler.cs
View file

@ -328,13 +328,13 @@ public class TaskScheduler : ITaskScheduler
} }
if (RunningAnyTasksByMethod(ScanTasks, ScanQueue)) if (RunningAnyTasksByMethod(ScanTasks, ScanQueue))
{ {
_logger.LogInformation("A Library Scan is already running, rescheduling ScanLibrary in 3 hours"); _logger.LogInformation("A Scan is already running, rescheduling ScanLibrary in 3 hours");
BackgroundJob.Schedule(() => ScanLibrary(libraryId, force), TimeSpan.FromHours(3)); BackgroundJob.Schedule(() => ScanLibrary(libraryId, force), TimeSpan.FromHours(3));
return; return;
} }
_logger.LogInformation("Enqueuing library scan for: {LibraryId}", libraryId); _logger.LogInformation("Enqueuing library scan for: {LibraryId}", libraryId);
BackgroundJob.Enqueue(() => _scannerService.ScanLibrary(libraryId, force)); BackgroundJob.Enqueue(() => _scannerService.ScanLibrary(libraryId, force, true));
// When we do a scan, force cache to re-unpack in case page numbers change // When we do a scan, force cache to re-unpack in case page numbers change
BackgroundJob.Enqueue(() => _cleanupService.CleanupCacheAndTempDirectories()); BackgroundJob.Enqueue(() => _cleanupService.CleanupCacheAndTempDirectories());
} }
@ -386,7 +386,7 @@ public class TaskScheduler : ITaskScheduler
} }
if (RunningAnyTasksByMethod(ScanTasks, ScanQueue)) if (RunningAnyTasksByMethod(ScanTasks, ScanQueue))
{ {
// BUG: This can end up triggering a ton of scan series calls // BUG: This can end up triggering a ton of scan series calls (but i haven't seen in practice)
_logger.LogInformation("A Scan is already running, rescheduling ScanSeries in 10 minutes"); _logger.LogInformation("A Scan is already running, rescheduling ScanSeries in 10 minutes");
BackgroundJob.Schedule(() => ScanSeries(libraryId, seriesId, forceUpdate), TimeSpan.FromMinutes(10)); BackgroundJob.Schedule(() => ScanSeries(libraryId, seriesId, forceUpdate), TimeSpan.FromMinutes(10));
return; return;
@ -428,8 +428,14 @@ public class TaskScheduler : ITaskScheduler
public static bool HasScanTaskRunningForLibrary(int libraryId, bool checkRunningJobs = true) public static bool HasScanTaskRunningForLibrary(int libraryId, bool checkRunningJobs = true)
{ {
return return
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, true}, ScanQueue, checkRunningJobs) || HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, true, true}, ScanQueue,
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, false}, ScanQueue, checkRunningJobs); checkRunningJobs) ||
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, false, true}, ScanQueue,
checkRunningJobs) ||
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, true, false}, ScanQueue,
checkRunningJobs) ||
HasAlreadyEnqueuedTask(ScannerService.Name, "ScanLibrary", new object[] {libraryId, false, false}, ScanQueue,
checkRunningJobs);
} }
/// <summary> /// <summary>

311
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -31,6 +31,49 @@ public class ParsedSeries
public required MangaFormat Format { get; init; } public required MangaFormat Format { get; init; }
} }
public class ScanResult
{
/// <summary>
/// A list of files in the Folder. Empty if HasChanged = false
/// </summary>
public IList<string> Files { get; set; }
/// <summary>
/// A nested folder from Library Root (at any level)
/// </summary>
public string Folder { get; set; }
/// <summary>
/// The library root
/// </summary>
public string LibraryRoot { get; set; }
/// <summary>
/// Was the Folder scanned or not. If not modified since last scan, this will be false and Files empty
/// </summary>
public bool HasChanged { get; set; }
/// <summary>
/// Set in Stage 2: Parsed Info from the Files
/// </summary>
public IList<ParserInfo> ParserInfos { get; set; }
}
/// <summary>
/// The final product of ParseScannedFiles. This has all the processed parserInfo and is ready for tracking/processing into entities
/// </summary>
public class ScannedSeriesResult
{
/// <summary>
/// Was the Folder scanned or not. If not modified since last scan, this will be false and indicates that upstream should count this as skipped
/// </summary>
public bool HasChanged { get; set; }
/// <summary>
/// The Parsed Series information used for tracking
/// </summary>
public ParsedSeries ParsedSeries { get; set; }
/// <summary>
/// Parsed files
/// </summary>
public IList<ParserInfo> ParsedInfos { get; set; }
}
public class SeriesModified public class SeriesModified
{ {
public required string FolderPath { get; set; } public required string FolderPath { get; set; }
@ -75,112 +118,79 @@ public class ParseScannedFiles
/// <param name="scanDirectoryByDirectory">Scan directory by directory and for each, call folderAction</param> /// <param name="scanDirectoryByDirectory">Scan directory by directory and for each, call folderAction</param>
/// <param name="seriesPaths">A dictionary mapping a normalized path to a list of <see cref="SeriesModified"/> to help scanner skip I/O</param> /// <param name="seriesPaths">A dictionary mapping a normalized path to a list of <see cref="SeriesModified"/> to help scanner skip I/O</param>
/// <param name="folderPath">A library folder or series folder</param> /// <param name="folderPath">A library folder or series folder</param>
/// <param name="folderAction">A callback async Task to be called once all files for each folder path are found</param>
/// <param name="forceCheck">If we should bypass any folder last write time checks on the scan and force I/O</param> /// <param name="forceCheck">If we should bypass any folder last write time checks on the scan and force I/O</param>
public async Task ProcessFiles(string folderPath, bool scanDirectoryByDirectory, public IList<ScanResult> ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<IList<string>, string, string, Task> folderAction, Library library, bool forceCheck = false) IDictionary<string, IList<SeriesModified>> seriesPaths, Library library, bool forceCheck = false)
{ {
string normalizedPath; string normalizedPath;
var result = new List<ScanResult>();
var fileExtensions = string.Join("|", library.LibraryFileTypes.Select(l => l.FileTypeGroup.GetRegex())); var fileExtensions = string.Join("|", library.LibraryFileTypes.Select(l => l.FileTypeGroup.GetRegex()));
if (scanDirectoryByDirectory) if (scanDirectoryByDirectory)
{ {
// This is used in library scan, so we should check first for a ignore file and use that here as well // This is used in library scan, so we should check first for a ignore file and use that here as well
var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(folderPath, DirectoryService.KavitaIgnoreFile); var matcher = new GlobMatcher();
var matcher = _directoryService.CreateMatcherFromFile(potentialIgnoreFile); foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
if (matcher != null)
{ {
_logger.LogWarning(".kavitaignore found! Ignore files is deprecated in favor of Library Settings. Please update and remove file at {Path}", potentialIgnoreFile); matcher.AddExclude(pattern.Pattern);
} }
if (library.LibraryExcludePatterns.Count != 0)
{
matcher ??= new GlobMatcher();
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
{
matcher.AddExclude(pattern.Pattern);
}
}
var directories = _directoryService.GetDirectories(folderPath, matcher).ToList(); var directories = _directoryService.GetDirectories(folderPath, matcher).ToList();
foreach (var directory in directories) foreach (var directory in directories)
{ {
// Since this is a loop, we need a list return
normalizedPath = Parser.Parser.NormalizePath(directory); normalizedPath = Parser.Parser.NormalizePath(directory);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck)) if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
{ {
await folderAction(new List<string>(), directory, folderPath); result.Add(new ScanResult()
{
Files = ArraySegment<string>.Empty,
Folder = directory,
LibraryRoot = folderPath,
HasChanged = false
});
} }
else else
{ {
// For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication // For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication
await folderAction(_directoryService.ScanFiles(directory, fileExtensions, matcher), directory, folderPath); result.Add(new ScanResult()
{
Files = _directoryService.ScanFiles(directory, fileExtensions, matcher),
Folder = directory,
LibraryRoot = folderPath,
HasChanged = true
});
} }
} }
return; return result;
} }
normalizedPath = Parser.Parser.NormalizePath(folderPath); normalizedPath = Parser.Parser.NormalizePath(folderPath);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck)) if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
{ {
await folderAction(new List<string>(), folderPath, folderPath); result.Add(new ScanResult()
return;
}
// We need to calculate all folders till library root and see if any kavitaignores
var seriesMatcher = BuildIgnoreFromLibraryRoot(folderPath, seriesPaths);
await folderAction(_directoryService.ScanFiles(folderPath, fileExtensions, seriesMatcher), folderPath, folderPath);
}
/// <summary>
/// Used in ScanSeries, which enters at a lower level folder and hence needs a .kavitaignore from higher (up to root) to be built before
/// the scan takes place.
/// </summary>
/// <param name="folderPath"></param>
/// <param name="seriesPaths"></param>
/// <returns>A GlobMatter. Empty if not applicable</returns>
private GlobMatcher BuildIgnoreFromLibraryRoot(string folderPath, IDictionary<string, IList<SeriesModified>> seriesPaths)
{
var seriesMatcher = new GlobMatcher();
try
{
var roots = seriesPaths[folderPath][0].LibraryRoots.Select(Parser.Parser.NormalizePath).ToList();
var libraryFolder = roots.SingleOrDefault(folderPath.Contains);
if (string.IsNullOrEmpty(libraryFolder) || !Directory.Exists(folderPath))
{ {
return seriesMatcher; Files = ArraySegment<string>.Empty,
} Folder = folderPath,
LibraryRoot = folderPath,
var allParents = _directoryService.GetFoldersTillRoot(libraryFolder, folderPath); HasChanged = false
var path = libraryFolder; });
// Apply the library root level kavitaignore
var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(path, DirectoryService.KavitaIgnoreFile);
seriesMatcher.Merge(_directoryService.CreateMatcherFromFile(potentialIgnoreFile));
// Then apply kavitaignores for each folder down to where the series folder is
foreach (var folderPart in allParents.Reverse())
{
path = Parser.Parser.NormalizePath(Path.Join(libraryFolder, folderPart));
potentialIgnoreFile = _directoryService.FileSystem.Path.Join(path, DirectoryService.KavitaIgnoreFile);
seriesMatcher.Merge(_directoryService.CreateMatcherFromFile(potentialIgnoreFile));
}
} }
catch (Exception ex)
result.Add(new ScanResult()
{ {
_logger.LogError(ex, Files = _directoryService.ScanFiles(folderPath, fileExtensions),
"[ScannerService] There was an error trying to find and apply .kavitaignores above the Series Folder. Scanning without them present"); Folder = folderPath,
} LibraryRoot = folderPath,
HasChanged = true
});
return seriesMatcher; return result;
} }
/// <summary> /// <summary>
/// Attempts to either add a new instance of a show mapping to the _scannedSeries bag or adds to an existing. /// Attempts to either add a new instance of a series mapping to the _scannedSeries bag or adds to an existing.
/// This will check if the name matches an existing series name (multiple fields) <see cref="MergeName"/> /// This will check if the name matches an existing series name (multiple fields) <see cref="MergeName"/>
/// </summary> /// </summary>
/// <param name="scannedSeries">A localized list of a series' parsed infos</param> /// <param name="scannedSeries">A localized list of a series' parsed infos</param>
@ -290,20 +300,62 @@ public class ParseScannedFiles
/// <param name="folders"></param> /// <param name="folders"></param>
/// <param name="isLibraryScan">If true, does a directory scan first (resulting in folders being tackled in parallel), else does an immediate scan files</param> /// <param name="isLibraryScan">If true, does a directory scan first (resulting in folders being tackled in parallel), else does an immediate scan files</param>
/// <param name="seriesPaths">A map of Series names -> existing folder paths to handle skipping folders</param> /// <param name="seriesPaths">A map of Series names -> existing folder paths to handle skipping folders</param>
/// <param name="processSeriesInfos">Action which returns if the folder was skipped and the infos from said folder</param>
/// <param name="forceCheck">Defaults to false</param> /// <param name="forceCheck">Defaults to false</param>
/// <returns></returns> /// <returns></returns>
public async Task ScanLibrariesForSeries(Library library, public async Task<IList<ScannedSeriesResult>> ScanLibrariesForSeries(Library library,
IEnumerable<string> folders, bool isLibraryScan, IEnumerable<string> folders, bool isLibraryScan,
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<Tuple<bool, IList<ParserInfo>>, Task>? processSeriesInfos, bool forceCheck = false) IDictionary<string, IList<SeriesModified>> seriesPaths, bool forceCheck = false)
{ {
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", library.Name, ProgressEventType.Started)); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", library.Name, ProgressEventType.Started));
var processedScannedSeries = new List<ScannedSeriesResult>();
foreach (var folderPath in folders) foreach (var folderPath in folders)
{ {
try try
{ {
await ProcessFiles(folderPath, isLibraryScan, seriesPaths, ProcessFolder, library, forceCheck); var scanResults = ProcessFiles(folderPath, isLibraryScan, seriesPaths, library, forceCheck);
foreach (var scanResult in scanResults)
{
// scanResult is updated with the parsed infos
await ProcessScanResult(scanResult, seriesPaths, library);
// We now have all the parsed infos from the scan result, perform any merging that is necessary and post processing steps
var scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
// Merge any series together (like Nagatoro/nagator.cbz, japanesename.cbz) -> Nagator series
MergeLocalizedSeriesWithSeries(scanResult.ParserInfos);
// Combine everything into scannedSeries
foreach (var info in scanResult.ParserInfos)
{
try
{
TrackSeries(scannedSeries, info);
}
catch (Exception ex)
{
_logger.LogError(ex,
"[ScannerService] There was an exception that occurred during tracking {FilePath}. Skipping this file",
info?.FullFilePath);
}
}
foreach (var series in scannedSeries.Keys)
{
if (scannedSeries[series].Count <= 0) continue;
UpdateSortOrder(scannedSeries, series);
processedScannedSeries.Add(new ScannedSeriesResult()
{
HasChanged = scanResult.HasChanged,
ParsedSeries = series,
ParsedInfos = scannedSeries[series]
});
}
}
} }
catch (ArgumentException ex) catch (ArgumentException ex)
{ {
@ -313,66 +365,61 @@ public class ParseScannedFiles
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Done", library.Name, ProgressEventType.Ended)); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Done", library.Name, ProgressEventType.Ended));
async Task ProcessFolder(IList<string> files, string folder, string libraryRoot) return processedScannedSeries;
{
var normalizedFolder = Parser.Parser.NormalizePath(folder);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedFolder, forceCheck))
{
var parsedInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
{
Series = fp.SeriesName,
Format = fp.Format,
}).ToList();
if (processSeriesInfos != null)
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent("Skipped " + normalizedFolder, library.Name, ProgressEventType.Updated));
return;
}
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
if (files.Count == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", folder);
return;
}
var scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
var infos = files
.Select(file => _readingItemService.ParseFile(file, folder, libraryRoot, library.Type))
.Where(info => info != null)
.ToList();
MergeLocalizedSeriesWithSeries(infos);
foreach (var info in infos)
{
try
{
TrackSeries(scannedSeries, info);
}
catch (Exception ex)
{
_logger.LogError(ex,
"[ScannerService] There was an exception that occurred during tracking {FilePath}. Skipping this file",
info?.FullFilePath);
}
}
foreach (var series in scannedSeries.Keys)
{
if (scannedSeries[series].Count <= 0 || processSeriesInfos == null) continue;
UpdateSortOrder(scannedSeries, series);
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
}
}
} }
/// <summary>
/// For a given ScanResult, sets the ParserInfos on the result
/// </summary>
/// <param name="result"></param>
/// <param name="seriesPaths"></param>
/// <param name="library"></param>
private async Task ProcessScanResult(ScanResult result, IDictionary<string, IList<SeriesModified>> seriesPaths, Library library)
{
// If the folder hasn't changed, generate fake ParserInfos for the Series that were in that folder.
if (!result.HasChanged)
{
var normalizedFolder = Parser.Parser.NormalizePath(result.Folder);
result.ParserInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
{
Series = fp.SeriesName,
Format = fp.Format,
}).ToList();
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", normalizedFolder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent("Skipped " + normalizedFolder, library.Name, ProgressEventType.Updated));
return;
}
var files = result.Files;
var folder = result.Folder;
var libraryRoot = result.LibraryRoot;
// When processing files for a folder and we do enter, we need to parse the information and combine parser infos
// NOTE: We might want to move the merge step later in the process, like return and combine.
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
if (files.Count == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty, no longer in this location, or has no file types that match Library File Types", folder);
result.ParserInfos = ArraySegment<ParserInfo>.Empty;
return;
}
// Multiple Series can exist within a folder. We should instead put these infos on the result and perform merging above
IList<ParserInfo> infos = files
.Select(file => _readingItemService.ParseFile(file, folder, libraryRoot, library.Type))
.Where(info => info != null)
.ToList()!;
result.ParserInfos = infos;
return;
}
private void UpdateSortOrder(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, ParsedSeries series) private void UpdateSortOrder(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, ParsedSeries series)
{ {
try try
@ -461,7 +508,7 @@ public class ParseScannedFiles
/// World of Acceleration v02.cbz having Series "Accel World" and Localized Series of "World of Acceleration" /// World of Acceleration v02.cbz having Series "Accel World" and Localized Series of "World of Acceleration"
/// </example> /// </example>
/// <param name="infos">A collection of ParserInfos</param> /// <param name="infos">A collection of ParserInfos</param>
private void MergeLocalizedSeriesWithSeries(IReadOnlyCollection<ParserInfo?> infos) private void MergeLocalizedSeriesWithSeries(IList<ParserInfo> infos)
{ {
var hasLocalizedSeries = infos.Any(i => !string.IsNullOrEmpty(i.LocalizedSeries)); var hasLocalizedSeries = infos.Any(i => !string.IsNullOrEmpty(i.LocalizedSeries));
if (!hasLocalizedSeries) return; if (!hasLocalizedSeries) return;

3
API/Services/Tasks/Scanner/Parser/BasicParser.cs
View file

@ -92,6 +92,9 @@ public class BasicParser(IDirectoryService directoryService, IDefaultParser imag
ret.Series = ret.Series.Substring(0, ret.Series.Length - ".pdf".Length); ret.Series = ret.Series.Substring(0, ret.Series.Length - ".pdf".Length);
} }
// Patch in other information from ComicInfo
UpdateFromComicInfo(ret);
// v0.8.x: Introducing a change where Specials will go in a separate Volume with a reserved number // v0.8.x: Introducing a change where Specials will go in a separate Volume with a reserved number
if (ret.IsSpecial) if (ret.IsSpecial)
{ {

29
API/Services/Tasks/Scanner/Parser/ComicVineParser.cs
View file

@ -98,33 +98,4 @@ public class ComicVineParser(IDirectoryService directoryService) : DefaultParser
{ {
return type == LibraryType.ComicVine; return type == LibraryType.ComicVine;
} }
private void UpdateFromComicInfo(ParserInfo info)
{
if (info.ComicInfo == null) return;
if (!string.IsNullOrEmpty(info.ComicInfo.Volume))
{
info.Volumes = info.ComicInfo.Volume;
}
if (string.IsNullOrEmpty(info.Series) && !string.IsNullOrEmpty(info.ComicInfo.Series))
{
info.Series = info.ComicInfo.Series.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
{
info.Chapters = info.ComicInfo.Number;
if (info.IsSpecial && Parser.DefaultChapter != info.Chapters)
{
info.IsSpecial = false;
info.Volumes = $"{Parser.SpecialVolumeNumber}";
}
}
// Patch is SeriesSort from ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo.TitleSort))
{
info.SeriesSort = info.ComicInfo.TitleSort.Trim();
}
}
} }

33
API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file

@ -99,6 +99,39 @@ public abstract class DefaultParser(IDirectoryService directoryService) : IDefau
} }
} }
protected void UpdateFromComicInfo(ParserInfo info)
{
if (info.ComicInfo == null) return;
if (!string.IsNullOrEmpty(info.ComicInfo.Volume))
{
info.Volumes = info.ComicInfo.Volume;
}
if (string.IsNullOrEmpty(info.Series) && !string.IsNullOrEmpty(info.ComicInfo.Series))
{
info.Series = info.ComicInfo.Series.Trim();
}
if (string.IsNullOrEmpty(info.LocalizedSeries) && !string.IsNullOrEmpty(info.ComicInfo.LocalizedSeries))
{
info.LocalizedSeries = info.ComicInfo.LocalizedSeries.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
{
info.Chapters = info.ComicInfo.Number;
if (info.IsSpecial && Parser.DefaultChapter != info.Chapters)
{
info.IsSpecial = false;
info.Volumes = $"{Parser.SpecialVolumeNumber}";
}
}
// Patch is SeriesSort from ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo.TitleSort))
{
info.SeriesSort = info.ComicInfo.TitleSort.Trim();
}
}
public abstract bool IsApplicable(string filePath, LibraryType type); public abstract bool IsApplicable(string filePath, LibraryType type);
protected static bool IsEmptyOrDefault(string volumes, string chapters) protected static bool IsEmptyOrDefault(string volumes, string chapters)

310
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -1,14 +1,11 @@
using System; using System;
using System.Collections.Generic; using System.Collections.Generic;
using System.Collections.Immutable;
using System.Diagnostics; using System.Diagnostics;
using System.Globalization; using System.Globalization;
using System.IO;
using System.Linq; using System.Linq;
using System.Threading.Tasks; using System.Threading.Tasks;
using API.Data; using API.Data;
using API.Data.Metadata; using API.Data.Metadata;
using API.Data.Repositories;
using API.Entities; using API.Entities;
using API.Entities.Enums; using API.Entities.Enums;
using API.Extensions; using API.Extensions;
@ -32,15 +29,9 @@ public interface IProcessSeries
/// </summary> /// </summary>
/// <returns></returns> /// <returns></returns>
Task Prime(); Task Prime();
Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false);
void EnqueuePostSeriesProcessTasks(int libraryId, int seriesId, bool forceUpdate = false);
// These exists only for Unit testing void Reset();
void UpdateSeriesMetadata(Series series, Library library); Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false);
void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false);
void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos, bool forceUpdate = false);
void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info, bool forceUpdate = false);
void UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? comicInfo, bool forceUpdate = false);
} }
/// <summary> /// <summary>
@ -60,16 +51,14 @@ public class ProcessSeries : IProcessSeries
private readonly ICollectionTagService _collectionTagService; private readonly ICollectionTagService _collectionTagService;
private readonly IReadingListService _readingListService; private readonly IReadingListService _readingListService;
private readonly IExternalMetadataService _externalMetadataService; private readonly IExternalMetadataService _externalMetadataService;
private readonly ITagManagerService _tagManagerService;
private Dictionary<string, Genre> _genres;
private IList<Person> _people;
private Dictionary<string, Tag> _tags;
private Dictionary<string, CollectionTag> _collectionTags;
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub, public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService, IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
IFileService fileService, IMetadataService metadataService, IWordCountAnalyzerService wordCountAnalyzerService, IFileService fileService, IMetadataService metadataService, IWordCountAnalyzerService wordCountAnalyzerService,
ICollectionTagService collectionTagService, IReadingListService readingListService, IExternalMetadataService externalMetadataService) ICollectionTagService collectionTagService, IReadingListService readingListService,
IExternalMetadataService externalMetadataService, ITagManagerService tagManagerService)
{ {
_unitOfWork = unitOfWork; _unitOfWork = unitOfWork;
_logger = logger; _logger = logger;
@ -83,12 +72,7 @@ public class ProcessSeries : IProcessSeries
_collectionTagService = collectionTagService; _collectionTagService = collectionTagService;
_readingListService = readingListService; _readingListService = readingListService;
_externalMetadataService = externalMetadataService; _externalMetadataService = externalMetadataService;
_tagManagerService = tagManagerService;
_genres = new Dictionary<string, Genre>();
_people = new List<Person>();
_tags = new Dictionary<string, Tag>();
_collectionTags = new Dictionary<string, CollectionTag>();
} }
/// <summary> /// <summary>
@ -96,12 +80,15 @@ public class ProcessSeries : IProcessSeries
/// </summary> /// </summary>
public async Task Prime() public async Task Prime()
{ {
_genres = (await _unitOfWork.GenreRepository.GetAllGenresAsync()).ToDictionary(t => t.NormalizedTitle); await _tagManagerService.Prime();
_people = await _unitOfWork.PersonRepository.GetAllPeople(); }
_tags = (await _unitOfWork.TagRepository.GetAllTagsAsync()).ToDictionary(t => t.NormalizedTitle);
_collectionTags = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync(CollectionTagIncludes.SeriesMetadata))
.ToDictionary(t => t.NormalizedTitle);
/// <summary>
/// Frees up memory
/// </summary>
public void Reset()
{
_tagManagerService.Reset();
} }
public async Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false) public async Task ProcessSeriesAsync(IList<ParserInfo> parsedInfos, Library library, bool forceUpdate = false)
@ -128,29 +115,7 @@ public class ProcessSeries : IProcessSeries
} }
catch (Exception ex) catch (Exception ex)
{ {
var seriesCollisions = await _unitOfWork.SeriesRepository.GetAllSeriesByAnyName(firstInfo.LocalizedSeries, string.Empty, library.Id, firstInfo.Format); await ReportDuplicateSeriesLookup(library, firstInfo, ex);
seriesCollisions = seriesCollisions.Where(collision =>
collision.Name != firstInfo.Series || collision.LocalizedName != firstInfo.LocalizedSeries).ToList();
if (seriesCollisions.Count > 1)
{
var firstCollision = seriesCollisions[0];
var secondCollision = seriesCollisions[1];
var tableRows = $"<tr><td>Name: {firstCollision.Name}</td><td>Name: {secondCollision.Name}</td></tr>" +
$"<tr><td>Localized: {firstCollision.LocalizedName}</td><td>Localized: {secondCollision.LocalizedName}</td></tr>" +
$"<tr><td>Filename: {Parser.Parser.NormalizePath(firstCollision.FolderPath)}</td><td>Filename: {Parser.Parser.NormalizePath(secondCollision.FolderPath)}</td></tr>";
var htmlTable = $"<table class='table table-striped'><thead><tr><th>Series 1</th><th>Series 2</th></tr></thead><tbody>{string.Join(string.Empty, tableRows)}</tbody></table>";
_logger.LogError(ex, "Scanner found a Series {SeriesName} which matched another Series {LocalizedName} in a different folder parallel to Library {LibraryName} root folder. This is not allowed. Please correct",
firstInfo.Series, firstInfo.LocalizedSeries, library.Name);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"Library {library.Name} Series collision on {firstInfo.Series}",
htmlTable));
}
return; return;
} }
@ -172,7 +137,7 @@ public class ProcessSeries : IProcessSeries
// parsedInfos[0] is not the first volume or chapter. We need to find it using a ComicInfo check (as it uses firstParsedInfo for series sort) // parsedInfos[0] is not the first volume or chapter. We need to find it using a ComicInfo check (as it uses firstParsedInfo for series sort)
var firstParsedInfo = parsedInfos.FirstOrDefault(p => p.ComicInfo != null, firstInfo); var firstParsedInfo = parsedInfos.FirstOrDefault(p => p.ComicInfo != null, firstInfo);
UpdateVolumes(series, parsedInfos, forceUpdate); await UpdateVolumes(series, parsedInfos, forceUpdate);
series.Pages = series.Volumes.Sum(v => v.Pages); series.Pages = series.Volumes.Sum(v => v.Pages);
series.NormalizedName = series.Name.ToNormalized(); series.NormalizedName = series.Name.ToNormalized();
@ -203,7 +168,7 @@ public class ProcessSeries : IProcessSeries
series.NormalizedLocalizedName = series.LocalizedName.ToNormalized(); series.NormalizedLocalizedName = series.LocalizedName.ToNormalized();
} }
UpdateSeriesMetadata(series, library); await UpdateSeriesMetadata(series, library);
// Update series FolderPath here // Update series FolderPath here
await UpdateSeriesFolderPath(parsedInfos, library, series); await UpdateSeriesFolderPath(parsedInfos, library, series);
@ -229,18 +194,25 @@ public class ProcessSeries : IProcessSeries
return; return;
} }
// Process reading list after commit as we need to commit per list // Process reading list after commit as we need to commit per list
await _readingListService.CreateReadingListsFromSeries(series, library); BackgroundJob.Enqueue(() => _readingListService.CreateReadingListsFromSeries(library.Id, series.Id));
if (seriesAdded) if (seriesAdded)
{ {
// See if any recommendations can link up to the series and pre-fetch external metadata for the series // See if any recommendations can link up to the series and pre-fetch external metadata for the series
_logger.LogInformation("Linking up External Recommendations new series (if applicable)"); _logger.LogInformation("Linking up External Recommendations new series (if applicable)");
await _externalMetadataService.GetNewSeriesData(series.Id, series.Library.Type);
await _unitOfWork.ExternalSeriesMetadataRepository.LinkRecommendationsToSeries(series); BackgroundJob.Enqueue(() =>
_externalMetadataService.GetNewSeriesData(series.Id, series.Library.Type));
await _eventHub.SendMessageAsync(MessageFactory.SeriesAdded, await _eventHub.SendMessageAsync(MessageFactory.SeriesAdded,
MessageFactory.SeriesAddedEvent(series.Id, series.Name, series.LibraryId), false); MessageFactory.SeriesAddedEvent(series.Id, series.Name, series.LibraryId), false);
} }
else
{
await _unitOfWork.ExternalSeriesMetadataRepository.LinkRecommendationsToSeries(series);
}
_logger.LogInformation("[ScannerService] Finished series update on {SeriesName} in {Milliseconds} ms", seriesName, scanWatch.ElapsedMilliseconds); _logger.LogInformation("[ScannerService] Finished series update on {SeriesName} in {Milliseconds} ms", seriesName, scanWatch.ElapsedMilliseconds);
} }
@ -253,7 +225,34 @@ public class ProcessSeries : IProcessSeries
var settings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync(); var settings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
await _metadataService.GenerateCoversForSeries(series, settings.EncodeMediaAs, settings.CoverImageSize); await _metadataService.GenerateCoversForSeries(series, settings.EncodeMediaAs, settings.CoverImageSize);
EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id); BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(series.LibraryId, series.Id, forceUpdate));
}
private async Task ReportDuplicateSeriesLookup(Library library, ParserInfo firstInfo, Exception ex)
{
var seriesCollisions = await _unitOfWork.SeriesRepository.GetAllSeriesByAnyName(firstInfo.LocalizedSeries, string.Empty, library.Id, firstInfo.Format);
seriesCollisions = seriesCollisions.Where(collision =>
collision.Name != firstInfo.Series || collision.LocalizedName != firstInfo.LocalizedSeries).ToList();
if (seriesCollisions.Count > 1)
{
var firstCollision = seriesCollisions[0];
var secondCollision = seriesCollisions[1];
var tableRows = $"<tr><td>Name: {firstCollision.Name}</td><td>Name: {secondCollision.Name}</td></tr>" +
$"<tr><td>Localized: {firstCollision.LocalizedName}</td><td>Localized: {secondCollision.LocalizedName}</td></tr>" +
$"<tr><td>Filename: {Parser.Parser.NormalizePath(firstCollision.FolderPath)}</td><td>Filename: {Parser.Parser.NormalizePath(secondCollision.FolderPath)}</td></tr>";
var htmlTable = $"<table class='table table-striped'><thead><tr><th>Series 1</th><th>Series 2</th></tr></thead><tbody>{string.Join(string.Empty, tableRows)}</tbody></table>";
_logger.LogError(ex, "Scanner found a Series {SeriesName} which matched another Series {LocalizedName} in a different folder parallel to Library {LibraryName} root folder. This is not allowed. Please correct",
firstInfo.Series, firstInfo.LocalizedSeries, library.Name);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"Library {library.Name} Series collision on {firstInfo.Series}",
htmlTable));
}
} }
@ -280,12 +279,8 @@ public class ProcessSeries : IProcessSeries
} }
} }
public void EnqueuePostSeriesProcessTasks(int libraryId, int seriesId, bool forceUpdate = false)
{
BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(libraryId, seriesId, forceUpdate));
}
public void UpdateSeriesMetadata(Series series, Library library) private async Task UpdateSeriesMetadata(Series series, Library library)
{ {
series.Metadata ??= new SeriesMetadataBuilder().Build(); series.Metadata ??= new SeriesMetadataBuilder().Build();
var firstChapter = SeriesService.GetFirstChapterForMetadata(series); var firstChapter = SeriesService.GetFirstChapterForMetadata(series);
@ -359,14 +354,9 @@ public class ProcessSeries : IProcessSeries
_logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name); _logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries)) foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{ {
var normalizedName = Parser.Parser.Normalize(collection); var t = await _tagManagerService.GetCollectionTag(collection);
if (!_collectionTags.TryGetValue(normalizedName, out var tag)) if (t == null) continue;
{ _collectionTagService.AddTagToSeriesMetadata(t, series.Metadata);
tag = _collectionTagService.CreateTag(collection);
_collectionTags.Add(normalizedName, tag);
}
_collectionTagService.AddTagToSeriesMetadata(tag, series.Metadata);
} }
} }
@ -541,7 +531,7 @@ public class ProcessSeries : IProcessSeries
} }
public void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false) private async Task UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
{ {
// Add new volumes and update chapters per volume // Add new volumes and update chapters per volume
var distinctVolumes = parsedInfos.DistinctVolumes(); var distinctVolumes = parsedInfos.DistinctVolumes();
@ -586,7 +576,7 @@ public class ProcessSeries : IProcessSeries
try try
{ {
var firstChapterInfo = infos.SingleOrDefault(i => i.FullFilePath.Equals(firstFile.FilePath)); var firstChapterInfo = infos.SingleOrDefault(i => i.FullFilePath.Equals(firstFile.FilePath));
UpdateChapterFromComicInfo(chapter, firstChapterInfo?.ComicInfo, forceUpdate); await UpdateChapterFromComicInfo(chapter, firstChapterInfo?.ComicInfo, forceUpdate);
} }
catch (Exception ex) catch (Exception ex)
{ {
@ -622,7 +612,7 @@ public class ProcessSeries : IProcessSeries
} }
} }
public void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos, bool forceUpdate = false) private void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
{ {
// Add new chapters // Add new chapters
foreach (var info in parsedInfos) foreach (var info in parsedInfos)
@ -689,7 +679,7 @@ public class ProcessSeries : IProcessSeries
} }
} }
public void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info, bool forceUpdate = false) private void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info, bool forceUpdate = false)
{ {
chapter.Files ??= new List<MangaFile>(); chapter.Files ??= new List<MangaFile>();
var existingFile = chapter.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath); var existingFile = chapter.Files.SingleOrDefault(f => f.FilePath == info.FullFilePath);
@ -700,7 +690,7 @@ public class ProcessSeries : IProcessSeries
if (!forceUpdate && !_fileService.HasFileBeenModifiedSince(existingFile.FilePath, existingFile.LastModified) && existingFile.Pages != 0) return; if (!forceUpdate && !_fileService.HasFileBeenModifiedSince(existingFile.FilePath, existingFile.LastModified) && existingFile.Pages != 0) return;
existingFile.Pages = _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format); existingFile.Pages = _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format);
existingFile.Extension = fileInfo.Extension.ToLowerInvariant(); existingFile.Extension = fileInfo.Extension.ToLowerInvariant();
existingFile.FileName = Path.GetFileNameWithoutExtension(existingFile.FilePath); existingFile.FileName = Parser.Parser.RemoveExtensionIfSupported(existingFile.FilePath);
existingFile.Bytes = fileInfo.Length; existingFile.Bytes = fileInfo.Length;
// We skip updating DB here with last modified time so that metadata refresh can do it // We skip updating DB here with last modified time so that metadata refresh can do it
} }
@ -715,7 +705,7 @@ public class ProcessSeries : IProcessSeries
} }
} }
public void UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? comicInfo, bool forceUpdate = false) private async Task UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? comicInfo, bool forceUpdate = false)
{ {
if (comicInfo == null) return; if (comicInfo == null) return;
var firstFile = chapter.Files.MinBy(x => x.Chapter); var firstFile = chapter.Files.MinBy(x => x.Chapter);
@ -774,9 +764,7 @@ public class ProcessSeries : IProcessSeries
if (!string.IsNullOrEmpty(comicInfo.Web)) if (!string.IsNullOrEmpty(comicInfo.Web))
{ {
chapter.WebLinks = string.Join(",", comicInfo.Web chapter.WebLinks = string.Join(",", comicInfo.Web
.Split(",") .Split(",", StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries)
.Where(s => !string.IsNullOrEmpty(s))
.Select(s => s.Trim())
); );
// For each weblink, try to parse out some MetadataIds and store in the Chapter directly for matching (CBL) // For each weblink, try to parse out some MetadataIds and store in the Chapter directly for matching (CBL)
@ -795,21 +783,6 @@ public class ProcessSeries : IProcessSeries
// This needs to check against both Number and Volume to calculate Count // This needs to check against both Number and Volume to calculate Count
chapter.Count = comicInfo.CalculatedCount(); chapter.Count = comicInfo.CalculatedCount();
void AddPerson(Person person)
{
PersonHelper.AddPersonIfNotExists(chapter.People, person);
}
void AddGenre(Genre genre, bool newTag)
{
chapter.Genres.Add(genre);
}
void AddTag(Tag tag, bool added)
{
chapter.Tags.Add(tag);
}
if (comicInfo.Year > 0) if (comicInfo.Year > 0)
{ {
@ -818,152 +791,79 @@ public class ProcessSeries : IProcessSeries
chapter.ReleaseDate = new DateTime(comicInfo.Year, month, day); chapter.ReleaseDate = new DateTime(comicInfo.Year, month, day);
} }
var people = GetTagValues(comicInfo.Colorist); var people = TagHelper.GetTagValues(comicInfo.Colorist);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Colorist); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Colorist);
UpdatePeople(people, PersonRole.Colorist, AddPerson); await UpdatePeople(chapter, people, PersonRole.Colorist);
people = GetTagValues(comicInfo.Characters); people = TagHelper.GetTagValues(comicInfo.Characters);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Character); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Character);
UpdatePeople(people, PersonRole.Character, AddPerson); await UpdatePeople(chapter, people, PersonRole.Character);
people = GetTagValues(comicInfo.Translator); people = TagHelper.GetTagValues(comicInfo.Translator);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Translator); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Translator);
UpdatePeople(people, PersonRole.Translator, AddPerson); await UpdatePeople(chapter, people, PersonRole.Translator);
people = GetTagValues(comicInfo.Writer); people = TagHelper.GetTagValues(comicInfo.Writer);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Writer); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Writer);
UpdatePeople(people, PersonRole.Writer, AddPerson); await UpdatePeople(chapter, people, PersonRole.Writer);
people = GetTagValues(comicInfo.Editor); people = TagHelper.GetTagValues(comicInfo.Editor);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Editor); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Editor);
UpdatePeople(people, PersonRole.Editor, AddPerson); await UpdatePeople(chapter, people, PersonRole.Editor);
people = GetTagValues(comicInfo.Inker); people = TagHelper.GetTagValues(comicInfo.Inker);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Inker); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Inker);
UpdatePeople(people, PersonRole.Inker, AddPerson); await UpdatePeople(chapter, people, PersonRole.Inker);
people = GetTagValues(comicInfo.Letterer); people = TagHelper.GetTagValues(comicInfo.Letterer);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Letterer); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Letterer);
UpdatePeople(people, PersonRole.Letterer, AddPerson); await UpdatePeople(chapter, people, PersonRole.Letterer);
people = GetTagValues(comicInfo.Penciller); people = TagHelper.GetTagValues(comicInfo.Penciller);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Penciller); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Penciller);
UpdatePeople(people, PersonRole.Penciller, AddPerson); await UpdatePeople(chapter, people, PersonRole.Penciller);
people = GetTagValues(comicInfo.CoverArtist); people = TagHelper.GetTagValues(comicInfo.CoverArtist);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.CoverArtist); PersonHelper.RemovePeople(chapter.People, people, PersonRole.CoverArtist);
UpdatePeople(people, PersonRole.CoverArtist, AddPerson); await UpdatePeople(chapter, people, PersonRole.CoverArtist);
people = GetTagValues(comicInfo.Publisher); people = TagHelper.GetTagValues(comicInfo.Publisher);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Publisher); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Publisher);
UpdatePeople(people, PersonRole.Publisher, AddPerson); await UpdatePeople(chapter, people, PersonRole.Publisher);
people = GetTagValues(comicInfo.Imprint); people = TagHelper.GetTagValues(comicInfo.Imprint);
PersonHelper.RemovePeople(chapter.People, people, PersonRole.Imprint); PersonHelper.RemovePeople(chapter.People, people, PersonRole.Imprint);
UpdatePeople(people, PersonRole.Imprint, AddPerson); await UpdatePeople(chapter, people, PersonRole.Imprint);
var genres = GetTagValues(comicInfo.Genre); var genres = TagHelper.GetTagValues(comicInfo.Genre);
GenreHelper.KeepOnlySameGenreBetweenLists(chapter.Genres, GenreHelper.KeepOnlySameGenreBetweenLists(chapter.Genres,
genres.Select(g => new GenreBuilder(g).Build()).ToList()); genres.Select(g => new GenreBuilder(g).Build()).ToList());
UpdateGenre(genres, AddGenre); foreach (var genre in genres)
{
var g = await _tagManagerService.GetGenre(genre);
if (g == null) continue;
chapter.Genres.Add(g);
}
var tags = GetTagValues(comicInfo.Tags); var tags = TagHelper.GetTagValues(comicInfo.Tags);
TagHelper.KeepOnlySameTagBetweenLists(chapter.Tags, tags.Select(t => new TagBuilder(t).Build()).ToList()); TagHelper.KeepOnlySameTagBetweenLists(chapter.Tags, tags.Select(t => new TagBuilder(t).Build()).ToList());
UpdateTag(tags, AddTag); foreach (var tag in tags)
}
private static IList<string> GetTagValues(string comicInfoTagSeparatedByComma)
{
// TODO: Move this to an extension and test it
if (string.IsNullOrEmpty(comicInfoTagSeparatedByComma))
{ {
return ImmutableList<string>.Empty; var t = await _tagManagerService.GetTag(tag);
} if (t == null) continue;
chapter.Tags.Add(t);
return comicInfoTagSeparatedByComma.Split(",")
.Select(s => s.Trim())
.DistinctBy(Parser.Parser.Normalize)
.ToList();
}
/// <summary>
/// Given a list of all existing people, this will check the new names and roles and if it doesn't exist in allPeople, will create and
/// add an entry. For each person in name, the callback will be executed.
/// </summary>
/// <remarks>This does not remove people if an empty list is passed into names</remarks>
/// <remarks>This is used to add new people to a list without worrying about duplicating rows in the DB</remarks>
/// <param name="names"></param>
/// <param name="role"></param>
/// <param name="action"></param>
private void UpdatePeople(IEnumerable<string> names, PersonRole role, Action<Person> action)
{
var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
foreach (var name in names)
{
var normalizedName = name.ToNormalized();
var person = allPeopleTypeRole.Find(p =>
p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
if (person == null)
{
person = new PersonBuilder(name, role).Build();
_people.Add(person);
}
action(person);
} }
} }
/// <summary> private async Task UpdatePeople(Chapter chapter, IList<string> people, PersonRole role)
///
/// </summary>
/// <param name="names"></param>
/// <param name="action">Executes for each tag</param>
private void UpdateGenre(IEnumerable<string> names, Action<Genre, bool> action)
{ {
foreach (var name in names) foreach (var person in people)
{ {
var normalizedName = name.ToNormalized(); var p = await _tagManagerService.GetPerson(person, role);
if (string.IsNullOrEmpty(normalizedName)) continue; if (p == null) continue;
chapter.People.Add(p);
_genres.TryGetValue(normalizedName, out var genre);
var newTag = genre == null;
if (newTag)
{
genre = new GenreBuilder(name).Build();
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}
action(genre!, newTag);
} }
} }
/// <summary>
///
/// </summary>
/// <param name="names"></param>
/// <param name="action">Callback for every item. Will give said item back and a bool if item was added</param>
private void UpdateTag(IEnumerable<string> names, Action<Tag, bool> action)
{
foreach (var name in names)
{
if (string.IsNullOrEmpty(name.Trim())) continue;
var normalizedName = name.ToNormalized();
_tags.TryGetValue(normalizedName, out var tag);
var added = tag == null;
if (tag == null)
{
tag = new TagBuilder(name).Build();
_tags.Add(normalizedName, tag);
}
action(tag, added);
}
}
} }

207
API/Services/Tasks/Scanner/TagManagerService.cs Normal file
View file

@ -0,0 +1,207 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers.Builders;
namespace API.Services.Tasks.Scanner;
#nullable enable
public interface ITagManagerService
{
/// <summary>
/// Should be called once before any usage
/// </summary>
/// <returns></returns>
Task Prime();
/// <summary>
/// Should be called after all work is done, will free up memory
/// </summary>
/// <returns></returns>
void Reset();
Task<Genre?> GetGenre(string genre);
Task<Tag?> GetTag(string tag);
Task<Person?> GetPerson(string name, PersonRole role);
Task<CollectionTag?> GetCollectionTag(string name);
}
/// <summary>
/// This is responsible for handling existing and new tags during the scan. When a new tag doesn't exist, it will create it.
/// This is Thread Safe.
/// </summary>
public class TagManagerService : ITagManagerService
{
private readonly IUnitOfWork _unitOfWork;
private Dictionary<string, Genre> _genres;
private Dictionary<string, Tag> _tags;
private Dictionary<string, Person> _people;
private Dictionary<string, CollectionTag> _collectionTags;
private readonly SemaphoreSlim _genreSemaphore = new SemaphoreSlim(1, 1);
private readonly SemaphoreSlim _tagSemaphore = new SemaphoreSlim(1, 1);
private readonly SemaphoreSlim _personSemaphore = new SemaphoreSlim(1, 1);
private readonly SemaphoreSlim _collectionTagSemaphore = new SemaphoreSlim(1, 1);
public TagManagerService(IUnitOfWork unitOfWork)
{
_unitOfWork = unitOfWork;
Reset();
}
public void Reset()
{
_genres = new Dictionary<string, Genre>();
_tags = new Dictionary<string, Tag>();
_people = new Dictionary<string, Person>();
_collectionTags = new Dictionary<string, CollectionTag>();
}
public async Task Prime()
{
_genres = (await _unitOfWork.GenreRepository.GetAllGenresAsync()).ToDictionary(t => t.NormalizedTitle);
_tags = (await _unitOfWork.TagRepository.GetAllTagsAsync()).ToDictionary(t => t.NormalizedTitle);
_people = (await _unitOfWork.PersonRepository.GetAllPeople()).ToDictionary(GetPersonKey);
_collectionTags = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync(CollectionTagIncludes.SeriesMetadata))
.ToDictionary(t => t.NormalizedTitle);
}
/// <summary>
/// Gets the Genre entity for the given string. If one doesn't exist, one will be created and committed.
/// </summary>
/// <param name="genre"></param>
/// <returns></returns>
public async Task<Genre?> GetGenre(string genre)
{
if (string.IsNullOrEmpty(genre)) return null;
await _genreSemaphore.WaitAsync();
try
{
if (_genres.TryGetValue(genre.ToNormalized(), out var result))
{
return result;
}
// We need to create a new Genre
result = new GenreBuilder(genre).Build();
_unitOfWork.GenreRepository.Attach(result);
await _unitOfWork.CommitAsync();
_genres.Add(result.NormalizedTitle, result);
return result;
}
finally
{
_genreSemaphore.Release();
}
}
/// <summary>
/// Gets the Tag entity for the given string. If one doesn't exist, one will be created and committed.
/// </summary>
/// <param name="tag"></param>
/// <returns></returns>
public async Task<Tag?> GetTag(string tag)
{
if (string.IsNullOrEmpty(tag)) return null;
await _tagSemaphore.WaitAsync();
try
{
if (_tags.TryGetValue(tag.ToNormalized(), out var result))
{
return result;
}
// We need to create a new Genre
result = new TagBuilder(tag).Build();
_unitOfWork.TagRepository.Attach(result);
await _unitOfWork.CommitAsync();
_tags.Add(result.NormalizedTitle, result);
return result;
}
finally
{
_tagSemaphore.Release();
}
}
/// <summary>
/// Gets the Person entity for the given string and role. If one doesn't exist, one will be created and committed.
/// </summary>
/// <param name="name">Person Name</param>
/// <param name="role"></param>
/// <returns></returns>
public async Task<Person?> GetPerson(string name, PersonRole role)
{
if (string.IsNullOrEmpty(name)) return null;
await _personSemaphore.WaitAsync();
try
{
var key = GetPersonKey(name.ToNormalized(), role);
if (_people.TryGetValue(key, out var result))
{
return result;
}
// We need to create a new Genre
result = new PersonBuilder(name, role).Build();
_unitOfWork.PersonRepository.Attach(result);
await _unitOfWork.CommitAsync();
_people.Add(key, result);
return result;
}
finally
{
_personSemaphore.Release();
}
}
private static string GetPersonKey(string normalizedName, PersonRole role)
{
return normalizedName + "_" + role;
}
private static string GetPersonKey(Person p)
{
return GetPersonKey(p.NormalizedName, p.Role);
}
/// <summary>
/// Gets the CollectionTag entity for the given string. If one doesn't exist, one will be created and committed.
/// </summary>
/// <param name="tag"></param>
/// <returns></returns>
public async Task<CollectionTag?> GetCollectionTag(string tag)
{
if (string.IsNullOrEmpty(tag)) return null;
await _collectionTagSemaphore.WaitAsync();
try
{
if (_collectionTags.TryGetValue(tag.ToNormalized(), out var result))
{
return result;
}
// We need to create a new Genre
result = new CollectionTagBuilder(tag).Build();
_unitOfWork.CollectionTagRepository.Add(result);
await _unitOfWork.CommitAsync();
_collectionTags.Add(result.NormalizedTitle, result);
return result;
}
finally
{
_collectionTagSemaphore.Release();
}
}
}

231
API/Services/Tasks/ScannerService.cs
View file

@ -33,7 +33,7 @@ public interface IScannerService
[Queue(TaskScheduler.ScanQueue)] [Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)] [DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task ScanLibrary(int libraryId, bool forceUpdate = false); Task ScanLibrary(int libraryId, bool forceUpdate = false, bool isSingleScan = true);
[Queue(TaskScheduler.ScanQueue)] [Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)] [DisableConcurrentExecution(60 * 60 * 60)]
@ -86,8 +86,6 @@ public class ScannerService : IScannerService
private readonly IProcessSeries _processSeries; private readonly IProcessSeries _processSeries;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService; private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly SemaphoreSlim _seriesProcessingSemaphore = new SemaphoreSlim(1, 1);
public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger, public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger,
IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub, IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub,
IDirectoryService directoryService, IReadingItemService readingItemService, IDirectoryService directoryService, IReadingItemService readingItemService,
@ -171,7 +169,7 @@ public class ScannerService : IScannerService
var libraries = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).ToList(); var libraries = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).ToList();
var libraryFolders = libraries.SelectMany(l => l.Folders); var libraryFolders = libraries.SelectMany(l => l.Folders);
var libraryFolder = libraryFolders.Select(Scanner.Parser.Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory)); var libraryFolder = libraryFolders.Select(Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory));
if (string.IsNullOrEmpty(libraryFolder)) return; if (string.IsNullOrEmpty(libraryFolder)) return;
var library = libraries.Find(l => l.Folders.Select(Parser.NormalizePath).Contains(libraryFolder)); var library = libraries.Find(l => l.Folders.Select(Parser.NormalizePath).Contains(libraryFolder));
@ -183,7 +181,7 @@ public class ScannerService : IScannerService
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder); _logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
return; return;
} }
BackgroundJob.Schedule(() => ScanLibrary(library.Id, false), TimeSpan.FromMinutes(1)); BackgroundJob.Schedule(() => ScanLibrary(library.Id, false, true), TimeSpan.FromMinutes(1));
} }
} }
@ -196,12 +194,14 @@ public class ScannerService : IScannerService
public async Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true) public async Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true)
{ {
var sw = Stopwatch.StartNew(); var sw = Stopwatch.StartNew();
var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId);
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId); var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update
var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId}); var existingChapterIdsToClean = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns); var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
if (library == null) return; if (library == null) return;
var libraryPaths = library.Folders.Select(f => f.Path).ToList(); var libraryPaths = library.Folders.Select(f => f.Path).ToList();
if (await ShouldScanSeries(seriesId, library, libraryPaths, series, true) != ScanCancelReason.NoCancel) if (await ShouldScanSeries(seriesId, library, libraryPaths, series, true) != ScanCancelReason.NoCancel)
{ {
@ -214,6 +214,7 @@ public class ScannerService : IScannerService
if (string.IsNullOrEmpty(folderPath) || !_directoryService.Exists(folderPath)) if (string.IsNullOrEmpty(folderPath) || !_directoryService.Exists(folderPath))
{ {
// We don't care if it's multiple due to new scan loop enforcing all in one root directory // We don't care if it's multiple due to new scan loop enforcing all in one root directory
var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId);
var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(libraryPaths, files.Select(f => f.FilePath).ToList()); var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(libraryPaths, files.Select(f => f.FilePath).ToList());
if (seriesDirs.Keys.Count == 0) if (seriesDirs.Keys.Count == 0)
{ {
@ -243,17 +244,19 @@ public class ScannerService : IScannerService
// If the series path doesn't exist anymore, it was either moved or renamed. We need to essentially delete it // If the series path doesn't exist anymore, it was either moved or renamed. We need to essentially delete it
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>(); var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name)); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
await _processSeries.Prime();
_logger.LogInformation("Beginning file scan on {SeriesName}", series.Name); _logger.LogInformation("Beginning file scan on {SeriesName}", series.Name);
var scanElapsedTime = await ScanFiles(library, new []{ folderPath }, false, TrackFiles, true); var (scanElapsedTime, processedSeries) = await ScanFiles(library, new []{ folderPath },
false, true);
// Transform seen series into the parsedSeries (I think we can actually just have processedSeries be used instead
TrackFoundSeriesAndFiles(parsedSeries, processedSeries);
_logger.LogInformation("ScanFiles for {Series} took {Time}", series.Name, scanElapsedTime); _logger.LogInformation("ScanFiles for {Series} took {Time}", series.Name, scanElapsedTime);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name)); // We now technically have all scannedSeries, we could invoke each Series to be scanned
// Remove any parsedSeries keys that don't belong to our series. This can occur when users store 2 series in the same folder // Remove any parsedSeries keys that don't belong to our series. This can occur when users store 2 series in the same folder
RemoveParsedInfosNotForSeries(parsedSeries, series); RemoveParsedInfosNotForSeries(parsedSeries, series);
@ -261,36 +264,57 @@ public class ScannerService : IScannerService
// If nothing was found, first validate any of the files still exist. If they don't then we have a deletion and can skip the rest of the logic flow // If nothing was found, first validate any of the files still exist. If they don't then we have a deletion and can skip the rest of the logic flow
if (parsedSeries.Count == 0) if (parsedSeries.Count == 0)
{ {
var seriesFiles = (await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id)); var seriesFiles = (await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id));
if (!string.IsNullOrEmpty(series.FolderPath) && !seriesFiles.Where(f => f.FilePath.Contains(series.FolderPath)).Any(m => File.Exists(m.FilePath))) if (!string.IsNullOrEmpty(series.FolderPath) && !seriesFiles.Where(f => f.FilePath.Contains(series.FolderPath)).Any(m => File.Exists(m.FilePath)))
{
try
{ {
_unitOfWork.SeriesRepository.Remove(series); try
await CommitAndSend(1, sw, scanElapsedTime, series); {
await _eventHub.SendMessageAsync(MessageFactory.SeriesRemoved, _unitOfWork.SeriesRepository.Remove(series);
MessageFactory.SeriesRemovedEvent(seriesId, string.Empty, series.LibraryId), false); await CommitAndSend(1, sw, scanElapsedTime, series);
await _eventHub.SendMessageAsync(MessageFactory.SeriesRemoved,
MessageFactory.SeriesRemovedEvent(seriesId, string.Empty, series.LibraryId), false);
}
catch (Exception ex)
{
_logger.LogCritical(ex, "There was an error during ScanSeries to delete the series as no files could be found. Aborting scan");
await _unitOfWork.RollbackAsync();
return;
}
} }
catch (Exception ex) else
{ {
_logger.LogCritical(ex, "There was an error during ScanSeries to delete the series as no files could be found. Aborting scan"); // I think we should just fail and tell user to fix their setup. This is extremely expensive for an edge case
_logger.LogCritical("We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan");
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"Error scanning {series.Name}", "We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan"));
await _unitOfWork.RollbackAsync(); await _unitOfWork.RollbackAsync();
return; return;
} }
}
else
{
// I think we should just fail and tell user to fix their setup. This is extremely expensive for an edge case
_logger.LogCritical("We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan");
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent($"Error scanning {series.Name}", "We weren't able to find any files in the series scan, but there should be. Please correct your naming convention or put Series in a dedicated folder. Aborting scan"));
await _unitOfWork.RollbackAsync();
return;
}
// At this point, parsedSeries will have at least one key and we can perform the update. If it still doesn't, just return and don't do anything
if (parsedSeries.Count == 0) return;
} }
// At this point, parsedSeries will have at least one key and we can perform the update. If it still doesn't, just return and don't do anything
if (parsedSeries.Count == 0) return;
// Don't allow any processing on files that aren't part of this series
var toProcess = parsedSeries.Keys.Where(key =>
key.NormalizedName.Equals(series.NormalizedName) ||
key.NormalizedName.Equals(series.OriginalName?.ToNormalized()))
.ToList();
if (toProcess.Count > 0)
{
await _processSeries.Prime();
}
foreach (var pSeries in toProcess)
{
// Process Series
await _processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, bypassFolderOptimizationChecks);
}
_processSeries.Reset();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name)); await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
// Tell UI that this series is done // Tell UI that this series is done
@ -298,32 +322,17 @@ public class ScannerService : IScannerService
MessageFactory.ScanSeriesEvent(library.Id, seriesId, series.Name)); MessageFactory.ScanSeriesEvent(library.Id, seriesId, series.Name));
await _metadataService.RemoveAbandonedMetadataKeys(); await _metadataService.RemoveAbandonedMetadataKeys();
//BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(series.LibraryId, seriesId, false));
//BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(library.Id, seriesId, false)); BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(existingChapterIdsToClean));
BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(chapterIds));
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory)); BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return; }
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo) private void TrackFoundSeriesAndFiles(Dictionary<ParsedSeries, IList<ParserInfo>> parsedSeries, IList<ScannedSeriesResult> seenSeries)
{
foreach (var series in seenSeries.Where(s => s.ParsedInfos.Count > 0))
{ {
var parsedFiles = parsedInfo.Item2; var parsedFiles = series.ParsedInfos;
if (parsedFiles.Count == 0) return; parsedSeries.Add(series.ParsedSeries, parsedFiles);
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles[0].Series,
NormalizedName = parsedFiles[0].Series.ToNormalized(),
Format = parsedFiles[0].Format
};
// For Scan Series, we need to filter out anything that isn't our Series
if (!foundParsedSeries.NormalizedName.Equals(series.NormalizedName) && !foundParsedSeries.NormalizedName.Equals(series.OriginalName?.ToNormalized()))
{
return;
}
await _processSeries.ProcessSeriesAsync(parsedFiles, library, bypassFolderOptimizationChecks);
parsedSeries.Add(foundParsedSeries, parsedFiles);
} }
} }
@ -451,11 +460,12 @@ public class ScannerService : IScannerService
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibraries(bool forceUpdate = false) public async Task ScanLibraries(bool forceUpdate = false)
{ {
_logger.LogInformation("Starting Scan of All Libraries"); _logger.LogInformation("Starting Scan of All Libraries, Forced: {Forced}", forceUpdate);
foreach (var lib in await _unitOfWork.LibraryRepository.GetLibrariesAsync()) foreach (var lib in await _unitOfWork.LibraryRepository.GetLibrariesAsync())
{ {
await ScanLibrary(lib.Id, forceUpdate); await ScanLibrary(lib.Id, forceUpdate, true);
} }
_processSeries.Reset();
_logger.LogInformation("Scan of All Libraries Finished"); _logger.LogInformation("Scan of All Libraries Finished");
} }
@ -467,10 +477,11 @@ public class ScannerService : IScannerService
/// </summary> /// </summary>
/// <param name="libraryId"></param> /// <param name="libraryId"></param>
/// <param name="forceUpdate">Defaults to false</param> /// <param name="forceUpdate">Defaults to false</param>
/// <param name="isSingleScan">Defaults to true. Is this a standalone invocation or is it in a loop?</param>
[Queue(TaskScheduler.ScanQueue)] [Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)] [DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)] [AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibrary(int libraryId, bool forceUpdate = false) public async Task ScanLibrary(int libraryId, bool forceUpdate = false, bool isSingleScan = true)
{ {
var sw = Stopwatch.StartNew(); var sw = Stopwatch.StartNew();
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns); var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
@ -490,19 +501,33 @@ public class ScannerService : IScannerService
var totalFiles = 0; var totalFiles = 0;
var seenSeries = new List<ParsedSeries>(); var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
var (scanElapsedTime, processedSeries) = await ScanFiles(library, libraryFolderPaths,
shouldUseLibraryScan, forceUpdate);
await _processSeries.Prime(); TrackFoundSeriesAndFiles(parsedSeries, processedSeries);
//var processTasks = new List<Func<Task>>();
var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate); // We need to remove any keys where there is no actual parser info
var toProcess = parsedSeries.Keys
.Where(k => parsedSeries[k].Any() && !string.IsNullOrEmpty(parsedSeries[k][0].Filename))
.ToList();
if (toProcess.Count > 0)
{
// This grabs all the shared entities, like tags, genre, people. To be solved later in this refactor on how to not have blocking access.
await _processSeries.Prime();
}
var tasks = new List<Task>();
foreach (var pSeries in toProcess)
{
totalFiles += parsedSeries[pSeries].Count;
tasks.Add(_processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, forceUpdate));
}
await Task.WhenAll(tasks);
// NOTE: This runs sync after every file is scanned
// foreach (var task in processTasks)
// {
// await task();
// }
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(string.Empty, library.Name, ProgressEventType.Ended)); MessageFactory.FileScanProgressEvent(string.Empty, library.Name, ProgressEventType.Ended));
@ -521,17 +546,22 @@ public class ScannerService : IScannerService
_unitOfWork.LibraryRepository.Update(library); _unitOfWork.LibraryRepository.Update(library);
if (await _unitOfWork.CommitAsync()) if (await _unitOfWork.CommitAsync())
{ {
if (isSingleScan)
{
_processSeries.Reset();
}
if (totalFiles == 0) if (totalFiles == 0)
{ {
_logger.LogInformation( _logger.LogInformation(
"[ScannerService] Finished library scan of {ParsedSeriesCount} series in {ElapsedScanTime} milliseconds for {LibraryName}. There were no changes", "[ScannerService] Finished library scan of {ParsedSeriesCount} series in {ElapsedScanTime} milliseconds for {LibraryName}. There were no changes",
seenSeries.Count, sw.ElapsedMilliseconds, library.Name); parsedSeries.Count, sw.ElapsedMilliseconds, library.Name);
} }
else else
{ {
_logger.LogInformation( _logger.LogInformation(
"[ScannerService] Finished library scan of {TotalFiles} files and {ParsedSeriesCount} series in {ElapsedScanTime} milliseconds for {LibraryName}", "[ScannerService] Finished library scan of {TotalFiles} files and {ParsedSeriesCount} series in {ElapsedScanTime} milliseconds for {LibraryName}",
totalFiles, seenSeries.Count, sw.ElapsedMilliseconds, library.Name); totalFiles, parsedSeries.Count, sw.ElapsedMilliseconds, library.Name);
} }
try try
@ -539,7 +569,7 @@ public class ScannerService : IScannerService
// Could I delete anything in a Library's Series where the LastScan date is before scanStart? // Could I delete anything in a Library's Series where the LastScan date is before scanStart?
// NOTE: This implementation is expensive // NOTE: This implementation is expensive
_logger.LogDebug("[ScannerService] Removing Series that were not found during the scan"); _logger.LogDebug("[ScannerService] Removing Series that were not found during the scan");
var removedSeries = await _unitOfWork.SeriesRepository.RemoveSeriesNotInList(seenSeries, library.Id); var removedSeries = await _unitOfWork.SeriesRepository.RemoveSeriesNotInList(parsedSeries.Keys.ToList(), library.Id);
_logger.LogDebug("[ScannerService] Found {Count} series that needs to be removed: {SeriesList}", _logger.LogDebug("[ScannerService] Found {Count} series that needs to be removed: {SeriesList}",
removedSeries.Count, removedSeries.Select(s => s.Name)); removedSeries.Count, removedSeries.Select(s => s.Name));
_logger.LogDebug("[ScannerService] Removing Series that were not found during the scan - complete"); _logger.LogDebug("[ScannerService] Removing Series that were not found during the scan - complete");
@ -567,63 +597,20 @@ public class ScannerService : IScannerService
await _metadataService.RemoveAbandonedMetadataKeys(); await _metadataService.RemoveAbandonedMetadataKeys();
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory)); BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return;
// Responsible for transforming parsedInfo into an actual ParsedSeries then calling the actual processing of the series
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return;
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles[0].Series,
NormalizedName = Parser.Normalize(parsedFiles[0].Series),
Format = parsedFiles[0].Format,
};
if (skippedScan)
{
seenSeries.AddRange(parsedFiles.Select(pf => new ParsedSeries()
{
Name = pf.Series,
NormalizedName = Parser.Normalize(pf.Series),
Format = pf.Format
}));
return;
}
totalFiles += parsedFiles.Count;
seenSeries.Add(foundParsedSeries);
// TODO: This is extremely expensive to lock the thread on this. We should instead move this onto Hangfire
// or in a queue to be processed.
await _seriesProcessingSemaphore.WaitAsync();
try
{
await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate);
}
finally
{
_seriesProcessingSemaphore.Release();
}
}
} }
private async Task<long> ScanFiles(Library library, IEnumerable<string> dirs, private async Task<Tuple<long, IList<ScannedSeriesResult>>> ScanFiles(Library library, IEnumerable<string> dirs,
bool isLibraryScan, Func<Tuple<bool, IList<ParserInfo>>, Task>? processSeriesInfos = null, bool forceChecks = false) bool isLibraryScan, bool forceChecks = false)
{ {
var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService, _eventHub); var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService, _eventHub);
var scanWatch = Stopwatch.StartNew(); var scanWatch = Stopwatch.StartNew();
await scanner.ScanLibrariesForSeries(library, dirs, var processedSeries = await scanner.ScanLibrariesForSeries(library, dirs,
isLibraryScan, await _unitOfWork.SeriesRepository.GetFolderPathMap(library.Id), processSeriesInfos, forceChecks); isLibraryScan, await _unitOfWork.SeriesRepository.GetFolderPathMap(library.Id), forceChecks);
var scanElapsedTime = scanWatch.ElapsedMilliseconds; var scanElapsedTime = scanWatch.ElapsedMilliseconds;
return scanElapsedTime; return Tuple.Create(scanElapsedTime, processedSeries);
} }
public static IEnumerable<Series> FindSeriesNotOnDisk(IEnumerable<Series> existingSeries, Dictionary<ParsedSeries, IList<ParserInfo>> parsedSeries) public static IEnumerable<Series> FindSeriesNotOnDisk(IEnumerable<Series> existingSeries, Dictionary<ParsedSeries, IList<ParserInfo>> parsedSeries)

2
UI/Web/src/assets/langs/en.json
View file

@ -1527,7 +1527,7 @@
"general-tab": "General", "general-tab": "General",
"cover-image-tab": "Cover Image", "cover-image-tab": "Cover Image",
"close": "{{common.close}}", "close": "{{common.close}}",
"save": "{common.save}}", "save": "{{common.save}}",
"year-validation": "Must be greater than 1000, 0 or blank", "year-validation": "Must be greater than 1000, 0 or blank",
"month-validation": "Must be between 1 and 12 or blank", "month-validation": "Must be between 1 and 12 or blank",
"name-unique-validation": "Name must be unique", "name-unique-validation": "Name must be unique",

2
openapi.json
View file

@ -7,7 +7,7 @@
"name": "GPL-3.0", "name": "GPL-3.0",
"url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE" "url": "https://github.com/Kareadita/Kavita/blob/develop/LICENSE"
}, },
"version": "0.7.14.7" "version": "0.7.14.8"
}, },
"servers": [ "servers": [
{ {