AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Image-only Libraries + Library Fixes (#2427)

Browse source
This commit is contained in:
Joe Milazzo 2023-11-11 13:50:11 -06:00 • committed by GitHub
parent 5963ea5f63
commit fe2b9b86bc
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
8 changed files with 195 additions and 107 deletions
Download patch file Download diff file Expand all files Collapse all files

5
API/Entities/Enums/LibraryType.cs
View file

@ -19,4 +19,9 @@ public enum LibraryType
/// </summary>
[Description("Book")]
Book = 2,
/// <summary>
/// Uses a different type of grouping and parsing mechanism
/// </summary>
[Description("Image")]
Image = 3,
}

3
API/Entities/Library.cs
View file

@ -41,6 +41,9 @@ public class Library : IEntityDate
/// </summary>
/// <remarks>Scrobbling requires a valid LicenseKey</remarks>
public bool AllowScrobbling { get; set; } = true;
public DateTime Created { get; set; }
public DateTime LastModified { get; set; }
public DateTime CreatedUtc { get; set; }

4
API/Helpers/Builders/ChapterBuilder.cs
View file

@ -28,8 +28,8 @@ public class ChapterBuilder : IEntityBuilder<Chapter>
{
var specialTreatment = info.IsSpecialInfo();
var specialTitle = specialTreatment ? info.Filename : info.Chapters;
var builder = new ChapterBuilder(Services.Tasks.Scanner.Parser.Parser.DefaultChapter);
return builder.WithNumber(specialTreatment ? Services.Tasks.Scanner.Parser.Parser.DefaultChapter : Services.Tasks.Scanner.Parser.Parser.MinNumberFromRange(info.Chapters) + string.Empty)
var builder = new ChapterBuilder(Parser.DefaultChapter);
return builder.WithNumber(specialTreatment ? Parser.DefaultChapter : Parser.MinNumberFromRange(info.Chapters) + string.Empty)
.WithRange(specialTreatment ? info.Filename : info.Chapters)
.WithTitle((specialTreatment && info.Format == MangaFormat.Epub)
? info.Title

80
API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file

@ -35,43 +35,38 @@ public class DefaultParser : IDefaultParser
{
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this. (we can probably remove this and have users use kavitaignore)
if (Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
if (type != LibraryType.Image && Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
ParserInfo ret;
if (Parser.IsEpub(filePath)) // NOTE: Will this ever be called? Because we use ReadingService to handle parse
var ret = new ParserInfo()
{
ret = new ParserInfo
{
Chapters = Parser.ParseChapter(fileName) ?? Parser.ParseComicChapter(fileName),
Series = Parser.ParseSeries(fileName) ?? Parser.ParseComicSeries(fileName),
Volumes = Parser.ParseVolume(fileName) ?? Parser.ParseComicVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
FullFilePath = filePath
};
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath,
Series = string.Empty
};
// If library type is Image or this is not a cover image in a non-image library, then use dedicated parsing mechanism
if (type == LibraryType.Image || Parser.IsImage(filePath))
{
return ParseImage(filePath, rootPath, ret);
}
// This will be called if the epub is already parsed once then we call and merge the information, if the
if (Parser.IsEpub(filePath))
{
ret.Chapters = Parser.ParseChapter(fileName);
ret.Series = Parser.ParseSeries(fileName);
ret.Volumes = Parser.ParseVolume(fileName);
}
else
{
ret = new ParserInfo
{
Chapters = type == LibraryType.Comic ? Parser.ParseComicChapter(fileName) : Parser.ParseChapter(fileName),
Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName),
Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName),
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath
};
}
if (Parser.IsImage(filePath))
{
// Reset Chapters, Volumes, and Series as images are not good to parse information out of. Better to use folders.
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
ret.Series = string.Empty;
ret.Chapters = type == LibraryType.Comic
? Parser.ParseComicChapter(fileName)
: Parser.ParseChapter(fileName);
ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName);
ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName);
}
if (ret.Series == string.Empty || Parser.IsImage(filePath))
@ -120,6 +115,23 @@ public class DefaultParser : IDefaultParser
return ret.Series == string.Empty ? null : ret;
}
private ParserInfo ParseImage(string filePath, string rootPath, ParserInfo ret)
{
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
// Next we need to see if the image has a folder between rootPath and filePath.
// if so, take that folder as a volume 0 chapter 0 special and group everything under there (if we can't parse a volume/chapter)
ParseFromFallbackFolders(filePath, rootPath, LibraryType.Image, ref ret);
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters == Parser.DefaultChapter) &&
(string.IsNullOrEmpty(ret.Volumes) || ret.Volumes == Parser.DefaultVolume))
{
ret.IsSpecial = true;
}
ret.Series = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
return ret;
}
/// <summary>
/// Fills out <see cref="ParserInfo"/> by trying to parse volume, chapters, and series from folders
/// </summary>
@ -160,11 +172,11 @@ public class DefaultParser : IDefaultParser
if (!parsedVolume.Equals(Parser.DefaultVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
{
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !parsedVolume.Equals(Parser.DefaultVolume))
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.DefaultVolume))
{
ret.Volumes = parsedVolume;
}
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !parsedChapter.Equals(Parser.DefaultChapter))
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}

38
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -62,9 +62,6 @@ public class ProcessSeries : IProcessSeries
private IList<Person> _people;
private Dictionary<string, Tag> _tags;
private Dictionary<string, CollectionTag> _collectionTags;
private readonly object _peopleLock = new object();
private readonly object _genreLock = new object();
private readonly object _tagLock = new object();
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
@ -845,23 +842,20 @@ public class ProcessSeries : IProcessSeries
/// <param name="action"></param>
private void UpdatePeople(IEnumerable<string> names, PersonRole role, Action<Person> action)
{
lock (_peopleLock)
var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
foreach (var name in names)
{
var allPeopleTypeRole = _people.Where(p => p.Role == role).ToList();
var normalizedName = name.ToNormalized();
var person = allPeopleTypeRole.Find(p =>
p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
foreach (var name in names)
if (person == null)
{
var normalizedName = name.ToNormalized();
var person = allPeopleTypeRole.Find(p =>
p.NormalizedName != null && p.NormalizedName.Equals(normalizedName));
if (person == null)
{
person = new PersonBuilder(name, role).Build();
_people.Add(person);
}
action(person);
person = new PersonBuilder(name, role).Build();
_people.Add(person);
}
action(person);
}
}
@ -882,11 +876,8 @@ public class ProcessSeries : IProcessSeries
if (newTag)
{
genre = new GenreBuilder(name).Build();
lock (_genreLock)
{
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
}
action(genre!, newTag);
@ -911,10 +902,7 @@ public class ProcessSeries : IProcessSeries
if (tag == null)
{
tag = new TagBuilder(name).Build();
lock (_tagLock)
{
_tags.Add(normalizedName, tag);
}
_tags.Add(normalizedName, tag);
}
action(tag, added);

31
API/Services/Tasks/ScannerService.cs
View file

@ -4,6 +4,7 @@ using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
@ -85,6 +86,8 @@ public class ScannerService : IScannerService
private readonly IProcessSeries _processSeries;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly SemaphoreSlim _seriesProcessingSemaphore = new SemaphoreSlim(1, 1);
public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger,
IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub,
IDirectoryService directoryService, IReadingItemService readingItemService,
@ -495,10 +498,10 @@ public class ScannerService : IScannerService
var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate);
// NOTE: This runs sync after every file is scanned
foreach (var task in processTasks)
{
await task();
}
// foreach (var task in processTasks)
// {
// await task();
// }
// TODO: We might be able to do Task.WhenAll
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
@ -566,17 +569,18 @@ public class ScannerService : IScannerService
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return;
Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
// Responsible for transforming parsedInfo into an actual ParsedSeries then calling the actual processing of the series
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return Task.CompletedTask;
if (parsedFiles.Count == 0) return;
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles[0].Series,
NormalizedName = Scanner.Parser.Parser.Normalize(parsedFiles[0].Series),
Format = parsedFiles[0].Format
Format = parsedFiles[0].Format,
};
if (skippedScan)
@ -587,15 +591,22 @@ public class ScannerService : IScannerService
NormalizedName = Scanner.Parser.Parser.Normalize(pf.Series),
Format = pf.Format
}));
return Task.CompletedTask;
return;
}
totalFiles += parsedFiles.Count;
seenSeries.Add(foundParsedSeries);
processTasks.Add(async () => await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate));
return Task.CompletedTask;
await _seriesProcessingSemaphore.WaitAsync();
try
{
await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate);
}
finally
{
_seriesProcessingSemaphore.Release();
}
}
}