AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Metadata Optimizations (#910)

Browse source 
* Added a tooltip to inform user that format and collection filter selections do not only show for the selected library.

* Refactored a lot of code around when we update chapter cover images. Applied an optimization for when we re-calculate volume/series covers, such that it only occurs when the first chapter's image updates.

* Updated code to ensure only lastmodified gets refreshed in metadata since it always follows a scan

* Optimized how metadata is populated on the series. Instead of re-reading the comicInfos, instead I read the data from the underlying chapter entities. This reduces N additional reads AND enables the ability in the future to show/edit chapter level metadata.

* Spelling mistake

* Fixed a concurency issue by not selecting Genres from DB. Added a test for long paths.

* Fixed a bug in filter where collection tag wasn't populating on load

* Cleaned up the logic for changelog to better compare against the installed verison. For nightly users, show the last stable as installed.

* Removed some demo code

* SplitQuery to allow loading tags much faster for series metadata load.
This commit is contained in:
Joseph Milazzo 2022-01-08 06:41:47 -08:00 • committed by GitHub
parent c215d5b7a8
commit 0be0e294aa
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
21 changed files with 1671 additions and 90 deletions
Download patch file Download diff file Expand all files Collapse all files

119
API/Services/MetadataService.cs
View file

@ -28,7 +28,7 @@ public interface IMetadataService
/// <param name="forceUpdate"></param>
Task RefreshMetadata(int libraryId, bool forceUpdate = false);
/// <summary>
/// Performs a forced refresh of metatdata just for a series and it's nested entities
/// Performs a forced refresh of metadata just for a series and it's nested entities
/// </summary>
/// <param name="libraryId"></param>
/// <param name="seriesId"></param>
@ -76,18 +76,17 @@ public class MetadataService : IMetadataService
return true;
}
private void UpdateChapterMetadata(Chapter chapter, ICollection<Person> allPeople, ICollection<Tag> allTags, bool forceUpdate)
private void UpdateChapterMetadata(Chapter chapter, ICollection<Person> allPeople, ICollection<Tag> allTags, ICollection<Genre> allGenres, bool forceUpdate)
{
var firstFile = chapter.Files.OrderBy(x => x.Chapter).FirstOrDefault();
if (firstFile == null || _cacheHelper.HasFileNotChangedSinceCreationOrLastScan(chapter, forceUpdate, firstFile)) return;
UpdateChapterFromComicInfo(chapter, allPeople, allTags, firstFile);
UpdateChapterFromComicInfo(chapter, allPeople, allTags, allGenres, firstFile);
firstFile.UpdateLastModified();
}
private void UpdateChapterFromComicInfo(Chapter chapter, ICollection<Person> allPeople, ICollection<Tag> allTags, MangaFile firstFile)
private void UpdateChapterFromComicInfo(Chapter chapter, ICollection<Person> allPeople, ICollection<Tag> allTags, ICollection<Genre> allGenres, MangaFile firstFile)
{
// TODO: Think about letting the higher level loop have access for series to avoid duplicate IO operations
var comicInfo = _readingItemService.GetComicInfo(firstFile.FilePath, firstFile.Format);
if (comicInfo == null) return;
@ -196,6 +195,14 @@ public class MetadataService : IMetadataService
PersonHelper.UpdatePeople(allPeople, people, PersonRole.Publisher,
person => PersonHelper.AddPersonIfNotExists(chapter.People, person));
}
if (!string.IsNullOrEmpty(comicInfo.Genre))
{
var genres = comicInfo.Genre.Split(",");
GenreHelper.KeepOnlySameGenreBetweenLists(chapter.Genres, genres.Select(g => DbFactory.Genre(g, false)).ToList());
GenreHelper.UpdateGenre(allGenres, genres, false,
genre => chapter.Genres.Add(genre));
}
}
@ -253,17 +260,44 @@ public class MetadataService : IMetadataService
series.CoverImage = firstCover?.CoverImage ?? coverImage;
}
private void UpdateSeriesMetadata(Series series, ICollection<Person> allPeople, ICollection<Genre> allGenres, ICollection<Tag> allTags, bool forceUpdate)
private static void UpdateSeriesMetadata(Series series, ICollection<Person> allPeople, ICollection<Genre> allGenres, ICollection<Tag> allTags, bool forceUpdate)
{
var isBook = series.Library.Type == LibraryType.Book;
var firstVolume = series.Volumes.OrderBy(c => c.Number, new ChapterSortComparer()).FirstWithChapters(isBook);
var firstChapter = firstVolume?.Chapters.GetFirstChapterWithFiles();
var firstFile = firstChapter?.Files.FirstOrDefault();
if (firstFile == null || _cacheHelper.HasFileNotChangedSinceCreationOrLastScan(firstChapter, forceUpdate, firstFile)) return;
if (firstFile == null) return;
if (Parser.Parser.IsPdf(firstFile.FilePath)) return;
foreach (var chapter in series.Volumes.SelectMany(volume => volume.Chapters))
var chapters = series.Volumes.SelectMany(volume => volume.Chapters).ToList();
// Update Metadata based on Chapter metadata
series.Metadata.ReleaseYear = chapters.Min(c => c.ReleaseDate.Year);
if (series.Metadata.ReleaseYear < 1000)
{
// Not a valid year, default to 0
series.Metadata.ReleaseYear = 0;
}
// Set the AgeRating as highest in all the comicInfos
series.Metadata.AgeRating = chapters.Max(chapter => chapter.AgeRating);
if (!string.IsNullOrEmpty(firstChapter.Summary))
{
series.Metadata.Summary = firstChapter.Summary;
}
if (!string.IsNullOrEmpty(firstChapter.Language))
{
series.Metadata.Language = firstChapter.Language;
}
// Handle People
foreach (var chapter in chapters)
{
PersonHelper.UpdatePeople(allPeople, chapter.People.Where(p => p.Role == PersonRole.Writer).Select(p => p.Name), PersonRole.Writer,
person => PersonHelper.AddPersonIfNotExists(series.Metadata.People, person));
@ -297,49 +331,14 @@ public class MetadataService : IMetadataService
TagHelper.UpdateTag(allTags, chapter.Tags.Select(t => t.Title), false, (tag, added) =>
TagHelper.AddTagIfNotExists(series.Metadata.Tags, tag));
GenreHelper.UpdateGenre(allGenres, chapter.Genres.Select(t => t.Title), false, genre =>
GenreHelper.AddGenreIfNotExists(series.Metadata.Genres, genre));
}
var comicInfos = series.Volumes
.SelectMany(volume => volume.Chapters)
.OrderBy(c => double.Parse(c.Number), new ChapterSortComparer())
.SelectMany(c => c.Files)
.Select(file => _readingItemService.GetComicInfo(file.FilePath, file.Format))
.Where(ci => ci != null)
.ToList();
var comicInfo = comicInfos.FirstOrDefault();
if (!string.IsNullOrEmpty(comicInfo?.Summary))
{
series.Metadata.Summary = comicInfo.Summary;
}
if (!string.IsNullOrEmpty(comicInfo?.LanguageISO))
{
series.Metadata.Language = comicInfo.LanguageISO;
}
// Set the AgeRating as highest in all the comicInfos
series.Metadata.AgeRating = comicInfos.Max(i => ComicInfo.ConvertAgeRatingToEnum(comicInfo?.AgeRating));
series.Metadata.ReleaseYear = series.Volumes
.SelectMany(volume => volume.Chapters).Min(c => c.ReleaseDate.Year);
if (series.Metadata.ReleaseYear < 1000)
{
// Not a valid year, default to 0
series.Metadata.ReleaseYear = 0;
}
var genres = comicInfos.SelectMany(i => i?.Genre.Split(",")).Distinct().ToList();
var tags = comicInfos.SelectMany(i => i?.Tags.Split(",")).Distinct().ToList();
var people = series.Volumes.SelectMany(volume => volume.Chapters).SelectMany(c => c.People).ToList();
var people = chapters.SelectMany(c => c.People).ToList();
PersonHelper.KeepOnlySamePeopleBetweenLists(series.Metadata.People,
people, person => series.Metadata.People.Remove(person));
GenreHelper.UpdateGenre(allGenres, genres, false, genre => GenreHelper.AddGenreIfNotExists(series.Metadata.Genres, genre));
GenreHelper.KeepOnlySameGenreBetweenLists(series.Metadata.Genres, genres.Select(g => DbFactory.Genre(g, false)).ToList(),
genre => series.Metadata.Genres.Remove(genre));
}
/// <summary>
@ -352,20 +351,34 @@ public class MetadataService : IMetadataService
_logger.LogDebug("[MetadataService] Processing series {SeriesName}", series.OriginalName);
try
{
var volumeUpdated = false;
var volumeIndex = 0;
var firstVolumeUpdated = false;
foreach (var volume in series.Volumes)
{
var chapterUpdated = false;
var firstChapterUpdated = false; // This only needs to be FirstChapter updated
var index = 0;
foreach (var chapter in volume.Chapters)
{
chapterUpdated = UpdateChapterCoverImage(chapter, forceUpdate);
UpdateChapterMetadata(chapter, allPeople, allTags, forceUpdate || chapterUpdated);
var chapterUpdated = UpdateChapterCoverImage(chapter, forceUpdate);
// If cover was update, either the file has changed or first scan and we should force a metadata update
UpdateChapterMetadata(chapter, allPeople, allTags, allGenres, forceUpdate || chapterUpdated);
if (index == 0 && chapterUpdated)
{
firstChapterUpdated = true;
}
index++;
}
volumeUpdated = UpdateVolumeCoverImage(volume, chapterUpdated || forceUpdate);
var volumeUpdated = UpdateVolumeCoverImage(volume, firstChapterUpdated || forceUpdate);
if (volumeIndex == 0 && volumeUpdated)
{
firstVolumeUpdated = true;
}
volumeIndex++;
}
UpdateSeriesCoverImage(series, volumeUpdated || forceUpdate);
UpdateSeriesCoverImage(series, firstVolumeUpdated || forceUpdate);
UpdateSeriesMetadata(series, allPeople, allGenres, allTags, forceUpdate);
}
catch (Exception ex)

1
API/Services/Tasks/ScannerService.cs
View file

@ -641,6 +641,7 @@ public class ScannerService : IScannerService
existingFile.Format = info.Format;
if (!_fileService.HasFileBeenModifiedSince(existingFile.FilePath, existingFile.LastModified) && existingFile.Pages != 0) return;
existingFile.Pages = _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format);
//existingFile.UpdateLastModified(); // We skip updating DB here so that metadata refresh can do it
}
else
{