AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scan Loop Fortification (#1573)

Browse source 
* Cleanup some messaging in the scan loop to be more context bearing

* Added Response Caching to Series Detail for 1 min, due to the heavy nature of the call.

* Refactored code to make it so that processing of series runs sync correctly.

Added a log to inform the user of corrupted volume from buggy code in v0.5.6.

* Moved folder watching out of experimental

* Fixed an issue where empty folders could break the scan loop

* Another fix for when dates aren't valid, the scanner wouldn't get the proper min and would throw an exception (develop)

* Implemented the ability to edit release year from the UI for a series.

* Added a unit test for some new logic

* Code smells

* Rewrote the handler for suspending watching to be more resilient and ensure no two threads have a race condition.

* More error handling for when a ScanFolder is invoked but multiple series belong to that folder, log it to the user and default to a library scan.

* ScanSeries now will check for kavitaignores higher than it's own folder and respect library level.

* Fixed an issue where image series with a folder name containing the word "folder" could get ignored as it thought the image was a cover image.

When a series folder is moved or deleted, skip parent ignore finding.

* Removed some old files, added in scanFolder a check if the series found for a folder is in a book library and if so to always do a library scan (as books are often nested into one folder with  multiple series). Added some unit tests

* Refactored some scan loop logic into ComicInfo, wrote tests and updated some documentation to make the fields more clear.

* Added a test for GetLastWriteTime based on recent bug

* Cleaned up some redundant code

* Fixed a bad merge

* Code smells

* Removed a package that's no longer used.

* Ensure we check against ScanQueue on ScanFolder enqueuing

* Documentation and more bullet proofing to ensure Hangfire checks work more as expected
This commit is contained in:
Joe Milazzo 2022-10-09 11:23:41 -05:00 • committed by GitHub
parent 5a75a204db
commit 6ea9f2c73e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
20 changed files with 391 additions and 250 deletions
Download patch file Download diff file Expand all files Collapse all files

82
API/Data/Repositories/SeriesRepository.cs
View file

@ -33,8 +33,7 @@ public enum SeriesIncludes
Volumes = 2,
Metadata = 4,
Related = 8,
//Related = 16,
//UserPreferences = 32
Library = 16,
}
internal class RecentlyAddedSeries
@ -120,8 +119,7 @@ public interface ISeriesRepository
Task<SeriesDto> GetSeriesForChapter(int chapterId, int userId);
Task<PagedList<SeriesDto>> GetWantToReadForUserAsync(int userId, UserParams userParams, FilterDto filter);
Task<int> GetSeriesIdByFolder(string folder);
Task<Series> GetSeriesByFolderPath(string folder);
Task<Series> GetFullSeriesByName(string series, int libraryId);
Task<Series> GetSeriesByFolderPath(string folder, SeriesIncludes includes = SeriesIncludes.None);
Task<Series> GetFullSeriesByAnyName(string seriesName, string localizedName, int libraryId, MangaFormat format, bool withFullIncludes = true);
Task<List<Series>> RemoveSeriesNotInList(IList<ParsedSeries> seenSeries, int libraryId);
Task<IDictionary<string, IList<SeriesModified>>> GetFolderPathMap(int libraryId);
@ -1173,52 +1171,16 @@ public class SeriesRepository : ISeriesRepository
/// Return a Series by Folder path. Null if not found.
/// </summary>
/// <param name="folder">This will be normalized in the query</param>
/// <param name="includes">Additional relationships to include with the base query</param>
/// <returns></returns>
public async Task<Series> GetSeriesByFolderPath(string folder)
public async Task<Series> GetSeriesByFolderPath(string folder, SeriesIncludes includes = SeriesIncludes.None)
{
var normalized = Services.Tasks.Scanner.Parser.Parser.NormalizePath(folder);
return await _context.Series.SingleOrDefaultAsync(s => s.FolderPath.Equals(normalized));
}
var query = _context.Series.Where(s => s.FolderPath.Equals(normalized));
/// <summary>
/// Finds a series by series name for a given library.
/// </summary>
/// <remarks>This pulls everything with the Series, so should be used only when needing tracking on all related tables</remarks>
/// <param name="series"></param>
/// <param name="libraryId"></param>
/// <returns></returns>
public Task<Series> GetFullSeriesByName(string series, int libraryId)
{
var localizedSeries = Services.Tasks.Scanner.Parser.Parser.Normalize(series);
return _context.Series
.Where(s => (s.NormalizedName.Equals(localizedSeries)
|| s.LocalizedName.Equals(series)) && s.LibraryId == libraryId)
.Include(s => s.Metadata)
.ThenInclude(m => m.People)
.Include(s => s.Metadata)
.ThenInclude(m => m.Genres)
.Include(s => s.Library)
.Include(s => s.Volumes)
.ThenInclude(v => v.Chapters)
.ThenInclude(cm => cm.People)
query = AddIncludesToQuery(query, includes);
.Include(s => s.Volumes)
.ThenInclude(v => v.Chapters)
.ThenInclude(c => c.Tags)
.Include(s => s.Volumes)
.ThenInclude(v => v.Chapters)
.ThenInclude(c => c.Genres)
.Include(s => s.Metadata)
.ThenInclude(m => m.Tags)
.Include(s => s.Volumes)
.ThenInclude(v => v.Chapters)
.ThenInclude(c => c.Files)
.AsSplitQuery()
.SingleOrDefaultAsync();
return await query.SingleOrDefaultAsync();
}
/// <summary>
@ -1240,6 +1202,7 @@ public class SeriesRepository : ISeriesRepository
.Where(s => s.Format == format && format != MangaFormat.Unknown)
.Where(s => s.NormalizedName.Equals(normalizedSeries)
|| (s.NormalizedLocalizedName.Equals(normalizedSeries) && s.NormalizedLocalizedName != string.Empty));
if (!string.IsNullOrEmpty(normalizedLocalized))
{
query = query.Where(s =>
@ -1516,7 +1479,8 @@ public class SeriesRepository : ISeriesRepository
LastScanned = s.LastFolderScanned,
SeriesName = s.Name,
FolderPath = s.FolderPath,
Format = s.Format
Format = s.Format,
LibraryRoots = s.Library.Folders.Select(f => f.Path)
}).ToListAsync();
var map = new Dictionary<string, IList<SeriesModified>>();
@ -1538,4 +1502,30 @@ public class SeriesRepository : ISeriesRepository
return map;
}
private static IQueryable<Series> AddIncludesToQuery(IQueryable<Series> query, SeriesIncludes includeFlags)
{
if (includeFlags.HasFlag(SeriesIncludes.Library))
{
query = query.Include(u => u.Library);
}
if (includeFlags.HasFlag(SeriesIncludes.Related))
{
query = query.Include(u => u.Relations);
}
if (includeFlags.HasFlag(SeriesIncludes.Metadata))
{
query = query.Include(u => u.Metadata);
}
if (includeFlags.HasFlag(SeriesIncludes.Volumes))
{
query = query.Include(u => u.Volumes);
}
return query;
}
}