AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Word Count (#1286)

Browse source 
* Adding some code for Robbie

* See more on series detail metadata area is now at the bottom on the section

* Cleaned up subtitle headings to use a single class for offset with actionables

* Added some markup for the new design, waiting for Robbie to finish it off

* styling age-rating badge

* Started hooking up basic analyze file service and hooks in the UI. Basic code to implement the count is implemented and in benchmarks.

* Hooked up analyze ui to backend

* Refactored Series Detail metadata area to use a new icon/title design

* Cleaned up the new design

* Pushing for robbie to do css

* Massive performance improvement to scan series where we only need to scan folders reported that have series in them, rather than the whole library.

* Removed theme page as we no longer need it. Added WordCount to DTOs so the UI can show them. Added new pipe to format numbers in compact mode.

* Hooked up actual reading time based on user's words per hour

* Refactor some magic numbers to consts

* Hooked in progress reporting for series word count

* Hooked up analyze files

* Re-implemented time to read on comics

* Removed the word Last Read

* Show proper language name instead of iso tag on series detail page. Added some error handling on word count code.

* Reworked error handling

* Fixed some security vulnerabilities in npm.

* Handle a case where there are no text nodes and instead of returning an empty list, htmlagilitypack returns null.

* Tweaked the styles a bit on the icon-and-title

* Code cleanup

Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>
This commit is contained in:
Joseph Milazzo 2022-05-25 16:53:39 -05:00 • committed by GitHub
parent 0a70ac35dc
commit c1490d6e86
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
48 changed files with 2354 additions and 408 deletions
Download patch file Download diff file Expand all files Collapse all files

37
API/Services/Tasks/ScannerService.cs
View file

@ -14,6 +14,7 @@ using API.Entities.Enums;
using API.Extensions;
using API.Helpers;
using API.Parser;
using API.Services.Tasks.Metadata;
using API.Services.Tasks.Scanner;
using API.SignalR;
using Hangfire;
@ -43,11 +44,12 @@ public class ScannerService : IScannerService
private readonly IDirectoryService _directoryService;
private readonly IReadingItemService _readingItemService;
private readonly ICacheHelper _cacheHelper;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger,
IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub,
IFileService fileService, IDirectoryService directoryService, IReadingItemService readingItemService,
ICacheHelper cacheHelper)
ICacheHelper cacheHelper, IWordCountAnalyzerService wordCountAnalyzerService)
{
_unitOfWork = unitOfWork;
_logger = logger;
@ -58,6 +60,7 @@ public class ScannerService : IScannerService
_directoryService = directoryService;
_readingItemService = readingItemService;
_cacheHelper = cacheHelper;
_wordCountAnalyzerService = wordCountAnalyzerService;
}
[DisableConcurrentExecution(timeoutInSeconds: 360)]
@ -71,6 +74,15 @@ public class ScannerService : IScannerService
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.Folders);
var folderPaths = library.Folders.Select(f => f.Path).ToList();
var seriesFolderPaths = (await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId))
.Select(f => _directoryService.FileSystem.FileInfo.FromFileName(f.FilePath).Directory.FullName)
.ToList();
if (!await CheckMounts(library.Name, seriesFolderPaths))
{
_logger.LogCritical("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted");
return;
}
if (!await CheckMounts(library.Name, library.Folders.Select(f => f.Path).ToList()))
{
@ -82,10 +94,15 @@ public class ScannerService : IScannerService
var allGenres = await _unitOfWork.GenreRepository.GetAllGenresAsync();
var allTags = await _unitOfWork.TagRepository.GetAllTagsAsync();
var dirs = _directoryService.FindHighestDirectoriesFromFiles(folderPaths, files.Select(f => f.FilePath).ToList());
var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(seriesFolderPaths, files.Select(f => f.FilePath).ToList());
if (seriesDirs.Keys.Count == 0)
{
_logger.LogDebug("Scan Series has files spread outside a main series folder. Defaulting to library folder");
seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(folderPaths, files.Select(f => f.FilePath).ToList());
}
_logger.LogInformation("Beginning file scan on {SeriesName}", series.Name);
var (totalFiles, scanElapsedTime, parsedSeries) = await ScanFiles(library, dirs.Keys);
var (totalFiles, scanElapsedTime, parsedSeries) = await ScanFiles(library, seriesDirs.Keys);
@ -117,10 +134,10 @@ public class ScannerService : IScannerService
// We need to do an additional check for an edge case: If the scan ran and the files do not match the existing Series name, then it is very likely,
// the files have crap naming and if we don't correct, the series will get deleted due to the parser not being able to fallback onto folder parsing as the root
// is the series folder.
var existingFolder = dirs.Keys.FirstOrDefault(key => key.Contains(series.OriginalName));
if (dirs.Keys.Count == 1 && !string.IsNullOrEmpty(existingFolder))
var existingFolder = seriesDirs.Keys.FirstOrDefault(key => key.Contains(series.OriginalName));
if (seriesDirs.Keys.Count == 1 && !string.IsNullOrEmpty(existingFolder))
{
dirs = new Dictionary<string, string>();
seriesDirs = new Dictionary<string, string>();
var path = Directory.GetParent(existingFolder)?.FullName;
if (!folderPaths.Contains(path) || !folderPaths.Any(p => p.Contains(path ?? string.Empty)))
{
@ -131,11 +148,11 @@ public class ScannerService : IScannerService
}
if (!string.IsNullOrEmpty(path))
{
dirs[path] = string.Empty;
seriesDirs[path] = string.Empty;
}
}
var (totalFiles2, scanElapsedTime2, parsedSeries2) = await ScanFiles(library, dirs.Keys);
var (totalFiles2, scanElapsedTime2, parsedSeries2) = await ScanFiles(library, seriesDirs.Keys);
_logger.LogInformation("{SeriesName} has bad naming convention, forcing rescan at a higher directory", series.OriginalName);
totalFiles += totalFiles2;
scanElapsedTime += scanElapsedTime2;
@ -303,10 +320,8 @@ public class ScannerService : IScannerService
await CleanupDbEntities();
// await _eventHub.SendMessageAsync(SignalREvents.NotificationProgress,
// MessageFactory.ScanLibraryProgressEvent(libraryId, 1F));
BackgroundJob.Enqueue(() => _metadataService.RefreshMetadata(libraryId, false));
BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanLibrary(libraryId, false));
}
private async Task<Tuple<int, long, Dictionary<ParsedSeries, List<ParserInfo>>>> ScanFiles(Library library, IEnumerable<string> dirs)