Event Widget Update (#1098)

* Took care of some notes in the code

* Fixed an issue where Extra might get flagged as special too early, if in a word like Extraordinary

* Moved Tag cleanup code into Scanner service. Added a SplitQuery to another heavy API. Refactored Scan loop to remove parallelism and use async instead.

* Lots of rework on the codebase to support detailed messages and easier management of message sending. Need to take a break on this work.

* Progress is being made, but slowly. Code is broken in this commit.

* Progress is being made, but slowly. Code is broken in this commit.

* Fixed merge issue

* Fixed unit tests

* CoverUpdate is now hooked into new ProgressEvent structure

* Refactored code to remove custom observables and have everything use standard messages$

* Refactored a ton of instances to NotificationProgressEvent style and tons of the UI to respect that too. UI is still a bit buggy, but wholistically the work is done.

* Working much better. Sometimes events come in too fast. Currently cover update progress doesn't display on UI

* Fixed unit tests

* Removed SignalREvent to minimize internal event types. Updated the UI to use progress bars. Finished SiteThemeService.

* Merged metadata refresh progress events and changed library scan events to merge cleaner in the UI

* Changed RefreshMetadataProgress to CoverUpdateProgress to reflect the event better.

* Theme Cleanup (#1089)

* Fixed e-ink theme not properly applying correctly

* Fixed some seed changes. Changed card checkboxes to use our themed ones

* Fixed recently added carousel not going to recently-added page

* Fixed an issue where no results found would show when searching for a library name

* Cleaned up list a bit, typeahead dropdown still needs work

* Added a TODO to streamline series-card component

* Removed ng-lazyload-image module since we don't use it. We use lazysizes

* Darken card on hover

* Fixing accordion focus style

* ux pass updates

- Fixed typeahead width
- Fixed changelog download buttons
- Fixed a select
- Fixed various input box-shadows
- Fixed all anchors to only have underline on hover
- Added navtab hover and active effects

* more ux pass

- Fixed spacing on theme cards
- Fixed some light theme issues
- Exposed text-muted-color for theme card subtitle color

* UX pass fixes

- Changed back to bright green for primary on dark theme
- Changed fa icon to black on e-ink

* Merged changelog component

* Fixed anchor buttons text decoration

* Changed nav tabs to have a background color instead of open active state

* When user is not authenticated, make sure we set default theme (dark)

* Cleanup on carousel

* Updated Users tab to use small buttons with icons to align with Library tab

* Cleaned up brand to not underline, removed default link underline on hover in dropdown and pill tabs

* Fixed collection detail posters not rendering

Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>

* Bump versions by dotnet-bump-version.

* Tweaked some of the emitting code

* Some css, but pretty bad. Robbie please save me

* Removed a todo

* styling update

* Only send filename on FileScanProgress

* Some console.log spam cleanup

* Various updates

* Show events widget activity based on activeEvents

* progress bar color updates

* Code cleanup

Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>
This commit is contained in:
Joseph Milazzo 2022-02-18 18:57:37 -08:00 committed by GitHub
parent d24620fd15
commit eddbb7ab18
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
49 changed files with 1022 additions and 463 deletions

View file

@ -17,7 +17,6 @@ using API.Parser;
using API.Services.Tasks.Scanner;
using API.SignalR;
using Hangfire;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Logging;
namespace API.Services.Tasks;
@ -39,14 +38,14 @@ public class ScannerService : IScannerService
private readonly ILogger<ScannerService> _logger;
private readonly IMetadataService _metadataService;
private readonly ICacheService _cacheService;
private readonly IHubContext<MessageHub> _messageHub;
private readonly IEventHub _eventHub;
private readonly IFileService _fileService;
private readonly IDirectoryService _directoryService;
private readonly IReadingItemService _readingItemService;
private readonly ICacheHelper _cacheHelper;
public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger,
IMetadataService metadataService, ICacheService cacheService, IHubContext<MessageHub> messageHub,
IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub,
IFileService fileService, IDirectoryService directoryService, IReadingItemService readingItemService,
ICacheHelper cacheHelper)
{
@ -54,7 +53,7 @@ public class ScannerService : IScannerService
_logger = logger;
_metadataService = metadataService;
_cacheService = cacheService;
_messageHub = messageHub;
_eventHub = eventHub;
_fileService = fileService;
_directoryService = directoryService;
_readingItemService = readingItemService;
@ -72,8 +71,8 @@ public class ScannerService : IScannerService
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.Folders);
var folderPaths = library.Folders.Select(f => f.Path).ToList();
// Check if any of the folder roots are not available (ie disconnected from network, etc) and fail if any of them are
if (folderPaths.Any(f => !_directoryService.IsDriveMounted(f)))
if (!await CheckMounts(library.Folders.Select(f => f.Path).ToList()))
{
_logger.LogError("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted");
return;
@ -86,8 +85,9 @@ public class ScannerService : IScannerService
var dirs = _directoryService.FindHighestDirectoriesFromFiles(folderPaths, files.Select(f => f.FilePath).ToList());
_logger.LogInformation("Beginning file scan on {SeriesName}", series.Name);
var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService);
var parsedSeries = scanner.ScanLibrariesForSeries(library.Type, dirs.Keys, out var totalFiles, out var scanElapsedTime);
var (totalFiles, scanElapsedTime, parsedSeries) = await ScanFiles(library, dirs.Keys);
// Remove any parsedSeries keys that don't belong to our series. This can occur when users store 2 series in the same folder
RemoveParsedInfosNotForSeries(parsedSeries, series);
@ -133,11 +133,11 @@ public class ScannerService : IScannerService
}
}
var (totalFiles2, scanElapsedTime2, parsedSeries2) = await ScanFiles(library, dirs.Keys);
_logger.LogInformation("{SeriesName} has bad naming convention, forcing rescan at a higher directory", series.OriginalName);
scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService);
parsedSeries = scanner.ScanLibrariesForSeries(library.Type, dirs.Keys, out var totalFiles2, out var scanElapsedTime2);
totalFiles += totalFiles2;
scanElapsedTime += scanElapsedTime2;
parsedSeries = parsedSeries2;
RemoveParsedInfosNotForSeries(parsedSeries, series);
}
}
@ -148,9 +148,12 @@ public class ScannerService : IScannerService
// Merge any series together that might have different ParsedSeries but belong to another group of ParsedSeries
try
{
UpdateSeries(series, parsedSeries, allPeople, allTags, allGenres, library.Type);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
await UpdateSeries(series, parsedSeries, allPeople, allTags, allGenres, library.Type);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
await CommitAndSend(totalFiles, parsedSeries, sw, scanElapsedTime, series);
await RemoveAbandonedMetadataKeys();
}
catch (Exception ex)
{
@ -158,7 +161,8 @@ public class ScannerService : IScannerService
await _unitOfWork.RollbackAsync();
}
// Tell UI that this series is done
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanSeries, MessageFactory.ScanSeriesEvent(seriesId, series.Name), token);
await _eventHub.SendMessageAsync(MessageFactory.ScanSeries,
MessageFactory.ScanSeriesEvent(seriesId, series.Name));
await CleanupDbEntities();
BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(chapterIds));
BackgroundJob.Enqueue(() => _metadataService.RefreshMetadataForSeries(libraryId, series.Id, false));
@ -186,6 +190,64 @@ public class ScannerService : IScannerService
}
}
private async Task<bool> CheckMounts(IList<string> folders)
{
// TODO: IF false, inform UI
// Check if any of the folder roots are not available (ie disconnected from network, etc) and fail if any of them are
if (folders.Any(f => !_directoryService.IsDriveMounted(f)))
{
_logger.LogError("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted");
await _eventHub.SendMessageAsync("library.scan.error", new SignalRMessage()
{
Name = "library.scan.error",
Body =
new {
Message =
"Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted",
Details = ""
},
Title = "Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted",
SubTitle = string.Join(", ", folders.Where(f => !_directoryService.IsDriveMounted(f)))
});
return false;
}
// For Docker instances check if any of the folder roots are not available (ie disconnected volumes, etc) and fail if any of them are
if (folders.Any(f => _directoryService.IsDirectoryEmpty(f)))
{
// TODO: Food for thought, move this to throw an exception and let a middleware inform the UI to keep the code clean. (We can throw a custom exception which
// will always propagate to the UI)
// That way logging and UI informing is all in one place with full context
_logger.LogError("Some of the root folders for the library are empty. " +
"Either your mount has been disconnected or you are trying to delete all series in the library. " +
"Scan will be aborted. " +
"Check that your mount is connected or change the library's root folder and rescan");
// TODO: Use a factory method
await _eventHub.SendMessageAsync(MessageFactory.Error, new SignalRMessage()
{
Name = MessageFactory.Error,
Title = "Some of the root folders for the library are empty.",
SubTitle = "Either your mount has been disconnected or you are trying to delete all series in the library. " +
"Scan will be aborted. " +
"Check that your mount is connected or change the library's root folder and rescan",
Body =
new {
Title =
"Some of the root folders for the library are empty.",
SubTitle = "Either your mount has been disconnected or you are trying to delete all series in the library. " +
"Scan will be aborted. " +
"Check that your mount is connected or change the library's root folder and rescan"
}
}, true);
return false;
}
return true;
}
[DisableConcurrentExecution(timeoutInSeconds: 360)]
[AutomaticRetry(Attempts = 0, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
@ -223,12 +285,11 @@ public class ScannerService : IScannerService
return;
}
// Check if any of the folder roots are not available (ie disconnected from network, etc) and fail if any of them are
if (library.Folders.Any(f => !_directoryService.IsDriveMounted(f.Path)))
if (!await CheckMounts(library.Folders.Select(f => f.Path).ToList()))
{
_logger.LogCritical("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted");
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(libraryId, 1F));
// await _eventHub.SendMessageAsync(SignalREvents.NotificationProgress,
// MessageFactory.ScanLibraryProgressEvent(libraryId, 1F));
return;
}
@ -239,17 +300,19 @@ public class ScannerService : IScannerService
"Either your mount has been disconnected or you are trying to delete all series in the library. " +
"Scan will be aborted. " +
"Check that your mount is connected or change the library's root folder and rescan");
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(libraryId, 1F));
// await _eventHub.SendMessageAsync(SignalREvents.NotificationProgress,
// MessageFactory.ScanLibraryProgressEvent(libraryId, 1F));
return;
}
_logger.LogInformation("[ScannerService] Beginning file scan on {LibraryName}", library.Name);
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(libraryId, 0));
// await _eventHub.SendMessageAsync(SignalREvents.NotificationProgress,
// MessageFactory.ScanLibraryProgressEvent(libraryId, 0F));
var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService);
var series = scanner.ScanLibrariesForSeries(library.Type, library.Folders.Select(fp => fp.Path), out var totalFiles, out var scanElapsedTime);
var (totalFiles, scanElapsedTime, series) = await ScanFiles(library, library.Folders.Select(fp => fp.Path));
// var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService);
// var series = scanner.ScanLibrariesForSeries(library.Type, library.Folders.Select(fp => fp.Path), out var totalFiles, out var scanElapsedTime);
_logger.LogInformation("[ScannerService] Finished file scan. Updating database");
foreach (var folderPath in library.Folders)
@ -276,11 +339,23 @@ public class ScannerService : IScannerService
await CleanupDbEntities();
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(libraryId, 1F));
// await _eventHub.SendMessageAsync(SignalREvents.NotificationProgress,
// MessageFactory.ScanLibraryProgressEvent(libraryId, 1F));
BackgroundJob.Enqueue(() => _metadataService.RefreshMetadata(libraryId, false));
}
private async Task<Tuple<int, long, Dictionary<ParsedSeries, List<ParserInfo>>>> ScanFiles(Library library, IEnumerable<string> dirs)
{
var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService, _eventHub);
var scanWatch = new Stopwatch();
var parsedSeries = await scanner.ScanLibrariesForSeries(library.Type, dirs, library.Name);
var totalFiles = parsedSeries.Keys.Sum(key => parsedSeries[key].Count);
var scanElapsedTime = scanWatch.ElapsedMilliseconds;
_logger.LogInformation("Scanned {TotalFiles} files in {ElapsedScanTime} milliseconds", totalFiles,
scanElapsedTime);
return new Tuple<int, long, Dictionary<ParsedSeries, List<ParserInfo>>>(totalFiles, scanElapsedTime, parsedSeries);
}
/// <summary>
/// Remove any user progress rows that no longer exist since scan library ran and deleted series/volumes/chapters
/// </summary>
@ -350,10 +425,16 @@ public class ScannerService : IScannerService
// Now, we only have to deal with series that exist on disk. Let's recalculate the volumes for each series
var librarySeries = cleanedSeries.ToList();
Parallel.ForEach(librarySeries, (series) =>
//var index = 0;
foreach (var series in librarySeries)
{
UpdateSeries(series, parsedSeries, allPeople, allTags, allGenres, library.Type);
});
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
await UpdateSeries(series, parsedSeries, allPeople, allTags, allGenres, library.Type);
// await _eventHub.SendMessageAsync(SignalREvents.NotificationProgress,
// MessageFactory.ScanLibraryProgressEvent(library.Id, (1F * index) / librarySeries.Count));
// index += 1;
}
try
{
@ -364,10 +445,10 @@ public class ScannerService : IScannerService
_logger.LogCritical(ex, "[ScannerService] There was an issue writing to the DB. Chunk {ChunkNumber} did not save to DB. If debug mode, series to check will be printed", chunk);
foreach (var series in nonLibrarySeries)
{
_logger.LogDebug("[ScannerService] There may be a constraint issue with {SeriesName}", series.OriginalName);
_logger.LogCritical("[ScannerService] There may be a constraint issue with {SeriesName}", series.OriginalName);
}
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryError,
MessageFactory.ScanLibraryError(library.Id));
await _eventHub.SendMessageAsync(MessageFactory.ScanLibraryError,
MessageFactory.ScanLibraryErrorEvent(library.Id, library.Name));
continue;
}
_logger.LogInformation(
@ -377,17 +458,19 @@ public class ScannerService : IScannerService
// Emit any series removed
foreach (var missing in missingSeries)
{
await _messageHub.Clients.All.SendAsync(SignalREvents.SeriesRemoved, MessageFactory.SeriesRemovedEvent(missing.Id, missing.Name, library.Id));
await _eventHub.SendMessageAsync(MessageFactory.SeriesRemoved, MessageFactory.SeriesRemovedEvent(missing.Id, missing.Name, library.Id));
}
foreach (var series in librarySeries)
{
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanSeries, MessageFactory.ScanSeriesEvent(series.Id, series.Name));
// TODO: Do I need this? Shouldn't this be NotificationProgress
// This is something more like, the series has finished updating in the backend. It may or may not have been modified.
await _eventHub.SendMessageAsync(MessageFactory.ScanSeries, MessageFactory.ScanSeriesEvent(series.Id, series.Name));
}
var progress = Math.Max(0, Math.Min(1, ((chunk + 1F) * chunkInfo.ChunkSize) / chunkInfo.TotalSize));
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(library.Id, progress));
//var progress = Math.Max(0, Math.Min(1, ((chunk + 1F) * chunkInfo.ChunkSize) / chunkInfo.TotalSize));
// await _eventHub.SendMessageAsync(SignalREvents.NotificationProgress,
// MessageFactory.ScanLibraryProgressEvent(library.Id, progress));
}
@ -435,7 +518,7 @@ public class ScannerService : IScannerService
foreach(var series in newSeries)
{
_logger.LogDebug("[ScannerService] Processing series {SeriesName}", series.OriginalName);
UpdateSeries(series, parsedSeries, allPeople, allTags, allGenres, library.Type);
await UpdateSeries(series, parsedSeries, allPeople, allTags, allGenres, library.Type);
_unitOfWork.SeriesRepository.Attach(series);
try
{
@ -445,7 +528,7 @@ public class ScannerService : IScannerService
newSeries.Count, stopwatch.ElapsedMilliseconds, library.Name);
// Inform UI of new series added
await _messageHub.Clients.All.SendAsync(SignalREvents.SeriesAdded, MessageFactory.SeriesAddedEvent(series.Id, series.Name, library.Id));
await _eventHub.SendMessageAsync(MessageFactory.SeriesAdded, MessageFactory.SeriesAddedEvent(series.Id, series.Name, library.Id));
}
catch (Exception ex)
{
@ -453,23 +536,28 @@ public class ScannerService : IScannerService
series.Name, $"{series.Name}_{series.NormalizedName}_{series.LocalizedName}_{series.LibraryId}_{series.Format}");
}
var progress = Math.Max(0F, Math.Min(1F, i * 1F / newSeries.Count));
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(library.Id, progress));
//var progress = Math.Max(0F, Math.Min(1F, i * 1F / newSeries.Count));
// await _eventHub.SendMessageAsync(SignalREvents.NotificationProgress,
// MessageFactory.ScanLibraryProgressEvent(library.Id, progress));
i++;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended));
_logger.LogInformation(
"[ScannerService] Added {NewSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}",
newSeries.Count, stopwatch.ElapsedMilliseconds, library.Name);
}
private void UpdateSeries(Series series, Dictionary<ParsedSeries, List<ParserInfo>> parsedSeries,
private async Task UpdateSeries(Series series, Dictionary<ParsedSeries, List<ParserInfo>> parsedSeries,
ICollection<Person> allPeople, ICollection<Tag> allTags, ICollection<Genre> allGenres, LibraryType libraryType)
{
try
{
_logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName);
//await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.DbUpdateProgressEvent(series, ProgressEventType.Started));
//await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.DbUpdateProgressEvent(series, ProgressEventType.Updated));
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(series.Library.Name, ProgressEventType.Ended, series.Name));
// Get all associated ParsedInfos to the series. This includes infos that use a different filename that matches Series LocalizedName
var parsedInfos = ParseScannedFiles.GetInfosByName(parsedSeries, series);
@ -484,6 +572,8 @@ public class ScannerService : IScannerService
}
series.OriginalName ??= parsedInfos[0].Series;
series.SortName ??= parsedInfos[0].SeriesSort;
//await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.DbUpdateProgressEvent(series, ProgressEventType.Updated));
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(series.Library.Name, ProgressEventType.Ended, series.Name));
UpdateSeriesMetadata(series, allPeople, allGenres, allTags, libraryType);
}
@ -491,6 +581,8 @@ public class ScannerService : IScannerService
{
_logger.LogError(ex, "[ScannerService] There was an exception updating volumes for {SeriesName}", series.Name);
}
//await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.DbUpdateProgressEvent(series, ProgressEventType.Ended));
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(series.Library.Name, ProgressEventType.Ended, series.Name));
}
public static IEnumerable<Series> FindSeriesNotOnDisk(IEnumerable<Series> existingSeries, Dictionary<ParsedSeries, List<ParserInfo>> parsedSeries)
@ -498,6 +590,13 @@ public class ScannerService : IScannerService
return existingSeries.Where(es => !ParserInfoHelpers.SeriesHasMatchingParserInfoFormat(es, parsedSeries));
}
private async Task RemoveAbandonedMetadataKeys()
{
await _unitOfWork.TagRepository.RemoveAllTagNoLongerAssociated();
await _unitOfWork.PersonRepository.RemoveAllPeopleNoLongerAssociated();
await _unitOfWork.GenreRepository.RemoveAllGenreNoLongerAssociated();
}
private static void UpdateSeriesMetadata(Series series, ICollection<Person> allPeople, ICollection<Genre> allGenres, ICollection<Tag> allTags, LibraryType libraryType)
{
@ -605,6 +704,7 @@ public class ScannerService : IScannerService
_unitOfWork.VolumeRepository.Add(volume);
}
// TODO: Here we can put a signalR update
_logger.LogDebug("[ScannerService] Parsing {SeriesName} - Volume {VolumeNumber}", series.Name, volume.Name);
var infos = parsedInfos.Where(p => p.Volumes == volumeNumber).ToArray();
UpdateChapters(volume, infos);