AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Event Widget Updates + Format Downloads + Scanner Work (#3024)

Browse source
This commit is contained in:
Joe Milazzo 2024-06-27 16:35:50 -05:00 • committed by GitHub
parent 30a8a2555f
commit a427d02ed1
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
34 changed files with 971 additions and 694 deletions
Download patch file Download diff file Expand all files Collapse all files

7
API/Controllers/LibraryController.cs
View file

@ -21,7 +21,6 @@ using AutoMapper;
using EasyCaching.Core;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration.UserSecrets;
using Microsoft.Extensions.Logging;
using TaskScheduler = API.Services.TaskScheduler;
@ -134,7 +133,7 @@ public class LibraryController : BaseApiController
if (!await _unitOfWork.CommitAsync()) return BadRequest(await _localizationService.Translate(User.GetUserId(), "generic-library"));
await _libraryWatcher.RestartWatching();
_taskScheduler.ScanLibrary(library.Id);
await _taskScheduler.ScanLibrary(library.Id);
await _eventHub.SendMessageAsync(MessageFactory.LibraryModified,
MessageFactory.LibraryModifiedEvent(library.Id, "create"), false);
await _eventHub.SendMessageAsync(MessageFactory.SideNavUpdate,
@ -292,7 +291,7 @@ public class LibraryController : BaseApiController
public async Task<ActionResult> Scan(int libraryId, bool force = false)
{
if (libraryId <= 0) return BadRequest(await _localizationService.Translate(User.GetUserId(), "greater-0", "libraryId"));
_taskScheduler.ScanLibrary(libraryId, force);
await _taskScheduler.ScanLibrary(libraryId, force);
return Ok();
}
@ -500,7 +499,7 @@ public class LibraryController : BaseApiController
if (originalFoldersCount != dto.Folders.Count() || typeUpdate)
{
await _libraryWatcher.RestartWatching();
_taskScheduler.ScanLibrary(library.Id);
await _taskScheduler.ScanLibrary(library.Id);
}
if (folderWatchingUpdate)

4
API/Controllers/OPDSController.cs
View file

@ -868,6 +868,7 @@ public class OpdsController : BaseApiController
SetFeedId(feed, $"series-{series.Id}");
feed.Links.Add(CreateLink(FeedLinkRelation.Image, FeedLinkType.Image, $"{baseUrl}api/image/series-cover?seriesId={seriesId}&apiKey={apiKey}"));
var chapterDict = new Dictionary<int, short>();
var seriesDetail = await _seriesService.GetSeriesDetail(seriesId, userId);
foreach (var volume in seriesDetail.Volumes)
{
@ -879,6 +880,7 @@ public class OpdsController : BaseApiController
var chapterDto = _mapper.Map<ChapterDto>(chapter);
foreach (var mangaFile in chapter.Files)
{
chapterDict.Add(chapterId, 0);
feed.Entries.Add(await CreateChapterWithFile(userId, seriesId, volume.Id, chapterId, _mapper.Map<MangaFileDto>(mangaFile), series,
chapterDto, apiKey, prefix, baseUrl));
}
@ -892,7 +894,7 @@ public class OpdsController : BaseApiController
chapters = seriesDetail.Chapters;
}
foreach (var chapter in chapters.Where(c => !c.IsSpecial))
foreach (var chapter in chapters.Where(c => !c.IsSpecial && !chapterDict.ContainsKey(c.Id)))
{
var files = await _unitOfWork.ChapterRepository.GetFilesForChapterAsync(chapter.Id);
var chapterDto = _mapper.Map<ChapterDto>(chapter);

40
API/Controllers/StatsController.cs
View file

@ -1,5 +1,8 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
@ -7,11 +10,15 @@ using API.DTOs.Statistics;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers;
using API.Services;
using API.Services.Plus;
using API.Services.Tasks.Scanner.Parser;
using CsvHelper;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc;
using MimeTypes;
namespace API.Controllers;
@ -24,15 +31,18 @@ public class StatsController : BaseApiController
private readonly UserManager<AppUser> _userManager;
private readonly ILocalizationService _localizationService;
private readonly ILicenseService _licenseService;
private readonly IDirectoryService _directoryService;
public StatsController(IStatisticService statService, IUnitOfWork unitOfWork,
UserManager<AppUser> userManager, ILocalizationService localizationService, ILicenseService licenseService)
UserManager<AppUser> userManager, ILocalizationService localizationService,
ILicenseService licenseService, IDirectoryService directoryService)
{
_statService = statService;
_unitOfWork = unitOfWork;
_userManager = userManager;
_localizationService = localizationService;
_licenseService = licenseService;
_directoryService = directoryService;
}
[HttpGet("user/{userId}/read")]
@ -111,6 +121,34 @@ public class StatsController : BaseApiController
return Ok(await _statService.GetFileBreakdown());
}
/// <summary>
/// Generates a csv of all file paths for a given extension
/// </summary>
/// <returns></returns>
[Authorize("RequireAdminRole")]
[HttpGet("server/file-extension")]
[ResponseCache(CacheProfileName = "Statistics")]
public async Task<ActionResult> DownloadFilesByExtension(string fileExtension)
{
if (!Regex.IsMatch(fileExtension, Parser.SupportedExtensions))
{
return BadRequest("Invalid file format");
}
var tempFile = Path.Join(_directoryService.TempDirectory,
$"file_breakdown_{fileExtension.Replace(".", string.Empty)}.csv");
if (!_directoryService.FileSystem.File.Exists(tempFile))
{
var results = await _statService.GetFilesByExtension(fileExtension);
await using var writer = new StreamWriter(tempFile);
await using var csv = new CsvWriter(writer, CultureInfo.InvariantCulture);
await csv.WriteRecordsAsync(results);
}
return PhysicalFile(tempFile, MimeTypeMap.GetMimeType(Path.GetExtension(tempFile)),
System.Web.HttpUtility.UrlEncode(Path.GetFileName(tempFile)), true);
}
/// <summary>
/// Returns reading history events for a give or all users, broken up by day, and format

15
API/DTOs/Stats/FileExtensionExportDto.cs Normal file
View file

@ -0,0 +1,15 @@
using CsvHelper.Configuration.Attributes;
namespace API.DTOs.Stats;
/// <summary>
/// Excel export for File Extension Report
/// </summary>
public class FileExtensionExportDto
{
[Name("Path")]
public string FilePath { get; set; }
[Name("Extension")]
public string Extension { get; set; }
}

4
API/Helpers/AutoMapperProfiles.cs
View file

@ -21,6 +21,7 @@ using API.DTOs.Search;
using API.DTOs.SeriesDetail;
using API.DTOs.Settings;
using API.DTOs.SideNav;
using API.DTOs.Stats;
using API.DTOs.Theme;
using API.Entities;
using API.Entities.Enums;
@ -326,5 +327,8 @@ public class AutoMapperProfiles : Profile
opt.MapFrom(src => ReviewService.GetCharacters(src.Body)));
CreateMap<ExternalRecommendation, ExternalSeriesDto>();
CreateMap<MangaFile, FileExtensionExportDto>();
}
}

4
API/Program.cs
View file

@ -97,7 +97,7 @@ public class Program
Task.Run(async () =>
{
// Apply all migrations on startup
logger.LogInformation("Running Migrations");
logger.LogInformation("Running Manual Migrations");
try
{
@ -113,7 +113,7 @@ public class Program
}
await unitOfWork.CommitAsync();
logger.LogInformation("Running Migrations - complete");
logger.LogInformation("Running Manual Migrations - complete");
}).GetAwaiter()
.GetResult();
}

38
API/Services/ImageService.cs
View file

@ -134,9 +134,16 @@ public class ImageService : IImageService
/// <returns></returns>
public static Enums.Size GetSizeForDimensions(Image image, int targetWidth, int targetHeight)
{
if (WillScaleWell(image, targetWidth, targetHeight) || IsLikelyWideImage(image.Width, image.Height))
try
{
return Enums.Size.Force;
if (WillScaleWell(image, targetWidth, targetHeight) || IsLikelyWideImage(image.Width, image.Height))
{
return Enums.Size.Force;
}
}
catch (Exception)
{
/* Swallow */
}
return Enums.Size.Both;
@ -144,9 +151,15 @@ public class ImageService : IImageService
public static Enums.Interesting? GetCropForDimensions(Image image, int targetWidth, int targetHeight)
{
if (WillScaleWell(image, targetWidth, targetHeight) || IsLikelyWideImage(image.Width, image.Height))
try
{
if (WillScaleWell(image, targetWidth, targetHeight) || IsLikelyWideImage(image.Width, image.Height))
{
return null;
}
} catch (Exception)
{
/* Swallow */
return null;
}
@ -166,8 +179,8 @@ public class ImageService : IImageService
}
// Calculate scaling factors
var widthScaleFactor = (double)targetWidth / sourceImage.Width;
var heightScaleFactor = (double)targetHeight / sourceImage.Height;
var widthScaleFactor = (double) targetWidth / sourceImage.Width;
var heightScaleFactor = (double) targetHeight / sourceImage.Height;
// Check resolution quality (example thresholds)
if (widthScaleFactor > 2.0 || heightScaleFactor > 2.0)
@ -219,14 +232,15 @@ public class ImageService : IImageService
/// <returns>File name with extension of the file. This will always write to <see cref="DirectoryService.CoverImageDirectory"/></returns>
public string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, EncodeFormat encodeFormat, CoverImageSize size = CoverImageSize.Default)
{
var (width, height) = size.GetDimensions();
stream.Position = 0;
var (targetWidth, targetHeight) = size.GetDimensions();
if (stream.CanSeek) stream.Position = 0;
using var sourceImage = Image.NewFromStream(stream);
stream.Position = 0;
if (stream.CanSeek) stream.Position = 0;
using var thumbnail = sourceImage.ThumbnailImage(targetWidth, targetHeight,
size: GetSizeForDimensions(sourceImage, targetWidth, targetHeight),
crop: GetCropForDimensions(sourceImage, targetWidth, targetHeight));
using var thumbnail = Image.ThumbnailStream(stream, width, height: height,
size: GetSizeForDimensions(sourceImage, width, height),
crop: GetCropForDimensions(sourceImage, width, height));
var filename = fileName + encodeFormat.GetExtension();
_directoryService.ExistOrCreate(outputDirectory);
try

1
API/Services/SeriesService.cs
View file

@ -501,6 +501,7 @@ public class SeriesService : ISeriesService
StorylineChapters = storylineChapters,
TotalCount = chapters.Count,
UnreadCount = chapters.Count(c => c.Pages > 0 && c.PagesRead < c.Pages),
// TODO: See if we can get the ContinueFrom here
};
}

13
API/Services/StatisticService.cs
View file

@ -5,10 +5,12 @@ using System.Threading.Tasks;
using API.Data;
using API.DTOs;
using API.DTOs.Statistics;
using API.DTOs.Stats;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Extensions.QueryExtensions;
using API.Helpers;
using API.Services.Plus;
using API.Services.Tasks.Scanner.Parser;
using AutoMapper;
@ -35,6 +37,7 @@ public interface IStatisticService
Task UpdateServerStatistics();
Task<long> TimeSpentReadingForUsersAsync(IList<int> userIds, IList<int> libraryIds);
Task<KavitaPlusMetadataBreakdownDto> GetKavitaPlusMetadataBreakdown();
Task<IEnumerable<FileExtensionExportDto>> GetFilesByExtension(string fileExtension);
}
/// <summary>
@ -559,6 +562,16 @@ public class StatisticService : IStatisticService
}
public async Task<IEnumerable<FileExtensionExportDto>> GetFilesByExtension(string fileExtension)
{
var query = _context.MangaFile
.Where(f => f.Extension == fileExtension)
.ProjectTo<FileExtensionExportDto>(_mapper.ConfigurationProvider)
.OrderBy(f => f.FilePath);
return await query.ToListAsync();
}
public async Task<IEnumerable<TopReadDto>> GetTopUsers(int days)
{
var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();

34
API/Services/TaskScheduler.cs
View file

@ -4,11 +4,13 @@ using System.Collections.Immutable;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
using API.Entities.Enums;
using API.Helpers.Converters;
using API.Services.Plus;
using API.Services.Tasks;
using API.Services.Tasks.Metadata;
using API.SignalR;
using Hangfire;
using Microsoft.Extensions.Logging;
@ -22,12 +24,12 @@ public interface ITaskScheduler
Task ScheduleKavitaPlusTasks();
void ScanFolder(string folderPath, string originalPath, TimeSpan delay);
void ScanFolder(string folderPath);
void ScanLibrary(int libraryId, bool force = false);
void ScanLibraries(bool force = false);
Task ScanLibrary(int libraryId, bool force = false);
Task ScanLibraries(bool force = false);
void CleanupChapters(int[] chapterIds);
void RefreshMetadata(int libraryId, bool forceUpdate = true);
void RefreshSeriesMetadata(int libraryId, int seriesId, bool forceUpdate = false);
void ScanSeries(int libraryId, int seriesId, bool forceUpdate = false);
Task ScanSeries(int libraryId, int seriesId, bool forceUpdate = false);
void AnalyzeFilesForSeries(int libraryId, int seriesId, bool forceUpdate = false);
void AnalyzeFilesForLibrary(int libraryId, bool forceUpdate = false);
void CancelStatsTasks();
@ -57,6 +59,7 @@ public class TaskScheduler : ITaskScheduler
private readonly ILicenseService _licenseService;
private readonly IExternalMetadataService _externalMetadataService;
private readonly ISmartCollectionSyncService _smartCollectionSyncService;
private readonly IEventHub _eventHub;
public static BackgroundJobServer Client => new ();
public const string ScanQueue = "scan";
@ -93,7 +96,7 @@ public class TaskScheduler : ITaskScheduler
ICleanupService cleanupService, IStatsService statsService, IVersionUpdaterService versionUpdaterService,
IThemeService themeService, IWordCountAnalyzerService wordCountAnalyzerService, IStatisticService statisticService,
IMediaConversionService mediaConversionService, IScrobblingService scrobblingService, ILicenseService licenseService,
IExternalMetadataService externalMetadataService, ISmartCollectionSyncService smartCollectionSyncService)
IExternalMetadataService externalMetadataService, ISmartCollectionSyncService smartCollectionSyncService, IEventHub eventHub)
{
_cacheService = cacheService;
_logger = logger;
@ -112,6 +115,7 @@ public class TaskScheduler : ITaskScheduler
_licenseService = licenseService;
_externalMetadataService = externalMetadataService;
_smartCollectionSyncService = smartCollectionSyncService;
_eventHub = eventHub;
}
public async Task ScheduleTasks()
@ -320,18 +324,21 @@ public class TaskScheduler : ITaskScheduler
/// Attempts to call ScanLibraries on ScannerService, but if another scan task is in progress, will reschedule the invocation for 3 hours in future.
/// </summary>
/// <param name="force"></param>
public void ScanLibraries(bool force = false)
public async Task ScanLibraries(bool force = false)
{
if (RunningAnyTasksByMethod(ScanTasks, ScanQueue))
{
_logger.LogInformation("A Scan is already running, rescheduling ScanLibraries in 3 hours");
// Send InfoEvent to UI as this is invoked my API
BackgroundJob.Schedule(() => ScanLibraries(force), TimeSpan.FromHours(3));
await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.InfoEvent($"Scan libraries task delayed",
$"A scan was ongoing during processing of the scan libraries task. Task has been rescheduled for 3 hours: {DateTime.Now.AddHours(3)}"));
return;
}
BackgroundJob.Enqueue(() => _scannerService.ScanLibraries(force));
}
public void ScanLibrary(int libraryId, bool force = false)
public async Task ScanLibrary(int libraryId, bool force = false)
{
if (HasScanTaskRunningForLibrary(libraryId))
{
@ -340,18 +347,18 @@ public class TaskScheduler : ITaskScheduler
}
if (RunningAnyTasksByMethod(ScanTasks, ScanQueue))
{
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId);
_logger.LogInformation("A Scan is already running, rescheduling ScanLibrary in 3 hours");
await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.InfoEvent($"Scan library task delayed",
$"A scan was ongoing during processing of the {library!.Name} scan task. Task has been rescheduled for 3 hours: {DateTime.Now.AddHours(3)}"));
BackgroundJob.Schedule(() => ScanLibrary(libraryId, force), TimeSpan.FromHours(3));
return;
}
// await _eventHub.SendMessageAsync(MessageFactory.Info,
// MessageFactory.InfoEvent($"Scan library invoked but a task is already running for {library.Name}. Rescheduling request for 10 mins", string.Empty));
_logger.LogInformation("Enqueuing library scan for: {LibraryId}", libraryId);
BackgroundJob.Enqueue(() => _scannerService.ScanLibrary(libraryId, force, true));
var jobId = BackgroundJob.Enqueue(() => _scannerService.ScanLibrary(libraryId, force, true));
// When we do a scan, force cache to re-unpack in case page numbers change
BackgroundJob.Enqueue(() => _cleanupService.CleanupCacheDirectory());
BackgroundJob.ContinueJobWith(jobId, () => _cleanupService.CleanupCacheDirectory());
}
public void TurnOnScrobbling(int userId = 0)
@ -392,7 +399,7 @@ public class TaskScheduler : ITaskScheduler
BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(libraryId, seriesId, forceUpdate));
}
public void ScanSeries(int libraryId, int seriesId, bool forceUpdate = false)
public async Task ScanSeries(int libraryId, int seriesId, bool forceUpdate = false)
{
if (HasAlreadyEnqueuedTask(ScannerService.Name, "ScanSeries", [seriesId, forceUpdate], ScanQueue))
{
@ -402,7 +409,10 @@ public class TaskScheduler : ITaskScheduler
if (RunningAnyTasksByMethod(ScanTasks, ScanQueue))
{
// BUG: This can end up triggering a ton of scan series calls (but i haven't seen in practice)
var series = await _unitOfWork.SeriesRepository.GetSeriesByIdAsync(seriesId, SeriesIncludes.None);
_logger.LogInformation("A Scan is already running, rescheduling ScanSeries in 10 minutes");
await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.InfoEvent($"Scan series task delayed: {series!.Name}",
$"A scan was ongoing during processing of the scan series task. Task has been rescheduled for 10 minutes: {DateTime.Now.AddMinutes(10)}"));
BackgroundJob.Schedule(() => ScanSeries(libraryId, seriesId, forceUpdate), TimeSpan.FromMinutes(10));
return;
}

26
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -150,13 +150,28 @@ public class LibraryWatcher : ILibraryWatcher
{
_logger.LogTrace("[LibraryWatcher] Changed: {FullPath}, {Name}, {ChangeType}", e.FullPath, e.Name, e.ChangeType);
if (e.ChangeType != WatcherChangeTypes.Changed) return;
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name))));
var isDirectoryChange = string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name));
if (TaskScheduler.HasAlreadyEnqueuedTask("LibraryWatcher", "ProcessChange", [e.FullPath, isDirectoryChange],
checkRunningJobs: true))
{
return;
}
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, isDirectoryChange));
}
private void OnCreated(object sender, FileSystemEventArgs e)
{
_logger.LogTrace("[LibraryWatcher] Created: {FullPath}, {Name}", e.FullPath, e.Name);
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name)));
var isDirectoryChange = !_directoryService.FileSystem.File.Exists(e.Name);
if (TaskScheduler.HasAlreadyEnqueuedTask("LibraryWatcher", "ProcessChange", [e.FullPath, isDirectoryChange],
checkRunningJobs: true))
{
return;
}
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, isDirectoryChange));
}
/// <summary>
@ -168,6 +183,11 @@ public class LibraryWatcher : ILibraryWatcher
var isDirectory = string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name));
if (!isDirectory) return;
_logger.LogTrace("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name);
if (TaskScheduler.HasAlreadyEnqueuedTask("LibraryWatcher", "ProcessChange", [e.FullPath, true],
checkRunningJobs: true))
{
return;
}
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, true));
}
@ -298,7 +318,7 @@ public class LibraryWatcher : ILibraryWatcher
/// This is called via Hangfire to decrement the counter. Must work around a lock
/// </summary>
// ReSharper disable once MemberCanBePrivate.Global
public void UpdateLastBufferOverflow()
public static void UpdateLastBufferOverflow()
{
lock (Lock)
{

3
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -351,14 +351,17 @@ public class ParseScannedFiles
{
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", library.Name, ProgressEventType.Started));
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 1.A: Process {FolderCount} folders", library.Name, folders.Count());
var processedScannedSeries = new List<ScannedSeriesResult>();
//var processedScannedSeries = new ConcurrentBag<ScannedSeriesResult>();
foreach (var folderPath in folders)
{
try
{
_logger.LogDebug("\t[ScannerService] Library {LibraryName} Step 1.B: Scan files in {Folder}", library.Name, folderPath);
var scanResults = await ProcessFiles(folderPath, isLibraryScan, seriesPaths, library, forceCheck);
_logger.LogDebug("\t[ScannerService] Library {LibraryName} Step 1.C: Process files in {Folder}", library.Name, folderPath);
foreach (var scanResult in scanResults)
{
await ParseAndTrackSeries(library, seriesPaths, scanResult, processedScannedSeries);

51
API/Services/Tasks/ScannerService.cs
View file

@ -76,7 +76,7 @@ public enum ScanCancelReason
public class ScannerService : IScannerService
{
public const string Name = "ScannerService";
public const int Timeout = 60 * 60 * 60;
private const int Timeout = 60 * 60 * 60; // 2.5 days
private readonly IUnitOfWork _unitOfWork;
private readonly ILogger<ScannerService> _logger;
private readonly IMetadataService _metadataService;
@ -157,11 +157,11 @@ public class ScannerService : IScannerService
}
// TODO: Figure out why we have the library type restriction here
if (series != null && series.Library.Type is not (LibraryType.Book or LibraryType.LightNovel))
if (series != null)// && series.Library.Type is not (LibraryType.Book or LibraryType.LightNovel)
{
if (TaskScheduler.HasScanTaskRunningForSeries(series.Id))
{
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder);
_logger.LogDebug("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder);
return;
}
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder}, Series matched to folder and ScanSeries enqueued for 1 minute", folder);
@ -185,7 +185,7 @@ public class ScannerService : IScannerService
{
if (TaskScheduler.HasScanTaskRunningForLibrary(library.Id))
{
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
_logger.LogDebug("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
return;
}
BackgroundJob.Schedule(() => ScanLibrary(library.Id, false, true), TimeSpan.FromMinutes(1));
@ -198,20 +198,21 @@ public class ScannerService : IScannerService
/// <param name="seriesId"></param>
/// <param name="bypassFolderOptimizationChecks">Not Used. Scan series will always force</param>
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(Timeout)]
[AutomaticRetry(Attempts = 200, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true)
{
if (TaskScheduler.HasAlreadyEnqueuedTask(Name, "ScanSeries", [seriesId, bypassFolderOptimizationChecks], TaskScheduler.ScanQueue))
{
_logger.LogInformation("[ScannerService] Scan series invoked but a task is already running/enqueued. Dropping request");
return;
}
var sw = Stopwatch.StartNew();
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update
// if (TaskScheduler.HasScanTaskRunningForSeries(seriesId))
// {
// _logger.LogInformation("[ScannerService] Scan series invoked but a task is already running/enqueued. Rescheduling request for 1 mins");
// BackgroundJob.Schedule(() => ScanSeries(seriesId, bypassFolderOptimizationChecks), TimeSpan.FromMinutes(1));
// return;
// }
var existingChapterIdsToClean = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
@ -444,7 +445,7 @@ public class ScannerService : IScannerService
// Check if any of the folder roots are not available (ie disconnected from network, etc) and fail if any of them are
if (folders.Any(f => !_directoryService.IsDriveMounted(f)))
{
_logger.LogCritical("Some of the root folders for library ({LibraryName} are not accessible. Please check that drives are connected and rescan. Scan will be aborted", libraryName);
_logger.LogCritical("[ScannerService] Some of the root folders for library ({LibraryName} are not accessible. Please check that drives are connected and rescan. Scan will be aborted", libraryName);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted",
@ -458,7 +459,7 @@ public class ScannerService : IScannerService
if (folders.Any(f => _directoryService.IsDirectoryEmpty(f)))
{
// That way logging and UI informing is all in one place with full context
_logger.LogError("Some of the root folders for the library are empty. " +
_logger.LogError("[ScannerService] Some of the root folders for the library are empty. " +
"Either your mount has been disconnected or you are trying to delete all series in the library. " +
"Scan has be aborted. " +
"Check that your mount is connected or change the library's root folder and rescan");
@ -479,22 +480,23 @@ public class ScannerService : IScannerService
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibraries(bool forceUpdate = false)
{
_logger.LogInformation("Starting Scan of All Libraries, Forced: {Forced}", forceUpdate);
_logger.LogInformation("[ScannerService] Starting Scan of All Libraries, Forced: {Forced}", forceUpdate);
foreach (var lib in await _unitOfWork.LibraryRepository.GetLibrariesAsync())
{
if (TaskScheduler.RunningAnyTasksByMethod(TaskScheduler.ScanTasks, TaskScheduler.ScanQueue))
// BUG: This will trigger the first N libraries to scan over and over if there is always an interruption later in the chain
if (TaskScheduler.HasScanTaskRunningForLibrary(lib.Id))
{
_logger.LogInformation("[ScannerService] Scan library invoked via nightly scan job but a task is already running. Rescheduling for 4 hours");
await _eventHub.SendMessageAsync(MessageFactory.Info, MessageFactory.InfoEvent($"Scan libraries task delayed",
$"A scan was ongoing during processing of the scan libraries task. Task has been rescheduled for {DateTime.UtcNow.AddHours(4)} UTC"));
BackgroundJob.Schedule(() => ScanLibraries(forceUpdate), TimeSpan.FromHours(4));
return;
// We don't need to send SignalR event as this is a background job that user doesn't need insight into
_logger.LogInformation("[ScannerService] Scan library invoked via nightly scan job but a task is already running for {LibraryName}. Rescheduling for 4 hours", lib.Name);
await Task.Delay(TimeSpan.FromHours(4));
//BackgroundJob.Schedule(() => ScanLibraries(forceUpdate), TimeSpan.FromHours(4));
//return;
}
await ScanLibrary(lib.Id, forceUpdate, true);
}
_processSeries.Reset();
_logger.LogInformation("Scan of All Libraries Finished");
_logger.LogInformation("[ScannerService] Scan of All Libraries Finished");
}
@ -526,23 +528,27 @@ public class ScannerService : IScannerService
var shouldUseLibraryScan = !(await _unitOfWork.LibraryRepository.DoAnySeriesFoldersMatch(libraryFolderPaths));
if (!shouldUseLibraryScan)
{
_logger.LogError("Library {LibraryName} consists of one or more Series folders, using series scan", library.Name);
_logger.LogError("[ScannerService] Library {LibraryName} consists of one or more Series folders, using series scan", library.Name);
}
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 1: Scan Files", library.Name);
var (scanElapsedTime, processedSeries) = await ScanFiles(library, libraryFolderPaths,
shouldUseLibraryScan, forceUpdate);
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 2: Track Found Series", library.Name);
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
TrackFoundSeriesAndFiles(parsedSeries, processedSeries);
// We need to remove any keys where there is no actual parser info
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 3: Process Parsed Series", library.Name);
var totalFiles = await ProcessParsedSeries(forceUpdate, parsedSeries, library, scanElapsedTime);
UpdateLastScanned(library);
_unitOfWork.LibraryRepository.Update(library);
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 4: Save Library", library.Name);
if (await _unitOfWork.CommitAsync())
{
if (isSingleScan)
@ -563,6 +569,7 @@ public class ScannerService : IScannerService
totalFiles, parsedSeries.Count, sw.ElapsedMilliseconds, library.Name);
}
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 5: Remove Deleted Series", library.Name);
await RemoveSeriesNotFound(parsedSeries, library);
}
else

2
API/SignalR/MessageFactory.cs
View file

@ -345,6 +345,7 @@ public static class MessageFactory
EventType = ProgressEventType.Single,
Body = new
{
Name = Error,
Title = title,
SubTitle = subtitle,
}
@ -362,6 +363,7 @@ public static class MessageFactory
EventType = ProgressEventType.Single,
Body = new
{
Name = Info,
Title = title,
SubTitle = subtitle,
}

2
API/Startup.cs
View file

@ -427,8 +427,8 @@ public class Startup
catch (Exception)
{
/* Swallow Exception */
Console.WriteLine($"Kavita - v{BuildInfo.Version}");
}
Console.WriteLine($"Kavita - v{BuildInfo.Version}");
});
logger.LogInformation("Starting with base url as {BaseUrl}", basePath);

2
API/config/appsettings.Development.json
View file

@ -2,7 +2,7 @@
"TokenKey": "super secret unguessable key that is longer because we require it",
"Port": 5000,
"IpAddresses": "0.0.0.0,::",
"BaseUrl": "/tes/",
"BaseUrl": "/",
"Cache": 75,
"AllowIFraming": false
}