AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Misc Polish and Fixes (#1542)

Browse source 
* Moved LibraryWatcher to utilize a queue for calculating the change event to ensure the Watcher doesn't get overwhelmed on large moves.

* Fixed a security vulnerability (https://huntr.dev/bounties/8a3e652f-d6bf-436e-877e-0eaf5c69ef95/). This will be disclosed in Stable release changelog.

* Tweaked the log message template

* Removed some dead code from Configuration json patcher

* Fixed a bug with the ComicInfo finding to properly handle root level.

Fixed a bug where sometimes scanner wouldn't choose the first file with ComicInfo for filling out information.

* Added new setting for managing how many logs files are allowed, just like how backups work.

* Added unit tests for new CleanupLogs code

* Fixed a bug where manga reader background color wasn't actually sending from the UI

* Added new stats for tracking to help understand usage in the app and what features are used or not.

* Fixed Stats url

* Fixed a bug where volumes that had larger than 1 difference wouldn't properly return next/prev chapter (for continuous reader)

* Remove a redundant test step in build pipeline, since it's already done at PR stage.

* Updated dockerfile to use the new Heath check endpoint

* Allow force to pass through to scan loop

* Removed some old config stuff from a safety check on config in entrypoint.sh

* Fixed broken unit tests due to new RBS check and how we setup mock data.
This commit is contained in:
Joseph Milazzo 2022-09-18 12:24:30 -05:00 • committed by GitHub
parent c58c7deaf9
commit e89a06865c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 702 additions and 308 deletions
Download patch file Download diff file Expand all files Collapse all files

26
API/Services/Tasks/CleanupService.cs
View file

@ -25,6 +25,7 @@ public interface ICleanupService
Task DeleteChapterCoverImages();
Task DeleteTagCoverImages();
Task CleanupBackups();
Task CleanupLogs();
void CleanupTemp();
/// <summary>
/// Responsible to remove Series from Want To Read when user's have fully read the series and the series has Publication Status of Completed or Cancelled.
@ -76,6 +77,8 @@ public class CleanupService : ICleanupService
await SendProgress(0.7F, "Cleaning deleted cover images");
await DeleteTagCoverImages();
await DeleteReadingListCoverImages();
await SendProgress(0.8F, "Cleaning old logs");
await CleanupLogs();
await SendProgress(1F, "Cleanup finished");
_logger.LogInformation("Cleanup finished");
}
@ -189,6 +192,29 @@ public class CleanupService : ICleanupService
_logger.LogInformation("Finished cleanup of Database backups at {Time}", DateTime.Now);
}
public async Task CleanupLogs()
{
_logger.LogInformation("Performing cleanup of logs directory");
var dayThreshold = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).TotalLogs;
var deltaTime = DateTime.Today.Subtract(TimeSpan.FromDays(dayThreshold));
var allLogs = _directoryService.GetFiles(_directoryService.LogDirectory).ToList();
var expiredLogs = allLogs.Select(filename => _directoryService.FileSystem.FileInfo.FromFileName(filename))
.Where(f => f.CreationTime < deltaTime)
.ToList();
if (expiredLogs.Count == allLogs.Count)
{
_logger.LogInformation("All expired backups are older than {Threshold} days. Removing all but last backup", dayThreshold);
var toDelete = expiredLogs.OrderBy(f => f.CreationTime).ToList();
_directoryService.DeleteFiles(toDelete.Take(toDelete.Count - 1).Select(f => f.FullName));
}
else
{
_directoryService.DeleteFiles(expiredLogs.Select(f => f.FullName));
}
_logger.LogInformation("Finished cleanup of logs at {Time}", DateTime.Now);
}
public void CleanupTemp()
{
_logger.LogInformation("Performing cleanup of Temp directory");

80
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -91,8 +91,11 @@ public class LibraryWatcher : ILibraryWatcher
/// This is just here to prevent GC from Disposing our watchers
/// </summary>
private readonly IList<FileSystemWatcher> _fileWatchers = new List<FileSystemWatcher>();
private IList<string> _libraryFolders = new List<string>();
private static IList<string> _libraryFolders = new List<string>();
/// <summary>
/// The amount of time until the Schedule ScanFolder task should be executed
/// </summary>
/// <remarks>The Job will be enqueued instantly</remarks>
private readonly TimeSpan _queueWaitTime;
@ -109,7 +112,7 @@ public class LibraryWatcher : ILibraryWatcher
public async Task StartWatching()
{
_logger.LogInformation("Starting file watchers");
_logger.LogInformation("[LibraryWatcher] Starting file watchers");
_libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
.SelectMany(l => l.Folders)
@ -119,7 +122,7 @@ public class LibraryWatcher : ILibraryWatcher
.ToList();
foreach (var libraryFolder in _libraryFolders)
{
_logger.LogDebug("Watching {FolderPath}", libraryFolder);
_logger.LogDebug("[LibraryWatcher] Watching {FolderPath}", libraryFolder);
var watcher = new FileSystemWatcher(libraryFolder);
watcher.Changed += OnChanged;
@ -138,17 +141,19 @@ public class LibraryWatcher : ILibraryWatcher
_watcherDictionary[libraryFolder].Add(watcher);
}
_logger.LogInformation("[LibraryWatcher] Watching {Count} folders", _fileWatchers.Count);
}
public void StopWatching()
{
_logger.LogInformation("Stopping watching folders");
_logger.LogInformation("[LibraryWatcher] Stopping watching folders");
foreach (var fileSystemWatcher in _watcherDictionary.Values.SelectMany(watcher => watcher))
{
fileSystemWatcher.EnableRaisingEvents = false;
fileSystemWatcher.Changed -= OnChanged;
fileSystemWatcher.Created -= OnCreated;
fileSystemWatcher.Deleted -= OnDeleted;
fileSystemWatcher.Error -= OnError;
fileSystemWatcher.Dispose();
}
_fileWatchers.Clear();
@ -165,13 +170,13 @@ public class LibraryWatcher : ILibraryWatcher
{
if (e.ChangeType != WatcherChangeTypes.Changed) return;
_logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}", e.FullPath, e.Name);
ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name)));
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name))));
}
private void OnCreated(object sender, FileSystemEventArgs e)
{
_logger.LogDebug("[LibraryWatcher] Created: {FullPath}, {Name}", e.FullPath, e.Name);
ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name));
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name)));
}
/// <summary>
@ -183,7 +188,7 @@ public class LibraryWatcher : ILibraryWatcher
var isDirectory = string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name));
if (!isDirectory) return;
_logger.LogDebug("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name);
ProcessChange(e.FullPath, true);
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, true));
}
@ -198,35 +203,42 @@ public class LibraryWatcher : ILibraryWatcher
/// Processes the file or folder change. If the change is a file change and not from a supported extension, it will be ignored.
/// </summary>
/// <remarks>This will ignore image files that are added to the system. However, they may still trigger scans due to folder changes.</remarks>
/// <remarks>This is public only because Hangfire will invoke it. Do not call external to this class.</remarks>
/// <param name="filePath">File or folder that changed</param>
/// <param name="isDirectoryChange">If the change is on a directory and not a file</param>
private void ProcessChange(string filePath, bool isDirectoryChange = false)
public void ProcessChange(string filePath, bool isDirectoryChange = false)
{
var sw = Stopwatch.StartNew();
_logger.LogDebug("[LibraryWatcher] Processing change of {FilePath}", filePath);
try
{
// We need to check if directory or not
// If not a directory change AND file is not an archive or book, ignore
if (!isDirectoryChange &&
!(Parser.Parser.IsArchive(filePath) || Parser.Parser.IsBook(filePath))) return;
!(Parser.Parser.IsArchive(filePath) || Parser.Parser.IsBook(filePath)))
{
_logger.LogDebug("[LibraryWatcher] Change from {FilePath} is not an archive or book, ignoring change", filePath);
return;
}
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
if (string.IsNullOrEmpty(parentDirectory)) return;
// var libraryFolders = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync())
// .SelectMany(l => l.Folders)
// .Distinct()
// .Select(Parser.Parser.NormalizePath)
// .Where(_directoryService.Exists)
// .ToList();
// We need to find the library this creation belongs to
// Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault
var libraryFolder = _libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f));
if (string.IsNullOrEmpty(libraryFolder)) return;
var fullPath = GetFolder(filePath, _libraryFolders);
if (string.IsNullOrEmpty(fullPath))
{
_logger.LogDebug("[LibraryWatcher] Change from {FilePath} could not find root level folder, ignoring change", filePath);
return;
}
var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList();
if (!rootFolder.Any()) return;
// Select the first folder and join with library folder, this should give us the folder to scan.
var fullPath =
Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()));
// Check if this task has already enqueued or is being processed, before enquing
var alreadyScheduled =
TaskScheduler.HasAlreadyEnqueuedTask(ScannerService.Name, "ScanFolder", new object[] {fullPath});
_logger.LogDebug("{FullPath} already enqueued: {Value}", fullPath, alreadyScheduled);
_logger.LogDebug("[LibraryWatcher] {FullPath} already enqueued: {Value}", fullPath, alreadyScheduled);
if (!alreadyScheduled)
{
_logger.LogDebug("[LibraryWatcher] Scheduling ScanFolder for {Folder}", fullPath);
@ -242,9 +254,27 @@ public class LibraryWatcher : ILibraryWatcher
{
_logger.LogError(ex, "[LibraryWatcher] An error occured when processing a watch event");
}
_logger.LogDebug("ProcessChange occured in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds);
_logger.LogDebug("[LibraryWatcher] ProcessChange ran in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds);
}
private string GetFolder(string filePath, IList<string> libraryFolders)
{
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
if (string.IsNullOrEmpty(parentDirectory))
{
return string.Empty;
}
if (string.IsNullOrEmpty(parentDirectory)) return string.Empty;
// We need to find the library this creation belongs to
// Multiple libraries can point to the same base folder. In this case, we need use FirstOrDefault
var libraryFolder = libraryFolders.FirstOrDefault(f => parentDirectory.Contains(f));
if (string.IsNullOrEmpty(libraryFolder)) return string.Empty;
var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList();
if (!rootFolder.Any()) return string.Empty;
// Select the first folder and join with library folder, this should give us the folder to scan.
return Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder.First()));
}
}

7
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -116,7 +116,8 @@ public class ProcessSeries : IProcessSeries
{
_logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName);
var firstParsedInfo = parsedInfos[0];
// parsedInfos[0] is not the first volume or chapter. We need to find it using a ComicInfo check (as it uses firstParsedInfo for series sort)
var firstParsedInfo = parsedInfos.FirstOrDefault(p => p.ComicInfo != null, parsedInfos[0]);
UpdateVolumes(series, parsedInfos);
series.Pages = series.Volumes.Sum(v => v.Pages);
@ -479,10 +480,10 @@ public class ProcessSeries : IProcessSeries
var deletedVolumes = series.Volumes.Except(nonDeletedVolumes);
foreach (var volume in deletedVolumes)
{
var file = volume.Chapters.FirstOrDefault()?.Files?.FirstOrDefault()?.FilePath ?? "";
var file = volume.Chapters.FirstOrDefault()?.Files?.FirstOrDefault()?.FilePath ?? string.Empty;
if (!string.IsNullOrEmpty(file) && _directoryService.FileSystem.File.Exists(file))
{
_logger.LogError(
_logger.LogInformation(
"[ScannerService] Volume cleanup code was trying to remove a volume with a file still existing on disk. File: {File}",
file);
}

2
API/Services/Tasks/ScannerService.cs
View file

@ -458,7 +458,7 @@ public class ScannerService : IScannerService
}
var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles);
var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate);
await Task.WhenAll(processTasks);

71
API/Services/Tasks/StatsService.cs
View file

@ -1,4 +1,6 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices;
@ -21,6 +23,7 @@ public interface IStatsService
{
Task Send();
Task<ServerInfoDto> GetServerInfo();
Task SendCancellation();
}
public class StatsService : IStatsService
{
@ -127,6 +130,10 @@ public class StatsService : IStatsService
MaxSeriesInALibrary = await MaxSeriesInAnyLibrary(),
MaxVolumesInASeries = await MaxVolumesInASeries(),
MaxChaptersInASeries = await MaxChaptersInASeries(),
MangaReaderBackgroundColors = await AllMangaReaderBackgroundColors(),
MangaReaderPageSplittingModes = await AllMangaReaderPageSplitting(),
MangaReaderLayoutModes = await AllMangaReaderLayoutModes(),
FileFormats = AllFormats(),
};
var usersWithPref = (await _unitOfWork.UserRepository.GetAllUsersAsync(AppUserIncludes.UserPreferences)).ToList();
@ -149,6 +156,39 @@ public class StatsService : IStatsService
return serverInfo;
}
public async Task SendCancellation()
{
_logger.LogInformation("Informing KavitaStats that this instance is no longer sending stats");
var installId = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).InstallId;
var responseContent = string.Empty;
try
{
var response = await (ApiUrl + "/api/v2/stats/opt-out?installId=" + installId)
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(30))
.PostAsync();
if (response.StatusCode != StatusCodes.Status200OK)
{
_logger.LogError("KavitaStats did not respond successfully. {Content}", response);
}
}
catch (HttpRequestException e)
{
_logger.LogError(e, "KavitaStats did not respond successfully. {Response}", responseContent);
}
catch (Exception e)
{
_logger.LogError(e, "An error happened during the request to KavitaStats");
}
}
private Task<bool> GetIfUsingSeriesRelationship()
{
return _context.SeriesRelation.AnyAsync();
@ -190,4 +230,35 @@ public class StatsService : IStatsService
.SelectMany(v => v.Chapters)
.Count());
}
private async Task<IEnumerable<string>> AllMangaReaderBackgroundColors()
{
return await _context.AppUserPreferences.Select(p => p.BackgroundColor).Distinct().ToListAsync();
}
private async Task<IEnumerable<PageSplitOption>> AllMangaReaderPageSplitting()
{
return await _context.AppUserPreferences.Select(p => p.PageSplitOption).Distinct().ToListAsync();
}
private async Task<IEnumerable<LayoutMode>> AllMangaReaderLayoutModes()
{
return await _context.AppUserPreferences.Select(p => p.LayoutMode).Distinct().ToListAsync();
}
private IEnumerable<FileFormatDto> AllFormats()
{
var results = _context.MangaFile
.AsNoTracking()
.AsEnumerable()
.Select(m => new FileFormatDto()
{
Format = m.Format,
Extension = Path.GetExtension(m.FilePath)?.ToLowerInvariant()
})
.DistinctBy(f => f.Extension)
.ToList();
return results;
}
}