AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Code cleanup. Implemented ability to schedule Library Backups.

Browse source
This commit is contained in:
Joseph Milazzo 2021-02-17 08:58:36 -06:00
parent 83b9394b17
commit b4ee16d8d1
35 changed files with 217 additions and 91 deletions
Download patch file Download diff file Expand all files Collapse all files

1
API/Services/ArchiveService.cs
View file

@ -5,6 +5,7 @@ using System.IO.Compression;
using System.Linq;
using API.Extensions;
using API.Interfaces;
using API.Interfaces.Services;
using Microsoft.Extensions.Logging;
using NetVips;

64
API/Services/BackupService.cs Normal file
View file

@ -0,0 +1,64 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using API.Entities.Enums;
using API.Interfaces;
using API.Interfaces.Services;
using Microsoft.Extensions.Logging;
namespace API.Services
{
public class BackupService : IBackupService
{
private readonly IUnitOfWork _unitOfWork;
private readonly ILogger<BackupService> _logger;
private readonly IDirectoryService _directoryService;
private readonly IList<string> _backupFiles = new List<string>()
{
"appsettings.json",
"Hangfire.db",
"Hangfire-log.db",
"kavita.db",
"kavita.db-shm",
"kavita.db-wal",
"kavita.log",
};
public BackupService(IUnitOfWork unitOfWork, ILogger<BackupService> logger, IDirectoryService directoryService)
{
_unitOfWork = unitOfWork;
_logger = logger;
_directoryService = directoryService;
}
public void BackupDatabase()
{
_logger.LogInformation("Beginning backup of Database at {BackupTime}", DateTime.Now);
var backupDirectory = Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BackupDirectory)).Result.Value;
_logger.LogDebug("Backing up to {BackupDirectory}", backupDirectory);
if (!_directoryService.ExistOrCreate(backupDirectory))
{
_logger.LogError("Could not write to {BackupDirectory}; aborting backup", backupDirectory);
return;
}
var fileInfos = _backupFiles.Select(file => new FileInfo(Path.Join(Directory.GetCurrentDirectory(), file))).ToList();
var zipPath = Path.Join(backupDirectory, $"kavita_backup_{DateTime.Now}.zip");
using (var zipArchive = ZipFile.Open(zipPath, ZipArchiveMode.Create))
{
foreach (var fileInfo in fileInfos)
{
zipArchive.CreateEntryFromFile(fileInfo.FullName, fileInfo.Name);
}
}
_logger.LogInformation("Database backup completed");
throw new System.NotImplementedException();
}
}
}

12
API/Services/CacheService.cs
View file

@ -6,6 +6,7 @@ using API.Comparators;
using API.Entities;
using API.Extensions;
using API.Interfaces;
using API.Interfaces.Services;
using Microsoft.Extensions.Logging;
namespace API.Services
@ -30,11 +31,12 @@ namespace API.Services
public void EnsureCacheDirectory()
{
_logger.LogDebug($"Checking if valid Cache directory: {CacheDirectory}");
// TODO: Replace with DirectoryService.ExistOrCreate()
_logger.LogDebug("Checking if valid Cache directory: {CacheDirectory}", CacheDirectory);
var di = new DirectoryInfo(CacheDirectory);
if (!di.Exists)
{
_logger.LogError($"Cache directory {CacheDirectory} is not accessible or does not exist. Creating...");
_logger.LogError("Cache directory {CacheDirectory} is not accessible or does not exist. Creating...", CacheDirectory);
Directory.CreateDirectory(CacheDirectory);
}
}
@ -66,15 +68,15 @@ namespace API.Services
}
catch (Exception ex)
{
_logger.LogError("There was an issue deleting one or more folders/files during cleanup.", ex);
_logger.LogError(ex, "There was an issue deleting one or more folders/files during cleanup");
}
_logger.LogInformation("Cache directory purged.");
_logger.LogInformation("Cache directory purged");
}
public void CleanupChapters(int[] chapterIds)
{
_logger.LogInformation($"Running Cache cleanup on Volumes");
_logger.LogInformation("Running Cache cleanup on Volumes");
foreach (var chapter in chapterIds)
{

43
API/Services/DirectoryService.cs
View file

@ -7,7 +7,7 @@ using System.Text.RegularExpressions;
using System.Threading;
using System.Threading.Tasks;
using API.DTOs;
using API.Interfaces;
using API.Interfaces.Services;
using Microsoft.Extensions.Logging;
using NetVips;
@ -49,7 +49,23 @@ namespace API.Services
return !Directory.Exists(path) ? Array.Empty<string>() : Directory.GetFiles(path);
}
public bool ExistOrCreate(string directoryPath)
{
var di = new DirectoryInfo(directoryPath);
if (di.Exists) return true;
try
{
Directory.CreateDirectory(directoryPath);
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an issue creating directory: {Directory}", directoryPath);
return false;
}
return true;
}
public IEnumerable<string> ListDirectory(string rootPath)
{
if (!Directory.Exists(rootPath)) return ImmutableList<string>.Empty;
@ -66,7 +82,7 @@ namespace API.Services
{
if (!File.Exists(imagePath))
{
_logger.LogError("Image does not exist on disk.");
_logger.LogError("Image does not exist on disk");
return null;
}
using var image = Image.NewFromFile(imagePath);
@ -82,16 +98,16 @@ namespace API.Services
};
}
/// <summary>
/// Recursively scans files and applies an action on them. This uses as many cores the underlying PC has to speed
/// up processing.
/// </summary>
/// <param name="root">Directory to scan</param>
/// <param name="action">Action to apply on file path</param>
/// <exception cref="ArgumentException"></exception>
public static int TraverseTreeParallelForEach(string root, Action<string> action, string searchPattern)
/// <summary>
/// Recursively scans files and applies an action on them. This uses as many cores the underlying PC has to speed
/// up processing.
/// </summary>
/// <param name="root">Directory to scan</param>
/// <param name="action">Action to apply on file path</param>
/// <param name="searchPattern">Regex pattern to search against</param>
/// <exception cref="ArgumentException"></exception>
public static int TraverseTreeParallelForEach(string root, Action<string> action, string searchPattern)
{
//Count of files traversed and timer for diagnostic output
var fileCount = 0;
@ -127,9 +143,6 @@ namespace API.Services
}
try {
// TODO: In future, we need to take LibraryType into consideration for what extensions to allow (RAW should allow images)
// or we need to move this filtering to another area (Process)
// or we can get all files and put a check in place during Process to abandon files
files = GetFilesWithCertainExtensions(currentDir, searchPattern)
.ToArray();
}

2
API/Services/ScannerService.cs
View file

@ -2,11 +2,9 @@
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Runtime.CompilerServices;
using System.Threading;
using System.Threading.Tasks;
using API.Entities;
using API.Entities.Enums;

57
API/Services/TaskScheduler.cs
View file

@ -13,60 +13,81 @@ namespace API.Services
private readonly ICacheService _cacheService;
private readonly ILogger<TaskScheduler> _logger;
private readonly IScannerService _scannerService;
private readonly IUnitOfWork _unitOfWork;
private readonly IMetadataService _metadataService;
private readonly IBackupService _backupService;
public BackgroundJobServer Client => new BackgroundJobServer(new BackgroundJobServerOptions()
{
WorkerCount = 1
});
public BackgroundJobServer Client => new BackgroundJobServer();
// new BackgroundJobServerOptions()
// {
// WorkerCount = 1
// }
public TaskScheduler(ICacheService cacheService, ILogger<TaskScheduler> logger, IScannerService scannerService,
IUnitOfWork unitOfWork, IMetadataService metadataService)
IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService)
{
_cacheService = cacheService;
_logger = logger;
_scannerService = scannerService;
_unitOfWork = unitOfWork;
_metadataService = metadataService;
_backupService = backupService;
_logger.LogInformation("Scheduling/Updating cache cleanup on a daily basis.");
var setting = Task.Run(() => unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskScan)).Result;
ScheduleTasks();
//JobStorage.Current.GetMonitoringApi().
}
public void ScheduleTasks()
{
_logger.LogInformation("Scheduling reoccurring tasks");
string setting = null;
setting = Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskScan)).Result.Value;
if (setting != null)
{
RecurringJob.AddOrUpdate(() => _scannerService.ScanLibraries(), () => CronConverter.ConvertToCronNotation(setting.Value));
_logger.LogDebug("Scheduling Scan Library Task for {Cron}", setting);
RecurringJob.AddOrUpdate(() => _scannerService.ScanLibraries(), () => CronConverter.ConvertToCronNotation(setting));
}
else
{
RecurringJob.AddOrUpdate(() => _cacheService.Cleanup(), Cron.Daily);
RecurringJob.AddOrUpdate(() => _scannerService.ScanLibraries(), Cron.Daily);
}
//JobStorage.Current.GetMonitoringApi().
setting = Task.Run(() => _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskBackup)).Result.Value;
if (setting != null)
{
_logger.LogDebug("Scheduling Backup Task for {Cron}", setting);
RecurringJob.AddOrUpdate(() => _backupService.BackupDatabase(), () => CronConverter.ConvertToCronNotation(setting2));
}
else
{
RecurringJob.AddOrUpdate(() => _backupService.BackupDatabase(), Cron.Weekly);
}
RecurringJob.AddOrUpdate(() => _cacheService.Cleanup(), Cron.Daily);
}
public void ScanLibrary(int libraryId, bool forceUpdate = false)
{
_logger.LogInformation($"Enqueuing library scan for: {libraryId}");
_logger.LogInformation("Enqueuing library scan for: {LibraryId}", libraryId);
BackgroundJob.Enqueue(() => _scannerService.ScanLibrary(libraryId, forceUpdate));
}
public void CleanupChapters(int[] chapterIds)
{
BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(chapterIds));
}
public void RefreshMetadata(int libraryId, bool forceUpdate = true)
{
_logger.LogInformation($"Enqueuing library metadata refresh for: {libraryId}");
_logger.LogInformation("Enqueuing library metadata refresh for: {LibraryId}", libraryId);
BackgroundJob.Enqueue((() => _metadataService.RefreshMetadata(libraryId, forceUpdate)));
}
public void ScanLibraryInternal(int libraryId, bool forceUpdate)
public void BackupDatabase()
{
_scannerService.ScanLibrary(libraryId, forceUpdate);
_metadataService.RefreshMetadata(libraryId, forceUpdate);
}
BackgroundJob.Enqueue(() => _backupService.BackupDatabase());
}
}
}

2
API/Services/TokenService.cs
View file

@ -6,7 +6,7 @@ using System.Security.Claims;
using System.Text;
using System.Threading.Tasks;
using API.Entities;
using API.Interfaces;
using API.Interfaces.Services;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Configuration;
using Microsoft.IdentityModel.Tokens;