AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Implemented download log files (not in service). Refactored backupservice to handle log file splitting. Improved a few interfaces and added some unit tests around them.

Browse source
This commit is contained in:
Joseph Milazzo 2021-02-24 11:59:16 -06:00
parent 30352403cf
commit bbb4240e20
22 changed files with 292 additions and 46 deletions
Download patch file Download diff file Expand all files Collapse all files

2
API/Controllers/ReaderController.cs
View file

@ -142,7 +142,7 @@ namespace API.Controllers
public async Task<ActionResult> Bookmark(BookmarkDto bookmarkDto)
{
var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(User.GetUsername());
_logger.LogInformation("Saving {UserName} progress for Chapter {ChapterId} to page {PageNum}", user.UserName, bookmarkDto.ChapterId, bookmarkDto.PageNum);
_logger.LogDebug("Saving {UserName} progress for Chapter {ChapterId} to page {PageNum}", user.UserName, bookmarkDto.ChapterId, bookmarkDto.PageNum);
// Don't let user bookmark past total pages.
var chapter = await _unitOfWork.VolumeRepository.GetChapterAsync(bookmarkDto.ChapterId);

53
API/Controllers/ServerController.cs
View file

@ -1,6 +1,11 @@
using API.Extensions;
using System;
using System.IO;
using System.Threading.Tasks;
using API.Extensions;
using API.Interfaces.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
@ -11,11 +16,16 @@ namespace API.Controllers
{
private readonly IHostApplicationLifetime _applicationLifetime;
private readonly ILogger<ServerController> _logger;
private readonly IConfiguration _config;
private readonly IDirectoryService _directoryService;
public ServerController(IHostApplicationLifetime applicationLifetime, ILogger<ServerController> logger)
public ServerController(IHostApplicationLifetime applicationLifetime, ILogger<ServerController> logger, IConfiguration config,
IDirectoryService directoryService)
{
_applicationLifetime = applicationLifetime;
_logger = logger;
_config = config;
_directoryService = directoryService;
}
[HttpPost("restart")]
@ -26,5 +36,44 @@ namespace API.Controllers
_applicationLifetime.StopApplication();
return Ok();
}
[HttpGet("logs")]
public async Task<ActionResult> GetLogs()
{
// TODO: Zip up the log files
var maxRollingFiles = int.Parse(_config.GetSection("Logging").GetSection("File").GetSection("MaxRollingFiles").Value);
var loggingSection = _config.GetSection("Logging").GetSection("File").GetSection("Path").Value;
var multipleFileRegex = maxRollingFiles > 0 ? @"\d*" : string.Empty;
FileInfo fi = new FileInfo(loggingSection);
var files = _directoryService.GetFilesWithExtension(Directory.GetCurrentDirectory(), $@"{fi.Name}{multipleFileRegex}\.log");
Console.WriteLine(files);
var logFile = Path.Join(Directory.GetCurrentDirectory(), loggingSection);
_logger.LogInformation("Fetching download of logs: {LogFile}", logFile);
// First, copy the file to temp
var originalFile = new FileInfo(logFile);
var tempDirectory = Path.Join(Directory.GetCurrentDirectory(), "temp");
_directoryService.ExistOrCreate(tempDirectory);
var tempLocation = Path.Join(tempDirectory, originalFile.Name);
originalFile.CopyTo(tempLocation); // TODO: Make this unique based on date
// Read into memory
await using var memory = new MemoryStream();
// We need to copy it else it will throw an exception
await using (var stream = new FileStream(tempLocation, FileMode.Open, FileAccess.Read))
{
await stream.CopyToAsync(memory);
}
memory.Position = 0;
// Delete temp
(new FileInfo(tempLocation)).Delete();
return File(memory, "text/plain", Path.GetFileName(logFile));
}
}
}

3
API/Data/DataContext.cs
View file

@ -16,8 +16,8 @@ namespace API.Data
{
ChangeTracker.Tracked += OnEntityTracked;
ChangeTracker.StateChanged += OnEntityStateChanged;
}
public DbSet<Library> Library { get; set; }
public DbSet<Series> Series { get; set; }
@ -33,6 +33,7 @@ namespace API.Data
protected override void OnModelCreating(ModelBuilder builder)
{
base.OnModelCreating(builder);
builder.Entity<AppUser>()
.HasMany(ur => ur.UserRoles)

5
API/Extensions/ApplicationServiceExtensions.cs
View file

@ -32,7 +32,10 @@ namespace API.Extensions
services.AddDbContext<DataContext>(options =>
{
options.UseSqlite(config.GetConnectionString("DefaultConnection"));
options.UseSqlite(config.GetConnectionString("DefaultConnection"), builder =>
{
//builder.UseQuerySplittingBehavior(QuerySplittingBehavior.SplitQuery);
});
});
services.AddLogging(loggingBuilder =>

16
API/Extensions/ConfigurationExtensions.cs Normal file
View file

@ -0,0 +1,16 @@
using Microsoft.Extensions.Configuration;
namespace API.Extensions
{
public static class ConfigurationExtensions
{
public static int GetMaxRollingFiles(this IConfiguration config)
{
return int.Parse(config.GetSection("Logging").GetSection("File").GetSection("MaxRollingFiles").Value);
}
public static string GetLoggingFileName(this IConfiguration config)
{
return config.GetSection("Logging").GetSection("File").GetSection("Path").Value;
}
}
}

18
API/Interfaces/Services/IDirectoryService.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System.IO;
using System.Threading.Tasks;
using API.DTOs;
@ -20,7 +21,7 @@ namespace API.Interfaces.Services
/// <param name="path"></param>
/// <param name="searchPatternExpression"></param>
/// <returns></returns>
string[] GetFiles(string path, string searchPatternExpression = "");
string[] GetFilesWithExtension(string path, string searchPatternExpression = "");
/// <summary>
/// Returns true if the path exists and is a directory. If path does not exist, this will create it. Returns false in all fail cases.
/// </summary>
@ -28,6 +29,21 @@ namespace API.Interfaces.Services
/// <returns></returns>
bool ExistOrCreate(string directoryPath);
/// <summary>
/// Deletes all files within the directory, then the directory itself.
/// </summary>
/// <param name="directoryPath"></param>
void ClearAndDeleteDirectory(string directoryPath);
/// <summary>
/// Deletes all files within the directory.
/// </summary>
/// <param name="directoryPath"></param>
/// <returns></returns>
void ClearDirectory(string directoryPath);
bool CopyFilesToDirectory(IEnumerable<string> filePaths, string directoryPath);
IEnumerable<string> GetFiles(string path, string searchPatternExpression = "",
SearchOption searchOption = SearchOption.TopDirectoryOnly);
}
}

7
API/Services/ArchiveService.cs
View file

@ -3,11 +3,8 @@ using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Xml;
using System.Xml.Linq;
using System.Xml.Serialization;
using API.Extensions;
using API.Interfaces;
using API.Interfaces.Services;
using Microsoft.Extensions.Logging;
using NetVips;
@ -20,7 +17,7 @@ namespace API.Services
public class ArchiveService : IArchiveService
{
private readonly ILogger<ArchiveService> _logger;
private const int ThumbnailWidth = 320;
private const int ThumbnailWidth = 320; // 153w x 230h TODO: Look into optimizing the images to be smaller
public ArchiveService(ILogger<ArchiveService> logger)
{
@ -94,7 +91,7 @@ namespace API.Services
{
using var stream = entry.Open();
using var ms = new MemoryStream();
stream.CopyTo(ms);
stream.CopyTo(ms); // TODO: Check if we can use CopyToAsync here
var data = ms.ToArray();
return data;

51
API/Services/BackupService.cs
View file

@ -5,8 +5,10 @@ using System.IO.Compression;
using System.Linq;
using System.Threading.Tasks;
using API.Entities.Enums;
using API.Extensions;
using API.Interfaces;
using API.Interfaces.Services;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
namespace API.Services
@ -18,22 +20,36 @@ namespace API.Services
private readonly IDirectoryService _directoryService;
private readonly string _tempDirectory = Path.Join(Directory.GetCurrentDirectory(), "temp");
private readonly IList<string> _backupFiles = new List<string>()
{
"appsettings.json",
"Hangfire.db",
"Hangfire-log.db",
"kavita.db",
"kavita.db-shm",
"kavita.db-wal",
"kavita.log",
};
private readonly IList<string> _backupFiles;
public BackupService(IUnitOfWork unitOfWork, ILogger<BackupService> logger, IDirectoryService directoryService)
public BackupService(IUnitOfWork unitOfWork, ILogger<BackupService> logger, IDirectoryService directoryService, IConfiguration config)
{
_unitOfWork = unitOfWork;
_logger = logger;
_directoryService = directoryService;
var maxRollingFiles = config.GetMaxRollingFiles();
var loggingSection = config.GetLoggingFileName();
var multipleFileRegex = maxRollingFiles > 0 ? @"\d*" : string.Empty;
var fi = new FileInfo(loggingSection);
var files = maxRollingFiles > 0
? _directoryService.GetFiles(Directory.GetCurrentDirectory(), $@"{fi.Name}{multipleFileRegex}\.log")
: new string[] {"kavita.log"};
_backupFiles = new List<string>()
{
"appsettings.json",
"Hangfire.db",
"Hangfire-log.db",
"kavita.db",
"kavita.db-shm", // This wont always be there
"kavita.db-wal", // This wont always be there
};
foreach (var file in files.Select(f => (new FileInfo(f)).Name).ToList())
{
_backupFiles.Add(file);
}
}
public void BackupDatabase()
@ -59,14 +75,10 @@ namespace API.Services
var tempDirectory = Path.Join(_tempDirectory, dateString);
_directoryService.ExistOrCreate(tempDirectory);
foreach (var file in _backupFiles)
{
var originalFile = new FileInfo(Path.Join(Directory.GetCurrentDirectory(), file));
originalFile.CopyTo(Path.Join(tempDirectory, originalFile.Name));
}
_directoryService.ClearDirectory(tempDirectory);
_directoryService.CopyFilesToDirectory(
_backupFiles.Select(file => Path.Join(Directory.GetCurrentDirectory(), file)).ToList(), tempDirectory);
try
{
ZipFile.CreateFromDirectory(tempDirectory, zipPath);
@ -79,5 +91,6 @@ namespace API.Services
_directoryService.ClearAndDeleteDirectory(tempDirectory);
_logger.LogInformation("Database backup completed");
}
}
}

2
API/Services/CacheService.cs
View file

@ -111,7 +111,7 @@ namespace API.Services
if (page <= (mangaFile.NumberOfPages + pagesSoFar))
{
var path = GetCachePath(chapter.Id);
var files = _directoryService.GetFiles(path, Parser.Parser.ImageFileExtensions);
var files = _directoryService.GetFilesWithExtension(path, Parser.Parser.ImageFileExtensions);
Array.Sort(files, _numericComparer);
// Since array is 0 based, we need to keep that in account (only affects last image)

57
API/Services/DirectoryService.cs
View file

@ -39,8 +39,23 @@ namespace API.Services
.Where(file =>
reSearchPattern.IsMatch(Path.GetExtension(file)));
}
public IEnumerable<string> GetFiles(string path, string searchPatternExpression = "",
SearchOption searchOption = SearchOption.TopDirectoryOnly)
{
if (searchPatternExpression != string.Empty)
{
if (!Directory.Exists(path)) return ImmutableList<string>.Empty;
var reSearchPattern = new Regex(searchPatternExpression, RegexOptions.IgnoreCase);
return Directory.EnumerateFiles(path, "*", searchOption)
.Where(file =>
reSearchPattern.IsMatch(file));
}
return !Directory.Exists(path) ? Array.Empty<string>() : Directory.GetFiles(path);
}
public string[] GetFiles(string path, string searchPatternExpression = "")
public string[] GetFilesWithExtension(string path, string searchPatternExpression = "")
{
if (searchPatternExpression != string.Empty)
{
@ -70,6 +85,15 @@ namespace API.Services
{
DirectoryInfo di = new DirectoryInfo(directoryPath);
ClearDirectory(directoryPath);
di.Delete(true);
}
public void ClearDirectory(string directoryPath)
{
DirectoryInfo di = new DirectoryInfo(directoryPath);
foreach (var file in di.EnumerateFiles())
{
file.Delete();
@ -78,8 +102,35 @@ namespace API.Services
{
dir.Delete(true);
}
di.Delete(true);
}
public bool CopyFilesToDirectory(IEnumerable<string> filePaths, string directoryPath)
{
string currentFile = null;
try
{
foreach (var file in filePaths)
{
currentFile = file;
var fileInfo = new FileInfo(file);
if (fileInfo.Exists)
{
fileInfo.CopyTo(Path.Join(directoryPath, fileInfo.Name));
}
else
{
_logger.LogWarning("Tried to copy {File} but it doesn't exist", file);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Unable to copy {File} to {DirectoryPath}", currentFile, directoryPath);
return false;
}
return true;
}
public IEnumerable<string> ListDirectory(string rootPath)

1
API/Services/TokenService.cs
View file

@ -22,6 +22,7 @@ namespace API.Services
public TokenService(IConfiguration config, UserManager<AppUser> userManager)
{
_userManager = userManager;
_key = new SymmetricSecurityKey(Encoding.UTF8.GetBytes(config["TokenKey"]));
}

3
API/Startup.cs
View file

@ -24,7 +24,6 @@ namespace API
// This method gets called by the runtime. Use this method to add services to the container.
public void ConfigureServices(IServiceCollection services)
{
services.AddApplicationServices(_config);
services.AddControllers();
services.Configure<ForwardedHeadersOptions>(options =>
@ -72,7 +71,7 @@ namespace API
app.UseStaticFiles(new StaticFileOptions
{
ContentTypeProvider = new FileExtensionContentTypeProvider() // this is not set by default
ContentTypeProvider = new FileExtensionContentTypeProvider()
});