AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Security Event Logging & Bugfixes (#1882)

Browse source 
* Fixed bookmarking failing to convert to webp

* Brought the ag-swipe/ng-swipe code into Kavita due to being abandoned by developer and angular requirements.

* Fixed average reading time per week finally

* Cleaned up some extra decimals on time duration pipe

* Don't try to update index.html for base url on local. Fixed ag-swipe on prod mode.

* Updated a link on theme manager to point to the new github

* Range knobs should be primary color on firefox too

* Implemented the ability to get thumbnails of pages inside an archive or pdf.

* Updated packages and fixed opds-ps 1.2 issue

* Fixed lock file

* Allow Kavita's Swagger to hit instances with CORS

* Added IP/Request logging for Security Audits

* Linked up Summary tag from CBL into Kavita.

* Redid the migration so SecurityEvent now has UTC date as well.

* Split security logging to a separate file

* Update to new versions of checkout and setup

* Added a PR check on PR body to ensure that it doesn't contain any characters that break our discord hook.

* Updating action

* optimize regex in action

* Fixed an issue where fit to width would cause the actual height of the image to be shown for pagination bars, instead of rendered.

* Added some new code in GetPageFromFiles to ensure pages that exceed array map down to last file.

* Added comment about robots

* Fixed up unit tests for new ReaderService signature

* Kavita now cleans up empty reading lists at night

* Don't allow nightly cleanup to run if we are running media conversion tasks

* Fixed some bugs in typeahead, it should behave much more reliably.

* Fix an issue where emulate comic book wasn't extending to the bottom properly

* Added support for Series Chapter 001 Volume 001

* Refactor XFrameOptions="SameOrigins" out to allow users to override in appsettings.json.

* Added a rate limiter for some endpoints, but it doesn't seem to be triggering

---------

Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>
This commit is contained in:
Joe Milazzo 2023-03-16 15:57:34 -05:00 • committed by GitHub
parent 21203414f0
commit c10acb1279
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
60 changed files with 2890 additions and 302 deletions
Download patch file Download diff file Expand all files Collapse all files

14
API/API.csproj
View file

@ -67,15 +67,15 @@
<PackageReference Include="Hangfire.Storage.SQLite" Version="0.3.3" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
<PackageReference Include="MarkdownDeep.NET.Core" Version="1.5.0.4" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="7.0.3" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="7.0.3" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.3" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="7.0.4" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="7.0.4" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="7.0.4" />
<PackageReference Include="Microsoft.AspNetCore.SignalR" Version="1.1.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.3">
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="7.0.4">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="7.0.3" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="7.0.4" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="2.3.2" />
<PackageReference Include="NetVips" Version="2.2.0" />
@ -92,14 +92,14 @@
<PackageReference Include="Serilog.Sinks.SignalR.Core" Version="0.1.2" />
<PackageReference Include="SharpCompress" Version="0.32.2" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.0.0" />
<PackageReference Include="SonarAnalyzer.CSharp" Version="8.53.0.62665">
<PackageReference Include="SonarAnalyzer.CSharp" Version="8.54.0.64047">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
<PackageReference Include="Swashbuckle.AspNetCore.Filters" Version="7.0.6" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.27.0" />
<PackageReference Include="System.IO.Abstractions" Version="19.2.1" />
<PackageReference Include="System.IO.Abstractions" Version="19.2.4" />
<PackageReference Include="System.Drawing.Common" Version="7.0.0" />
<PackageReference Include="VersOne.Epub" Version="3.3.0-alpha1" />
</ItemGroup>

4
API/Controllers/AccountController.cs
View file

@ -13,6 +13,7 @@ using API.Entities;
using API.Entities.Enums;
using API.Errors;
using API.Extensions;
using API.Middleware.RateLimit;
using API.Services;
using API.SignalR;
using AutoMapper;
@ -22,6 +23,7 @@ using Kavita.Common.EnvironmentInfo;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Identity;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.RateLimiting;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
@ -769,6 +771,7 @@ public class AccountController : BaseApiController
/// <returns></returns>
[AllowAnonymous]
[HttpPost("forgot-password")]
[EnableRateLimiting("Authentication")]
public async Task<ActionResult<string>> ForgotPassword([FromQuery] string email)
{
var user = await _unitOfWork.UserRepository.GetUserByEmailAsync(email);
@ -847,6 +850,7 @@ public class AccountController : BaseApiController
/// <param name="userId"></param>
/// <returns></returns>
[HttpPost("resend-confirmation-email")]
[EnableRateLimiting("Authentication")]
public async Task<ActionResult<string>> ResendConfirmationSendEmail([FromQuery] int userId)
{
var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId);

7
API/Controllers/OPDSController.cs
View file

@ -904,8 +904,11 @@ public class OpdsController : BaseApiController
var link = CreateLink(FeedLinkRelation.Stream, "image/jpeg",
$"{Prefix}{apiKey}/image?libraryId={libraryId}&seriesId={seriesId}&volumeId={volumeId}&chapterId={chapterId}&pageNumber=" + "{pageNumber}");
link.TotalPages = mangaFile.Pages;
link.LastRead = progress.PageNum;
link.LastReadDate = progress.LastModifiedUtc;
if (progress != null)
{
link.LastRead = progress.PageNum;
link.LastReadDate = progress.LastModifiedUtc;
}
link.IsPageStream = true;
return link;
}

25
API/Controllers/ReaderController.cs
View file

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@ -34,12 +34,14 @@ public class ReaderController : BaseApiController
private readonly IBookmarkService _bookmarkService;
private readonly IAccountService _accountService;
private readonly IEventHub _eventHub;
private readonly IImageService _imageService;
private readonly IDirectoryService _directoryService;
/// <inheritdoc />
public ReaderController(ICacheService cacheService,
IUnitOfWork unitOfWork, ILogger<ReaderController> logger,
IReaderService readerService, IBookmarkService bookmarkService,
IAccountService accountService, IEventHub eventHub)
IAccountService accountService, IEventHub eventHub, IImageService imageService, IDirectoryService directoryService)
{
_cacheService = cacheService;
_unitOfWork = unitOfWork;
@ -48,6 +50,8 @@ public class ReaderController : BaseApiController
_bookmarkService = bookmarkService;
_accountService = accountService;
_eventHub = eventHub;
_imageService = imageService;
_directoryService = directoryService;
}
/// <summary>
@ -114,6 +118,20 @@ public class ReaderController : BaseApiController
}
}
[HttpGet("thumbnail")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Hour)]
[AllowAnonymous]
public async Task<ActionResult> GetThumbnail(int chapterId, int pageNum)
{
var chapter = await _cacheService.Ensure(chapterId, true);
if (chapter == null) return BadRequest("There was an issue extracting images from chapter");
var images = _cacheService.GetCachedPages(chapterId);
var path = await _readerService.GetThumbnail(chapter, pageNum, images);
var format = Path.GetExtension(path).Replace(".", string.Empty); // TODO: Make this an extension
return PhysicalFile(path, "image/" + format, Path.GetFileName(path), true);
}
/// <summary>
/// Returns an image for a given bookmark series. Side effect: This will cache the bookmark images for reading.
/// </summary>
@ -172,13 +190,14 @@ public class ReaderController : BaseApiController
/// <summary>
/// Returns various information about a Chapter. Side effect: This will cache the chapter images for reading.
/// </summary>
/// <remarks>This is generally the first call when attempting to read to allow pre-generation of assets needed for reading</remarks>
/// <param name="chapterId"></param>
/// <param name="extractPdf">Should Kavita extract pdf into images. Defaults to false.</param>
/// <param name="includeDimensions">Include file dimensions. Only useful for image based reading</param>
/// <returns></returns>
[HttpGet("chapter-info")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Hour, VaryByQueryKeys = new []{"chapterId", "extractPdf", "includeDimensions"})]
public async Task<ActionResult<ChapterInfoDto?>> GetChapterInfo(int chapterId, bool extractPdf = false, bool includeDimensions = false)
public async Task<ActionResult<ChapterInfoDto>> GetChapterInfo(int chapterId, bool extractPdf = false, bool includeDimensions = false)
{
if (chapterId <= 0) return Ok(null); // This can happen occasionally from UI, we should just ignore
var chapter = await _cacheService.Ensure(chapterId, extractPdf);

1
API/Data/DataContext.cs
View file

@ -47,6 +47,7 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
public DbSet<FolderPath> FolderPath { get; set; } = null!;
public DbSet<Device> Device { get; set; } = null!;
public DbSet<ServerStatistics> ServerStatistics { get; set; } = null!;
public DbSet<SecurityEvent> SecurityEvent { get; set; } = null!;
protected override void OnModelCreating(ModelBuilder builder)

1901
API/Data/Migrations/20230316123908_SecurityEvent.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

40
API/Data/Migrations/20230316123908_SecurityEvent.cs Normal file
View file

@ -0,0 +1,40 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class SecurityEvent : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "SecurityEvent",
columns: table => new
{
Id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
IpAddress = table.Column<string>(type: "TEXT", nullable: true),
RequestMethod = table.Column<string>(type: "TEXT", nullable: true),
RequestPath = table.Column<string>(type: "TEXT", nullable: true),
UserAgent = table.Column<string>(type: "TEXT", nullable: true),
CreatedAt = table.Column<DateTime>(type: "TEXT", nullable: false),
CreatedAtUtc = table.Column<DateTime>(type: "TEXT", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_SecurityEvent", x => x.Id);
});
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "SecurityEvent");
}
}
}

31
API/Data/Migrations/DataContextModelSnapshot.cs
View file

@ -15,7 +15,7 @@ namespace API.Data.Migrations
protected override void BuildModel(ModelBuilder modelBuilder)
{
#pragma warning disable 612, 618
modelBuilder.HasAnnotation("ProductVersion", "7.0.3");
modelBuilder.HasAnnotation("ProductVersion", "7.0.4");
modelBuilder.Entity("API.Entities.AppRole", b =>
{
@ -944,6 +944,35 @@ namespace API.Data.Migrations
b.ToTable("ReadingListItem");
});
modelBuilder.Entity("API.Entities.SecurityEvent", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<DateTime>("CreatedAt")
.HasColumnType("TEXT");
b.Property<DateTime>("CreatedAtUtc")
.HasColumnType("TEXT");
b.Property<string>("IpAddress")
.HasColumnType("TEXT");
b.Property<string>("RequestMethod")
.HasColumnType("TEXT");
b.Property<string>("RequestPath")
.HasColumnType("TEXT");
b.Property<string>("UserAgent")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("SecurityEvent");
});
modelBuilder.Entity("API.Entities.Series", b =>
{
b.Property<int>("Id")

13
API/Data/Repositories/ReadingListRepository.cs
View file

@ -47,6 +47,7 @@ public interface IReadingListRepository
IEnumerable<PersonDto> GetReadingListCharactersAsync(int readingListId);
Task<IList<ReadingList>> GetAllWithNonWebPCovers();
Task<IList<string>> GetFirstFourCoverImagesByReadingListId(int readingListId);
Task<int> RemoveReadingListsWithoutSeries();
}
public class ReadingListRepository : IReadingListRepository
@ -132,6 +133,18 @@ public class ReadingListRepository : IReadingListRepository
.ToListAsync();
}
public async Task<int> RemoveReadingListsWithoutSeries()
{
var listsToDelete = await _context.ReadingList
.Include(c => c.Items)
.Where(c => c.Items.Count == 0)
.AsSplitQuery()
.ToListAsync();
_context.RemoveRange(listsToDelete);
return await _context.SaveChangesAsync();
}
public void Remove(ReadingListItem item)
{
_context.ReadingListItem.Remove(item);

27
API/Data/Repositories/SecurityEventRepository.cs Normal file
View file

@ -0,0 +1,27 @@
using API.Entities;
using AutoMapper;
using Microsoft.EntityFrameworkCore;
namespace API.Data.Repositories;
public interface ISecurityEventRepository
{
void Add(SecurityEvent securityEvent);
}
public class SecurityEventRepository : ISecurityEventRepository
{
private readonly DataContext _context;
private readonly IMapper _mapper;
public SecurityEventRepository(DataContext context, IMapper mapper)
{
_context = context;
_mapper = mapper;
}
public void Add(SecurityEvent securityEvent)
{
_context.SecurityEvent.Add(securityEvent);
}
}

2
API/Data/UnitOfWork.cs
View file

@ -25,6 +25,7 @@ public interface IUnitOfWork
ISiteThemeRepository SiteThemeRepository { get; }
IMangaFileRepository MangaFileRepository { get; }
IDeviceRepository DeviceRepository { get; }
ISecurityEventRepository SecurityEventRepository { get; }
bool Commit();
Task<bool> CommitAsync();
bool HasChanges();
@ -62,6 +63,7 @@ public class UnitOfWork : IUnitOfWork
public ISiteThemeRepository SiteThemeRepository => new SiteThemeRepository(_context, _mapper);
public IMangaFileRepository MangaFileRepository => new MangaFileRepository(_context);
public IDeviceRepository DeviceRepository => new DeviceRepository(_context, _mapper);
public ISecurityEventRepository SecurityEventRepository => new SecurityEventRepository(_context, _mapper);
/// <summary>
/// Commits changes to the DB. Completes the open transaction.

14
API/Entities/SecurityEvent.cs Normal file
View file

@ -0,0 +1,14 @@
using System;
namespace API.Entities;
public class SecurityEvent
{
public int Id { get; set; }
public string IpAddress { get; set; }
public string RequestMethod { get; set; }
public string RequestPath { get; set; }
public string UserAgent { get; set; }
public DateTime CreatedAt { get; set; }
public DateTime CreatedAtUtc { get; set; }
}

3
API/Helpers/PersonHelper.cs
View file

@ -7,6 +7,7 @@ using API.DTOs;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers.Builders;
namespace API.Helpers;
@ -156,7 +157,7 @@ public static class PersonHelper
else
{
// Add new tag
handleAdd(DbFactory.Person(tag.Name, role));
handleAdd(new PersonBuilder(tag.Name, role).Build());
isModified = true;
}
}

2
API/Logging/LogLevelOptions.cs
View file

@ -1,6 +1,7 @@
using Serilog;
using Serilog.Core;
using Serilog.Events;
using Serilog.Filters;
using Serilog.Formatting.Display;
namespace API.Logging;
@ -12,6 +13,7 @@ namespace API.Logging;
public static class LogLevelOptions
{
public const string LogFile = "config/logs/kavita.log";
public const string SecurityLogFile = "config/logs/security.log";
public const bool LogRollingEnabled = true;
/// <summary>
/// Controls the Logging Level of the Application

36
API/Middleware/RateLimit/AuthenticationRateLimiterPolicy.cs Normal file
View file

@ -0,0 +1,36 @@
using System;
using System.Globalization;
using System.Threading;
using System.Threading.RateLimiting;
using System.Threading.Tasks;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.RateLimiting;
namespace API.Middleware.RateLimit;
public class AuthenticationRateLimiterPolicy : IRateLimiterPolicy<string>
{
public RateLimitPartition<string> GetPartition(HttpContext httpContext)
{
return RateLimitPartition.GetFixedWindowLimiter(httpContext.Request.Headers.Host.ToString(),
partition => new FixedWindowRateLimiterOptions
{
AutoReplenishment = true,
PermitLimit = 1,
Window = TimeSpan.FromMinutes(10),
});
}
public Func<OnRejectedContext, CancellationToken, ValueTask>? OnRejected { get; } =
(context, _) =>
{
if (context.Lease.TryGetMetadata(MetadataName.RetryAfter, out var retryAfter))
{
context.HttpContext.Response.Headers.RetryAfter =
((int) retryAfter.TotalSeconds).ToString(NumberFormatInfo.InvariantInfo);
}
context.HttpContext.Response.StatusCode = StatusCodes.Status429TooManyRequests;
return new ValueTask();
};
}

62
API/Middleware/SecurityEventMiddleware.cs Normal file
View file

@ -0,0 +1,62 @@
using System;
using System.IO;
using System.Security.AccessControl;
using System.Threading.Tasks;
using API.Data;
using API.DTOs;
using API.Entities;
using API.Logging;
using API.Services;
using Microsoft.AspNetCore.Http;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Logging;
using Serilog;
using Serilog.Core;
using ILogger = Serilog.ILogger;
namespace API.Middleware;
public class SecurityEventMiddleware
{
private readonly RequestDelegate _next;
private readonly ILogger _logger;
public SecurityEventMiddleware(RequestDelegate next)
{
_next = next;
_logger = new LoggerConfiguration()
.MinimumLevel.Debug()
.WriteTo.File(Path.Join(Directory.GetCurrentDirectory(), "config/logs/", "security.log"), rollingInterval: RollingInterval.Day)
.CreateLogger();
}
public async Task InvokeAsync(HttpContext context)
{
var ipAddress = context.Connection.RemoteIpAddress?.ToString();
var requestMethod = context.Request.Method;
var requestPath = context.Request.Path;
var userAgent = context.Request.Headers["User-Agent"];
var securityEvent = new SecurityEvent
{
IpAddress = ipAddress,
RequestMethod = requestMethod,
RequestPath = requestPath,
UserAgent = userAgent,
CreatedAt = DateTime.Now,
CreatedAtUtc = DateTime.UtcNow,
};
using (var scope = context.RequestServices.CreateScope())
{
var dbContext = scope.ServiceProvider.GetRequiredService<DataContext>();
dbContext.Add(securityEvent);
await dbContext.SaveChangesAsync();
_logger.Debug("Request Processed: {@SecurityEvent}", securityEvent);
}
await _next(context);
}
}

3
API/Services/BookmarkService.cs
View file

@ -76,7 +76,8 @@ public class BookmarkService : IBookmarkService
/// <summary>
/// This is a job that runs after a bookmark is saved
/// </summary>
private async Task ConvertBookmarkToWebP(int bookmarkId)
/// <remarks>This must be public</remarks>
public async Task ConvertBookmarkToWebP(int bookmarkId)
{
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;

47
API/Services/CacheService.cs
View file

@ -32,6 +32,7 @@ public interface ICacheService
void CleanupChapters(IEnumerable<int> chapterIds);
void CleanupBookmarks(IEnumerable<int> seriesIds);
string GetCachedPagePath(int chapterId, int page);
IEnumerable<string> GetCachedPages(int chapterId);
IEnumerable<FileDimensionDto> GetCachedFileDimensions(int chapterId);
string GetCachedBookmarkPagePath(int seriesId, int page);
string GetCachedFile(Chapter chapter);
@ -58,6 +59,13 @@ public class CacheService : ICacheService
_bookmarkService = bookmarkService;
}
public IEnumerable<string> GetCachedPages(int chapterId)
{
var path = GetCachePath(chapterId);
return _directoryService.GetFilesWithExtension(path, Tasks.Scanner.Parser.Parser.ImageFileExtensions)
.OrderByNatural(Path.GetFileNameWithoutExtension);
}
public IEnumerable<FileDimensionDto> GetCachedFileDimensions(int chapterId)
{
var sw = Stopwatch.StartNew();
@ -276,15 +284,7 @@ public class CacheService : ICacheService
.OrderByNatural(Path.GetFileNameWithoutExtension)
.ToArray();
if (files.Length == 0)
{
return string.Empty;
}
if (page > files.Length) page = files.Length;
// Since array is 0 based, we need to keep that in account (only affects last image)
return page == files.Length ? files.ElementAt(page - 1) : files.ElementAt(page);
return GetPageFromFiles(files, page);
}
public async Task<int> CacheBookmarkForSeries(int userId, int seriesId)
@ -310,4 +310,33 @@ public class CacheService : ICacheService
_directoryService.ClearAndDeleteDirectory(destDirectory);
}
/// <summary>
/// Returns either the file or an empty string
/// </summary>
/// <param name="files"></param>
/// <param name="pageNum"></param>
/// <returns></returns>
public static string GetPageFromFiles(string[] files, int pageNum)
{
files = files
.AsEnumerable()
.OrderByNatural(Path.GetFileNameWithoutExtension)
.ToArray();
if (files.Length == 0)
{
return string.Empty;
}
if (pageNum < 0)
{
pageNum = 0;
}
// Since array is 0 based, we need to keep that in account (only affects last image)
return pageNum >= files.Length ? files.ElementAt(Math.Min(pageNum - 1, files.Length - 1)) : files.ElementAt(pageNum);
}
}

41
API/Services/ImageService.cs
View file

@ -22,9 +22,25 @@ public interface IImageService
/// <param name="thumbnailWidth">Width of thumbnail</param>
/// <returns>File name with extension of the file. This will always write to <see cref="DirectoryService.CoverImageDirectory"/></returns>
string CreateThumbnailFromBase64(string encodedImage, string fileName, bool saveAsWebP = false, int thumbnailWidth = 320);
/// <summary>
/// Writes out a thumbnail by stream input
/// </summary>
/// <param name="stream"></param>
/// <param name="fileName"></param>
/// <param name="outputDirectory"></param>
/// <param name="saveAsWebP"></param>
/// <returns></returns>
string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, bool saveAsWebP = false);
/// <summary>
/// Writes out a thumbnail by file path input
/// </summary>
/// <param name="sourceFile"></param>
/// <param name="fileName"></param>
/// <param name="outputDirectory"></param>
/// <param name="saveAsWebP"></param>
/// <returns></returns>
string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, bool saveAsWebP = false);
/// <summary>
/// Converts the passed image to webP and outputs it in the same directory
/// </summary>
/// <param name="filePath">Full path to the image to convert</param>
@ -115,6 +131,19 @@ public class ImageService : IImageService
return filename;
}
public string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, bool saveAsWebP = false)
{
using var thumbnail = Image.Thumbnail(sourceFile, ThumbnailWidth);
var filename = fileName + (saveAsWebP ? ".webp" : ".png");
_directoryService.ExistOrCreate(outputDirectory);
try
{
_directoryService.FileSystem.File.Delete(_directoryService.FileSystem.Path.Join(outputDirectory, filename));
} catch (Exception) {/* Swallow exception */}
thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(outputDirectory, filename));
return filename;
}
public Task<string> ConvertToWebP(string filePath, string outputPath)
{
var file = _directoryService.FileSystem.FileInfo.New(filePath);
@ -218,6 +247,16 @@ public class ImageService : IImageService
return $"readinglist{readingListId}";
}
/// <summary>
/// Returns the name format for a thumbnail (temp thumbnail)
/// </summary>
/// <param name="chapterId"></param>
/// <returns></returns>
public static string GetThumbnailFormat(int chapterId)
{
return $"thumbnail{chapterId}";
}
public static string CreateMergedImage(List<string> coverImages, string dest)
{

51
API/Services/ReaderService.cs
View file

@ -1,4 +1,5 @@
using System;
using System.Collections;
using System.Collections.Generic;
using System.IO;
using System.Linq;
@ -33,6 +34,7 @@ public interface IReaderService
Task MarkVolumesUntilAsRead(AppUser user, int seriesId, int volumeNumber);
HourEstimateRangeDto GetTimeEstimate(long wordCount, int pageCount, bool isEpub);
IDictionary<int, int> GetPairs(IEnumerable<FileDimensionDto> dimensions);
Task<string> GetThumbnail(Chapter chapter, int pageNum, IEnumerable<string> cachedImages);
}
public class ReaderService : IReaderService
@ -40,6 +42,8 @@ public class ReaderService : IReaderService
private readonly IUnitOfWork _unitOfWork;
private readonly ILogger<ReaderService> _logger;
private readonly IEventHub _eventHub;
private readonly IImageService _imageService;
private readonly IDirectoryService _directoryService;
private readonly ChapterSortComparer _chapterSortComparer = ChapterSortComparer.Default;
private readonly ChapterSortComparerZeroFirst _chapterSortComparerForInChapterSorting = ChapterSortComparerZeroFirst.Default;
@ -48,14 +52,17 @@ public class ReaderService : IReaderService
public const float AvgWordsPerHour = (MaxWordsPerHour + MinWordsPerHour) / 2F;
private const float MinPagesPerMinute = 3.33F;
private const float MaxPagesPerMinute = 2.75F;
public const float AvgPagesPerMinute = (MaxPagesPerMinute + MinPagesPerMinute) / 2F;
public const float AvgPagesPerMinute = (MaxPagesPerMinute + MinPagesPerMinute) / 2F; //3.04
public ReaderService(IUnitOfWork unitOfWork, ILogger<ReaderService> logger, IEventHub eventHub)
public ReaderService(IUnitOfWork unitOfWork, ILogger<ReaderService> logger, IEventHub eventHub, IImageService imageService,
IDirectoryService directoryService)
{
_unitOfWork = unitOfWork;
_logger = logger;
_eventHub = eventHub;
_imageService = imageService;
_directoryService = directoryService;
}
public static string FormatBookmarkFolderPath(string baseDirectory, int userId, int seriesId, int chapterId)
@ -644,6 +651,44 @@ public class ReaderService : IReaderService
return pairs;
}
/// <summary>
///
/// </summary>
/// <param name="chapter"></param>
/// <param name="pageNum"></param>
/// <param name="cachedImages"></param>
/// <returns>Full path of thumbnail</returns>
public async Task<string> GetThumbnail(Chapter chapter, int pageNum, IEnumerable<string> cachedImages)
{
var outputDirectory =
_directoryService.FileSystem.Path.Join(_directoryService.TempDirectory, ImageService.GetThumbnailFormat(chapter.Id));
try
{
var saveAsWebp =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertBookmarkToWebP;
if (!Directory.Exists(outputDirectory))
{
var outputtedThumbnails = cachedImages
.Select((img, idx) =>
_directoryService.FileSystem.Path.Join(outputDirectory,
_imageService.WriteCoverThumbnail(img, $"{idx}", outputDirectory, saveAsWebp)))
.ToArray();
return CacheService.GetPageFromFiles(outputtedThumbnails, pageNum);
}
var files = _directoryService.GetFilesWithExtension(outputDirectory,
Tasks.Scanner.Parser.Parser.ImageFileExtensions);
return CacheService.GetPageFromFiles(files, pageNum);
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an error when trying to get thumbnail for Chapter {ChapterId}, Page {PageNum}", chapter.Id, pageNum);
_directoryService.ClearAndDeleteDirectory(outputDirectory);
throw;
}
}
/// <summary>
/// Formats a Chapter name based on the library it's in
/// </summary>
@ -668,4 +713,6 @@ public class ReaderService : IReaderService
throw new ArgumentOutOfRangeException(nameof(libraryType), libraryType, null);
}
}
}

2
API/Services/ReadingListService.cs
View file

@ -509,7 +509,7 @@ public class ReadingListService : IReadingListService
var allReadingLists = (user.ReadingLists).ToDictionary(s => s.NormalizedTitle);
if (!allReadingLists.TryGetValue(readingListNameNormalized, out var readingList))
{
readingList = DbFactory.ReadingList(cblReading.Name, string.Empty, false);
readingList = DbFactory.ReadingList(cblReading.Name, cblReading.Summary, false);
user.ReadingLists.Add(readingList);
}
else

3
API/Services/StatisticService.cs
View file

@ -103,7 +103,6 @@ public class StatisticService : IStatisticService
})
.ToListAsync();
var averageReadingTimePerWeek = _context.AppUserProgresses
.Where(p => p.AppUserId == userId)
.Join(_context.Chapter, p => p.ChapterId, c => c.Id,
@ -112,7 +111,7 @@ public class StatisticService : IStatisticService
AverageReadingHours = Math.Min((float) p.PagesRead / (float) c.Pages, 1.0) * ((float) c.AvgHoursToRead)
})
.Select(x => x.AverageReadingHours)
.Average() / 7.0;
.Average() * 7.0;
return new UserReadStatistics()
{

12
API/Services/Tasks/CleanupService.cs
View file

@ -58,6 +58,17 @@ public class CleanupService : ICleanupService
[AutomaticRetry(Attempts = 3, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Fail)]
public async Task Cleanup()
{
if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllCoverToWebP", Array.Empty<object>(),
TaskScheduler.DefaultQueue, true) ||
TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllBookmarkToWebP", Array.Empty<object>(),
TaskScheduler.DefaultQueue, true))
{
_logger.LogInformation("Cleanup put on hold as a conversion to WebP in progress");
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ErrorEvent("Cleanup", "Cleanup put on hold as a conversion to WebP in progress"));
return;
}
_logger.LogInformation("Starting Cleanup");
await SendProgress(0F, "Starting cleanup");
_logger.LogInformation("Cleaning temp directory");
@ -90,6 +101,7 @@ public class CleanupService : ICleanupService
await _unitOfWork.PersonRepository.RemoveAllPeopleNoLongerAssociated();
await _unitOfWork.GenreRepository.RemoveAllGenreNoLongerAssociated();
await _unitOfWork.CollectionTagRepository.RemoveTagsWithoutSeries();
await _unitOfWork.ReadingListRepository.RemoveReadingListsWithoutSeries();
}
private async Task SendProgress(float progress, string subtitle)

5
API/Services/Tasks/Scanner/Parser/Parser.cs
View file

@ -225,6 +225,11 @@ public static class Parser
new Regex(
@"(?<Series>.+?):? (\b|_|-)(vol)\.?(\s|-|_)?\d+",
MatchOptions, RegexTimeout),
// [xPearse] Kyochuu Rettou Chapter 001 Volume 1 [English] [Manga] [Volume Scans]
new Regex(
@"(?<Series>.+?):?(\s|\b|_|-)Chapter(\s|\b|_|-)\d+(\s|\b|_|-)(vol)(ume)",
MatchOptions,
RegexTimeout),
// [xPearse] Kyochuu Rettou Volume 1 [English] [Manga] [Volume Scans]
new Regex(
@"(?<Series>.+?):? (\b|_|-)(vol)(ume)",

9
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -51,6 +51,9 @@ public class ProcessSeries : IProcessSeries
private IList<Person> _people;
private Dictionary<string, Tag> _tags;
private Dictionary<string, CollectionTag> _collectionTags;
private readonly object _peopleLock;
private readonly object _genreLock;
private readonly object _tagLock;
public ProcessSeries(IUnitOfWork unitOfWork, ILogger<ProcessSeries> logger, IEventHub eventHub,
IDirectoryService directoryService, ICacheHelper cacheHelper, IReadingItemService readingItemService,
@ -838,7 +841,7 @@ public class ProcessSeries : IProcessSeries
if (person == null)
{
person = DbFactory.Person(name, role);
lock (_people)
lock (_peopleLock)
{
_people.Add(person);
}
@ -865,7 +868,7 @@ public class ProcessSeries : IProcessSeries
if (newTag)
{
genre = DbFactory.Genre(name);
lock (_genres)
lock (_genreLock)
{
_genres.Add(normalizedName, genre);
_unitOfWork.GenreRepository.Attach(genre);
@ -894,7 +897,7 @@ public class ProcessSeries : IProcessSeries
if (tag == null)
{
tag = DbFactory.Tag(name);
lock (_tags)
lock (_tagLock)
{
_tags.Add(normalizedName, tag);
}

64
API/Startup.cs
View file

@ -6,6 +6,7 @@ using System.Linq;
using System.Net;
using System.Net.Sockets;
using System.Reflection;
using System.Threading.RateLimiting;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
@ -14,6 +15,7 @@ using API.Entities.Enums;
using API.Extensions;
using API.Logging;
using API.Middleware;
using API.Middleware.RateLimit;
using API.Services;
using API.Services.HostedServices;
using API.Services.Tasks;
@ -179,6 +181,19 @@ public class Startup
services.AddResponseCaching();
services.AddRateLimiter(options =>
{
options.AddPolicy("Authentication", httpContext =>
new AuthenticationRateLimiterPolicy().GetPartition(httpContext));
// RateLimitPartition.GetFixedWindowLimiter(httpContext.Connection.RemoteIpAddress?.ToString(),
// partition => new FixedWindowRateLimiterOptions
// {
// AutoReplenishment = true,
// PermitLimit = 1,
// Window = TimeSpan.FromMinutes(1),
// }));
});
services.AddHangfire(configuration => configuration
.UseSimpleAssemblyNameTypeSerializer()
.UseRecommendedSerializerSettings()
@ -259,6 +274,7 @@ public class Startup
app.UseMiddleware<ExceptionMiddleware>();
app.UseMiddleware<SecurityEventMiddleware>();
if (env.IsDevelopment())
{
@ -278,10 +294,16 @@ public class Startup
app.UseForwardedHeaders();
var basePath = Configuration.BaseUrl;
app.UseRateLimiter();
var basePath = Configuration.BaseUrl;
app.UsePathBase(basePath);
UpdateBaseUrlInIndex(basePath);
if (!env.IsDevelopment())
{
// We don't update the index.html in local as we don't serve from there
UpdateBaseUrlInIndex(basePath);
}
app.UseRouting();
@ -292,7 +314,17 @@ public class Startup
.AllowAnyHeader()
.AllowAnyMethod()
.AllowCredentials() // For SignalR token query param
.WithOrigins("http://localhost:4200", $"http://{GetLocalIpAddress()}:4200", $"http://{GetLocalIpAddress()}:5000")
.WithOrigins("http://localhost:4200", $"http://{GetLocalIpAddress()}:4200", $"http://{GetLocalIpAddress()}:5000", "https://kavita.majora2007.duckdns.org")
.WithExposedHeaders("Content-Disposition", "Pagination"));
}
else
{
// Allow CORS for Kavita's url
app.UseCors(policy => policy
.AllowAnyHeader()
.AllowAnyMethod()
.AllowCredentials() // For SignalR token query param
.WithOrigins("https://kavita.majora2007.duckdns.org")
.WithExposedHeaders("Content-Disposition", "Pagination"));
}
@ -311,6 +343,7 @@ public class Startup
OnPrepareResponse = ctx =>
{
ctx.Context.Response.Headers[HeaderNames.CacheControl] = "public,max-age=" + TimeSpan.FromHours(24);
ctx.Context.Response.Headers["X-Robots-Tag"] = "noindex,nofollow";
}
});
@ -326,7 +359,7 @@ public class Startup
new[] { "Accept-Encoding" };
// Don't let the site be iframed outside the same origin (clickjacking)
context.Response.Headers.XFrameOptions = "SAMEORIGIN";
context.Response.Headers.XFrameOptions = Configuration.XFrameOptions;
// Setup CSP to ensure we load assets only from these origins
context.Response.Headers.Add("Content-Security-Policy", "frame-ancestors 'none';");
@ -359,19 +392,26 @@ public class Startup
});
var _logger = serviceProvider.GetRequiredService<ILogger<Startup>>();
_logger.LogInformation("Starting with base url as {baseUrl}", basePath);
_logger.LogInformation("Starting with base url as {BaseUrl}", basePath);
}
private static void UpdateBaseUrlInIndex(string baseUrl)
{
if (new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker) return;
var htmlDoc = new HtmlDocument();
var indexHtmlPath = Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "index.html");
htmlDoc.Load(indexHtmlPath);
try
{
if (new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker) return;
var htmlDoc = new HtmlDocument();
var indexHtmlPath = Path.Combine(Directory.GetCurrentDirectory(), "wwwroot", "index.html");
htmlDoc.Load(indexHtmlPath);
var baseNode = htmlDoc.DocumentNode.SelectSingleNode("/html/head/base");
baseNode.SetAttributeValue("href", baseUrl);
htmlDoc.Save(indexHtmlPath);
var baseNode = htmlDoc.DocumentNode.SelectSingleNode("/html/head/base");
baseNode.SetAttributeValue("href", baseUrl);
htmlDoc.Save(indexHtmlPath);
}
catch (Exception ex)
{
Log.Error(ex, "There was an error setting base url");
}
}
private static void OnShutdown()