AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Polish 2 (#3555)

Browse source 
Co-authored-by: Robbie Davis <robbie@therobbiedavis.com>
Co-authored-by: Fesaa <77553571+Fesaa@users.noreply.github.com>
This commit is contained in:
Joe Milazzo 2025-02-19 15:06:54 -06:00 • committed by GitHub
parent b858729c9e
commit 9565fe7360
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
57 changed files with 777 additions and 314 deletions
Download patch file Download diff file Expand all files Collapse all files

3
API/API.csproj
View file

@ -197,6 +197,9 @@
<Content Include="EmailTemplates\**">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Content Include="Assets\**">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Folder Include="Data\ManualMigrations\v0.8.3\" />
<Folder Include="Extensions\KavitaPlus\" />
<None Include="I18N\**" />

BIN
API/Assets/anilist-no-image-placeholder.jpg Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 22 KiB

12
API/Controllers/OPDSController.cs
View file

@ -1370,7 +1370,9 @@ public class OpdsController : BaseApiController
using var sm = new StringWriter();
_xmlSerializer.Serialize(sm, feed);
return sm.ToString().Replace("utf-16", "utf-8"); // Chunky cannot accept UTF-16 feeds
var ret = sm.ToString().Replace("utf-16", "utf-8"); // Chunky cannot accept UTF-16 feeds
return ret;
}
// Recursively sanitize all string properties in the object
@ -1381,6 +1383,10 @@ public class OpdsController : BaseApiController
var properties = obj.GetType().GetProperties();
foreach (var property in properties)
{
// Skip properties that require an index (e.g., indexed collections)
if (property.GetIndexParameters().Length > 0)
continue;
if (property.PropertyType == typeof(string) && property.CanWrite)
{
var value = (string?)property.GetValue(obj);
@ -1391,7 +1397,9 @@ public class OpdsController : BaseApiController
}
else if (property.PropertyType.IsClass) // Handle nested objects
{
SanitizeFeed(property.GetValue(obj));
var nestedObject = property.GetValue(obj);
if (nestedObject != null)
SanitizeFeed(nestedObject);
}
}
}

3
API/Controllers/SettingsController.cs
View file

@ -567,14 +567,15 @@ public class SettingsController : BaseApiController
existingMetadataSetting.EnableStartDate = dto.EnableStartDate;
existingMetadataSetting.EnableGenres = dto.EnableGenres;
existingMetadataSetting.EnableTags = dto.EnableTags;
existingMetadataSetting.PersonRoles = dto.PersonRoles;
existingMetadataSetting.FirstLastPeopleNaming = dto.FirstLastPeopleNaming;
existingMetadataSetting.EnableCoverImage = dto.EnableCoverImage;
existingMetadataSetting.AgeRatingMappings = dto.AgeRatingMappings ?? [];
existingMetadataSetting.Blacklist = dto.Blacklist.Where(s => !string.IsNullOrWhiteSpace(s)).DistinctBy(d => d.ToNormalized()).ToList() ?? [];
existingMetadataSetting.Whitelist = dto.Whitelist.Where(s => !string.IsNullOrWhiteSpace(s)).DistinctBy(d => d.ToNormalized()).ToList() ?? [];
existingMetadataSetting.Overrides = dto.Overrides.ToList() ?? [];
existingMetadataSetting.PersonRoles = dto.PersonRoles ?? [];
// Handle Field Mappings
if (dto.FieldMappings != null)

11
API/Controllers/UploadController.cs
View file

@ -110,13 +110,10 @@ public class UploadController : BaseApiController
lockState = uploadFileDto.LockCover;
}
if (!string.IsNullOrEmpty(filePath))
{
series.CoverImage = filePath;
series.CoverImageLocked = lockState;
_imageService.UpdateColorScape(series);
_unitOfWork.SeriesRepository.Update(series);
}
series.CoverImage = filePath;
series.CoverImageLocked = lockState;
_imageService.UpdateColorScape(series);
_unitOfWork.SeriesRepository.Update(series);
if (_unitOfWork.HasChanges())
{

1
API/DTOs/Update/UpdateNotificationDto.cs
View file

@ -62,6 +62,7 @@ public class UpdateNotificationDto
public IList<string> Theme { get; set; }
public IList<string> Developer { get; set; }
public IList<string> Api { get; set; }
public IList<string> FeatureRequests { get; set; }
/// <summary>
/// The part above the changelog part
/// </summary>

6
API/Data/Repositories/AppUserProgressRepository.cs
View file

@ -19,6 +19,7 @@ namespace API.Data.Repositories;
public interface IAppUserProgressRepository
{
void Update(AppUserProgress userProgress);
void Remove(AppUserProgress userProgress);
Task<int> CleanupAbandonedChapters();
Task<bool> UserHasProgress(LibraryType libraryType, int userId);
Task<AppUserProgress?> GetUserProgressAsync(int chapterId, int userId);
@ -57,6 +58,11 @@ public class AppUserProgressRepository : IAppUserProgressRepository
_context.Entry(userProgress).State = EntityState.Modified;
}
public void Remove(AppUserProgress userProgress)
{
_context.Remove(userProgress);
}
/// <summary>
/// This will remove any entries that have chapterIds that no longer exists. This will execute the save as well.
/// </summary>

2
API/Data/Repositories/SeriesRepository.cs
View file

@ -1848,7 +1848,7 @@ public class SeriesRepository : ISeriesRepository
.ToList();
// Prefer the first match or handle duplicates by choosing the last one
if (matchingSeries.Any())
if (matchingSeries.Count != 0)
{
ids.Add(matchingSeries.Last().Id);
}

122
API/Extensions/ImageExtensions.cs Normal file
View file

@ -0,0 +1,122 @@
using System;
using System.IO;
using NetVips;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using Image = NetVips.Image;
namespace API.Extensions;
public static class ImageExtensions
{
public static int GetResolution(this Image image)
{
return image.Width * image.Height;
}
/// <summary>
/// Smaller is better
/// </summary>
/// <param name="img1"></param>
/// <param name="img2"></param>
/// <returns></returns>
public static float GetMeanSquaredError(this Image<Rgba32> img1, Image<Rgba32> img2)
{
if (img1.Width != img2.Width || img1.Height != img2.Height)
{
img2.Mutate(x => x.Resize(img1.Width, img1.Height));
}
double totalDiff = 0;
for (var y = 0; y < img1.Height; y++)
{
for (var x = 0; x < img1.Width; x++)
{
var pixel1 = img1[x, y];
var pixel2 = img2[x, y];
var diff = Math.Pow(pixel1.R - pixel2.R, 2) +
Math.Pow(pixel1.G - pixel2.G, 2) +
Math.Pow(pixel1.B - pixel2.B, 2);
totalDiff += diff;
}
}
return (float)(totalDiff / (img1.Width * img1.Height));
}
public static float GetSimilarity(this string imagePath1, string imagePath2)
{
if (!File.Exists(imagePath1) || !File.Exists(imagePath2))
{
throw new FileNotFoundException("One or both image files do not exist");
}
// Calculate similarity score
return CalculateSimilarity(imagePath1, imagePath2);
}
/// <summary>
/// Determines which image is "better" based on similarity and resolution.
/// </summary>
/// <param name="imagePath1">Path to first image</param>
/// <param name="imagePath2">Path to second image</param>
/// <param name="similarityThreshold">Minimum similarity to consider images similar</param>
/// <returns>The path of the better image</returns>
public static string GetBetterImage(this string imagePath1, string imagePath2, float similarityThreshold = 0.7f)
{
if (!File.Exists(imagePath1) || !File.Exists(imagePath2))
{
throw new FileNotFoundException("One or both image files do not exist");
}
// Calculate similarity score
var similarity = CalculateSimilarity(imagePath1, imagePath2);
using var img1 = Image.NewFromFile(imagePath1, access: Enums.Access.Sequential);
using var img2 = Image.NewFromFile(imagePath2, access: Enums.Access.Sequential);
var resolution1 = img1.Width * img1.Height;
var resolution2 = img2.Width * img2.Height;
// If images are similar, choose the one with higher resolution
if (similarity >= similarityThreshold)
{
return resolution1 >= resolution2 ? imagePath1 : imagePath2;
}
// If images are not similar, allow the new image
return imagePath2;
}
/// <summary>
/// Calculate a similarity score (0-1f) based on resolution difference and MSE.
/// </summary>
/// <param name="imagePath1"></param>
/// <param name="imagePath2"></param>
/// <returns></returns>
private static float CalculateSimilarity(string imagePath1, string imagePath2)
{
if (!File.Exists(imagePath1) || !File.Exists(imagePath2))
{
return -1;
}
using var img1 = Image.NewFromFile(imagePath1, access: Enums.Access.Sequential);
using var img2 = Image.NewFromFile(imagePath2, access: Enums.Access.Sequential);
var res1 = img1.Width * img1.Height;
var res2 = img2.Width * img2.Height;
var resolutionDiff = Math.Abs(res1 - res2) / (float)Math.Max(res1, res2);
using var imgSharp1 = SixLabors.ImageSharp.Image.Load<Rgba32>(imagePath1);
using var imgSharp2 = SixLabors.ImageSharp.Image.Load<Rgba32>(imagePath2);
var mse = imgSharp1.GetMeanSquaredError(imgSharp2);
var normalizedMse = 1f - Math.Min(1f, mse / 65025f); // Normalize based on max color diff
// Final similarity score (weighted)
return Math.Max(0f, 1f - (resolutionDiff * 0.5f) - (1f - normalizedMse) * 0.5f);
}
}

49
API/Helpers/PdfComicInfoExtractor.cs
View file

@ -1,16 +1,11 @@
/// Translate PDF metadata (See PdfMetadataExtractor.cs) into ComicInfo structure.
// Contributed by https://github.com/microtherion
// All references to the "PDF Spec" (section numbers, etc) refer to the
// PDF 1.7 Specification a.k.a. PDF32000-1:2008
// https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/PDF32000_2008.pdf
/**
* Contributed by https://github.com/microtherion
*
* All references to the "PDF Spec" (section numbers, etc) refer to the
* PDF 1.7 Specification a.k.a. PDF32000-1:2008
* https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/PDF32000_2008.pdf
*/
using System;
using System.Xml;
using System.Text;
using System.IO;
using System.Diagnostics;
using API.Data.Metadata;
using API.Entities.Enums;
using API.Services;
@ -18,6 +13,7 @@ using API.Services.Tasks.Scanner.Parser;
using Microsoft.Extensions.Logging;
using Nager.ArticleNumber;
using System.Collections.Generic;
using System.Globalization;
namespace API.Helpers;
#nullable enable
@ -27,6 +23,9 @@ public interface IPdfComicInfoExtractor
ComicInfo? GetComicInfo(string filePath);
}
/// <summary>
/// Translate PDF metadata (See PdfMetadataExtractor.cs) into ComicInfo structure.
/// </summary>
public class PdfComicInfoExtractor : IPdfComicInfoExtractor
{
private readonly ILogger<BookService> _logger;
@ -44,7 +43,7 @@ public class PdfComicInfoExtractor : IPdfComicInfoExtractor
_mediaErrorService = mediaErrorService;
}
private float? GetFloatFromText(string? text)
private static float? GetFloatFromText(string? text)
{
if (string.IsNullOrEmpty(text)) return null;
@ -78,9 +77,9 @@ public class PdfComicInfoExtractor : IPdfComicInfoExtractor
return null;
}
private string? MaybeGetMetadata(Dictionary<string, string> metadata, string key)
private static string? MaybeGetMetadata(Dictionary<string, string> metadata, string key)
{
return metadata.ContainsKey(key) ? metadata[key] : null;
return metadata.TryGetValue(key, out var value) ? value : null;
}
private ComicInfo? GetComicInfoFromMetadata(Dictionary<string, string> metadata, string filePath)
@ -100,6 +99,7 @@ public class PdfComicInfoExtractor : IPdfComicInfoExtractor
info.Publisher = MaybeGetMetadata(metadata, "Publisher") ?? string.Empty;
info.Writer = MaybeGetMetadata(metadata, "Author") ?? string.Empty;
info.Title = MaybeGetMetadata(metadata, "Title") ?? string.Empty;
info.TitleSort = MaybeGetMetadata(metadata, "TitleSort") ?? string.Empty;
info.Genre = MaybeGetMetadata(metadata, "Subject") ?? string.Empty;
info.LanguageISO = BookService.ValidateLanguage(MaybeGetMetadata(metadata, "Language"));
info.Isbn = MaybeGetMetadata(metadata, "ISBN") ?? string.Empty;
@ -111,10 +111,9 @@ public class PdfComicInfoExtractor : IPdfComicInfoExtractor
}
info.UserRating = GetFloatFromText(MaybeGetMetadata(metadata, "UserRating")) ?? 0.0f;
info.TitleSort = MaybeGetMetadata(metadata, "TitleSort") ?? string.Empty;
info.Series = MaybeGetMetadata(metadata, "Series") ?? info.TitleSort;
info.Series = MaybeGetMetadata(metadata, "Series") ?? info.Title;
info.SeriesSort = info.Series;
info.Volume = (GetFloatFromText(MaybeGetMetadata(metadata, "Volume")) ?? 0.0f).ToString();
info.Volume = MaybeGetMetadata(metadata, "Volume") ?? string.Empty;
// If this is a single book and not a collection, set publication status to Completed
if (string.IsNullOrEmpty(info.Volume) && Parser.ParseVolume(filePath, LibraryType.Manga).Equals(Parser.LooseLeafVolume))
@ -122,18 +121,6 @@ public class PdfComicInfoExtractor : IPdfComicInfoExtractor
info.Count = 1;
}
// Removed as probably unneeded per discussion in https://github.com/Kareadita/Kavita/pull/3108#discussion_r1956747782
//
// var hasVolumeInSeries = !Parser.ParseVolume(info.Title, LibraryType.Manga)
// .Equals(Parser.LooseLeafVolume);
// if (string.IsNullOrEmpty(info.Volume) && hasVolumeInSeries && (!info.Series.Equals(info.Title) || string.IsNullOrEmpty(info.Series)))
// {
// // This is likely a light novel for which we can set series from parsed title
// info.Series = Parser.ParseSeries(info.Title, LibraryType.Manga);
// info.Volume = Parser.ParseVolume(info.Title, LibraryType.Manga);
// }
ComicInfo.CleanComicInfo(info);
return info;
@ -156,4 +143,4 @@ public class PdfComicInfoExtractor : IPdfComicInfoExtractor
return null;
}
}
}

363
API/Helpers/PdfMetadataExtractor.cs
View file

@ -1,21 +1,14 @@
/// Parse PDF file and try to extract as much metadata as possible.
/// Supports both text based XRef tables and compressed XRef streams (Deflate only).
/// Supports both UTF-16 and PDFDocEncoding for strings.
/// Lacks support for many PDF configurations that are theoretically possible, but should handle most common cases.
// Contributed by https://github.com/microtherion
// All references to the "PDF Spec" (section numbers, etc) refer to the
// PDF 1.7 Specification a.k.a. PDF32000-1:2008
// https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/PDF32000_2008.pdf
/**
* Contributed by https://github.com/microtherion
*
* All references to the "PDF Spec" (section numbers, etc) refer to the
* PDF 1.7 Specification a.k.a. PDF32000-1:2008
* https://opensource.adobe.com/dc-acrobat-sdk-docs/pdfstandards/PDF32000_2008.pdf
*/
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO.Compression;
using System.Reflection.Metadata.Ecma335;
using System.Runtime.InteropServices;
using System.Security.Principal;
using System.Text;
using System.Xml;
using System.IO;
@ -25,6 +18,12 @@ using API.Services;
namespace API.Helpers;
#nullable enable
/// <summary>
/// Parse PDF file and try to extract as much metadata as possible.
/// Supports both text based XRef tables and compressed XRef streams (Deflate only).
/// Supports both UTF-16 and PDFDocEncoding for strings.
/// Lacks support for many PDF configurations that are theoretically possible, but should handle most common cases.
/// </summary>
public class PdfMetadataExtractorException : Exception
{
public PdfMetadataExtractorException()
@ -56,19 +55,21 @@ class PdfStringBuilder
// PDFDocEncoding defined in PDF Spec D.1
private readonly char[] _pdfDocMappingLow = new char[] {
'\u02D8', '\u02C7', '\u02C6', '\u02D9', '\u02DD', '\u02DB', '\u02DA', '\u02DC',
};
private readonly char[] _pdfDocMappingLow =
[
'\u02D8', '\u02C7', '\u02C6', '\u02D9', '\u02DD', '\u02DB', '\u02DA', '\u02DC'
];
private readonly char[] _pdfDocMappingHigh = new char[] {
private readonly char[] _pdfDocMappingHigh =
[
'\u2022', '\u2020', '\u2021', '\u2026', '\u2014', '\u2013', '\u0192', '\u2044',
'\u2039', '\u203A', '\u2212', '\u2030', '\u201E', '\u201C', '\u201D', '\u2018',
'\u2019', '\u201A', '\u2122', '\uFB01', '\uFB02', '\u0141', '\u0152', '\u0160',
'\u0178', '\u017D', '\u0131', '\u0142', '\u0153', '\u0161', '\u017E', ' ',
'\u20AC',
};
'\u20AC'
];
public void AppendPdfDocByte(byte b)
private void AppendPdfDocByte(byte b)
{
if (b >= 0x18 && b < 0x20)
{
@ -148,8 +149,13 @@ class PdfStringBuilder
}
}
class PdfLexer(Stream stream)
internal class PdfLexer(Stream stream)
{
private const int BufferSize = 1024;
private readonly byte[] _buffer = new byte[BufferSize];
private int _pos = 0;
private int _valid = 0;
public enum TokenType
{
None,
@ -171,16 +177,10 @@ class PdfLexer(Stream stream)
Newline,
}
public struct Token
public struct Token(TokenType type, object value)
{
public TokenType type;
public object value;
public Token(TokenType type, object value)
{
this.type = type;
this.value = value;
}
public TokenType Type = type;
public object Value = value;
}
public Token NextToken(bool reportNewlines = false)
@ -273,7 +273,7 @@ class PdfLexer(Stream stream)
{
while (true)
{
byte b = NextByte();
var b = NextByte();
switch ((char)b)
{
case ' ':
@ -303,7 +303,7 @@ class PdfLexer(Stream stream)
// Look for the startxref element as per PDF Spec 7.5.5
while (true)
{
byte b = NextByte();
var b = NextByte();
switch ((char)b)
{
@ -345,13 +345,13 @@ class PdfLexer(Stream stream)
var token = NextToken(true);
if (token.type == TokenType.Keyword && (string)token.value == "startxref")
if (token.Type == TokenType.Keyword && (string)token.Value == "startxref")
{
token = NextToken();
if (token.type == TokenType.Int)
if (token.Type == TokenType.Int)
{
return (long)token.value;
return (long)token.Value;
}
else
{
@ -382,8 +382,8 @@ class PdfLexer(Stream stream)
if (obj == 0)
{
obj = Convert.ToInt64(System.Text.Encoding.ASCII.GetString(_buffer, _pos, 10));
generation = Convert.ToInt32(System.Text.Encoding.ASCII.GetString(_buffer, _pos + 11, 5));
obj = Convert.ToInt64(Encoding.ASCII.GetString(_buffer, _pos, 10));
generation = Convert.ToInt32(Encoding.ASCII.GetString(_buffer, _pos + 11, 5));
inUse = _buffer[_pos + 17] == 'n';
}
@ -404,7 +404,7 @@ class PdfLexer(Stream stream)
if (_pos < _valid)
{
int buffered = Math.Min(_valid - _pos, length);
var buffered = Math.Min(_valid - _pos, length);
rawData.Write(_buffer, _pos, buffered);
length -= buffered;
_pos += buffered;
@ -412,8 +412,8 @@ class PdfLexer(Stream stream)
while (length > 0)
{
int buffered = Math.Min(length, _bufferSize);
stream.Read(_buffer, 0, buffered);
var buffered = Math.Min(length, BufferSize);
stream.ReadExactly(_buffer, 0, buffered);
rawData.Write(_buffer, 0, buffered);
_pos = 0;
_valid = 0;
@ -432,17 +432,12 @@ class PdfLexer(Stream stream)
}
}
private const int _bufferSize = 1024;
private readonly byte[] _buffer = new byte[_bufferSize];
private int _pos = 0;
private int _valid = 0;
private byte NextByte()
{
if (_pos >= _valid)
{
_pos = 0;
_valid = stream.Read(_buffer, 0, _bufferSize);
_valid = stream.Read(_buffer, 0, BufferSize);
if (_valid <= 0)
{
@ -478,7 +473,7 @@ class PdfLexer(Stream stream)
Buffer.BlockCopy(_buffer, _pos, _buffer, 0, _valid - _pos);
_valid -= _pos;
_pos = 0;
_valid += stream.Read(_buffer, _valid, _bufferSize - _valid);
_valid += stream.Read(_buffer, _valid, BufferSize - _valid);
}
}
@ -486,7 +481,7 @@ class PdfLexer(Stream stream)
{
while (true)
{
byte b = NextByte();
var b = NextByte();
if (b == '\n')
{
@ -507,14 +502,14 @@ class PdfLexer(Stream stream)
private Token ScanNumber()
{
StringBuilder sb = new();
bool hasDot = LastByte() == '.';
bool followedBySpace = false;
var hasDot = LastByte() == '.';
var followedBySpace = false;
sb.Append((char)LastByte());
while (true)
{
byte b = NextByte();
var b = NextByte();
if (b == '.' || b >= '0' && b <= '9')
{
@ -533,17 +528,19 @@ class PdfLexer(Stream stream)
break;
}
}
if (hasDot)
{
return new Token(TokenType.Double, double.Parse(sb.ToString()));
}
if (followedBySpace)
{
// Look ahead to see if it's an object reference (PDF Spec 7.3.10)
WantLookahead(32);
var savedPos = _pos;
byte b = NextByte();
var b = NextByte();
while (b == ' ' || b == '\t')
{
@ -578,32 +575,25 @@ class PdfLexer(Stream stream)
return new Token(TokenType.Int, long.Parse(sb.ToString()));
}
private int HexDigit(byte b)
private static int HexDigit(byte b)
{
switch ((char)b)
return (char) b switch
{
case >= '0' and <= '9':
return b - (byte)'0';
case >= 'a' and <= 'f':
return b - (byte)'a' + 10;
case >= 'A' and <= 'F':
return b - (byte)'A' + 10;
default:
throw new PdfMetadataExtractorException("Invalid hex digit, got {b}");
}
>= '0' and <= '9' => b - (byte) '0',
>= 'a' and <= 'f' => b - (byte) 'a' + 10,
>= 'A' and <= 'F' => b - (byte) 'A' + 10,
_ => throw new PdfMetadataExtractorException("Invalid hex digit, got {b}")
};
}
private Token ScanName()
{
// PDF Spec 7.3.5
StringBuilder sb = new StringBuilder();
var sb = new StringBuilder();
while (true)
{
byte b = NextByte();
var b = NextByte();
switch ((char)b)
{
case '(':
@ -628,8 +618,8 @@ class PdfLexer(Stream stream)
return new Token(TokenType.Name, sb.ToString());
case '#':
byte b1 = NextByte();
byte b2 = NextByte();
var b1 = NextByte();
var b2 = NextByte();
b = (byte)((HexDigit(b1) << 4) | HexDigit(b2));
goto default;
@ -646,11 +636,11 @@ class PdfLexer(Stream stream)
// PDF Spec 7.3.4.2
PdfStringBuilder sb = new();
int parenLevel = 1;
var parenLevel = 1;
while (true)
{
byte b = NextByte();
var b = NextByte();
switch ((char)b)
{
@ -698,9 +688,9 @@ class PdfLexer(Stream stream)
break;
case >= '0' and <= '7':
byte b1 = b;
byte b2 = NextByte();
byte b3 = NextByte();
var b1 = b;
var b2 = NextByte();
var b3 = NextByte();
if (b2 < '0' || b2 > '7' || b3 < '0' || b3 > '7')
{
@ -728,12 +718,12 @@ class PdfLexer(Stream stream)
while (true)
{
byte b = NextByte();
var b = NextByte();
switch ((char)b)
{
case (>= '0' and <= '9') or (>= 'a' and <= 'f') or (>= 'A' and <= 'F'):
byte b1 = NextByte();
var b1 = NextByte();
if (b1 == '>')
{
PutBack();
@ -760,7 +750,7 @@ class PdfLexer(Stream stream)
while (true)
{
byte b = NextByte();
var b = NextByte();
if ((b >= 'a' && b <= 'z') || (b >= 'A' && b <= 'Z'))
{
sb.Append((char)b);
@ -796,38 +786,25 @@ class PdfLexer(Stream stream)
}
}
class PdfMetadataExtractor : IPdfMetadataExtractor
internal class PdfMetadataExtractor : IPdfMetadataExtractor
{
private readonly ILogger<BookService> _logger;
private readonly PdfLexer _lexer;
private readonly FileStream _stream;
private long[] _objectOffsets = new long[0];
private readonly Dictionary<string, string> _metadata = new();
private readonly Dictionary<string, string> _metadata = [];
private readonly Stack<MetadataRef> _metadataRef = new();
private struct MetadataRef
private struct MetadataRef(long root, long info)
{
public long root;
public long info;
public MetadataRef(long root, long info)
{
this.root = root;
this.info = info;
}
public long Root = root;
public long Info = info;
}
private readonly Stack<MetadataRef> metadataRef = new();
private struct XRefSection
private struct XRefSection(long first, long count)
{
public long first;
public long count;
public XRefSection(long first, long count)
{
this.first = first;
this.count = count;
}
public readonly long First = first;
public readonly long Count = count;
}
public PdfMetadataExtractor(ILogger<BookService> logger, string filename)
@ -887,7 +864,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
var token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.Keyword || (string)token.value != "xref")
if (token.Type != PdfLexer.TokenType.Keyword || (string)token.Value != "xref")
{
throw new PdfMetadataExtractorException("Expected xref keyword");
}
@ -896,17 +873,17 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
token = _lexer.NextToken();
if (token.type == PdfLexer.TokenType.Int)
if (token.Type == PdfLexer.TokenType.Int)
{
var startObj = (long)token.value;
var startObj = (long)token.Value;
token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.Int)
if (token.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected number of objects in xref subsection");
}
var numObj = (long)token.value;
var numObj = (long)token.Value;
if (_objectOffsets.Length < startObj + numObj)
{
@ -927,7 +904,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
}
}
}
else if (token.type == PdfLexer.TokenType.Keyword && (string)token.value == "trailer")
else if (token.Type == PdfLexer.TokenType.Keyword && (string)token.Value == "trailer")
{
break;
}
@ -946,7 +923,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
var token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.ObjectStart)
if (token.Type != PdfLexer.TokenType.ObjectStart)
{
throw new PdfMetadataExtractorException("Expected obj keyword");
}
@ -967,7 +944,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
switch (key)
{
case "Type":
if (value.type != PdfLexer.TokenType.Name || (string)value.value != "XRef")
if (value.Type != PdfLexer.TokenType.Name || (string)value.Value != "XRef")
{
throw new PdfMetadataExtractorException("Expected /Type to be /XRef");
}
@ -975,37 +952,37 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
return true;
case "Length":
if (value.type != PdfLexer.TokenType.Int)
if (value.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected integer after /Length");
}
length = (long)value.value;
length = (long)value.Value;
return true;
case "Size":
if (value.type != PdfLexer.TokenType.Int)
if (value.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected integer after /Size");
}
size = (long)value.value;
size = (long)value.Value;
return true;
case "Prev":
if (value.type != PdfLexer.TokenType.Int)
if (value.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected offset after /Prev");
}
prev = (long)value.value;
prev = (long)value.Value;
return true;
case "Index":
if (value.type != PdfLexer.TokenType.ArrayStart)
if (value.Type != PdfLexer.TokenType.ArrayStart)
{
throw new PdfMetadataExtractorException("Expected array after /Index");
}
@ -1014,31 +991,31 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
token = _lexer.NextToken();
if (token.type == PdfLexer.TokenType.ArrayEnd)
if (token.Type == PdfLexer.TokenType.ArrayEnd)
{
break;
}
else if (token.type != PdfLexer.TokenType.Int)
else if (token.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected integer in /Index array");
}
var first = (long)token.value;
var first = (long)token.Value;
token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.Int)
if (token.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected integer pair in /Index array");
}
var count = (long)token.value;
var count = (long)token.Value;
sections.Enqueue(new XRefSection(first, count));
}
return true;
case "W":
if (value.type != PdfLexer.TokenType.ArrayStart)
if (value.Type != PdfLexer.TokenType.ArrayStart)
{
throw new PdfMetadataExtractorException("Expected array after /W");
}
@ -1049,17 +1026,17 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.Int)
if (token.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected integer in /W array");
}
widths[i] = (long)token.value;
widths[i] = (long)token.Value;
}
token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.ArrayEnd)
if (token.Type != PdfLexer.TokenType.ArrayEnd)
{
throw new PdfMetadataExtractorException("Unclosed array after /W");
}
@ -1071,12 +1048,12 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
return true;
case "Filter":
if (value.type != PdfLexer.TokenType.Name)
if (value.Type != PdfLexer.TokenType.Name)
{
throw new PdfMetadataExtractorException("Expected name after /Filter");
}
if ((string)value.value != "FlateDecode")
if ((string)value.Value != "FlateDecode")
{
throw new PdfMetadataExtractorException("Unsupported filter, only FlateDecode is supported");
}
@ -1086,22 +1063,22 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
return true;
case "Root":
if (value.type != PdfLexer.TokenType.ObjectRef)
if (value.Type != PdfLexer.TokenType.ObjectRef)
{
throw new PdfMetadataExtractorException("Expected object reference after /Root");
}
meta.root = (long)value.value;
meta.Root = (long)value.Value;
return true;
case "Info":
if (value.type != PdfLexer.TokenType.ObjectRef)
if (value.Type != PdfLexer.TokenType.ObjectRef)
{
throw new PdfMetadataExtractorException("Expected object reference after /Info");
}
meta.info = (long)value.value;
meta.Info = (long)value.Value;
return true;
@ -1112,7 +1089,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.StreamStart)
if (token.Type != PdfLexer.TokenType.StreamStart)
{
throw new PdfMetadataExtractorException("Expected xref stream after dictionary");
}
@ -1133,7 +1110,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
Array.Resize(ref _objectOffsets, (int)size);
}
for (var i = section.first; i < section.first + section.count; ++i)
for (var i = section.First; i < section.First + section.Count; ++i)
{
long type = 0;
long offset = 0;
@ -1146,17 +1123,17 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
for (var j = 0; j < typeWidth; ++j)
{
type = (type << 8) | (UInt16)stream.ReadByte();
type = (type << 8) | (ushort)stream.ReadByte();
}
for (var j = 0; j < offsetWidth; ++j)
{
offset = (offset << 8) | (UInt16)stream.ReadByte();
offset = (offset << 8) | (ushort)stream.ReadByte();
}
for (var j = 0; j < generationWidth; ++j)
{
generation = (generation << 8) | (UInt16)stream.ReadByte();
generation = (generation << 8) | (ushort)stream.ReadByte();
}
if (type == 1 && _objectOffsets[i] == 0)
@ -1176,22 +1153,22 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
private void PushMetadataRef(MetadataRef meta)
{
if (metadataRef.Count > 0)
if (_metadataRef.Count > 0)
{
if (meta.root == metadataRef.Peek().root)
if (meta.Root == _metadataRef.Peek().Root)
{
meta.root = -1;
meta.Root = -1;
}
if (meta.info == metadataRef.Peek().info)
if (meta.Info == _metadataRef.Peek().Info)
{
meta.info = -1;
meta.Info = -1;
}
}
if (meta.root != -1 || meta.info != -1)
if (meta.Root != -1 || meta.Info != -1)
{
metadataRef.Push(meta);
_metadataRef.Push(meta);
}
}
@ -1209,40 +1186,40 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
switch (key)
{
case "Root":
if (value.type != PdfLexer.TokenType.ObjectRef)
if (value.Type != PdfLexer.TokenType.ObjectRef)
{
throw new PdfMetadataExtractorException("Expected object reference after /Root");
}
meta.root = (long)value.value;
meta.Root = (long)value.Value;
return true;
case "Prev":
if (value.type != PdfLexer.TokenType.Int)
if (value.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected offset after /Prev");
}
prev = (long)value.value;
prev = (long)value.Value;
return true;
case "Info":
if (value.type != PdfLexer.TokenType.ObjectRef)
if (value.Type != PdfLexer.TokenType.ObjectRef)
{
throw new PdfMetadataExtractorException("Expected object reference after /Info");
}
meta.info = (long)value.value;
meta.Info = (long)value.Value;
return true;
case "XRefStm":
// Prefer encoded xref stream over xref table
if (value.type != PdfLexer.TokenType.Int)
if (value.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected offset after /XRefStm");
}
xrefStm = (long)value.value;
xrefStm = (long)value.Value;
return true;
@ -1272,14 +1249,14 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
// We read potential metadata sources in backwards historical order, so
// we can overwrite to our heart's content
while (metadataRef.Count > 0)
while (_metadataRef.Count > 0)
{
var meta = metadataRef.Pop();
var meta = _metadataRef.Pop();
_logger.LogTrace("DocumentCatalog for {Path}: {Root}, Info: {Info}", filename, meta.root, meta.info);
//_logger.LogTrace("DocumentCatalog for {Path}: {Root}, Info: {Info}", filename, meta.root, meta.info);
ReadMetadataFromInfo(meta.info);
ReadMetadataFromXML(MetadataObjInObjectCatalog(meta.root));
ReadMetadataFromInfo(meta.Info);
ReadMetadataFromXml(MetadataObjInObjectCatalog(meta.Root));
}
}
@ -1298,12 +1275,12 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
var token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.ObjectStart)
if (token.Type != PdfLexer.TokenType.ObjectStart)
{
throw new PdfMetadataExtractorException("Expected object header");
}
Dictionary<String, long> indirectObjects = new();
Dictionary<string, long> indirectObjects = [];
ParseDictionary(delegate(string key, PdfLexer.Token value)
{
@ -1317,16 +1294,16 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
case "Producer":
case "CreationDate":
case "ModDate":
if (value.type == PdfLexer.TokenType.ObjectRef) {
indirectObjects[key] = (long)value.value;
if (value.Type == PdfLexer.TokenType.ObjectRef) {
indirectObjects[key] = (long)value.Value;
}
else if (value.type != PdfLexer.TokenType.String)
else if (value.Type != PdfLexer.TokenType.String)
{
throw new PdfMetadataExtractorException("Expected string value");
}
else
{
_metadata[key] = (string)value.value;
_metadata[key] = (string)value.Value;
}
return true;
@ -1343,17 +1320,17 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.ObjectStart) {
if (token.Type != PdfLexer.TokenType.ObjectStart) {
throw new PdfMetadataExtractorException("Expected object here");
}
token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.String) {
if (token.Type != PdfLexer.TokenType.String) {
throw new PdfMetadataExtractorException("Expected string");
}
_metadata[key] = (string)token.value;
_metadata[key] = (string) token.Value;
}
}
@ -1371,7 +1348,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
var token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.ObjectStart)
if (token.Type != PdfLexer.TokenType.ObjectStart)
{
throw new PdfMetadataExtractorException("Expected object header");
}
@ -1382,12 +1359,12 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
switch (key) {
case "Metadata":
if (value.type != PdfLexer.TokenType.ObjectRef)
if (value.Type != PdfLexer.TokenType.ObjectRef)
{
throw new PdfMetadataExtractorException("Expected object number after /Metadata");
}
meta = (long)value.value;
meta = (long)value.Value;
return true;
@ -1403,13 +1380,13 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
// See XMP specification: https://developer.adobe.com/xmp/docs/XMPSpecifications/
// and Dublin Core: https://www.dublincore.org/specifications/dublin-core/
private string? GetTextFromXmlNode(XmlDocument doc, XmlNamespaceManager ns, string path)
private static string? GetTextFromXmlNode(XmlDocument doc, XmlNamespaceManager ns, string path)
{
return (doc.DocumentElement?.SelectSingleNode(path + "//rdf:li", ns)
?? doc.DocumentElement?.SelectSingleNode(path, ns))?.InnerText;
}
private string? GetListFromXmlNode(XmlDocument doc, XmlNamespaceManager ns, string path)
private static string? GetListFromXmlNode(XmlDocument doc, XmlNamespaceManager ns, string path)
{
var nodes = doc.DocumentElement?.SelectNodes(path + "//rdf:li", ns);
@ -1421,7 +1398,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
if (list.Length > 0)
{
list.Append(",");
list.Append(',');
}
list.Append(n.InnerText);
@ -1437,7 +1414,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
_metadata[key] = value;
}
private void ReadMetadataFromXML(long meta)
private void ReadMetadataFromXml(long meta)
{
if (meta < 1 || meta >= _objectOffsets.Length || _objectOffsets[meta] == 0) return;
@ -1446,7 +1423,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
var token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.ObjectStart)
if (token.Type != PdfLexer.TokenType.ObjectStart)
{
throw new PdfMetadataExtractorException("Expected object header");
}
@ -1460,7 +1437,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
switch (key) {
case "Type":
if (value.type != PdfLexer.TokenType.Name || (string)value.value != "Metadata")
if (value.Type != PdfLexer.TokenType.Name || (string)value.Value != "Metadata")
{
throw new PdfMetadataExtractorException("Expected /Type to be /Metadata");
}
@ -1468,7 +1445,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
return true;
case "Subtype":
if (value.type != PdfLexer.TokenType.Name || (string)value.value != "XML")
if (value.Type != PdfLexer.TokenType.Name || (string)value.Value != "XML")
{
throw new PdfMetadataExtractorException("Expected /Subtype to be /XML");
}
@ -1476,22 +1453,22 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
return true;
case "Length":
if (value.type != PdfLexer.TokenType.Int)
if (value.Type != PdfLexer.TokenType.Int)
{
throw new PdfMetadataExtractorException("Expected integer after /Length");
}
length = (long)value.value;
length = (long)value.Value;
return true;
case "Filter":
if (value.type != PdfLexer.TokenType.Name)
if (value.Type != PdfLexer.TokenType.Name)
{
throw new PdfMetadataExtractorException("Expected name after /Filter");
}
if ((string)value.value != "FlateDecode")
if ((string)value.Value != "FlateDecode")
{
throw new PdfMetadataExtractorException("Unsupported filter, only FlateDecode is supported");
}
@ -1507,7 +1484,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.StreamStart)
if (token.Type != PdfLexer.TokenType.StreamStart)
{
throw new PdfMetadataExtractorException("Expected xref stream after dictionary");
}
@ -1567,7 +1544,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
var token = _lexer.NextToken();
if (token.type != PdfLexer.TokenType.DictionaryStart)
if (token.Type != PdfLexer.TokenType.DictionaryStart)
{
throw new PdfMetadataExtractorException("Expected dictionary");
}
@ -1576,15 +1553,16 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
token = _lexer.NextToken();
if (token.type == PdfLexer.TokenType.DictionaryEnd)
if (token.Type == PdfLexer.TokenType.DictionaryEnd)
{
return;
}
else if (token.type == PdfLexer.TokenType.Name)
if (token.Type == PdfLexer.TokenType.Name)
{
var value = _lexer.NextToken();
if (!handler((string)token.value, value)) {
if (!handler((string)token.Value, value)) {
SkipValue(value);
}
}
@ -1599,7 +1577,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
var token = existingToken ?? _lexer.NextToken();
switch (token.type)
switch (token.Type)
{
case PdfLexer.TokenType.Bool:
case PdfLexer.TokenType.Int:
@ -1608,17 +1586,16 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
case PdfLexer.TokenType.String:
case PdfLexer.TokenType.ObjectRef:
break;
case PdfLexer.TokenType.ArrayStart:
{
SkipArray();
break;
}
case PdfLexer.TokenType.DictionaryStart:
{
SkipDictionary();
break;
}
default:
throw new PdfMetadataExtractorException("Unexpected token in SkipValue");
}
@ -1630,7 +1607,7 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
var token = _lexer.NextToken();
if (token.type == PdfLexer.TokenType.ArrayEnd)
if (token.Type == PdfLexer.TokenType.ArrayEnd)
{
break;
}
@ -1645,11 +1622,11 @@ class PdfMetadataExtractor : IPdfMetadataExtractor
{
var token = _lexer.NextToken();
if (token.type == PdfLexer.TokenType.DictionaryEnd)
if (token.Type == PdfLexer.TokenType.DictionaryEnd)
{
break;
}
else if (token.type != PdfLexer.TokenType.Name)
if (token.Type != PdfLexer.TokenType.Name)
{
throw new PdfMetadataExtractorException("Expected name in dictionary");
}

2
API/Program.cs
View file

@ -58,7 +58,7 @@ public class Program
}
Configuration.KavitaPlusApiUrl = Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") == Environments.Development
? "http://localhost:5020" : "https://plus-next.kavitareader.com";
? "http://localhost:5020" : "https://plus.kavitareader.com";
try
{

7
API/Services/DirectoryService.cs
View file

@ -39,6 +39,10 @@ public interface IDirectoryService
/// </summary>
string BookmarkDirectory { get; }
/// <summary>
/// Used for random files needed, like images to check against, list of countries, etc
/// </summary>
string AssetsDirectory { get; }
/// <summary>
/// Lists out top-level folders for a given directory. Filters out System and Hidden folders.
/// </summary>
/// <param name="rootPath">Absolute path of directory to scan.</param>
@ -87,6 +91,7 @@ public class DirectoryService : IDirectoryService
public string TempDirectory { get; }
public string ConfigDirectory { get; }
public string BookmarkDirectory { get; }
public string AssetsDirectory { get; }
public string SiteThemeDirectory { get; }
public string FaviconDirectory { get; }
public string LocalizationDirectory { get; }
@ -120,6 +125,8 @@ public class DirectoryService : IDirectoryService
ExistOrCreate(TempDirectory);
BookmarkDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "bookmarks");
ExistOrCreate(BookmarkDirectory);
AssetsDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "Assets");
ExistOrCreate(AssetsDirectory);
SiteThemeDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "themes");
ExistOrCreate(SiteThemeDirectory);
FaviconDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "favicons");

4
API/Services/ImageService.cs
View file

@ -1,6 +1,5 @@
using System;
using System.Collections.Generic;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Numerics;
@ -11,9 +10,11 @@ using API.Entities.Interfaces;
using API.Extensions;
using Microsoft.Extensions.Logging;
using NetVips;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using SixLabors.ImageSharp.Processing;
using SixLabors.ImageSharp.Processing.Processors.Quantization;
using Color = System.Drawing.Color;
using Image = NetVips.Image;
namespace API.Services;
@ -748,6 +749,7 @@ public class ImageService : IImageService
entity.SecondaryColor = colors.Secondary;
}
public static Color HexToRgb(string? hex)
{
if (string.IsNullOrEmpty(hex)) throw new ArgumentException("Hex cannot be null");

16
API/Services/Plus/ExternalMetadataService.cs
View file

@ -752,7 +752,7 @@ public class ExternalMetadataService : IExternalMetadataService
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
await DownloadAndSetCovers(upstreamArtists);
await DownloadAndSetPersonCovers(upstreamArtists);
return true;
}
@ -809,7 +809,7 @@ public class ExternalMetadataService : IExternalMetadataService
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
await DownloadAndSetCovers(upstreamWriters);
await DownloadAndSetPersonCovers(upstreamWriters);
return true;
}
@ -1058,7 +1058,7 @@ public class ExternalMetadataService : IExternalMetadataService
{
try
{
await _coverDbService.SetSeriesCoverByUrl(series, coverUrl, false);
await _coverDbService.SetSeriesCoverByUrl(series, coverUrl, false, true);
}
catch (Exception ex)
{
@ -1066,7 +1066,7 @@ public class ExternalMetadataService : IExternalMetadataService
}
}
private async Task DownloadAndSetCovers(List<SeriesStaffDto> people)
private async Task DownloadAndSetPersonCovers(List<SeriesStaffDto> people)
{
foreach (var staff in people)
{
@ -1075,7 +1075,7 @@ public class ExternalMetadataService : IExternalMetadataService
var person = await _unitOfWork.PersonRepository.GetPersonByAniListId(aniListId.Value);
if (person != null && !string.IsNullOrEmpty(staff.ImageUrl) && string.IsNullOrEmpty(person.CoverImage))
{
await _coverDbService.SetPersonCoverByUrl(person, staff.ImageUrl, false);
await _coverDbService.SetPersonCoverByUrl(person, staff.ImageUrl, false, true);
}
}
}
@ -1326,11 +1326,15 @@ public class ExternalMetadataService : IExternalMetadataService
}
try
{
return await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-by-ids")
var ret = await (Configuration.KavitaPlusApiUrl + "/api/metadata/v2/series-by-ids")
.WithKavitaPlusHeaders(license)
.PostJsonAsync(payload)
.ReceiveJson<ExternalSeriesDetailDto>();
ret.Summary = StringHelper.SquashBreaklines(ret.Summary);
return ret;
}
catch (Exception e)
{

65
API/Services/Tasks/CleanupService.cs
View file

@ -33,6 +33,8 @@ public interface ICleanupService
/// </summary>
/// <returns></returns>
Task CleanupWantToRead();
Task ConsolidateProgress();
}
/// <summary>
/// Cleans up after operations on reoccurring basis
@ -74,13 +76,21 @@ public class CleanupService : ICleanupService
_logger.LogInformation("Starting Cleanup");
await SendProgress(0F, "Starting cleanup");
_logger.LogInformation("Cleaning temp directory");
_directoryService.ClearDirectory(_directoryService.TempDirectory);
await SendProgress(0.1F, "Cleaning temp directory");
CleanupCacheAndTempDirectories();
await SendProgress(0.25F, "Cleaning old database backups");
_logger.LogInformation("Cleaning old database backups");
await CleanupBackups();
await SendProgress(0.35F, "Consolidating Progress Events");
_logger.LogInformation("Consolidating Progress Events");
await ConsolidateProgress();
await SendProgress(0.50F, "Cleaning deleted cover images");
_logger.LogInformation("Cleaning deleted cover images");
await DeleteSeriesCoverImages();
@ -226,6 +236,61 @@ public class CleanupService : ICleanupService
_logger.LogInformation("Finished cleanup of Database backups at {Time}", DateTime.Now);
}
/// <summary>
/// Find any progress events that have duplicate, find the highest page read event, then copy over information from that and delete others, to leave one.
/// </summary>
public async Task ConsolidateProgress()
{
// AppUserProgress
var allProgress = await _unitOfWork.AppUserProgressRepository.GetAllProgress();
// Group by the unique identifiers that would make a progress entry unique
var duplicateGroups = allProgress
.GroupBy(p => new
{
p.AppUserId,
p.ChapterId,
})
.Where(g => g.Count() > 1);
foreach (var group in duplicateGroups)
{
// Find the entry with the highest pages read
var highestProgress = group
.OrderByDescending(p => p.PagesRead)
.ThenByDescending(p => p.LastModifiedUtc)
.First();
// Get the duplicate entries to remove (all except the highest progress)
var duplicatesToRemove = group
.Where(p => p.Id != highestProgress.Id)
.ToList();
// Copy over any non-null BookScrollId if the highest progress entry doesn't have one
if (string.IsNullOrEmpty(highestProgress.BookScrollId))
{
var firstValidScrollId = duplicatesToRemove
.FirstOrDefault(p => !string.IsNullOrEmpty(p.BookScrollId))
?.BookScrollId;
if (firstValidScrollId != null)
{
highestProgress.BookScrollId = firstValidScrollId;
highestProgress.MarkModified();
}
}
// Remove the duplicates
foreach (var duplicate in duplicatesToRemove)
{
_unitOfWork.AppUserProgressRepository.Remove(duplicate);
}
}
// Save changes
await _unitOfWork.CommitAsync();
}
public async Task CleanupLogs()
{
_logger.LogInformation("Performing cleanup of logs directory");

47
API/Services/Tasks/Metadata/CoverDbService.cs
View file

@ -19,6 +19,7 @@ using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using NetVips;
namespace API.Services.Tasks.Metadata;
#nullable enable
@ -28,8 +29,8 @@ public interface ICoverDbService
Task<string> DownloadPublisherImageAsync(string publisherName, EncodeFormat encodeFormat);
Task<string?> DownloadPersonImageAsync(Person person, EncodeFormat encodeFormat);
Task<string?> DownloadPersonImageAsync(Person person, EncodeFormat encodeFormat, string url);
Task SetPersonCoverByUrl(Person person, string url, bool fromBase64 = true);
Task SetSeriesCoverByUrl(Series series, string url, bool fromBase64 = true);
Task SetPersonCoverByUrl(Person person, string url, bool fromBase64 = true, bool checkNoImagePlaceholder = false);
Task SetSeriesCoverByUrl(Series series, string url, bool fromBase64 = true, bool chooseBetterImage = false);
}
@ -461,13 +462,39 @@ public class CoverDbService : ICoverDbService
return null;
}
public async Task SetPersonCoverByUrl(Person person, string url, bool fromBase64 = true)
/// <summary>
///
/// </summary>
/// <param name="person"></param>
/// <param name="url"></param>
/// <param name="fromBase64"></param>
/// <param name="checkNoImagePlaceholder">Will check against all known null image placeholders to avoid writing it</param>
public async Task SetPersonCoverByUrl(Person person, string url, bool fromBase64 = true, bool checkNoImagePlaceholder = false)
{
// TODO: Refactor checkNoImagePlaceholder bool to an action that evaluates how to process Image
if (!string.IsNullOrEmpty(url))
{
var filePath = await CreateThumbnail(url, $"{ImageService.GetPersonFormat(person.Id)}", fromBase64);
// Additional check to see if downloaded image is similar and we have a higher resolution
if (checkNoImagePlaceholder)
{
var matchRating = Path.Join(_directoryService.AssetsDirectory, "anilist-no-image-placeholder.jpg").GetSimilarity(Path.Join(_directoryService.CoverImageDirectory, filePath))!;
if (matchRating >= 0.9f)
{
if (string.IsNullOrEmpty(person.CoverImage))
{
filePath = null;
}
else
{
filePath = Path.GetFileName(Path.Join(_directoryService.CoverImageDirectory, person.CoverImage));
}
}
}
if (!string.IsNullOrEmpty(filePath))
{
person.CoverImage = filePath;
@ -498,7 +525,8 @@ public class CoverDbService : ICoverDbService
/// <param name="series"></param>
/// <param name="url"></param>
/// <param name="fromBase64"></param>
public async Task SetSeriesCoverByUrl(Series series, string url, bool fromBase64 = true)
/// <param name="chooseBetterImage">If images are similar, will choose the higher quality image</param>
public async Task SetSeriesCoverByUrl(Series series, string url, bool fromBase64 = true, bool chooseBetterImage = false)
{
if (!string.IsNullOrEmpty(url))
{
@ -506,6 +534,13 @@ public class CoverDbService : ICoverDbService
if (!string.IsNullOrEmpty(filePath))
{
// Additional check to see if downloaded image is similar and we have a higher resolution
if (chooseBetterImage)
{
var betterImage = Path.Join(_directoryService.CoverImageDirectory, series.CoverImage).GetBetterImage(Path.Join(_directoryService.CoverImageDirectory, filePath))!;
filePath = Path.GetFileName(betterImage);
}
series.CoverImage = filePath;
series.CoverImageLocked = true;
_imageService.UpdateColorScape(series);
@ -540,6 +575,6 @@ public class CoverDbService : ICoverDbService
filename, encodeFormat, coverImageSize.GetDimensions().Width);
}
return await DownloadImageFromUrl(filename, encodeFormat, url);
return await DownloadImageFromUrl(filename, encodeFormat, url);
}
}

2
API/Services/Tasks/ScannerService.cs
View file

@ -515,7 +515,7 @@ public class ScannerService : IScannerService
var shouldUseLibraryScan = !(await _unitOfWork.LibraryRepository.DoAnySeriesFoldersMatch(libraryFolderPaths));
if (!shouldUseLibraryScan)
{
_logger.LogError("[ScannerService] Library {LibraryName} consists of one or more Series folders, using series scan", library.Name);
_logger.LogError("[ScannerService] Library {LibraryName} consists of one or more Series folders as a library root, using series scan", library.Name);
}

16
API/Services/Tasks/VersionUpdaterService.cs
View file

@ -67,7 +67,7 @@ public partial class VersionUpdaterService : IVersionUpdaterService
[GeneratedRegex(@"^\n*(.*?)\n+#{1,2}\s", RegexOptions.Singleline)]
private static partial Regex BlogPartRegex();
private static string _cacheFilePath;
private readonly string _cacheFilePath;
private static readonly TimeSpan CacheDuration = TimeSpan.FromHours(1);
public VersionUpdaterService(ILogger<VersionUpdaterService> logger, IEventHub eventHub, IDirectoryService directoryService)
@ -131,6 +131,7 @@ public partial class VersionUpdaterService : IVersionUpdaterService
Theme = sections.TryGetValue("Theme", out var theme) ? theme : [],
Developer = sections.TryGetValue("Developer", out var developer) ? developer : [],
Api = sections.TryGetValue("Api", out var api) ? api : [],
FeatureRequests = sections.TryGetValue("Feature Requests", out var frs) ? frs : [],
BlogPart = _markdown.Transform(blogPart.Trim()),
UpdateBody = _markdown.Transform(prInfo.Body.Trim())
};
@ -305,7 +306,7 @@ public partial class VersionUpdaterService : IVersionUpdaterService
return updateDtos;
}
private static async Task<IList<UpdateNotificationDto>?> TryGetCachedReleases()
private async Task<IList<UpdateNotificationDto>?> TryGetCachedReleases()
{
if (!File.Exists(_cacheFilePath)) return null;
@ -376,6 +377,7 @@ public partial class VersionUpdaterService : IVersionUpdaterService
Theme = parsedSections.TryGetValue("Theme", out var theme) ? theme : [],
Developer = parsedSections.TryGetValue("Developer", out var developer) ? developer : [],
Api = parsedSections.TryGetValue("Api", out var api) ? api : [],
FeatureRequests = parsedSections.TryGetValue("Feature Requests", out var frs) ? frs : [],
BlogPart = blogPart
};
}
@ -492,7 +494,7 @@ public partial class VersionUpdaterService : IVersionUpdaterService
return item;
}
sealed class PullRequestInfo
private sealed class PullRequestInfo
{
public required string Title { get; init; }
public required string Body { get; init; }
@ -501,25 +503,25 @@ public partial class VersionUpdaterService : IVersionUpdaterService
public required int Number { get; init; }
}
sealed class CommitInfo
private sealed class CommitInfo
{
public required string Sha { get; init; }
public required CommitDetail Commit { get; init; }
public required string Html_Url { get; init; }
}
sealed class CommitDetail
private sealed class CommitDetail
{
public required string Message { get; init; }
public required CommitAuthor Author { get; init; }
}
sealed class CommitAuthor
private sealed class CommitAuthor
{
public required string Date { get; init; }
}
sealed class NightlyInfo
private sealed class NightlyInfo
{
public required string Version { get; init; }
public required int PrNumber { get; init; }

20
API/Services/TokenService.cs
View file

@ -4,6 +4,7 @@ using System.IdentityModel.Tokens.Jwt;
using System.Linq;
using System.Security.Claims;
using System.Text;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Account;
@ -36,6 +37,7 @@ public class TokenService : ITokenService
private readonly IUnitOfWork _unitOfWork;
private readonly SymmetricSecurityKey _key;
private const string RefreshTokenName = "RefreshToken";
private static readonly SemaphoreSlim _refreshTokenLock = new SemaphoreSlim(1, 1);
public TokenService(IConfiguration config, UserManager<AppUser> userManager, ILogger<TokenService> logger, IUnitOfWork unitOfWork)
{
@ -81,6 +83,8 @@ public class TokenService : ITokenService
public async Task<TokenRequestDto?> ValidateRefreshToken(TokenRequestDto request)
{
await _refreshTokenLock.WaitAsync();
try
{
var tokenHandler = new JwtSecurityTokenHandler();
@ -91,6 +95,7 @@ public class TokenService : ITokenService
_logger.LogDebug("[RefreshToken] failed to validate due to not finding user in RefreshToken");
return null;
}
var user = await _userManager.FindByNameAsync(username);
if (user == null)
{
@ -98,13 +103,19 @@ public class TokenService : ITokenService
return null;
}
var validated = await _userManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider, RefreshTokenName, request.RefreshToken);
var validated = await _userManager.VerifyUserTokenAsync(user, TokenOptions.DefaultProvider,
RefreshTokenName, request.RefreshToken);
if (!validated && tokenContent.ValidTo <= DateTime.UtcNow.Add(TimeSpan.FromHours(1)))
{
_logger.LogDebug("[RefreshToken] failed to validate due to invalid refresh token");
return null;
}
// Remove the old refresh token first
await _userManager.RemoveAuthenticationTokenAsync(user,
TokenOptions.DefaultProvider,
RefreshTokenName);
try
{
user.UpdateLastActive();
@ -121,7 +132,8 @@ public class TokenService : ITokenService
Token = await CreateToken(user),
RefreshToken = await CreateRefreshToken(user)
};
} catch (SecurityTokenExpiredException ex)
}
catch (SecurityTokenExpiredException ex)
{
// Handle expired token
_logger.LogError(ex, "Failed to validate refresh token");
@ -133,6 +145,10 @@ public class TokenService : ITokenService
_logger.LogError(ex, "Failed to validate refresh token");
return null;
}
finally
{
_refreshTokenLock.Release();
}
}
public async Task<string?> GetJwtFromUser(AppUser user)