AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

v0.7.3 - The Quality of Life Update (#2036)

Browse source 
* Version bump

* Okay this should be the last (#2037)

* Fixed improper date visualization for reading list detail page.

* Correct not-read badge position (#2034)

---------

Co-authored-by: Andre Smith <Hobogrammer@users.noreply.github.com>

* Bump versions by dotnet-bump-version.

* Merged develop in

---------

Co-authored-by: Andre Smith <Hobogrammer@users.noreply.github.com>
This commit is contained in:
Joe Milazzo 2023-06-07 07:55:53 -05:00 • committed by GitHub
parent 51e23b7eca
commit 1b3866568f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
235 changed files with 14827 additions and 21948 deletions
Download patch file Download diff file Expand all files Collapse all files

8
.github/workflows/sonar-scan.yml vendored
View file

@ -156,11 +156,11 @@ jobs:
- name: NodeJS to Compile WebUI
uses: actions/setup-node@v2.1.5
with:
node-version: '14'
node-version: '16'
- run: |
cd UI/Web || exit
echo 'Installing web dependencies'
npm ci
npm install --legacy-peer-deps
echo 'Building UI'
npm run prod
@ -280,12 +280,12 @@ jobs:
- name: NodeJS to Compile WebUI
uses: actions/setup-node@v2.1.5
with:
node-version: '14'
node-version: '16'
- run: |
cd UI/Web || exit
echo 'Installing web dependencies'
npm install
npm ci
echo 'Building UI'
npm run prod

1
.gitignore vendored
View file

@ -512,6 +512,7 @@ UI/Web/dist/
/API/config/themes/
/API/config/stats/
/API/config/bookmarks/
/API/config/favicons/
/API/config/kavita.db
/API/config/kavita.db-shm
/API/config/kavita.db-wal

6
API.Benchmark/ArchiveServiceBenchmark.cs
View file

@ -5,6 +5,8 @@ using Microsoft.Extensions.Logging.Abstractions;
using API.Services;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Order;
using EasyCaching.Core;
using NSubstitute;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.Formats.Png;
using SixLabors.ImageSharp.Formats.Webp;
@ -30,8 +32,8 @@ public class ArchiveServiceBenchmark
public ArchiveServiceBenchmark()
{
_directoryService = new DirectoryService(null, new FileSystem());
_imageService = new ImageService(null, _directoryService);
_archiveService = new ArchiveService(new NullLogger<ArchiveService>(), _directoryService, _imageService);
_imageService = new ImageService(null, _directoryService, Substitute.For<IEasyCachingProviderFactory>());
_archiveService = new ArchiveService(new NullLogger<ArchiveService>(), _directoryService, _imageService, Substitute.For<IMediaErrorService>());
}
[Benchmark(Baseline = true)]

105
API.Benchmark/EpubBenchmark.cs
View file

@ -1,105 +0,0 @@
using System;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using API.Services;
using BenchmarkDotNet.Attributes;
using BenchmarkDotNet.Order;
using HtmlAgilityPack;
using VersOne.Epub;
namespace API.Benchmark;
[StopOnFirstError]
[MemoryDiagnoser]
[RankColumn]
[Orderer(SummaryOrderPolicy.FastestToSlowest)]
[SimpleJob(launchCount: 1, warmupCount: 5, invocationCount: 20)]
public class EpubBenchmark
{
private const string FilePath = @"E:\Books\Invaders of the Rokujouma\Invaders of the Rokujouma - Volume 01.epub";
private readonly Regex _wordRegex = new Regex(@"\b\w+\b", RegexOptions.Compiled | RegexOptions.IgnoreCase);
[Benchmark]
public async Task GetWordCount_PassByRef()
{
using var book = await EpubReader.OpenBookAsync(FilePath, BookService.BookReaderOptions);
foreach (var bookFile in book.Content.Html.Values)
{
await GetBookWordCount_PassByRef(bookFile);
}
}
[Benchmark]
public async Task GetBookWordCount_SumEarlier()
{
using var book = await EpubReader.OpenBookAsync(FilePath, BookService.BookReaderOptions);
foreach (var bookFile in book.Content.Html.Values)
{
await GetBookWordCount_SumEarlier(bookFile);
}
}
[Benchmark]
public async Task GetBookWordCount_Regex()
{
using var book = await EpubReader.OpenBookAsync(FilePath, BookService.BookReaderOptions);
foreach (var bookFile in book.Content.Html.Values)
{
await GetBookWordCount_Regex(bookFile);
}
}
private int GetBookWordCount_PassByString(string fileContents)
{
var doc = new HtmlDocument();
doc.LoadHtml(fileContents);
var delimiter = new char[] {' '};
return doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]")
.Select(node => node.InnerText)
.Select(text => text.Split(delimiter, StringSplitOptions.RemoveEmptyEntries)
.Where(s => char.IsLetter(s[0])))
.Select(words => words.Count())
.Where(wordCount => wordCount > 0)
.Sum();
}
private async Task<int> GetBookWordCount_PassByRef(EpubContentFileRef bookFile)
{
var doc = new HtmlDocument();
doc.LoadHtml(await bookFile.ReadContentAsTextAsync());
var delimiter = new char[] {' '};
var textNodes = doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]");
if (textNodes == null) return 0;
return textNodes.Select(node => node.InnerText)
.Select(text => text.Split(delimiter, StringSplitOptions.RemoveEmptyEntries)
.Where(s => char.IsLetter(s[0])))
.Select(words => words.Count())
.Where(wordCount => wordCount > 0)
.Sum();
}
private async Task<int> GetBookWordCount_SumEarlier(EpubContentFileRef bookFile)
{
var doc = new HtmlDocument();
doc.LoadHtml(await bookFile.ReadContentAsTextAsync());
return doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]")
.DefaultIfEmpty()
.Select(node => node.InnerText.Split(' ', StringSplitOptions.RemoveEmptyEntries)
.Where(s => char.IsLetter(s[0])))
.Sum(words => words.Count());
}
private async Task<int> GetBookWordCount_Regex(EpubContentFileRef bookFile)
{
var doc = new HtmlDocument();
doc.LoadHtml(await bookFile.ReadContentAsTextAsync());
return doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]")
.Sum(node => _wordRegex.Matches(node.InnerText).Count);
}
}

6
API.Tests/API.Tests.csproj
View file

@ -7,11 +7,11 @@
<ItemGroup>
<PackageReference Include="Microsoft.EntityFrameworkCore.InMemory" Version="7.0.5" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.5.0" />
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0" />
<PackageReference Include="Moq" Version="4.18.4" />
<PackageReference Include="NSubstitute" Version="4.4.0" />
<PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="19.2.11" />
<PackageReference Include="TestableIO.System.IO.Abstractions.Wrappers" Version="19.2.11" />
<PackageReference Include="System.IO.Abstractions.TestingHelpers" Version="19.2.29" />
<PackageReference Include="TestableIO.System.IO.Abstractions.Wrappers" Version="19.2.29" />
<PackageReference Include="xunit" Version="2.4.2" />
<PackageReference Include="xunit.runner.visualstudio" Version="2.4.5">
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>

1
API.Tests/Parser/BookParserTests.cs
View file

@ -39,4 +39,5 @@ public class BookParserTests
// var actual = API.Parser.Parser.CssImportUrlRegex.Replace(input, "$1" + apiBase + "$2" + "$3");
// Assert.Equal(expected, actual);
// }
}

2
API.Tests/Parser/MangaParserTests.cs
View file

@ -197,6 +197,7 @@ public class MangaParserTests
[InlineData("Esquire 6권 2021년 10월호", "Esquire")]
[InlineData("Accel World: Vol 1", "Accel World")]
[InlineData("Accel World Chapter 001 Volume 002", "Accel World")]
[InlineData("Bleach 001-003", "Bleach")]
public void ParseSeriesTest(string filename, string expected)
{
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.ParseSeries(filename));
@ -281,6 +282,7 @@ public class MangaParserTests
[InlineData("Манга 2 Глава", "2")]
[InlineData("Манга Том 1 2 Глава", "2")]
[InlineData("Accel World Chapter 001 Volume 002", "1")]
[InlineData("Bleach 001-003", "1-3")]
public void ParseChaptersTest(string filename, string expected)
{
Assert.Equal(expected, API.Services.Tasks.Scanner.Parser.Parser.ParseChapter(filename));

26
API.Tests/Services/ArchiveServiceTests.cs
View file

@ -5,7 +5,9 @@ using System.IO.Abstractions.TestingHelpers;
using System.IO.Compression;
using System.Linq;
using API.Archive;
using API.Entities.Enums;
using API.Services;
using EasyCaching.Core;
using Microsoft.Extensions.Logging;
using NetVips;
using NSubstitute;
@ -26,7 +28,9 @@ public class ArchiveServiceTests
public ArchiveServiceTests(ITestOutputHelper testOutputHelper)
{
_testOutputHelper = testOutputHelper;
_archiveService = new ArchiveService(_logger, _directoryService, new ImageService(Substitute.For<ILogger<ImageService>>(), _directoryService));
_archiveService = new ArchiveService(_logger, _directoryService,
new ImageService(Substitute.For<ILogger<ImageService>>(), _directoryService, Substitute.For<IEasyCachingProviderFactory>()),
Substitute.For<IMediaErrorService>());
}
[Theory]
@ -163,8 +167,8 @@ public class ArchiveServiceTests
public void GetCoverImage_Default_Test(string inputFile, string expectedOutputFile)
{
var ds = Substitute.For<DirectoryService>(_directoryServiceLogger, new FileSystem());
var imageService = new ImageService(Substitute.For<ILogger<ImageService>>(), ds);
var archiveService = Substitute.For<ArchiveService>(_logger, ds, imageService);
var imageService = new ImageService(Substitute.For<ILogger<ImageService>>(), ds, Substitute.For<IEasyCachingProviderFactory>());
var archiveService = Substitute.For<ArchiveService>(_logger, ds, imageService, Substitute.For<IMediaErrorService>());
var testDirectory = Path.GetFullPath(Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/CoverImages"));
var expectedBytes = Image.Thumbnail(Path.Join(testDirectory, expectedOutputFile), 320).WriteToBuffer(".png");
@ -176,7 +180,7 @@ public class ArchiveServiceTests
_directoryService.ExistOrCreate(outputDir);
var coverImagePath = archiveService.GetCoverImage(Path.Join(testDirectory, inputFile),
Path.GetFileNameWithoutExtension(inputFile) + "_output", outputDir);
Path.GetFileNameWithoutExtension(inputFile) + "_output", outputDir, EncodeFormat.PNG);
var actual = File.ReadAllBytes(Path.Join(outputDir, coverImagePath));
@ -194,9 +198,10 @@ public class ArchiveServiceTests
[InlineData("sorting.zip", "sorting.expected.png")]
public void GetCoverImage_SharpCompress_Test(string inputFile, string expectedOutputFile)
{
var imageService = new ImageService(Substitute.For<ILogger<ImageService>>(), _directoryService);
var imageService = new ImageService(Substitute.For<ILogger<ImageService>>(), _directoryService, Substitute.For<IEasyCachingProviderFactory>());
var archiveService = Substitute.For<ArchiveService>(_logger,
new DirectoryService(_directoryServiceLogger, new FileSystem()), imageService);
new DirectoryService(_directoryServiceLogger, new FileSystem()), imageService,
Substitute.For<IMediaErrorService>());
var testDirectory = API.Services.Tasks.Scanner.Parser.Parser.NormalizePath(Path.GetFullPath(Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/CoverImages")));
var outputDir = Path.Join(testDirectory, "output");
@ -205,7 +210,7 @@ public class ArchiveServiceTests
archiveService.Configure().CanOpen(Path.Join(testDirectory, inputFile)).Returns(ArchiveLibrary.SharpCompress);
var coverOutputFile = archiveService.GetCoverImage(Path.Join(testDirectory, inputFile),
Path.GetFileNameWithoutExtension(inputFile), outputDir);
Path.GetFileNameWithoutExtension(inputFile), outputDir, EncodeFormat.PNG);
var actualBytes = File.ReadAllBytes(Path.Join(outputDir, coverOutputFile));
var expectedBytes = File.ReadAllBytes(Path.Join(testDirectory, expectedOutputFile));
Assert.Equal(expectedBytes, actualBytes);
@ -219,13 +224,14 @@ public class ArchiveServiceTests
public void CanParseCoverImage(string inputFile)
{
var imageService = Substitute.For<IImageService>();
imageService.WriteCoverThumbnail(Arg.Any<Stream>(), Arg.Any<string>(), Arg.Any<string>()).Returns(x => "cover.jpg");
var archiveService = new ArchiveService(_logger, _directoryService, imageService);
imageService.WriteCoverThumbnail(Arg.Any<Stream>(), Arg.Any<string>(), Arg.Any<string>(), Arg.Any<EncodeFormat>())
.Returns(x => "cover.jpg");
var archiveService = new ArchiveService(_logger, _directoryService, imageService, Substitute.For<IMediaErrorService>());
var testDirectory = Path.Join(Directory.GetCurrentDirectory(), "../../../Services/Test Data/ArchiveService/");
var inputPath = Path.GetFullPath(Path.Join(testDirectory, inputFile));
var outputPath = Path.Join(testDirectory, Path.GetFileNameWithoutExtension(inputFile) + "_output");
new DirectoryInfo(outputPath).Create();
var expectedImage = archiveService.GetCoverImage(inputPath, inputFile, outputPath);
var expectedImage = archiveService.GetCoverImage(inputPath, inputFile, outputPath, EncodeFormat.PNG);
Assert.Equal("cover.jpg", expectedImage);
new DirectoryInfo(outputPath).Delete();
}

5
API.Tests/Services/BookServiceTests.cs
View file

@ -1,6 +1,7 @@
using System.IO;
using System.IO.Abstractions;
using API.Services;
using EasyCaching.Core;
using Microsoft.Extensions.Logging;
using NSubstitute;
using Xunit;
@ -15,7 +16,9 @@ public class BookServiceTests
public BookServiceTests()
{
var directoryService = new DirectoryService(Substitute.For<ILogger<DirectoryService>>(), new FileSystem());
_bookService = new BookService(_logger, directoryService, new ImageService(Substitute.For<ILogger<ImageService>>(), directoryService));
_bookService = new BookService(_logger, directoryService,
new ImageService(Substitute.For<ILogger<ImageService>>(), directoryService, Substitute.For<IEasyCachingProviderFactory>())
, Substitute.For<IMediaErrorService>());
}
[Theory]

2
API.Tests/Services/BookmarkServiceTests.cs
View file

@ -55,7 +55,7 @@ public class BookmarkServiceTests
private BookmarkService Create(IDirectoryService ds)
{
return new BookmarkService(Substitute.For<ILogger<BookmarkService>>(), _unitOfWork, ds,
Substitute.For<IImageService>(), Substitute.For<IEventHub>());
Substitute.For<IMediaConversionService>());
}
#region Setup

2
API.Tests/Services/CacheServiceTests.cs
View file

@ -42,7 +42,7 @@ internal class MockReadingItemServiceForCacheService : IReadingItemService
return 1;
}
public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format, bool saveAsWebP)
public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format, EncodeFormat encodeFormat)
{
return string.Empty;
}

2
API.Tests/Services/ParseScannedFilesTests.cs
View file

@ -43,7 +43,7 @@ internal class MockReadingItemService : IReadingItemService
return 1;
}
public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format, bool saveAsWebP)
public string GetCoverImage(string fileFilePath, string fileName, MangaFormat format, EncodeFormat encodeFormat)
{
return string.Empty;
}

178
API.Tests/Services/ReaderServiceTests.cs
View file

@ -1986,6 +1986,184 @@ public class ReaderServiceTests
Assert.Equal(4, nextChapter.VolumeId);
}
/// <summary>
/// Volume 1-10 are fully read (single volumes),
/// Special 1 is fully read
/// Chapters 56-90 are read
/// Chapter 91 has partial progress on
/// </summary>
[Fact]
public async Task GetContinuePoint_ShouldReturnLastLooseChapter()
{
await ResetDb();
var series = new SeriesBuilder("Test")
.WithVolume(new VolumeBuilder("1")
.WithChapter(new ChapterBuilder("1").WithPages(1).Build())
.Build())
.WithVolume(new VolumeBuilder("2")
.WithChapter(new ChapterBuilder("21").WithPages(1).Build())
.WithChapter(new ChapterBuilder("22").WithPages(1).Build())
.Build())
.WithVolume(new VolumeBuilder("0")
.WithChapter(new ChapterBuilder("51").WithPages(1).Build())
.WithChapter(new ChapterBuilder("52").WithPages(1).Build())
.WithChapter(new ChapterBuilder("91").WithPages(2).Build())
.WithChapter(new ChapterBuilder("Special").WithIsSpecial(true).WithPages(1).Build())
.Build())
.Build();
series.Library = new LibraryBuilder("Test LIb", LibraryType.Manga).Build();
_context.Series.Add(series);
_context.AppUser.Add(new AppUser()
{
UserName = "majora2007"
});
await _context.SaveChangesAsync();
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 1,
SeriesId = 1,
VolumeId = 1
}, 1);
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 2,
SeriesId = 1,
VolumeId = 1
}, 1);
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 3,
SeriesId = 1,
VolumeId = 2
}, 1);
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 4,
SeriesId = 1,
VolumeId = 2
}, 1);
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 5,
SeriesId = 1,
VolumeId = 2
}, 1);
// Chapter 91 has partial progress, hence it should resume there
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 6,
SeriesId = 1,
VolumeId = 2
}, 1);
// Special is fully read
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 7,
SeriesId = 1,
VolumeId = 2
}, 1);
await _context.SaveChangesAsync();
var nextChapter = await _readerService.GetContinuePoint(1, 1);
Assert.Equal("91", nextChapter.Range);
}
[Fact]
public async Task GetContinuePoint_DuplicateIssueNumberBetweenChapters()
{
await ResetDb();
var series = new SeriesBuilder("Test")
.WithVolume(new VolumeBuilder("1")
.WithChapter(new ChapterBuilder("1").WithPages(1).Build())
.WithChapter(new ChapterBuilder("2").WithPages(1).Build())
.WithChapter(new ChapterBuilder("21").WithPages(1).Build())
.WithChapter(new ChapterBuilder("22").WithPages(1).Build())
.WithChapter(new ChapterBuilder("32").WithPages(1).Build())
.Build())
.WithVolume(new VolumeBuilder("2")
.WithChapter(new ChapterBuilder("1").WithPages(1).Build())
.WithChapter(new ChapterBuilder("2").WithPages(1).Build())
.WithChapter(new ChapterBuilder("21").WithPages(1).Build())
.WithChapter(new ChapterBuilder("22").WithPages(1).Build())
.WithChapter(new ChapterBuilder("32").WithPages(1).Build())
.Build())
.Build();
series.Library = new LibraryBuilder("Test LIb", LibraryType.Manga).Build();
_context.Series.Add(series);
_context.AppUser.Add(new AppUser()
{
UserName = "majora2007"
});
await _context.SaveChangesAsync();
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 1,
SeriesId = 1,
VolumeId = 1
}, 1);
await _context.SaveChangesAsync();
var nextChapter = await _readerService.GetContinuePoint(1, 1);
Assert.Equal("2", nextChapter.Range);
Assert.Equal(1, nextChapter.VolumeId);
// Mark chapter 2 as read
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 2,
SeriesId = 1,
VolumeId = 1
}, 1);
await _context.SaveChangesAsync();
nextChapter = await _readerService.GetContinuePoint(1, 1);
Assert.Equal("21", nextChapter.Range);
Assert.Equal(1, nextChapter.VolumeId);
// Mark chapter 21 as read
await _readerService.SaveReadingProgress(new ProgressDto()
{
PageNum = 1,
ChapterId = 3,
SeriesId = 1,
VolumeId = 1
}, 1);
await _context.SaveChangesAsync();
nextChapter = await _readerService.GetContinuePoint(1, 1);
Assert.Equal("22", nextChapter.Range);
Assert.Equal(1, nextChapter.VolumeId);
}
#endregion
#region MarkChaptersUntilAsRead

33
API.Tests/Services/SeriesServiceTests.cs
View file

@ -343,7 +343,7 @@ public class SeriesServiceTests : AbstractDbTest
Assert.True(result);
var ratings = (await _unitOfWork.UserRepository.GetUserByUsernameAsync("majora2007", AppUserIncludes.Ratings))
var ratings = (await _unitOfWork.UserRepository.GetUserByUsernameAsync("majora2007", AppUserIncludes.Ratings))!
.Ratings;
Assert.NotEmpty(ratings);
Assert.Equal(3, ratings.First().Rating);
@ -780,7 +780,7 @@ public class SeriesServiceTests : AbstractDbTest
{
var series = CreateSeriesMock();
var firstChapter = SeriesService.GetFirstChapterForMetadata(series, true);
var firstChapter = SeriesService.GetFirstChapterForMetadata(series);
Assert.Same("1", firstChapter.Range);
}
@ -789,7 +789,7 @@ public class SeriesServiceTests : AbstractDbTest
{
var series = CreateSeriesMock();
var firstChapter = SeriesService.GetFirstChapterForMetadata(series, false);
var firstChapter = SeriesService.GetFirstChapterForMetadata(series);
Assert.Same("1", firstChapter.Range);
}
@ -808,10 +808,35 @@ public class SeriesServiceTests : AbstractDbTest
new ChapterBuilder("1.2").WithFiles(files).WithPages(1).Build(),
};
var firstChapter = SeriesService.GetFirstChapterForMetadata(series, false);
var firstChapter = SeriesService.GetFirstChapterForMetadata(series);
Assert.Same("1.1", firstChapter.Range);
}
[Fact]
public void GetFirstChapterForMetadata_NonBook_ShouldReturnChapter1_WhenFirstVolumeIs3()
{
var file = new MangaFileBuilder("Test.cbz", MangaFormat.Archive, 1).Build();
var series = new SeriesBuilder("Test")
.WithVolume(new VolumeBuilder("0")
.WithChapter(new ChapterBuilder("1").WithPages(1).WithFile(file).Build())
.WithChapter(new ChapterBuilder("2").WithPages(1).WithFile(file).Build())
.Build())
.WithVolume(new VolumeBuilder("2")
.WithChapter(new ChapterBuilder("21").WithPages(1).WithFile(file).Build())
.WithChapter(new ChapterBuilder("22").WithPages(1).WithFile(file).Build())
.Build())
.WithVolume(new VolumeBuilder("3")
.WithChapter(new ChapterBuilder("31").WithPages(1).WithFile(file).Build())
.WithChapter(new ChapterBuilder("32").WithPages(1).WithFile(file).Build())
.Build())
.Build();
series.Library = new LibraryBuilder("Test LIb", LibraryType.Book).Build();
var firstChapter = SeriesService.GetFirstChapterForMetadata(series);
Assert.Same("1", firstChapter.Range);
}
#endregion
#region SeriesRelation

24
API/API.csproj
View file

@ -56,15 +56,16 @@
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="12.0.1" />
<PackageReference Include="CsvHelper" Version="30.0.1" />
<PackageReference Include="Docnet.Core" Version="2.4.0-alpha.4" />
<PackageReference Include="EasyCaching.InMemory" Version="1.9.0" />
<PackageReference Include="ExCSS" Version="4.1.0" />
<PackageReference Include="Flurl" Version="3.0.7" />
<PackageReference Include="Flurl.Http" Version="3.2.4" />
<PackageReference Include="Hangfire" Version="1.7.34" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.7.34" />
<PackageReference Include="Hangfire.InMemory" Version="0.3.7" />
<PackageReference Include="Hangfire" Version="1.8.1" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.8.1" />
<PackageReference Include="Hangfire.InMemory" Version="0.4.0" />
<PackageReference Include="Hangfire.MaximumConcurrentExecutions" Version="1.1.0" />
<PackageReference Include="Hangfire.MemoryStorage.Core" Version="1.4.0" />
<PackageReference Include="Hangfire.Storage.SQLite" Version="0.3.3" />
<PackageReference Include="Hangfire.Storage.SQLite" Version="0.3.4" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.46" />
<PackageReference Include="MarkdownDeep.NET.Core" Version="1.5.0.4" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="7.0.5" />
@ -79,30 +80,31 @@
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="7.0.0" />
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="2.3.2" />
<PackageReference Include="MimeTypeMapOfficial" Version="1.0.17" />
<PackageReference Include="Nager.ArticleNumber" Version="1.0.7" />
<PackageReference Include="NetVips" Version="2.3.0" />
<PackageReference Include="NetVips.Native" Version="8.14.2" />
<PackageReference Include="NReco.Logging.File" Version="1.1.6" />
<PackageReference Include="Serilog" Version="2.12.0" />
<PackageReference Include="Serilog.AspNetCore" Version="6.1.0" />
<PackageReference Include="Serilog.AspNetCore" Version="7.0.0" />
<PackageReference Include="Serilog.Enrichers.Thread" Version="3.2.0-dev-00752" />
<PackageReference Include="Serilog.Extensions.Hosting" Version="5.0.1" />
<PackageReference Include="Serilog.Settings.Configuration" Version="3.4.0" />
<PackageReference Include="Serilog.Extensions.Hosting" Version="7.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="7.0.0" />
<PackageReference Include="Serilog.Sinks.AspNetCore.SignalR" Version="0.4.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="4.1.0" />
<PackageReference Include="Serilog.Sinks.File" Version="5.0.0" />
<PackageReference Include="Serilog.Sinks.SignalR.Core" Version="0.1.2" />
<PackageReference Include="SharpCompress" Version="0.33.0" />
<PackageReference Include="SixLabors.ImageSharp" Version="3.0.1" />
<PackageReference Include="SonarAnalyzer.CSharp" Version="8.55.0.65544">
<PackageReference Include="SonarAnalyzer.CSharp" Version="9.0.0.68202">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" />
<PackageReference Include="Swashbuckle.AspNetCore.Filters" Version="7.0.6" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.29.0" />
<PackageReference Include="System.IO.Abstractions" Version="19.2.11" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="6.30.1" />
<PackageReference Include="System.IO.Abstractions" Version="19.2.29" />
<PackageReference Include="System.Drawing.Common" Version="7.0.0" />
<PackageReference Include="VersOne.Epub" Version="3.3.0-alpha1" />
<PackageReference Include="VersOne.Epub" Version="3.3.1" />
</ItemGroup>
<ItemGroup>

10
API/Constants/CacheProfiles.cs Normal file
View file

@ -0,0 +1,10 @@
namespace API.Constants;
public static class EasyCacheProfiles
{
/// <summary>
/// Not in use
/// </summary>
public const string RevokedJwt = "revokedJWT";
public const string Favicon = "favicon";
}

6
API/Constants/ControllerConstants.cs Normal file
View file

@ -0,0 +1,6 @@
namespace API.Constants;
public abstract class ControllerConstants
{
public const int MaxUploadSizeBytes = 8_000_000;
}

12
API/Controllers/AccountController.cs
View file

@ -18,6 +18,7 @@ using API.Middleware.RateLimit;
using API.Services;
using API.SignalR;
using AutoMapper;
using EasyCaching.Core;
using Hangfire;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
@ -44,6 +45,7 @@ public class AccountController : BaseApiController
private readonly IAccountService _accountService;
private readonly IEmailService _emailService;
private readonly IEventHub _eventHub;
private readonly IEasyCachingProviderFactory _cacheFactory;
/// <inheritdoc />
public AccountController(UserManager<AppUser> userManager,
@ -51,7 +53,8 @@ public class AccountController : BaseApiController
ITokenService tokenService, IUnitOfWork unitOfWork,
ILogger<AccountController> logger,
IMapper mapper, IAccountService accountService,
IEmailService emailService, IEventHub eventHub)
IEmailService emailService, IEventHub eventHub,
IEasyCachingProviderFactory cacheFactory)
{
_userManager = userManager;
_signInManager = signInManager;
@ -62,6 +65,7 @@ public class AccountController : BaseApiController
_accountService = accountService;
_emailService = emailService;
_eventHub = eventHub;
_cacheFactory = cacheFactory;
}
/// <summary>
@ -187,8 +191,9 @@ public class AccountController : BaseApiController
var result = await _signInManager
.CheckPasswordSignInAsync(user, loginDto.Password, true);
if (result.IsLockedOut)
if (result.IsLockedOut) // result.IsLockedOut
{
await _userManager.UpdateSecurityStampAsync(user);
return Unauthorized("You've been locked out from too many authorization attempts. Please wait 10 minutes.");
}
@ -443,6 +448,9 @@ public class AccountController : BaseApiController
if (!roleResult.Succeeded) return BadRequest(roleResult.Errors);
}
// We might want to check if they had admin and no longer, if so:
// await _userManager.UpdateSecurityStampAsync(user); to force them to re-authenticate
var allLibraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();
List<Library> libraries;

5
API/Controllers/BookController.cs
View file

@ -98,9 +98,10 @@ public class BookController : BaseApiController
using var book = await EpubReader.OpenBookAsync(chapter.Files.ElementAt(0).FilePath, BookService.BookReaderOptions);
var key = BookService.CoalesceKeyForAnyFile(book, file);
if (!book.Content.AllFiles.ContainsKey(key)) return BadRequest("File was not found in book");
var bookFile = book.Content.AllFiles[key];
if (!book.Content.AllFiles.ContainsLocalFileRefWithKey(key)) return BadRequest("File was not found in book");
var bookFile = book.Content.AllFiles.GetLocalFileRefByKey(key);
var content = await bookFile.ReadContentAsBytesAsync();
var contentType = BookService.GetContentType(bookFile.ContentType);

6
API/Controllers/DownloadController.cs
View file

@ -117,7 +117,7 @@ public class DownloadController : BaseApiController
private ActionResult GetFirstFileDownload(IEnumerable<MangaFile> files)
{
var (zipFile, contentType, fileDownloadName) = _downloadService.GetFirstFileDownload(files);
return PhysicalFile(zipFile, contentType, fileDownloadName, true);
return PhysicalFile(zipFile, contentType, Uri.EscapeDataString(fileDownloadName), true);
}
/// <summary>
@ -163,7 +163,7 @@ public class DownloadController : BaseApiController
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.DownloadProgressEvent(User.GetUsername(),
Path.GetFileNameWithoutExtension(downloadName), 1F, "ended"));
return PhysicalFile(filePath, DefaultContentType, downloadName, true);
return PhysicalFile(filePath, DefaultContentType, Uri.EscapeDataString(downloadName), true);
}
catch (Exception ex)
{
@ -220,7 +220,7 @@ public class DownloadController : BaseApiController
MessageFactory.DownloadProgressEvent(username, Path.GetFileNameWithoutExtension(filename), 1F));
return PhysicalFile(filePath, DefaultContentType, filename, true);
return PhysicalFile(filePath, DefaultContentType, System.Web.HttpUtility.UrlEncode(filename), true);
}
}

43
API/Controllers/ImageController.cs
View file

@ -1,4 +1,5 @@
using System.IO;
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Constants;
@ -20,12 +21,14 @@ public class ImageController : BaseApiController
{
private readonly IUnitOfWork _unitOfWork;
private readonly IDirectoryService _directoryService;
private readonly IImageService _imageService;
/// <inheritdoc />
public ImageController(IUnitOfWork unitOfWork, IDirectoryService directoryService)
public ImageController(IUnitOfWork unitOfWork, IDirectoryService directoryService, IImageService imageService)
{
_unitOfWork = unitOfWork;
_directoryService = directoryService;
_imageService = imageService;
}
/// <summary>
@ -157,6 +160,42 @@ public class ImageController : BaseApiController
return PhysicalFile(file.FullName, MimeTypeMap.GetMimeType(format), Path.GetFileName(file.FullName));
}
/// <summary>
/// Returns the image associated with a web-link
/// </summary>
/// <param name="apiKey"></param>
/// <returns></returns>
[HttpGet("web-link")]
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Month, VaryByQueryKeys = new []{"url", "apiKey"})]
public async Task<ActionResult> GetWebLinkImage(string url, string apiKey)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey);
if (userId == 0) return BadRequest();
if (string.IsNullOrEmpty(url)) return BadRequest("Url cannot be null");
var encodeFormat = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
// Check if the domain exists
var domainFilePath = _directoryService.FileSystem.Path.Join(_directoryService.FaviconDirectory, ImageService.GetWebLinkFormat(url, encodeFormat));
if (!_directoryService.FileSystem.File.Exists(domainFilePath))
{
// We need to request the favicon and save it
try
{
domainFilePath = _directoryService.FileSystem.Path.Join(_directoryService.FaviconDirectory,
await _imageService.DownloadFaviconAsync(url, encodeFormat));
}
catch (Exception)
{
return BadRequest("There was an issue fetching favicon for domain");
}
}
var file = new FileInfo(domainFilePath);
var format = Path.GetExtension(file.FullName);
return PhysicalFile(file.FullName, MimeTypeMap.GetMimeType(format), Path.GetFileName(file.FullName));
}
/// <summary>
/// Returns a temp coverupload image
/// </summary>

25
API/Controllers/LibraryController.cs
View file

@ -50,22 +50,28 @@ public class LibraryController : BaseApiController
/// <summary>
/// Creates a new Library. Upon library creation, adds new library to all Admin accounts.
/// </summary>
/// <param name="createLibraryDto"></param>
/// <param name="dto"></param>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[HttpPost("create")]
public async Task<ActionResult> AddLibrary(CreateLibraryDto createLibraryDto)
public async Task<ActionResult> AddLibrary(UpdateLibraryDto dto)
{
if (await _unitOfWork.LibraryRepository.LibraryExists(createLibraryDto.Name))
if (await _unitOfWork.LibraryRepository.LibraryExists(dto.Name))
{
return BadRequest("Library name already exists. Please choose a unique name to the server.");
}
var library = new Library
{
Name = createLibraryDto.Name,
Type = createLibraryDto.Type,
Folders = createLibraryDto.Folders.Select(x => new FolderPath {Path = x}).ToList()
Name = dto.Name,
Type = dto.Type,
Folders = dto.Folders.Select(x => new FolderPath {Path = x}).Distinct().ToList(),
FolderWatching = dto.FolderWatching,
IncludeInDashboard = dto.IncludeInDashboard,
IncludeInRecommended = dto.IncludeInRecommended,
IncludeInSearch = dto.IncludeInSearch,
ManageCollections = dto.ManageCollections,
ManageReadingLists = dto.ManageReadingLists,
};
_unitOfWork.LibraryRepository.Add(library);
@ -335,9 +341,8 @@ public class LibraryController : BaseApiController
[HttpGet("name-exists")]
public async Task<ActionResult<bool>> IsLibraryNameValid(string name)
{
var trimmed = name.Trim();
if (string.IsNullOrEmpty(trimmed)) return Ok(true);
return Ok(await _unitOfWork.LibraryRepository.LibraryExists(trimmed));
if (string.IsNullOrWhiteSpace(name)) return Ok(true);
return Ok(await _unitOfWork.LibraryRepository.LibraryExists(name.Trim()));
}
/// <summary>
@ -360,7 +365,7 @@ public class LibraryController : BaseApiController
var originalFolders = library.Folders.Select(x => x.Path).ToList();
library.Name = newName;
library.Folders = dto.Folders.Select(s => new FolderPath() {Path = s}).ToList();
library.Folders = dto.Folders.Select(s => new FolderPath() {Path = s}).Distinct().ToList();
var typeUpdate = library.Type != dto.Type;
var folderWatchingUpdate = library.FolderWatching != dto.FolderWatching;

12
API/Controllers/MetadataController.cs
View file

@ -35,7 +35,7 @@ public class MetadataController : BaseApiController
public async Task<ActionResult<IList<GenreTagDto>>> GetAllGenres(string? libraryIds)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
var ids = libraryIds?.Split(",").Select(int.Parse).ToList();
var ids = libraryIds?.Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
if (ids != null && ids.Count > 0)
{
return Ok(await _unitOfWork.GenreRepository.GetAllGenreDtosForLibrariesAsync(ids, userId));
@ -56,7 +56,7 @@ public class MetadataController : BaseApiController
public async Task<ActionResult<IList<PersonDto>>> GetAllPeople(string? libraryIds)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
var ids = libraryIds?.Split(",").Select(int.Parse).ToList();
var ids = libraryIds?.Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
if (ids != null && ids.Count > 0)
{
return Ok(await _unitOfWork.PersonRepository.GetAllPeopleDtosForLibrariesAsync(ids, userId));
@ -74,7 +74,7 @@ public class MetadataController : BaseApiController
public async Task<ActionResult<IList<TagDto>>> GetAllTags(string? libraryIds)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
var ids = libraryIds?.Split(",").Select(int.Parse).ToList();
var ids = libraryIds?.Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
if (ids != null && ids.Count > 0)
{
return Ok(await _unitOfWork.TagRepository.GetAllTagDtosForLibrariesAsync(ids, userId));
@ -92,7 +92,7 @@ public class MetadataController : BaseApiController
[HttpGet("age-ratings")]
public async Task<ActionResult<IList<AgeRatingDto>>> GetAllAgeRatings(string? libraryIds)
{
var ids = libraryIds?.Split(",").Select(int.Parse).ToList();
var ids = libraryIds?.Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
if (ids != null && ids.Count > 0)
{
return Ok(await _unitOfWork.LibraryRepository.GetAllAgeRatingsDtosForLibrariesAsync(ids));
@ -115,7 +115,7 @@ public class MetadataController : BaseApiController
[HttpGet("publication-status")]
public ActionResult<IList<AgeRatingDto>> GetAllPublicationStatus(string? libraryIds)
{
var ids = libraryIds?.Split(",").Select(int.Parse).ToList();
var ids = libraryIds?.Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
if (ids is {Count: > 0})
{
return Ok(_unitOfWork.LibraryRepository.GetAllPublicationStatusesDtosForLibrariesAsync(ids));
@ -138,7 +138,7 @@ public class MetadataController : BaseApiController
[ResponseCache(CacheProfileName = ResponseCacheProfiles.Instant, VaryByQueryKeys = new []{"libraryIds"})]
public async Task<ActionResult<IList<LanguageDto>>> GetAllLanguages(string? libraryIds)
{
var ids = libraryIds?.Split(",").Select(int.Parse).ToList();
var ids = libraryIds?.Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries).Select(int.Parse).ToList();
if (ids is {Count: > 0})
{
return Ok(await _unitOfWork.LibraryRepository.GetAllLanguagesForLibrariesAsync(ids));

12
API/Controllers/ReaderController.cs
View file

@ -62,7 +62,7 @@ public class ReaderController : BaseApiController
{
if (await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey) == 0) return BadRequest();
var chapter = await _cacheService.Ensure(chapterId);
if (chapter == null) return BadRequest("There was an issue finding pdf file for reading");
if (chapter == null) return NoContent();
// Validate the user has access to the PDF
var series = await _unitOfWork.SeriesRepository.GetSeriesForChapter(chapter.Id,
@ -101,7 +101,7 @@ public class ReaderController : BaseApiController
if (page < 0) page = 0;
if (await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey) == 0) return BadRequest();
var chapter = await _cacheService.Ensure(chapterId, extractPdf);
if (chapter == null) return BadRequest("There was an issue finding image file for reading");
if (chapter == null) return NoContent();
try
{
@ -125,7 +125,7 @@ public class ReaderController : BaseApiController
{
if (await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey) == 0) return BadRequest();
var chapter = await _cacheService.Ensure(chapterId, true);
if (chapter == null) return BadRequest("There was an issue extracting images from chapter");
if (chapter == null) return NoContent();
var images = _cacheService.GetCachedPages(chapterId);
var path = await _readerService.GetThumbnail(chapter, pageNum, images);
@ -148,7 +148,7 @@ public class ReaderController : BaseApiController
{
if (page < 0) page = 0;
var userId = await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey);
if (userId == 0) return BadRequest();
if (userId == 0) return Unauthorized();
var totalPages = await _cacheService.CacheBookmarkForSeries(userId, seriesId);
if (page > totalPages)
@ -185,7 +185,7 @@ public class ReaderController : BaseApiController
{
if (chapterId <= 0) return ArraySegment<FileDimensionDto>.Empty;
var chapter = await _cacheService.Ensure(chapterId, extractPdf);
if (chapter == null) return BadRequest("Could not find Chapter");
if (chapter == null) return NoContent();
return Ok(_cacheService.GetCachedFileDimensions(_cacheService.GetCachePath(chapterId)));
}
@ -203,7 +203,7 @@ public class ReaderController : BaseApiController
{
if (chapterId <= 0) return Ok(null); // This can happen occasionally from UI, we should just ignore
var chapter = await _cacheService.Ensure(chapterId, extractPdf);
if (chapter == null) return BadRequest("Could not find Chapter");
if (chapter == null) return NoContent();
var dto = await _unitOfWork.ChapterRepository.GetChapterInfoDtoAsync(chapterId);
if (dto == null) return BadRequest("Please perform a scan on this series or library and try again");

18
API/Controllers/SeriesController.cs
View file

@ -64,16 +64,9 @@ public class SeriesController : BaseApiController
public async Task<ActionResult<SeriesDto>> GetSeries(int seriesId)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
try
{
return Ok(await _unitOfWork.SeriesRepository.GetSeriesDtoByIdAsync(seriesId, userId));
}
catch (Exception e)
{
_logger.LogError(e, "There was an issue fetching {SeriesId}", seriesId);
throw new KavitaException("This series does not exist");
}
var series = await _unitOfWork.SeriesRepository.GetSeriesDtoByIdAsync(seriesId, userId);
if (series == null) return NoContent();
return Ok(series);
}
[Authorize(Policy = "RequireAdminRole")]
@ -114,13 +107,16 @@ public class SeriesController : BaseApiController
public async Task<ActionResult<VolumeDto?>> GetVolume(int volumeId)
{
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
return Ok(await _unitOfWork.VolumeRepository.GetVolumeDtoAsync(volumeId, userId));
var vol = await _unitOfWork.VolumeRepository.GetVolumeDtoAsync(volumeId, userId);
if (vol == null) return NoContent();
return Ok(vol);
}
[HttpGet("chapter")]
public async Task<ActionResult<ChapterDto>> GetChapter(int chapterId)
{
var chapter = await _unitOfWork.ChapterRepository.GetChapterDtoAsync(chapterId);
if (chapter == null) return NoContent();
return Ok(await _unitOfWork.ChapterRepository.AddChapterModifiers(User.GetUserId(), chapter));
}

68
API/Controllers/ServerController.cs
View file

@ -3,10 +3,14 @@ using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Jobs;
using API.DTOs.MediaErrors;
using API.DTOs.Stats;
using API.DTOs.Update;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers;
using API.Services;
using API.Services.Tasks;
using Hangfire;
@ -14,7 +18,6 @@ using Hangfire.Storage;
using Kavita.Common;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using TaskScheduler = API.Services.TaskScheduler;
@ -23,7 +26,6 @@ namespace API.Controllers;
[Authorize(Policy = "RequireAdminRole")]
public class ServerController : BaseApiController
{
private readonly IHostApplicationLifetime _applicationLifetime;
private readonly ILogger<ServerController> _logger;
private readonly IBackupService _backupService;
private readonly IArchiveService _archiveService;
@ -34,13 +36,13 @@ public class ServerController : BaseApiController
private readonly IScannerService _scannerService;
private readonly IAccountService _accountService;
private readonly ITaskScheduler _taskScheduler;
private readonly IUnitOfWork _unitOfWork;
public ServerController(IHostApplicationLifetime applicationLifetime, ILogger<ServerController> logger,
public ServerController(ILogger<ServerController> logger,
IBackupService backupService, IArchiveService archiveService, IVersionUpdaterService versionUpdaterService, IStatsService statsService,
ICleanupService cleanupService, IBookmarkService bookmarkService, IScannerService scannerService, IAccountService accountService,
ITaskScheduler taskScheduler)
ITaskScheduler taskScheduler, IUnitOfWork unitOfWork)
{
_applicationLifetime = applicationLifetime;
_logger = logger;
_backupService = backupService;
_archiveService = archiveService;
@ -51,6 +53,7 @@ public class ServerController : BaseApiController
_scannerService = scannerService;
_accountService = accountService;
_taskScheduler = taskScheduler;
_unitOfWork = unitOfWork;
}
/// <summary>
@ -117,29 +120,22 @@ public class ServerController : BaseApiController
return Ok(await _statsService.GetServerInfo());
}
/// <summary>
/// Triggers the scheduling of the convert bookmarks job. Only one job will run at a time.
/// </summary>
/// <returns></returns>
[HttpPost("convert-bookmarks")]
public ActionResult ScheduleConvertBookmarks()
{
if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllBookmarkToWebP", Array.Empty<object>(),
TaskScheduler.DefaultQueue, true)) return Ok();
BackgroundJob.Enqueue(() => _bookmarkService.ConvertAllBookmarkToWebP());
return Ok();
}
/// <summary>
/// Triggers the scheduling of the convert covers job. Only one job will run at a time.
/// Triggers the scheduling of the convert media job. This will convert all media to the target encoding (except for PNG). Only one job will run at a time.
/// </summary>
/// <returns></returns>
[HttpPost("convert-covers")]
public ActionResult ScheduleConvertCovers()
[HttpPost("convert-media")]
public async Task<ActionResult> ScheduleConvertCovers()
{
if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllCoverToWebP", Array.Empty<object>(),
TaskScheduler.DefaultQueue, true)) return Ok();
BackgroundJob.Enqueue(() => _taskScheduler.CovertAllCoversToWebP());
var encoding = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
if (encoding == EncodeFormat.PNG)
{
return BadRequest(
"You cannot convert to PNG. For covers, use Refresh Covers. Bookmarks and favicons cannot be encoded back.");
}
BackgroundJob.Enqueue(() => _taskScheduler.CovertAllCoversToEncoding());
return Ok();
}
@ -154,7 +150,8 @@ public class ServerController : BaseApiController
try
{
var zipPath = _archiveService.CreateZipForDownload(files, "logs");
return PhysicalFile(zipPath, "application/zip", Path.GetFileName(zipPath), true);
return PhysicalFile(zipPath, "application/zip",
System.Web.HttpUtility.UrlEncode(Path.GetFileName(zipPath)), true);
}
catch (KavitaException ex)
{
@ -213,5 +210,28 @@ public class ServerController : BaseApiController
return Ok(recurringJobs);
}
/// <summary>
/// Returns a list of issues found during scanning or reading in which files may have corruption or bad metadata (structural metadata)
/// </summary>
/// <returns></returns>
[Authorize("RequireAdminRole")]
[HttpGet("media-errors")]
public ActionResult<PagedList<MediaErrorDto>> GetMediaErrors()
{
return Ok(_unitOfWork.MediaErrorRepository.GetAllErrorDtosAsync());
}
/// <summary>
/// Deletes all media errors
/// </summary>
/// <returns></returns>
[Authorize("RequireAdminRole")]
[HttpPost("clear-media-alerts")]
public async Task<ActionResult> ClearMediaErrors()
{
await _unitOfWork.MediaErrorRepository.DeleteAll();
return Ok();
}
}

15
API/Controllers/SettingsController.cs
View file

@ -15,6 +15,7 @@ using API.Services.Tasks.Scanner;
using AutoMapper;
using Flurl.Http;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Kavita.Common.Extensions;
using Kavita.Common.Helpers;
using Microsoft.AspNetCore.Authorization;
@ -183,6 +184,7 @@ public class SettingsController : BaseApiController
if (setting.Key == ServerSettingKey.Port && updateSettingsDto.Port + string.Empty != setting.Value)
{
if (OsInfo.IsDocker) continue;
setting.Value = updateSettingsDto.Port + string.Empty;
// Port is managed in appSetting.json
Configuration.Port = updateSettingsDto.Port;
@ -191,8 +193,9 @@ public class SettingsController : BaseApiController
if (setting.Key == ServerSettingKey.IpAddresses && updateSettingsDto.IpAddresses != setting.Value)
{
if (OsInfo.IsDocker) continue;
// Validate IP addresses
foreach (var ipAddress in updateSettingsDto.IpAddresses.Split(','))
foreach (var ipAddress in updateSettingsDto.IpAddresses.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
if (!IPAddress.TryParse(ipAddress.Trim(), out _)) {
return BadRequest($"IP Address '{ipAddress}' is invalid");
@ -231,15 +234,9 @@ public class SettingsController : BaseApiController
_unitOfWork.SettingsRepository.Update(setting);
}
if (setting.Key == ServerSettingKey.ConvertBookmarkToWebP && updateSettingsDto.ConvertBookmarkToWebP + string.Empty != setting.Value)
if (setting.Key == ServerSettingKey.EncodeMediaAs && updateSettingsDto.EncodeMediaAs + string.Empty != setting.Value)
{
setting.Value = updateSettingsDto.ConvertBookmarkToWebP + string.Empty;
_unitOfWork.SettingsRepository.Update(setting);
}
if (setting.Key == ServerSettingKey.ConvertCoverToWebP && updateSettingsDto.ConvertCoverToWebP + string.Empty != setting.Value)
{
setting.Value = updateSettingsDto.ConvertCoverToWebP + string.Empty;
setting.Value = updateSettingsDto.EncodeMediaAs + string.Empty;
_unitOfWork.SettingsRepository.Update(setting);
}

17
API/Controllers/UploadController.cs
View file

@ -1,5 +1,6 @@
using System;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
using API.DTOs.Uploads;
using API.Extensions;
@ -78,7 +79,7 @@ public class UploadController : BaseApiController
/// <param name="uploadFileDto"></param>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[RequestSizeLimit(8_000_000)]
[RequestSizeLimit(ControllerConstants.MaxUploadSizeBytes)]
[HttpPost("series")]
public async Task<ActionResult> UploadSeriesCoverImageFromUrl(UploadFileDto uploadFileDto)
{
@ -126,7 +127,7 @@ public class UploadController : BaseApiController
/// <param name="uploadFileDto"></param>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[RequestSizeLimit(8_000_000)]
[RequestSizeLimit(ControllerConstants.MaxUploadSizeBytes)]
[HttpPost("collection")]
public async Task<ActionResult> UploadCollectionCoverImageFromUrl(UploadFileDto uploadFileDto)
{
@ -174,7 +175,7 @@ public class UploadController : BaseApiController
/// <remarks>This is the only API that can be called by non-admins, but the authenticated user must have a readinglist permission</remarks>
/// <param name="uploadFileDto"></param>
/// <returns></returns>
[RequestSizeLimit(8_000_000)]
[RequestSizeLimit(ControllerConstants.MaxUploadSizeBytes)]
[HttpPost("reading-list")]
public async Task<ActionResult> UploadReadingListCoverImageFromUrl(UploadFileDto uploadFileDto)
{
@ -221,15 +222,15 @@ public class UploadController : BaseApiController
private async Task<string> CreateThumbnail(UploadFileDto uploadFileDto, string filename, int thumbnailSize = 0)
{
var convertToWebP = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP;
var encodeFormat = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
if (thumbnailSize > 0)
{
return _imageService.CreateThumbnailFromBase64(uploadFileDto.Url,
filename, convertToWebP, thumbnailSize);
filename, encodeFormat, thumbnailSize);
}
return _imageService.CreateThumbnailFromBase64(uploadFileDto.Url,
filename, convertToWebP);
filename, encodeFormat);
}
/// <summary>
@ -238,7 +239,7 @@ public class UploadController : BaseApiController
/// <param name="uploadFileDto"></param>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[RequestSizeLimit(8_000_000)]
[RequestSizeLimit(ControllerConstants.MaxUploadSizeBytes)]
[HttpPost("chapter")]
public async Task<ActionResult> UploadChapterCoverImageFromUrl(UploadFileDto uploadFileDto)
{
@ -294,7 +295,7 @@ public class UploadController : BaseApiController
/// <param name="uploadFileDto"></param>
/// <returns></returns>
[Authorize(Policy = "RequireAdminRole")]
[RequestSizeLimit(8_000_000)]
[RequestSizeLimit(ControllerConstants.MaxUploadSizeBytes)]
[HttpPost("library")]
public async Task<ActionResult> UploadLibraryCoverImageFromUrl(UploadFileDto uploadFileDto)
{

3
API/Controllers/UsersController.cs
View file

@ -33,6 +33,9 @@ public class UsersController : BaseApiController
var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(username);
_unitOfWork.UserRepository.Delete(user);
//(TODO: After updating a role or removing a user, delete their token)
// await _userManager.RemoveAuthenticationTokenAsync(user, TokenOptions.DefaultProvider, RefreshTokenName);
if (await _unitOfWork.CommitAsync()) return Ok();
return BadRequest("Could not delete the user.");

9
API/DTOs/ChapterDto.cs
View file

@ -93,4 +93,13 @@ public class ChapterDto : IHasReadTimeEstimate, IEntityDate
public int MaxHoursToRead { get; set; }
/// <inheritdoc cref="IHasReadTimeEstimate.AvgHoursToRead"/>
public int AvgHoursToRead { get; set; }
/// <summary>
/// Comma-separated link of urls to external services that have some relation to the Chapter
/// </summary>
public string WebLinks { get; set; }
/// <summary>
/// ISBN-13 (usually) of the Chapter
/// </summary>
/// <remarks>This is guaranteed to be Valid</remarks>
public string ISBN { get; set; }
}

25
API/DTOs/MediaErrors/MediaErrorDto.cs Normal file
View file

@ -0,0 +1,25 @@
using System;
namespace API.DTOs.MediaErrors;
public class MediaErrorDto
{
/// <summary>
/// Format Type (RAR, ZIP, 7Zip, Epub, PDF)
/// </summary>
public required string Extension { get; set; }
/// <summary>
/// Full Filepath to the file that has some issue
/// </summary>
public required string FilePath { get; set; }
/// <summary>
/// Developer defined string
/// </summary>
public string Comment { get; set; }
/// <summary>
/// Exception message
/// </summary>
public string Details { get; set; }
public DateTime Created { get; set; }
public DateTime CreatedUtc { get; set; }
}

4
API/DTOs/SeriesMetadataDto.cs
View file

@ -58,6 +58,10 @@ public class SeriesMetadataDto
/// Publication status of the Series
/// </summary>
public PublicationStatus PublicationStatus { get; set; }
/// <summary>
/// A comma-separated list of Urls
/// </summary>
public string WebLinks { get; set; }
public bool LanguageLocked { get; set; }
public bool SummaryLocked { get; set; }

14
API/DTOs/Settings/ServerSettingDTO.cs
View file

@ -1,9 +1,11 @@
using API.Services;
using API.Entities.Enums;
using API.Services;
namespace API.DTOs.Settings;
public class ServerSettingDto
{
public string CacheDirectory { get; set; } = default!;
public string TaskScan { get; set; } = default!;
/// <summary>
@ -47,9 +49,11 @@ public class ServerSettingDto
/// </summary>
public string InstallId { get; set; } = default!;
/// <summary>
/// If the server should save bookmarks as WebP encoding
/// The format that should be used when saving media for Kavita
/// </summary>
public bool ConvertBookmarkToWebP { get; set; }
/// <example>This includes things like: Covers, Bookmarks, Favicons</example>
public EncodeFormat EncodeMediaAs { get; set; }
/// <summary>
/// The amount of Backups before cleanup
/// </summary>
@ -65,10 +69,6 @@ public class ServerSettingDto
/// <remarks>Value should be between 1 and 30</remarks>
public int TotalLogs { get; set; }
/// <summary>
/// If the server should save covers as WebP encoding
/// </summary>
public bool ConvertCoverToWebP { get; set; }
/// <summary>
/// The Host name (ie Reverse proxy domain name) for the server
/// </summary>
public string HostName { get; set; }

11
API/DTOs/Stats/ServerInfoDto.cs
View file

@ -85,11 +85,6 @@ public class ServerInfoDto
/// <remarks>Introduced in v0.5.4</remarks>
public int TotalPeople { get; set; }
/// <summary>
/// Is this instance storing bookmarks as WebP
/// </summary>
/// <remarks>Introduced in v0.5.4</remarks>
public bool StoreBookmarksAsWebP { get; set; }
/// <summary>
/// Number of users on this instance using Card Layout
/// </summary>
/// <remarks>Introduced in v0.5.4</remarks>
@ -175,8 +170,8 @@ public class ServerInfoDto
/// <remarks>Introduced in v0.7.0</remarks>
public long TotalReadingHours { get; set; }
/// <summary>
/// Is the Server saving covers as WebP
/// The encoding the server is using to save media
/// </summary>
/// <remarks>Added in v0.7.0</remarks>
public bool StoreCoversAsWebP { get; set; }
/// <remarks>Added in v0.7.3</remarks>
public EncodeFormat EncodeMediaAs { get; set; }
}

12
API/Data/DataContext.cs
View file

@ -47,6 +47,7 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
public DbSet<FolderPath> FolderPath { get; set; } = null!;
public DbSet<Device> Device { get; set; } = null!;
public DbSet<ServerStatistics> ServerStatistics { get; set; } = null!;
public DbSet<MediaError> MediaError { get; set; } = null!;
protected override void OnModelCreating(ModelBuilder builder)
@ -113,6 +114,17 @@ public sealed class DataContext : IdentityDbContext<AppUser, AppRole, int,
builder.Entity<Library>()
.Property(b => b.ManageReadingLists)
.HasDefaultValue(true);
builder.Entity<Chapter>()
.Property(b => b.WebLinks)
.HasDefaultValue(string.Empty);
builder.Entity<SeriesMetadata>()
.Property(b => b.WebLinks)
.HasDefaultValue(string.Empty);
builder.Entity<Chapter>()
.Property(b => b.ISBN)
.HasDefaultValue(string.Empty);
}

2
API/Data/MigrateBrokenGMT1Dates.cs → API/Data/ManualMigrations/MigrateBrokenGMT1Dates.cs
View file

@ -3,7 +3,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.7 introduced UTC dates and GMT+1 users would sometimes have dates stored as '0000-12-31 23:00:00'.

2
API/Data/MigrateChangePasswordRoles.cs → API/Data/ManualMigrations/MigrateChangePasswordRoles.cs
View file

@ -3,7 +3,7 @@ using API.Constants;
using API.Entities;
using Microsoft.AspNetCore.Identity;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// New role introduced in v0.5.1. Adds the role to all users.

2
API/Data/MigrateChangeRestrictionRoles.cs → API/Data/ManualMigrations/MigrateChangeRestrictionRoles.cs
View file

@ -4,7 +4,7 @@ using API.Entities;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// New role introduced in v0.6. Adds the role to all users.

2
API/Data/MigrateLoginRole.cs → API/Data/ManualMigrations/MigrateLoginRole.cs
View file

@ -4,7 +4,7 @@ using API.Entities;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// Added in v0.7.1.18

2
API/Data/MigrateNormalizedEverything.cs → API/Data/ManualMigrations/MigrateNormalizedEverything.cs
View file

@ -3,7 +3,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.6.0 introduced a change in how Normalization works and hence every normalized field needs to be re-calculated

2
API/Data/MigrateNormalizedLocalizedName.cs → API/Data/ManualMigrations/MigrateNormalizedLocalizedName.cs
View file

@ -3,7 +3,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// v0.5.6 introduced Normalized Localized Name, which allows for faster lookups and less memory usage. This migration will calculate them once

2
API/Data/MigrateReadingListAgeRating.cs → API/Data/ManualMigrations/MigrateReadingListAgeRating.cs
View file

@ -4,7 +4,7 @@ using API.Services;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// New role introduced in v0.6. Calculates the Age Rating on all Reading Lists

2
API/Data/MigrateRemoveExtraThemes.cs → API/Data/ManualMigrations/MigrateRemoveExtraThemes.cs
View file

@ -3,7 +3,7 @@ using System.Linq;
using System.Threading.Tasks;
using API.Services.Tasks;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// In v0.5.3, we removed Light and E-Ink themes. This migration will remove the themes from the DB and default anyone on

31
API/Data/ManualMigrations/MigrateRemoveWebPSettingRows.cs Normal file
View file

@ -0,0 +1,31 @@
using System.Threading.Tasks;
using API.Entities.Enums;
using Microsoft.Extensions.Logging;
namespace API.Data.ManualMigrations;
/// <summary>
/// Added in v0.7.2.7/v0.7.3 in which the ConvertXToWebP Setting keys were removed. This migration will remove them.
/// </summary>
public static class MigrateRemoveWebPSettingRows
{
public static async Task Migrate(IUnitOfWork unitOfWork, ILogger<Program> logger)
{
logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - Please be patient, this may take some time. This is not an error");
var key = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.ConvertBookmarkToWebP);
var key2 = await unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.ConvertCoverToWebP);
if (key == null && key2 == null)
{
logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - complete. Nothing to do");
return;
}
unitOfWork.SettingsRepository.Remove(key);
unitOfWork.SettingsRepository.Remove(key2);
await unitOfWork.CommitAsync();
logger.LogCritical("Running MigrateRemoveWebPSettingRows migration - Completed. This is not an error");
}
}

2
API/Data/MigrateSeriesRelationsExport.cs → API/Data/ManualMigrations/MigrateSeriesRelationsExport.cs
View file

@ -10,7 +10,7 @@ using Kavita.Common.EnvironmentInfo;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
internal sealed class SeriesRelationMigrationOutput
{

2
API/Data/MigrateSeriesRelationsImport.cs → API/Data/ManualMigrations/MigrateSeriesRelationsImport.cs
View file

@ -8,7 +8,7 @@ using CsvHelper;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.6.1.2 and v0.7, this imports to a temp file the existing series relationships. It is a 3 part migration.

2
API/Data/MigrateToUtcDates.cs → API/Data/ManualMigrations/MigrateToUtcDates.cs
View file

@ -3,7 +3,7 @@ using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.6.1.38 or v0.7.0,

2
API/Data/MigrateUserProgressLibraryId.cs → API/Data/ManualMigrations/MigrateUserProgressLibraryId.cs
View file

@ -1,7 +1,7 @@
using System.Threading.Tasks;
using Microsoft.Extensions.Logging;
namespace API.Data;
namespace API.Data.ManualMigrations;
/// <summary>
/// Introduced in v0.6.1.8 and v0.7, this adds library ids to all User Progress to allow for easier queries against progress

31
API/Data/Metadata/ComicInfo.cs
View file

@ -2,7 +2,9 @@
using System.Linq;
using API.Entities;
using API.Entities.Enums;
using API.Services;
using Kavita.Common.Extensions;
using Nager.ArticleNumber;
namespace API.Data.Metadata;
@ -35,9 +37,21 @@ public class ComicInfo
/// IETF BCP 47 Code to represent the language of the content
/// </summary>
public string LanguageISO { get; set; } = string.Empty;
// ReSharper disable once InconsistentNaming
/// <summary>
/// ISBN for the underlying document
/// </summary>
/// <remarks>ComicInfo.xml will actually output a GTIN (Global Trade Item Number) and it is the responsibility of the Parser to extract the ISBN. EPub will return ISBN.</remarks>
public string Isbn { get; set; } = string.Empty;
/// <summary>
/// This is only for deserialization and used within <see cref="ArchiveService"/>. Use <see cref="Isbn"/> for the actual value.
/// </summary>
public string GTIN { get; set; } = string.Empty;
/// <summary>
/// This is the link to where the data was scraped from
/// </summary>
/// <remarks>This can be comma-separated</remarks>
public string Web { get; set; } = string.Empty;
[System.ComponentModel.DefaultValueAttribute(0)]
public int Day { get; set; } = 0;
@ -137,6 +151,23 @@ public class ComicInfo
info.Characters = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Characters);
info.Translator = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.Translator);
info.CoverArtist = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.CoverArtist);
// We need to convert GTIN to ISBN
if (!string.IsNullOrEmpty(info.GTIN))
{
// This is likely a valid ISBN
if (info.GTIN[0] == '0')
{
var potentialISBN = info.GTIN.Substring(1, info.GTIN.Length - 1);
if (ArticleNumberHelper.IsValidIsbn13(potentialISBN))
{
info.Isbn = potentialISBN;
}
} else if (ArticleNumberHelper.IsValidIsbn10(info.GTIN) || ArticleNumberHelper.IsValidIsbn13(info.GTIN))
{
info.Isbn = info.GTIN;
}
}
}
/// <summary>

1912
API/Data/Migrations/20230505124430_MediaError.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

42
API/Data/Migrations/20230505124430_MediaError.cs Normal file
View file

@ -0,0 +1,42 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class MediaError : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.CreateTable(
name: "MediaError",
columns: table => new
{
Id = table.Column<int>(type: "INTEGER", nullable: false)
.Annotation("Sqlite:Autoincrement", true),
Extension = table.Column<string>(type: "TEXT", nullable: true),
FilePath = table.Column<string>(type: "TEXT", nullable: true),
Comment = table.Column<string>(type: "TEXT", nullable: true),
Details = table.Column<string>(type: "TEXT", nullable: true),
Created = table.Column<DateTime>(type: "TEXT", nullable: false),
LastModified = table.Column<DateTime>(type: "TEXT", nullable: false),
CreatedUtc = table.Column<DateTime>(type: "TEXT", nullable: false),
LastModifiedUtc = table.Column<DateTime>(type: "TEXT", nullable: false)
},
constraints: table =>
{
table.PrimaryKey("PK_MediaError", x => x.Id);
});
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropTable(
name: "MediaError");
}
}
}

1917
API/Data/Migrations/20230511165427_WebLinksForChapter.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

29
API/Data/Migrations/20230511165427_WebLinksForChapter.cs Normal file
View file

@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class WebLinksForChapter : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "WebLinks",
table: "Chapter",
type: "TEXT",
nullable: true,
defaultValue: string.Empty);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "WebLinks",
table: "Chapter");
}
}
}

1922
API/Data/Migrations/20230511183339_WebLinksForSeries.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

29
API/Data/Migrations/20230511183339_WebLinksForSeries.cs Normal file
View file

@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class WebLinksForSeries : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "WebLinks",
table: "SeriesMetadata",
type: "TEXT",
nullable: true,
defaultValue: string.Empty);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "WebLinks",
table: "SeriesMetadata");
}
}
}

1927
API/Data/Migrations/20230512004545_ChapterISBN.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

29
API/Data/Migrations/20230512004545_ChapterISBN.cs Normal file
View file

@ -0,0 +1,29 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class ChapterISBN : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "ISBN",
table: "Chapter",
type: "TEXT",
nullable: true,
defaultValue: string.Empty);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ISBN",
table: "Chapter");
}
}
}

110
API/Data/Migrations/DataContextModelSnapshot.cs
View file

@ -177,7 +177,7 @@ namespace API.Data.Migrations
b.HasIndex("AppUserId");
b.ToTable("AppUserBookmark");
b.ToTable("AppUserBookmark", (string)null);
});
modelBuilder.Entity("API.Entities.AppUserPreferences", b =>
@ -282,7 +282,7 @@ namespace API.Data.Migrations
b.HasIndex("ThemeId");
b.ToTable("AppUserPreferences");
b.ToTable("AppUserPreferences", (string)null);
});
modelBuilder.Entity("API.Entities.AppUserProgress", b =>
@ -332,7 +332,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesId");
b.ToTable("AppUserProgresses");
b.ToTable("AppUserProgresses", (string)null);
});
modelBuilder.Entity("API.Entities.AppUserRating", b =>
@ -359,7 +359,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesId");
b.ToTable("AppUserRating");
b.ToTable("AppUserRating", (string)null);
});
modelBuilder.Entity("API.Entities.AppUserRole", b =>
@ -413,6 +413,11 @@ namespace API.Data.Migrations
b.Property<DateTime>("CreatedUtc")
.HasColumnType("TEXT");
b.Property<string>("ISBN")
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("");
b.Property<bool>("IsSpecial")
.HasColumnType("INTEGER");
@ -467,6 +472,11 @@ namespace API.Data.Migrations
b.Property<int>("VolumeId")
.HasColumnType("INTEGER");
b.Property<string>("WebLinks")
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("");
b.Property<long>("WordCount")
.HasColumnType("INTEGER");
@ -474,7 +484,7 @@ namespace API.Data.Migrations
b.HasIndex("VolumeId");
b.ToTable("Chapter");
b.ToTable("Chapter", (string)null);
});
modelBuilder.Entity("API.Entities.CollectionTag", b =>
@ -509,7 +519,7 @@ namespace API.Data.Migrations
b.HasIndex("Id", "Promoted")
.IsUnique();
b.ToTable("CollectionTag");
b.ToTable("CollectionTag", (string)null);
});
modelBuilder.Entity("API.Entities.Device", b =>
@ -555,7 +565,7 @@ namespace API.Data.Migrations
b.HasIndex("AppUserId");
b.ToTable("Device");
b.ToTable("Device", (string)null);
});
modelBuilder.Entity("API.Entities.FolderPath", b =>
@ -577,7 +587,7 @@ namespace API.Data.Migrations
b.HasIndex("LibraryId");
b.ToTable("FolderPath");
b.ToTable("FolderPath", (string)null);
});
modelBuilder.Entity("API.Entities.Genre", b =>
@ -597,7 +607,7 @@ namespace API.Data.Migrations
b.HasIndex("NormalizedTitle")
.IsUnique();
b.ToTable("Genre");
b.ToTable("Genre", (string)null);
});
modelBuilder.Entity("API.Entities.Library", b =>
@ -662,7 +672,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.ToTable("Library");
b.ToTable("Library", (string)null);
});
modelBuilder.Entity("API.Entities.MangaFile", b =>
@ -711,7 +721,42 @@ namespace API.Data.Migrations
b.HasIndex("ChapterId");
b.ToTable("MangaFile");
b.ToTable("MangaFile", (string)null);
});
modelBuilder.Entity("API.Entities.MediaError", b =>
{
b.Property<int>("Id")
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<string>("Comment")
.HasColumnType("TEXT");
b.Property<DateTime>("Created")
.HasColumnType("TEXT");
b.Property<DateTime>("CreatedUtc")
.HasColumnType("TEXT");
b.Property<string>("Details")
.HasColumnType("TEXT");
b.Property<string>("Extension")
.HasColumnType("TEXT");
b.Property<string>("FilePath")
.HasColumnType("TEXT");
b.Property<DateTime>("LastModified")
.HasColumnType("TEXT");
b.Property<DateTime>("LastModifiedUtc")
.HasColumnType("TEXT");
b.HasKey("Id");
b.ToTable("MediaError", (string)null);
});
modelBuilder.Entity("API.Entities.Metadata.SeriesMetadata", b =>
@ -796,6 +841,11 @@ namespace API.Data.Migrations
b.Property<bool>("TranslatorLocked")
.HasColumnType("INTEGER");
b.Property<string>("WebLinks")
.ValueGeneratedOnAdd()
.HasColumnType("TEXT")
.HasDefaultValue("");
b.Property<bool>("WriterLocked")
.HasColumnType("INTEGER");
@ -807,7 +857,7 @@ namespace API.Data.Migrations
b.HasIndex("Id", "SeriesId")
.IsUnique();
b.ToTable("SeriesMetadata");
b.ToTable("SeriesMetadata", (string)null);
});
modelBuilder.Entity("API.Entities.Metadata.SeriesRelation", b =>
@ -831,7 +881,7 @@ namespace API.Data.Migrations
b.HasIndex("TargetSeriesId");
b.ToTable("SeriesRelation");
b.ToTable("SeriesRelation", (string)null);
});
modelBuilder.Entity("API.Entities.Person", b =>
@ -851,7 +901,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.ToTable("Person");
b.ToTable("Person", (string)null);
});
modelBuilder.Entity("API.Entities.ReadingList", b =>
@ -912,7 +962,7 @@ namespace API.Data.Migrations
b.HasIndex("AppUserId");
b.ToTable("ReadingList");
b.ToTable("ReadingList", (string)null);
});
modelBuilder.Entity("API.Entities.ReadingListItem", b =>
@ -946,7 +996,7 @@ namespace API.Data.Migrations
b.HasIndex("VolumeId");
b.ToTable("ReadingListItem");
b.ToTable("ReadingListItem", (string)null);
});
modelBuilder.Entity("API.Entities.Series", b =>
@ -1045,7 +1095,7 @@ namespace API.Data.Migrations
b.HasIndex("LibraryId");
b.ToTable("Series");
b.ToTable("Series", (string)null);
});
modelBuilder.Entity("API.Entities.ServerSetting", b =>
@ -1062,7 +1112,7 @@ namespace API.Data.Migrations
b.HasKey("Key");
b.ToTable("ServerSetting");
b.ToTable("ServerSetting", (string)null);
});
modelBuilder.Entity("API.Entities.ServerStatistics", b =>
@ -1100,7 +1150,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.ToTable("ServerStatistics");
b.ToTable("ServerStatistics", (string)null);
});
modelBuilder.Entity("API.Entities.SiteTheme", b =>
@ -1138,7 +1188,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.ToTable("SiteTheme");
b.ToTable("SiteTheme", (string)null);
});
modelBuilder.Entity("API.Entities.Tag", b =>
@ -1158,7 +1208,7 @@ namespace API.Data.Migrations
b.HasIndex("NormalizedTitle")
.IsUnique();
b.ToTable("Tag");
b.ToTable("Tag", (string)null);
});
modelBuilder.Entity("API.Entities.Volume", b =>
@ -1210,7 +1260,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesId");
b.ToTable("Volume");
b.ToTable("Volume", (string)null);
});
modelBuilder.Entity("AppUserLibrary", b =>
@ -1225,7 +1275,7 @@ namespace API.Data.Migrations
b.HasIndex("LibrariesId");
b.ToTable("AppUserLibrary");
b.ToTable("AppUserLibrary", (string)null);
});
modelBuilder.Entity("ChapterGenre", b =>
@ -1240,7 +1290,7 @@ namespace API.Data.Migrations
b.HasIndex("GenresId");
b.ToTable("ChapterGenre");
b.ToTable("ChapterGenre", (string)null);
});
modelBuilder.Entity("ChapterPerson", b =>
@ -1255,7 +1305,7 @@ namespace API.Data.Migrations
b.HasIndex("PeopleId");
b.ToTable("ChapterPerson");
b.ToTable("ChapterPerson", (string)null);
});
modelBuilder.Entity("ChapterTag", b =>
@ -1270,7 +1320,7 @@ namespace API.Data.Migrations
b.HasIndex("TagsId");
b.ToTable("ChapterTag");
b.ToTable("ChapterTag", (string)null);
});
modelBuilder.Entity("CollectionTagSeriesMetadata", b =>
@ -1285,7 +1335,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesMetadatasId");
b.ToTable("CollectionTagSeriesMetadata");
b.ToTable("CollectionTagSeriesMetadata", (string)null);
});
modelBuilder.Entity("GenreSeriesMetadata", b =>
@ -1300,7 +1350,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesMetadatasId");
b.ToTable("GenreSeriesMetadata");
b.ToTable("GenreSeriesMetadata", (string)null);
});
modelBuilder.Entity("Microsoft.AspNetCore.Identity.IdentityRoleClaim<int>", b =>
@ -1399,7 +1449,7 @@ namespace API.Data.Migrations
b.HasIndex("SeriesMetadatasId");
b.ToTable("PersonSeriesMetadata");
b.ToTable("PersonSeriesMetadata", (string)null);
});
modelBuilder.Entity("SeriesMetadataTag", b =>
@ -1414,7 +1464,7 @@ namespace API.Data.Migrations
b.HasIndex("TagsId");
b.ToTable("SeriesMetadataTag");
b.ToTable("SeriesMetadataTag", (string)null);
});
modelBuilder.Entity("API.Entities.AppUserBookmark", b =>

9
API/Data/Repositories/AppUserProgressRepository.cs
View file

@ -1,6 +1,7 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using API.Data.ManualMigrations;
using API.DTOs;
using API.Entities;
using API.Entities.Enums;
@ -25,6 +26,7 @@ public interface IAppUserProgressRepository
Task<IEnumerable<AppUserProgress>> GetUserProgressForSeriesAsync(int seriesId, int userId);
Task<IEnumerable<AppUserProgress>> GetAllProgress();
Task<ProgressDto> GetUserProgressDtoAsync(int chapterId, int userId);
Task<bool> AnyUserProgressForSeriesAsync(int seriesId, int userId);
}
public class AppUserProgressRepository : IAppUserProgressRepository
@ -128,6 +130,13 @@ public class AppUserProgressRepository : IAppUserProgressRepository
.FirstOrDefaultAsync();
}
public async Task<bool> AnyUserProgressForSeriesAsync(int seriesId, int userId)
{
return await _context.AppUserProgresses
.Where(p => p.SeriesId == seriesId && p.AppUserId == userId && p.PagesRead > 0)
.AnyAsync();
}
public async Task<AppUserProgress?> GetUserProgressAsync(int chapterId, int userId)
{
return await _context.AppUserProgresses

8
API/Data/Repositories/ChapterRepository.cs
View file

@ -6,6 +6,7 @@ using API.DTOs;
using API.DTOs.Metadata;
using API.DTOs.Reader;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Extensions.QueryExtensions;
using AutoMapper;
@ -36,7 +37,7 @@ public interface IChapterRepository
Task<IList<MangaFile>> GetFilesForChaptersAsync(IReadOnlyList<int> chapterIds);
Task<string?> GetChapterCoverImageAsync(int chapterId);
Task<IList<string>> GetAllCoverImagesAsync();
Task<IList<Chapter>> GetAllChaptersWithNonWebPCovers();
Task<IList<Chapter>> GetAllChaptersWithCoversInDifferentEncoding(EncodeFormat format);
Task<IEnumerable<string>> GetCoverImagesForLockedChaptersAsync();
Task<ChapterDto> AddChapterModifiers(int userId, ChapterDto chapter);
}
@ -208,10 +209,11 @@ public class ChapterRepository : IChapterRepository
.ToListAsync())!;
}
public async Task<IList<Chapter>> GetAllChaptersWithNonWebPCovers()
public async Task<IList<Chapter>> GetAllChaptersWithCoversInDifferentEncoding(EncodeFormat format)
{
var extension = format.GetExtension();
return await _context.Chapter
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp"))
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension))
.ToListAsync();
}

8
API/Data/Repositories/CollectionTagRepository.cs
View file

@ -5,6 +5,7 @@ using System.Threading.Tasks;
using API.Data.Misc;
using API.DTOs.CollectionTags;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Extensions.QueryExtensions;
using AutoMapper;
@ -34,7 +35,7 @@ public interface ICollectionTagRepository
Task<IEnumerable<CollectionTag>> GetAllTagsAsync(CollectionTagIncludes includes = CollectionTagIncludes.None);
Task<IList<string>> GetAllCoverImagesAsync();
Task<bool> TagExists(string title);
Task<IList<CollectionTag>> GetAllWithNonWebPCovers();
Task<IList<CollectionTag>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat);
}
public class CollectionTagRepository : ICollectionTagRepository
{
@ -108,10 +109,11 @@ public class CollectionTagRepository : ICollectionTagRepository
.AnyAsync(x => x.NormalizedTitle != null && x.NormalizedTitle.Equals(normalized));
}
public async Task<IList<CollectionTag>> GetAllWithNonWebPCovers()
public async Task<IList<CollectionTag>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat)
{
var extension = encodeFormat.GetExtension();
return await _context.CollectionTag
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp"))
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension))
.ToListAsync();
}

12
API/Data/Repositories/LibraryRepository.cs
View file

@ -52,7 +52,7 @@ public interface ILibraryRepository
Task<string?> GetLibraryCoverImageAsync(int libraryId);
Task<IList<string>> GetAllCoverImagesAsync();
Task<IDictionary<int, LibraryType>> GetLibraryTypesForIdsAsync(IEnumerable<int> libraryIds);
Task<IList<Library>> GetAllWithNonWebPCovers();
Task<IList<Library>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat);
}
public class LibraryRepository : ILibraryRepository
@ -170,10 +170,7 @@ public class LibraryRepository : ILibraryRepository
var c = sortChar;
var isAlpha = char.IsLetter(sortChar);
if (!isAlpha) c = '#';
if (!firstCharacterMap.ContainsKey(c))
{
firstCharacterMap[c] = 0;
}
firstCharacterMap.TryAdd(c, 0);
firstCharacterMap[c] += 1;
}
@ -371,10 +368,11 @@ public class LibraryRepository : ILibraryRepository
return dict;
}
public async Task<IList<Library>> GetAllWithNonWebPCovers()
public async Task<IList<Library>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat)
{
var extension = encodeFormat.GetExtension();
return await _context.Library
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp"))
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension))
.ToListAsync();
}
}

83
API/Data/Repositories/MediaErrorRepository.cs Normal file
View file

@ -0,0 +1,83 @@
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using API.DTOs.MediaErrors;
using API.Entities;
using API.Helpers;
using AutoMapper;
using AutoMapper.QueryableExtensions;
using Microsoft.EntityFrameworkCore;
namespace API.Data.Repositories;
public interface IMediaErrorRepository
{
void Attach(MediaError error);
void Remove(MediaError error);
Task<MediaError> Find(string filename);
Task<PagedList<MediaErrorDto>> GetAllErrorDtosAsync(UserParams userParams);
IEnumerable<MediaErrorDto> GetAllErrorDtosAsync();
Task<bool> ExistsAsync(MediaError error);
Task DeleteAll();
}
public class MediaErrorRepository : IMediaErrorRepository
{
private readonly DataContext _context;
private readonly IMapper _mapper;
public MediaErrorRepository(DataContext context, IMapper mapper)
{
_context = context;
_mapper = mapper;
}
public void Attach(MediaError? error)
{
if (error == null) return;
_context.MediaError.Attach(error);
}
public void Remove(MediaError? error)
{
if (error == null) return;
_context.MediaError.Remove(error);
}
public Task<MediaError?> Find(string filename)
{
return _context.MediaError.Where(e => e.FilePath == filename).SingleOrDefaultAsync();
}
public Task<PagedList<MediaErrorDto>> GetAllErrorDtosAsync(UserParams userParams)
{
var query = _context.MediaError
.OrderByDescending(m => m.Created)
.ProjectTo<MediaErrorDto>(_mapper.ConfigurationProvider)
.AsNoTracking();
return PagedList<MediaErrorDto>.CreateAsync(query, userParams.PageNumber, userParams.PageSize);
}
public IEnumerable<MediaErrorDto> GetAllErrorDtosAsync()
{
var query = _context.MediaError
.OrderByDescending(m => m.Created)
.ProjectTo<MediaErrorDto>(_mapper.ConfigurationProvider)
.AsNoTracking();
return query.AsEnumerable();
}
public Task<bool> ExistsAsync(MediaError error)
{
return _context.MediaError.AnyAsync(m => m.FilePath.Equals(error.FilePath)
&& m.Comment.Equals(error.Comment)
&& m.Details.Equals(error.Details)
);
}
public async Task DeleteAll()
{
_context.MediaError.RemoveRange(await _context.MediaError.ToListAsync());
await _context.SaveChangesAsync();
}
}

7
API/Data/Repositories/ReadingListRepository.cs
View file

@ -45,7 +45,7 @@ public interface IReadingListRepository
Task<IList<string>> GetAllCoverImagesAsync();
Task<bool> ReadingListExists(string name);
IEnumerable<PersonDto> GetReadingListCharactersAsync(int readingListId);
Task<IList<ReadingList>> GetAllWithNonWebPCovers();
Task<IList<ReadingList>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat);
Task<IList<string>> GetFirstFourCoverImagesByReadingListId(int readingListId);
Task<int> RemoveReadingListsWithoutSeries();
Task<ReadingList?> GetReadingListByTitleAsync(string name, int userId, ReadingListIncludes includes = ReadingListIncludes.Items);
@ -110,10 +110,11 @@ public class ReadingListRepository : IReadingListRepository
.AsEnumerable();
}
public async Task<IList<ReadingList>> GetAllWithNonWebPCovers()
public async Task<IList<ReadingList>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat)
{
var extension = encodeFormat.GetExtension();
return await _context.ReadingList
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp"))
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension))
.ToListAsync();
}

25
API/Data/Repositories/SeriesRepository.cs
View file

@ -4,6 +4,7 @@ using System.Drawing;
using System.Linq;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using API.Data.ManualMigrations;
using API.Data.Misc;
using API.Data.Scanner;
using API.DTOs;
@ -79,7 +80,7 @@ public interface ISeriesRepository
/// <returns></returns>
Task<SearchResultGroupDto> SearchSeries(int userId, bool isAdmin, IList<int> libraryIds, string searchQuery);
Task<IEnumerable<Series>> GetSeriesForLibraryIdAsync(int libraryId, SeriesIncludes includes = SeriesIncludes.None);
Task<SeriesDto> GetSeriesDtoByIdAsync(int seriesId, int userId);
Task<SeriesDto?> GetSeriesDtoByIdAsync(int seriesId, int userId);
Task<Series?> GetSeriesByIdAsync(int seriesId, SeriesIncludes includes = SeriesIncludes.Volumes | SeriesIncludes.Metadata);
Task<IList<Series>> GetSeriesByIdsAsync(IList<int> seriesIds);
Task<int[]> GetChapterIdsForSeriesAsync(IList<int> seriesIds);
@ -132,7 +133,7 @@ public interface ISeriesRepository
Task<IDictionary<int, int>> GetLibraryIdsForSeriesAsync();
Task<IList<SeriesMetadataDto>> GetSeriesMetadataForIds(IEnumerable<int> seriesIds);
Task<IList<Series>> GetAllWithNonWebPCovers(bool customOnly = true);
Task<IList<Series>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat, bool customOnly = true);
}
public class SeriesRepository : ISeriesRepository
@ -347,11 +348,11 @@ public class SeriesRepository : ISeriesRepository
result.Series = _context.Series
.Where(s => libraryIds.Contains(s.LibraryId))
.Where(s => (EF.Functions.Like(s.Name, $"%{searchQuery}%")
.Where(s => EF.Functions.Like(s.Name, $"%{searchQuery}%")
|| (s.OriginalName != null && EF.Functions.Like(s.OriginalName, $"%{searchQuery}%"))
|| (s.LocalizedName != null && EF.Functions.Like(s.LocalizedName, $"%{searchQuery}%"))
|| (EF.Functions.Like(s.NormalizedName, $"%{searchQueryNormalized}%"))
|| (hasYearInQuery && s.Metadata.ReleaseYear == yearComparison)))
|| (hasYearInQuery && s.Metadata.ReleaseYear == yearComparison))
.RestrictAgainstAgeRestriction(userRating)
.Include(s => s.Library)
.OrderBy(s => s.SortName!.ToLower())
@ -429,7 +430,9 @@ public class SeriesRepository : ISeriesRepository
result.Chapters = await _context.Chapter
.Include(c => c.Files)
.Where(c => EF.Functions.Like(c.TitleName, $"%{searchQuery}%"))
.Where(c => EF.Functions.Like(c.TitleName, $"%{searchQuery}%")
|| EF.Functions.Like(c.ISBN, $"%{searchQuery}%")
)
.Where(c => c.Files.All(f => fileIds.Contains(f.Id)))
.AsSplitQuery()
.Take(maxRecords)
@ -439,11 +442,13 @@ public class SeriesRepository : ISeriesRepository
return result;
}
public async Task<SeriesDto> GetSeriesDtoByIdAsync(int seriesId, int userId)
public async Task<SeriesDto?> GetSeriesDtoByIdAsync(int seriesId, int userId)
{
var series = await _context.Series.Where(x => x.Id == seriesId)
.ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
.SingleAsync();
.SingleOrDefaultAsync();
if (series == null) return null;
var seriesList = new List<SeriesDto>() {series};
await AddSeriesModifiers(userId, seriesList);
@ -565,12 +570,14 @@ public class SeriesRepository : ISeriesRepository
/// Returns custom images only
/// </summary>
/// <returns></returns>
public async Task<IList<Series>> GetAllWithNonWebPCovers(bool customOnly = true)
public async Task<IList<Series>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat,
bool customOnly = true)
{
var extension = encodeFormat.GetExtension();
var prefix = ImageService.GetSeriesFormat(0).Replace("0", string.Empty);
return await _context.Series
.Where(c => !string.IsNullOrEmpty(c.CoverImage)
&& !c.CoverImage.EndsWith(".webp")
&& !c.CoverImage.EndsWith(extension)
&& (!customOnly || c.CoverImage.StartsWith(prefix)))
.ToListAsync();
}

6
API/Data/Repositories/SettingsRepository.cs
View file

@ -15,6 +15,7 @@ public interface ISettingsRepository
Task<ServerSettingDto> GetSettingsDtoAsync();
Task<ServerSetting> GetSettingAsync(ServerSettingKey key);
Task<IEnumerable<ServerSetting>> GetSettingsAsync();
void Remove(ServerSetting setting);
}
public class SettingsRepository : ISettingsRepository
{
@ -32,6 +33,11 @@ public class SettingsRepository : ISettingsRepository
_context.Entry(settings).State = EntityState.Modified;
}
public void Remove(ServerSetting setting)
{
_context.Remove(setting);
}
public async Task<ServerSettingDto> GetSettingsDtoAsync()
{
var settings = await _context.ServerSetting

9
API/Data/Repositories/VolumeRepository.cs
View file

@ -4,10 +4,12 @@ using System.Linq;
using System.Threading.Tasks;
using API.DTOs;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Services;
using AutoMapper;
using AutoMapper.QueryableExtensions;
using Kavita.Common;
using Microsoft.EntityFrameworkCore;
namespace API.Data.Repositories;
@ -26,7 +28,7 @@ public interface IVolumeRepository
Task<IEnumerable<Volume>> GetVolumesForSeriesAsync(IList<int> seriesIds, bool includeChapters = false);
Task<IEnumerable<Volume>> GetVolumes(int seriesId);
Task<Volume?> GetVolumeByIdAsync(int volumeId);
Task<IList<Volume>> GetAllWithNonWebPCovers();
Task<IList<Volume>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat);
}
public class VolumeRepository : IVolumeRepository
{
@ -200,10 +202,11 @@ public class VolumeRepository : IVolumeRepository
return await _context.Volume.SingleOrDefaultAsync(x => x.Id == volumeId);
}
public async Task<IList<Volume>> GetAllWithNonWebPCovers()
public async Task<IList<Volume>> GetAllWithCoversInDifferentEncoding(EncodeFormat encodeFormat)
{
var extension = encodeFormat.GetExtension();
return await _context.Volume
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(".webp"))
.Where(c => !string.IsNullOrEmpty(c.CoverImage) && !c.CoverImage.EndsWith(extension))
.ToListAsync();
}

3
API/Data/Seed.cs
View file

@ -101,12 +101,11 @@ public static class Seed
new() {Key = ServerSettingKey.InstallVersion, Value = BuildInfo.Version.ToString()},
new() {Key = ServerSettingKey.BookmarkDirectory, Value = directoryService.BookmarkDirectory},
new() {Key = ServerSettingKey.EmailServiceUrl, Value = EmailService.DefaultApiUrl},
new() {Key = ServerSettingKey.ConvertBookmarkToWebP, Value = "false"},
new() {Key = ServerSettingKey.TotalBackups, Value = "30"},
new() {Key = ServerSettingKey.TotalLogs, Value = "30"},
new() {Key = ServerSettingKey.EnableFolderWatching, Value = "false"},
new() {Key = ServerSettingKey.ConvertCoverToWebP, Value = "false"},
new() {Key = ServerSettingKey.HostName, Value = string.Empty},
new() {Key = ServerSettingKey.EncodeMediaAs, Value = EncodeFormat.PNG.ToString()},
}.ToArray());
foreach (var defaultSetting in DefaultSettings)

2
API/Data/UnitOfWork.cs
View file

@ -25,6 +25,7 @@ public interface IUnitOfWork
ISiteThemeRepository SiteThemeRepository { get; }
IMangaFileRepository MangaFileRepository { get; }
IDeviceRepository DeviceRepository { get; }
IMediaErrorRepository MediaErrorRepository { get; }
bool Commit();
Task<bool> CommitAsync();
bool HasChanges();
@ -62,6 +63,7 @@ public class UnitOfWork : IUnitOfWork
public ISiteThemeRepository SiteThemeRepository => new SiteThemeRepository(_context, _mapper);
public IMangaFileRepository MangaFileRepository => new MangaFileRepository(_context);
public IDeviceRepository DeviceRepository => new DeviceRepository(_context, _mapper);
public IMediaErrorRepository MediaErrorRepository => new MediaErrorRepository(_context, _mapper);
/// <summary>
/// Commits changes to the DB. Completes the open transaction.

7
API/Entities/Chapter.cs
View file

@ -100,7 +100,11 @@ public class Chapter : IEntityDate, IHasReadTimeEstimate
public int MaxHoursToRead { get; set; }
/// <inheritdoc cref="IHasReadTimeEstimate"/>
public int AvgHoursToRead { get; set; }
/// <summary>
/// Comma-separated link of urls to external services that have some relation to the Chapter
/// </summary>
public string WebLinks { get; set; } = string.Empty;
public string ISBN { get; set; } = string.Empty;
/// <summary>
/// All people attached at a Chapter level. Usually Comics will have different people per issue.
@ -115,7 +119,6 @@ public class Chapter : IEntityDate, IHasReadTimeEstimate
public ICollection<AppUserProgress> UserProgress { get; set; }
// Relationships
public Volume Volume { get; set; } = null!;
public int VolumeId { get; set; }

13
API/Entities/Enums/EncodeFormat.cs Normal file
View file

@ -0,0 +1,13 @@
using System.ComponentModel;
namespace API.Entities.Enums;
public enum EncodeFormat
{
[Description("PNG")]
PNG = 0,
[Description("WebP")]
WEBP = 1,
[Description("AVIF")]
AVIF = 2
}

12
API/Entities/Enums/ServerSettingKey.cs
View file

@ -1,4 +1,5 @@
using System.ComponentModel;
using System;
using System.ComponentModel;
namespace API.Entities.Enums;
@ -82,6 +83,7 @@ public enum ServerSettingKey
/// <summary>
/// If Kavita should save bookmarks as WebP images
/// </summary>
[Obsolete("Use EncodeMediaAs instead")]
[Description("ConvertBookmarkToWebP")]
ConvertBookmarkToWebP = 14,
/// <summary>
@ -102,6 +104,7 @@ public enum ServerSettingKey
/// <summary>
/// If Kavita should save covers as WebP images
/// </summary>
[Obsolete("Use EncodeMediaAs instead")]
[Description("ConvertCoverToWebP")]
ConvertCoverToWebP = 19,
/// <summary>
@ -114,4 +117,11 @@ public enum ServerSettingKey
/// </summary>
[Description("IpAddresses")]
IpAddresses = 21,
/// <summary>
/// Encode all media as PNG/WebP/AVIF/etc.
/// </summary>
/// <remarks>As of v0.7.3 this replaced ConvertCoverToWebP and ConvertBookmarkToWebP</remarks>
[Description("EncodeMediaAs")]
EncodeMediaAs = 22,
}

36
API/Entities/MediaError.cs Normal file
View file

@ -0,0 +1,36 @@
using System;
using API.Entities.Interfaces;
namespace API.Entities;
/// <summary>
/// Represents issues found during scanning or interacting with media. For example) Can't open file, corrupt media, missing content in epub.
/// </summary>
public class MediaError : IEntityDate
{
public int Id { get; set; }
/// <summary>
/// Format Type (RAR, ZIP, 7Zip, Epub, PDF)
/// </summary>
public required string Extension { get; set; }
/// <summary>
/// Full Filepath to the file that has some issue
/// </summary>
public required string FilePath { get; set; }
/// <summary>
/// Developer defined string
/// </summary>
public string Comment { get; set; }
/// <summary>
/// Exception message
/// </summary>
public string Details { get; set; }
/// <summary>
/// Was the file imported or not
/// </summary>
//public bool Imported { get; set; }
public DateTime Created { get; set; }
public DateTime LastModified { get; set; }
public DateTime CreatedUtc { get; set; }
public DateTime LastModifiedUtc { get; set; }
}

8
API/Entities/Metadata/SeriesMetadata.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using API.Entities.Enums;
using API.Entities.Interfaces;
@ -43,6 +44,11 @@ public class SeriesMetadata : IHasConcurrencyToken
/// </summary>
public int MaxCount { get; set; } = 0;
public PublicationStatus PublicationStatus { get; set; }
/// <summary>
/// A Comma-separated list of strings representing links from the series
/// </summary>
/// <remarks>This is not populated from Chapters of the Series</remarks>
public string WebLinks { get; set; } = string.Empty;
// Locks
public bool LanguageLocked { get; set; }

17
API/Extensions/ApplicationServiceExtensions.cs
View file

@ -1,4 +1,5 @@
using System.IO.Abstractions;
using API.Constants;
using API.Data;
using API.Helpers;
using API.Services;
@ -22,9 +23,7 @@ public static class ApplicationServiceExtensions
services.AddAutoMapper(typeof(AutoMapperProfiles).Assembly);
services.AddScoped<IUnitOfWork, UnitOfWork>();
services.AddScoped<IDirectoryService, DirectoryService>();
services.AddScoped<ITokenService, TokenService>();
services.AddScoped<IFileSystem, FileSystem>();
services.AddScoped<IFileService, FileService>();
services.AddScoped<ICacheHelper, CacheHelper>();
@ -35,7 +34,6 @@ public static class ApplicationServiceExtensions
services.AddScoped<IBackupService, BackupService>();
services.AddScoped<ICleanupService, CleanupService>();
services.AddScoped<IBookService, BookService>();
services.AddScoped<IImageService, ImageService>();
services.AddScoped<IVersionUpdaterService, VersionUpdaterService>();
services.AddScoped<IDownloadService, DownloadService>();
services.AddScoped<IReaderService, ReaderService>();
@ -49,6 +47,8 @@ public static class ApplicationServiceExtensions
services.AddScoped<IReadingListService, ReadingListService>();
services.AddScoped<IDeviceService, DeviceService>();
services.AddScoped<IStatisticService, StatisticService>();
services.AddScoped<IMediaErrorService, MediaErrorService>();
services.AddScoped<IMediaConversionService, MediaConversionService>();
services.AddScoped<IScannerService, ScannerService>();
services.AddScoped<IMetadataService, MetadataService>();
@ -57,11 +57,20 @@ public static class ApplicationServiceExtensions
services.AddScoped<ITachiyomiService, TachiyomiService>();
services.AddScoped<ICollectionTagService, CollectionTagService>();
services.AddScoped<IPresenceTracker, PresenceTracker>();
services.AddScoped<IFileSystem, FileSystem>();
services.AddScoped<IDirectoryService, DirectoryService>();
services.AddScoped<IEventHub, EventHub>();
services.AddScoped<IPresenceTracker, PresenceTracker>();
services.AddScoped<IImageService, ImageService>();
services.AddSqLite(env);
services.AddSignalR(opt => opt.EnableDetailedErrors = true);
services.AddEasyCaching(options =>
{
options.UseInMemory(EasyCacheProfiles.Favicon);
});
}
private static void AddSqLite(this IServiceCollection services, IHostEnvironment env)

18
API/Extensions/EncodeFormatExtensions.cs Normal file
View file

@ -0,0 +1,18 @@
using System;
using API.Entities.Enums;
namespace API.Extensions;
public static class EncodeFormatExtensions
{
public static string GetExtension(this EncodeFormat encodeFormat)
{
return encodeFormat switch
{
EncodeFormat.PNG => ".png",
EncodeFormat.WEBP => ".webp",
EncodeFormat.AVIF => ".avif",
_ => throw new ArgumentOutOfRangeException(nameof(encodeFormat), encodeFormat, null)
};
}
}

2
API/Helpers/AutoMapperProfiles.cs
View file

@ -4,6 +4,7 @@ using API.DTOs;
using API.DTOs.Account;
using API.DTOs.CollectionTags;
using API.DTOs.Device;
using API.DTOs.MediaErrors;
using API.DTOs.Metadata;
using API.DTOs.Reader;
using API.DTOs.ReadingLists;
@ -33,6 +34,7 @@ public class AutoMapperProfiles : Profile
CreateMap<Tag, TagDto>();
CreateMap<AgeRating, AgeRatingDto>();
CreateMap<PublicationStatus, PublicationStatusDto>();
CreateMap<MediaError, MediaErrorDto>();
CreateMap<AppUserProgress, ProgressDto>()
.ForMember(dest => dest.PageNum,

31
API/Helpers/Builders/MediaErrorBuilder.cs Normal file
View file

@ -0,0 +1,31 @@
using System.IO;
using API.Entities;
namespace API.Helpers.Builders;
public class MediaErrorBuilder : IEntityBuilder<MediaError>
{
private readonly MediaError _mediaError;
public MediaError Build() => _mediaError;
public MediaErrorBuilder(string filePath)
{
_mediaError = new MediaError()
{
FilePath = filePath,
Extension = Path.GetExtension(filePath).Replace(".", string.Empty).ToUpperInvariant()
};
}
public MediaErrorBuilder WithComment(string comment)
{
_mediaError.Comment = comment.Trim();
return this;
}
public MediaErrorBuilder WithDetails(string details)
{
_mediaError.Details = details.Trim();
return this;
}
}

10
API/Helpers/Converters/ServerSettingConverter.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using API.DTOs.Settings;
using API.Entities;
using API.Entities.Enums;
@ -51,11 +52,8 @@ public class ServerSettingConverter : ITypeConverter<IEnumerable<ServerSetting>,
case ServerSettingKey.InstallVersion:
destination.InstallVersion = row.Value;
break;
case ServerSettingKey.ConvertBookmarkToWebP:
destination.ConvertBookmarkToWebP = bool.Parse(row.Value);
break;
case ServerSettingKey.ConvertCoverToWebP:
destination.ConvertCoverToWebP = bool.Parse(row.Value);
case ServerSettingKey.EncodeMediaAs:
destination.EncodeMediaAs = Enum.Parse<EncodeFormat>(row.Value);
break;
case ServerSettingKey.TotalBackups:
destination.TotalBackups = int.Parse(row.Value);

14
API/Helpers/PersonHelper.cs
View file

@ -115,21 +115,21 @@ public static class PersonHelper
/// For a given role and people dtos, update a series
/// </summary>
/// <param name="role"></param>
/// <param name="tags"></param>
/// <param name="people"></param>
/// <param name="series"></param>
/// <param name="allTags"></param>
/// <param name="allPeople"></param>
/// <param name="handleAdd">This will call with an existing or new tag, but the method does not update the series Metadata</param>
/// <param name="onModified"></param>
public static void UpdatePeopleList(PersonRole role, ICollection<PersonDto>? tags, Series series, IReadOnlyCollection<Person> allTags,
public static void UpdatePeopleList(PersonRole role, ICollection<PersonDto>? people, Series series, IReadOnlyCollection<Person> allPeople,
Action<Person> handleAdd, Action onModified)
{
if (tags == null) return;
if (people == null) return;
var isModified = false;
// I want a union of these 2 lists. Return only elements that are in both lists, but the list types are different
var existingTags = series.Metadata.People.Where(p => p.Role == role).ToList();
foreach (var existing in existingTags)
{
if (tags.SingleOrDefault(t => t.Id == existing.Id) == null) // This needs to check against role
if (people.SingleOrDefault(t => t.Id == existing.Id) == null) // This needs to check against role
{
// Remove tag
series.Metadata.People.Remove(existing);
@ -138,9 +138,9 @@ public static class PersonHelper
}
// At this point, all tags that aren't in dto have been removed.
foreach (var tag in tags)
foreach (var tag in people)
{
var existingTag = allTags.SingleOrDefault(t => t.Name == tag.Name && t.Role == tag.Role);
var existingTag = allPeople.FirstOrDefault(t => t.Name == tag.Name && t.Role == tag.Role);
if (existingTag != null)
{
if (series.Metadata.People.Where(t => t.Role == tag.Role).All(t => t.Name != null && !t.Name.Equals(tag.Name)))

57
API/Middleware/JWTRevocationMiddleware.cs Normal file
View file

@ -0,0 +1,57 @@
using System.Threading.Tasks;
using API.Constants;
using EasyCaching.Core;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace API.Middleware;
/// <summary>
/// Responsible for maintaining an in-memory. Not in use
/// </summary>
public class JwtRevocationMiddleware
{
private readonly RequestDelegate _next;
private readonly IEasyCachingProviderFactory _cacheFactory;
private readonly ILogger<JwtRevocationMiddleware> _logger;
public JwtRevocationMiddleware(RequestDelegate next, IEasyCachingProviderFactory cacheFactory, ILogger<JwtRevocationMiddleware> logger)
{
_next = next;
_cacheFactory = cacheFactory;
_logger = logger;
}
public async Task InvokeAsync(HttpContext context)
{
if (context.User.Identity is {IsAuthenticated: false})
{
await _next(context);
return;
}
// Get the JWT from the request headers or wherever you store it
var token = context.Request.Headers["Authorization"].ToString()?.Replace("Bearer ", string.Empty);
// Check if the token is revoked
if (await IsTokenRevoked(token))
{
_logger.LogWarning("Revoked token detected: {Token}", token);
context.Response.StatusCode = StatusCodes.Status401Unauthorized;
return;
}
await _next(context);
}
private async Task<bool> IsTokenRevoked(string token)
{
// Check if the token exists in the revocation list stored in the cache
var isRevoked = await _cacheFactory.GetCachingProvider(EasyCacheProfiles.RevokedJwt)
.GetAsync<string>(token);
return isRevoked.HasValue;
}
}

12
API/Program.cs
View file

@ -4,6 +4,7 @@ using System.Linq;
using System.Security.Cryptography;
using System.Threading.Tasks;
using API.Data;
using API.Data.ManualMigrations;
using API.Entities;
using API.Entities.Enums;
using API.Logging;
@ -49,7 +50,7 @@ public class Program
Environment.GetEnvironmentVariable("ASPNETCORE_ENVIRONMENT") != Environments.Development)
{
Log.Logger.Information("Generating JWT TokenKey for encrypting user sessions...");
var rBytes = new byte[128];
var rBytes = new byte[256];
RandomNumberGenerator.Create().GetBytes(rBytes);
Configuration.JwtToken = Convert.ToBase64String(rBytes).Replace("/", string.Empty);
}
@ -173,13 +174,13 @@ public class Program
webBuilder.UseKestrel((opts) =>
{
var ipAddresses = Configuration.IpAddresses;
if (new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker || string.IsNullOrEmpty(ipAddresses) || ipAddresses.Equals(Configuration.DefaultIpAddresses))
if (OsInfo.IsDocker || string.IsNullOrEmpty(ipAddresses) || ipAddresses.Equals(Configuration.DefaultIpAddresses))
{
opts.ListenAnyIP(HttpPort, options => { options.Protocols = HttpProtocols.Http1AndHttp2; });
}
else
{
foreach (var ipAddress in ipAddresses.Split(','))
foreach (var ipAddress in ipAddresses.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
try
{
@ -194,9 +195,6 @@ public class Program
}
});
webBuilder.UseStartup<Startup>();
webBuilder.UseStartup<Startup>();
});
}

62
API/Services/ArchiveService.cs
View file

@ -4,9 +4,11 @@ using System.Diagnostics;
using System.IO;
using System.IO.Compression;
using System.Linq;
using System.Xml.Linq;
using System.Xml.Serialization;
using API.Archive;
using API.Data.Metadata;
using API.Entities.Enums;
using API.Extensions;
using API.Services.Tasks;
using Kavita.Common;
@ -20,7 +22,7 @@ public interface IArchiveService
{
void ExtractArchive(string archivePath, string extractPath);
int GetNumberOfPagesFromArchive(string archivePath);
string GetCoverImage(string archivePath, string fileName, string outputDirectory, bool saveAsWebP = false);
string GetCoverImage(string archivePath, string fileName, string outputDirectory, EncodeFormat format);
bool IsValidArchive(string archivePath);
ComicInfo? GetComicInfo(string archivePath);
ArchiveLibrary CanOpen(string archivePath);
@ -44,13 +46,16 @@ public class ArchiveService : IArchiveService
private readonly ILogger<ArchiveService> _logger;
private readonly IDirectoryService _directoryService;
private readonly IImageService _imageService;
private readonly IMediaErrorService _mediaErrorService;
private const string ComicInfoFilename = "ComicInfo.xml";
public ArchiveService(ILogger<ArchiveService> logger, IDirectoryService directoryService, IImageService imageService)
public ArchiveService(ILogger<ArchiveService> logger, IDirectoryService directoryService,
IImageService imageService, IMediaErrorService mediaErrorService)
{
_logger = logger;
_directoryService = directoryService;
_imageService = imageService;
_mediaErrorService = mediaErrorService;
}
/// <summary>
@ -120,6 +125,8 @@ public class ArchiveService : IArchiveService
catch (Exception ex)
{
_logger.LogWarning(ex, "[GetNumberOfPagesFromArchive] There was an exception when reading archive stream: {ArchivePath}. Defaulting to 0 pages", archivePath);
_mediaErrorService.ReportMediaIssue(archivePath, MediaErrorProducer.ArchiveService,
"This archive cannot be read or not supported", ex);
return 0;
}
}
@ -196,9 +203,9 @@ public class ArchiveService : IArchiveService
/// <param name="archivePath"></param>
/// <param name="fileName">File name to use based on context of entity.</param>
/// <param name="outputDirectory">Where to output the file, defaults to covers directory</param>
/// <param name="saveAsWebP">When saving the file, use WebP encoding instead of PNG</param>
/// <param name="encodeFormat">When saving the file, use encoding</param>
/// <returns></returns>
public string GetCoverImage(string archivePath, string fileName, string outputDirectory, bool saveAsWebP = false)
public string GetCoverImage(string archivePath, string fileName, string outputDirectory, EncodeFormat encodeFormat)
{
if (archivePath == null || !IsValidArchive(archivePath)) return string.Empty;
try
@ -214,7 +221,7 @@ public class ArchiveService : IArchiveService
var entry = archive.Entries.Single(e => e.FullName == entryName);
using var stream = entry.Open();
return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, saveAsWebP);
return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, encodeFormat);
}
case ArchiveLibrary.SharpCompress:
{
@ -225,7 +232,7 @@ public class ArchiveService : IArchiveService
var entry = archive.Entries.Single(e => e.Key == entryName);
using var stream = entry.OpenEntryStream();
return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, saveAsWebP);
return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, encodeFormat);
}
case ArchiveLibrary.NotSupported:
_logger.LogWarning("[GetCoverImage] This archive cannot be read: {ArchivePath}. Defaulting to no cover image", archivePath);
@ -238,6 +245,8 @@ public class ArchiveService : IArchiveService
catch (Exception ex)
{
_logger.LogWarning(ex, "[GetCoverImage] There was an exception when reading archive stream: {ArchivePath}. Defaulting to no cover image", archivePath);
_mediaErrorService.ReportMediaIssue(archivePath, MediaErrorProducer.ArchiveService,
"This archive cannot be read or not supported", ex);
}
return string.Empty;
@ -364,10 +373,7 @@ public class ArchiveService : IArchiveService
if (entry != null)
{
using var stream = entry.Open();
var serializer = new XmlSerializer(typeof(ComicInfo));
var info = (ComicInfo?) serializer.Deserialize(stream);
ComicInfo.CleanComicInfo(info);
return info;
return Deserialize(stream);
}
break;
@ -382,9 +388,7 @@ public class ArchiveService : IArchiveService
if (entry != null)
{
using var stream = entry.OpenEntryStream();
var serializer = new XmlSerializer(typeof(ComicInfo));
var info = (ComicInfo?) serializer.Deserialize(stream);
ComicInfo.CleanComicInfo(info);
var info = Deserialize(stream);
return info;
}
@ -403,11 +407,35 @@ public class ArchiveService : IArchiveService
catch (Exception ex)
{
_logger.LogWarning(ex, "[GetComicInfo] There was an exception when reading archive stream: {Filepath}", archivePath);
_mediaErrorService.ReportMediaIssue(archivePath, MediaErrorProducer.ArchiveService,
"This archive cannot be read or not supported", ex);
}
return null;
}
/// <summary>
/// Strips out empty tags before deserializing
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
private static ComicInfo? Deserialize(Stream stream)
{
var comicInfoXml = XDocument.Load(stream);
comicInfoXml.Descendants()
.Where(e => e.IsEmpty || string.IsNullOrWhiteSpace(e.Value))
.Remove();
var serializer = new XmlSerializer(typeof(ComicInfo));
using var reader = comicInfoXml.Root?.CreateReader();
if (reader == null) return null;
var info = (ComicInfo?) serializer.Deserialize(reader);
ComicInfo.CleanComicInfo(info);
return info;
}
private void ExtractArchiveEntities(IEnumerable<IArchiveEntry> entries, string extractPath)
{
@ -417,7 +445,7 @@ public class ArchiveService : IArchiveService
{
entry.WriteToDirectory(extractPath, new ExtractionOptions()
{
ExtractFullPath = true, // Don't flatten, let the flatterner ensure correct order of nested folders
ExtractFullPath = true, // Don't flatten, let the flattener ensure correct order of nested folders
Overwrite = false
});
}
@ -485,9 +513,11 @@ public class ArchiveService : IArchiveService
}
}
catch (Exception e)
catch (Exception ex)
{
_logger.LogWarning(e, "[ExtractArchive] There was a problem extracting {ArchivePath} to {ExtractPath}",archivePath, extractPath);
_logger.LogWarning(ex, "[ExtractArchive] There was a problem extracting {ArchivePath} to {ExtractPath}",archivePath, extractPath);
_mediaErrorService.ReportMediaIssue(archivePath, MediaErrorProducer.ArchiveService,
"This archive cannot be read or not supported", ex);
throw new KavitaException(
$"There was an error when extracting {archivePath}. Check the file exists, has read permissions or the server OS can support all path characters.");
}

385
API/Services/BookService.cs
View file

@ -21,17 +21,19 @@ using HtmlAgilityPack;
using Kavita.Common;
using Microsoft.Extensions.Logging;
using Microsoft.IO;
using Nager.ArticleNumber;
using SixLabors.ImageSharp;
using SixLabors.ImageSharp.PixelFormats;
using VersOne.Epub;
using VersOne.Epub.Options;
using VersOne.Epub.Schema;
namespace API.Services;
public interface IBookService
{
int GetNumberOfPages(string filePath);
string GetCoverImage(string fileFilePath, string fileName, string outputDirectory, bool saveAsWebP = false);
string GetCoverImage(string fileFilePath, string fileName, string outputDirectory, EncodeFormat encodeFormat);
ComicInfo? GetComicInfo(string filePath);
ParserInfo? ParseInfo(string filePath);
/// <summary>
@ -60,6 +62,7 @@ public class BookService : IBookService
private readonly ILogger<BookService> _logger;
private readonly IDirectoryService _directoryService;
private readonly IImageService _imageService;
private readonly IMediaErrorService _mediaErrorService;
private readonly StylesheetParser _cssParser = new ();
private static readonly RecyclableMemoryStreamManager StreamManager = new ();
private const string CssScopeClass = ".book-content";
@ -72,11 +75,12 @@ public class BookService : IBookService
}
};
public BookService(ILogger<BookService> logger, IDirectoryService directoryService, IImageService imageService)
public BookService(ILogger<BookService> logger, IDirectoryService directoryService, IImageService imageService, IMediaErrorService mediaErrorService)
{
_logger = logger;
_directoryService = directoryService;
_imageService = imageService;
_mediaErrorService = mediaErrorService;
}
private static bool HasClickableHrefPart(HtmlNode anchor)
@ -123,7 +127,7 @@ public class BookService : IBookService
var hrefParts = CleanContentKeys(anchor.GetAttributeValue("href", string.Empty))
.Split("#");
// Some keys get uri encoded when parsed, so replace any of those characters with original
var mappingKey = HttpUtility.UrlDecode(hrefParts[0]);
var mappingKey = Uri.UnescapeDataString(hrefParts[0]);
if (!mappings.ContainsKey(mappingKey))
{
@ -132,6 +136,15 @@ public class BookService : IBookService
var part = hrefParts.Length > 1
? hrefParts[1]
: anchor.GetAttributeValue("href", string.Empty);
// hrefParts[0] might not have path from mappings
var pageKey = mappings.Keys.FirstOrDefault(mKey => mKey.EndsWith(hrefParts[0]));
if (!string.IsNullOrEmpty(pageKey))
{
mappings.TryGetValue(pageKey, out currentPage);
}
anchor.Attributes.Add("kavita-page", $"{currentPage}");
anchor.Attributes.Add("kavita-part", part);
anchor.Attributes.Remove("href");
@ -171,20 +184,20 @@ public class BookService : IBookService
// @Import statements will be handled by browser, so we must inline the css into the original file that request it, so they can be Scoped
var prepend = filename.Length > 0 ? filename.Replace(Path.GetFileName(filename), string.Empty) : string.Empty;
var importBuilder = new StringBuilder();
//foreach (Match match in Tasks.Scanner.Parser.Parser.CssImportUrlRegex().Matches(stylesheetHtml))
foreach (Match match in Parser.CssImportUrlRegex.Matches(stylesheetHtml))
{
if (!match.Success) continue;
var importFile = match.Groups["Filename"].Value;
var key = CleanContentKeys(importFile);
var key = CleanContentKeys(importFile); // Validate if CoalesceKey works well here
if (!key.Contains(prepend))
{
key = prepend + key;
}
if (!book.Content.AllFiles.ContainsKey(key)) continue;
if (!book.Content.AllFiles.TryGetLocalFileRefByKey(key, out var bookFile)) continue;
var bookFile = book.Content.AllFiles[key];
//var bookFile = book.Content.AllFiles.Local[key];
var content = await bookFile.ReadContentAsBytesAsync();
importBuilder.Append(Encoding.UTF8.GetString(content));
}
@ -218,12 +231,20 @@ public class BookService : IBookService
}
styleRule.Text = $"{CssScopeClass} " + styleRule.Text;
}
return RemoveWhiteSpaceFromStylesheets(stylesheet.ToCss());
try
{
return RemoveWhiteSpaceFromStylesheets(stylesheet.ToCss());
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an issue escaping css, likely due to an unsupported css rule");
}
return RemoveWhiteSpaceFromStylesheets($"{CssScopeClass} {styleContent}");
}
private static void EscapeCssImportReferences(ref string stylesheetHtml, string apiBase, string prepend)
{
//foreach (Match match in Tasks.Scanner.Parser.Parser.CssImportUrlRegex().Matches(stylesheetHtml))
foreach (Match match in Parser.CssImportUrlRegex.Matches(stylesheetHtml))
{
if (!match.Success) continue;
@ -234,7 +255,6 @@ public class BookService : IBookService
private static void EscapeFontFamilyReferences(ref string stylesheetHtml, string apiBase, string prepend)
{
//foreach (Match match in Tasks.Scanner.Parser.Parser.FontSrcUrlRegex().Matches(stylesheetHtml))
foreach (Match match in Parser.FontSrcUrlRegex.Matches(stylesheetHtml))
{
if (!match.Success) continue;
@ -245,7 +265,6 @@ public class BookService : IBookService
private static void EscapeCssImageReferences(ref string stylesheetHtml, string apiBase, EpubBookRef book)
{
//var matches = Tasks.Scanner.Parser.Parser.CssImageUrlRegex().Matches(stylesheetHtml);
var matches = Parser.CssImageUrlRegex.Matches(stylesheetHtml);
foreach (Match match in matches)
{
@ -253,7 +272,7 @@ public class BookService : IBookService
var importFile = match.Groups["Filename"].Value;
var key = CleanContentKeys(importFile);
if (!book.Content.AllFiles.ContainsKey(key)) continue;
if (!book.Content.AllFiles.ContainsLocalFileRefWithKey(key)) continue;
stylesheetHtml = stylesheetHtml.Replace(importFile, apiBase + key);
}
@ -286,7 +305,7 @@ public class BookService : IBookService
var imageFile = GetKeyForImage(book, image.Attributes[key].Value);
image.Attributes.Remove(key);
// UrlEncode here to transform ../ into an escaped version, which avoids blocking on nginx
image.Attributes.Add(key, $"{apiBase}" + HttpUtility.UrlEncode(imageFile));
image.Attributes.Add(key, $"{apiBase}" + Uri.EscapeDataString(imageFile));
// Add a custom class that the reader uses to ensure images stay within reader
parent.AddClass("kavita-scale-width-container");
@ -303,9 +322,9 @@ public class BookService : IBookService
/// <returns></returns>
private static string GetKeyForImage(EpubBookRef book, string imageFile)
{
if (book.Content.Images.ContainsKey(imageFile)) return imageFile;
if (book.Content.Images.ContainsLocalFileRefWithKey(imageFile)) return imageFile;
var correctedKey = book.Content.Images.Keys.SingleOrDefault(s => s.EndsWith(imageFile));
var correctedKey = book.Content.Images.Local.Select(s => s.Key).SingleOrDefault(s => s.EndsWith(imageFile));
if (correctedKey != null)
{
imageFile = correctedKey;
@ -314,13 +333,14 @@ public class BookService : IBookService
{
// There are cases where the key is defined static like OEBPS/Images/1-4.jpg but reference is ../Images/1-4.jpg
correctedKey =
book.Content.Images.Keys.SingleOrDefault(s => s.EndsWith(imageFile.Replace("..", string.Empty)));
book.Content.Images.Local.Select(s => s.Key).SingleOrDefault(s => s.EndsWith(imageFile.Replace("..", string.Empty)));
if (correctedKey != null)
{
imageFile = correctedKey;
}
}
return imageFile;
}
@ -338,6 +358,7 @@ public class BookService : IBookService
}
private static void RewriteAnchors(int page, HtmlDocument doc, Dictionary<string, int> mappings)
{
var anchors = doc.DocumentNode.SelectNodes("//a");
if (anchors == null) return;
@ -368,9 +389,9 @@ public class BookService : IBookService
var key = CleanContentKeys(styleLinks.Attributes["href"].Value);
// Some epubs are malformed the key in content.opf might be: content/resources/filelist_0_0.xml but the actual html links to resources/filelist_0_0.xml
// In this case, we will do a search for the key that ends with
if (!book.Content.Css.ContainsKey(key))
if (!book.Content.Css.ContainsLocalFileRefWithKey(key))
{
var correctedKey = book.Content.Css.Keys.SingleOrDefault(s => s.EndsWith(key));
var correctedKey = book.Content.Css.Local.Select(s => s.Key).SingleOrDefault(s => s.EndsWith(key));
if (correctedKey == null)
{
_logger.LogError("Epub is Malformed, key: {Key} is not matching OPF file", key);
@ -382,10 +403,11 @@ public class BookService : IBookService
try
{
var cssFile = book.Content.Css[key];
var cssFile = book.Content.Css.GetLocalFileRefByKey(key);
var styleContent = await ScopeStyles(await cssFile.ReadContentAsync(), apiBase,
cssFile.FileName, book);
var stylesheetHtml = await cssFile.ReadContentAsync();
var styleContent = await ScopeStyles(stylesheetHtml, apiBase,
cssFile.FilePath, book);
if (styleContent != null)
{
body.PrependChild(HtmlNode.CreateNode($"<style>{styleContent}</style>"));
@ -394,6 +416,8 @@ public class BookService : IBookService
catch (Exception ex)
{
_logger.LogError(ex, "There was an error reading css file for inlining likely due to a key mismatch in metadata");
await _mediaErrorService.ReportMediaIssueAsync(book.FilePath, MediaErrorProducer.BookService,
"There was an error reading css file for inlining likely due to a key mismatch in metadata", ex);
}
}
}
@ -415,20 +439,35 @@ public class BookService : IBookService
}
var (year, month, day) = GetPublicationDate(publicationDate);
var summary = epubBook.Schema.Package.Metadata.Descriptions.FirstOrDefault();
var info = new ComicInfo
{
Summary = epubBook.Schema.Package.Metadata.Description,
Writer = string.Join(",", epubBook.Schema.Package.Metadata.Creators.Select(c => Parser.CleanAuthor(c.Creator))),
Publisher = string.Join(",", epubBook.Schema.Package.Metadata.Publishers),
Summary = string.IsNullOrEmpty(summary?.Description) ? string.Empty : summary.Description,
Publisher = string.Join(",", epubBook.Schema.Package.Metadata.Publishers.Select(p => p.Publisher)),
Month = month,
Day = day,
Year = year,
Title = epubBook.Title,
Genre = string.Join(",", epubBook.Schema.Package.Metadata.Subjects.Select(s => s.ToLower().Trim())),
LanguageISO = ValidateLanguage(epubBook.Schema.Package.Metadata.Languages.FirstOrDefault())
Genre = string.Join(",", epubBook.Schema.Package.Metadata.Subjects.Select(s => s.Subject.ToLower().Trim())),
LanguageISO = ValidateLanguage(epubBook.Schema.Package.Metadata.Languages
.Select(l => l.Language)
.FirstOrDefault())
};
ComicInfo.CleanComicInfo(info);
foreach (var identifier in epubBook.Schema.Package.Metadata.Identifiers.Where(id => !string.IsNullOrEmpty(id.Scheme) && id.Scheme.Equals("ISBN")))
{
if (string.IsNullOrEmpty(identifier.Identifier)) continue;
var isbn = identifier.Identifier.Replace("urn:isbn:", string.Empty).Replace("isbn:", string.Empty);
if (!ArticleNumberHelper.IsValidIsbn10(isbn) && !ArticleNumberHelper.IsValidIsbn13(isbn))
{
_logger.LogDebug("[BookService] {File} has invalid ISBN number", filePath);
continue;
}
info.Isbn = isbn;
break;
}
// Parse tags not exposed via Library
foreach (var metadataItem in epubBook.Schema.Package.Metadata.MetaItems)
{
@ -443,13 +482,17 @@ public class BookService : IBookService
break;
case "calibre:series":
info.Series = metadataItem.Content;
info.SeriesSort = metadataItem.Content;
if (string.IsNullOrEmpty(info.SeriesSort))
{
info.SeriesSort = metadataItem.Content;
}
break;
case "calibre:series_index":
info.Volume = metadataItem.Content;
break;
}
// EPUB 3.2+ only
switch (metadataItem.Property)
{
@ -458,14 +501,47 @@ public class BookService : IBookService
break;
case "belongs-to-collection":
info.Series = metadataItem.Content;
info.SeriesSort = metadataItem.Content;
if (string.IsNullOrEmpty(info.SeriesSort))
{
info.SeriesSort = metadataItem.Content;
}
break;
case "collection-type":
// These look to be genres from https://manual.calibre-ebook.com/sub_groups.html or can be "series"
break;
case "role":
if (metadataItem.Scheme != null && !metadataItem.Scheme.Equals("marc:relators")) break;
var creatorId = metadataItem.Refines?.Replace("#", string.Empty);
var person = epubBook.Schema.Package.Metadata.Creators
.SingleOrDefault(c => c.Id == creatorId);
if (person == null) break;
PopulatePerson(metadataItem, info, person);
break;
case "title-type":
if (metadataItem.Content.Equals("collection"))
{
ExtractCollectionOrReadingList(metadataItem, epubBook, info);
}
if (metadataItem.Content.Equals("main"))
{
ExtractSortTitle(metadataItem, epubBook, info);
}
break;
}
}
// Check if there is a SortTitle
// Include regular Writer as well, for cases where there is no special tag
info.Writer = string.Join(",",
epubBook.Schema.Package.Metadata.Creators.Select(c => Parser.CleanAuthor(c.Creator)));
var hasVolumeInSeries = !Parser.ParseVolume(info.Title)
.Equals(Parser.DefaultVolume);
@ -480,12 +556,94 @@ public class BookService : IBookService
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[GetComicInfo] There was an exception getting metadata");
_logger.LogWarning(ex, "[GetComicInfo] There was an exception parsing metadata");
_mediaErrorService.ReportMediaIssue(filePath, MediaErrorProducer.BookService,
"There was an exception parsing metadata", ex);
}
return null;
}
private static void ExtractSortTitle(EpubMetadataMeta metadataItem, EpubBookRef epubBook, ComicInfo info)
{
var titleId = metadataItem.Refines?.Replace("#", string.Empty);
var titleElem = epubBook.Schema.Package.Metadata.Titles
.FirstOrDefault(item => item.Id == titleId);
if (titleElem == null) return;
var sortTitleElem = epubBook.Schema.Package.Metadata.MetaItems
.FirstOrDefault(item =>
item.Property == "file-as" && item.Refines == metadataItem.Refines);
if (sortTitleElem == null || string.IsNullOrWhiteSpace(sortTitleElem.Content)) return;
info.SeriesSort = sortTitleElem.Content;
}
private static void ExtractCollectionOrReadingList(EpubMetadataMeta metadataItem, EpubBookRef epubBook, ComicInfo info)
{
var titleId = metadataItem.Refines?.Replace("#", string.Empty);
var readingListElem = epubBook.Schema.Package.Metadata.Titles
.FirstOrDefault(item => item.Id == titleId);
if (readingListElem == null) return;
var count = epubBook.Schema.Package.Metadata.MetaItems
.FirstOrDefault(item =>
item.Property == "display-seq" && item.Refines == metadataItem.Refines);
if (count == null || count.Content == "0")
{
// TODO: Rewrite this to use a StringBuilder
// Treat this as a Collection
info.SeriesGroup += (string.IsNullOrEmpty(info.StoryArc) ? string.Empty : ",") +
readingListElem.Title.Replace(",", "_");
}
else
{
// Treat as a reading list
info.AlternateSeries += (string.IsNullOrEmpty(info.AlternateSeries) ? string.Empty : ",") +
readingListElem.Title.Replace(",", "_");
info.AlternateNumber += (string.IsNullOrEmpty(info.AlternateNumber) ? string.Empty : ",") + count.Content;
}
}
private static void PopulatePerson(EpubMetadataMeta metadataItem, ComicInfo info, EpubMetadataCreator person)
{
switch (metadataItem.Content)
{
case "art":
case "artist":
info.CoverArtist += AppendAuthor(person);
return;
case "aut":
case "author":
info.Writer += AppendAuthor(person);
return;
case "pbl":
case "publisher":
info.Publisher += AppendAuthor(person);
return;
case "trl":
case "translator":
info.Translator += AppendAuthor(person);
return;
case "edt":
case "editor":
info.Editor += AppendAuthor(person);
return;
case "ill":
case "illustrator":
info.Letterer += AppendAuthor(person);
return;
case "clr":
case "colorist":
info.Colorist += AppendAuthor(person);
return;
}
}
private static string AppendAuthor(EpubMetadataCreator person)
{
return Parser.CleanAuthor(person.Creator) + ",";
}
private static (int year, int month, int day) GetPublicationDate(string publicationDate)
{
var dateParsed = DateTime.TryParse(publicationDate, out var date);
@ -553,6 +711,8 @@ public class BookService : IBookService
catch (Exception ex)
{
_logger.LogWarning(ex, "[BookService] There was an exception getting number of pages, defaulting to 0");
_mediaErrorService.ReportMediaIssue(filePath, MediaErrorProducer.BookService,
"There was an exception getting number of pages, defaulting to 0", ex);
}
return 0;
@ -584,7 +744,9 @@ public class BookService : IBookService
foreach (var contentFileRef in await book.GetReadingOrderAsync())
{
if (contentFileRef.ContentType != EpubContentType.XHTML_1_1) continue;
dict.Add(contentFileRef.FileName, pageCount);
// Some keys are different than FilePath, so we add both to ease loookup
dict.Add(contentFileRef.FilePath, pageCount); // FileName -> FilePath
dict.TryAdd(contentFileRef.Key, pageCount); // FileName -> FilePath
pageCount += 1;
}
@ -697,6 +859,8 @@ public class BookService : IBookService
catch (Exception ex)
{
_logger.LogWarning(ex, "[BookService] There was an exception when opening epub book: {FileName}", filePath);
_mediaErrorService.ReportMediaIssue(filePath, MediaErrorProducer.BookService,
"There was an exception when opening epub book", ex);
}
return null;
@ -751,33 +915,47 @@ public class BookService : IBookService
/// <param name="mappings"></param>
/// <param name="key"></param>
/// <returns></returns>
private static string CoalesceKey(EpubBookRef book, IDictionary<string, int> mappings, string key)
private static string CoalesceKey(EpubBookRef book, IReadOnlyDictionary<string, int> mappings, string key)
{
if (mappings.ContainsKey(CleanContentKeys(key))) return key;
// Fallback to searching for key (bad epub metadata)
var correctedKey = book.Content.Html.Keys.FirstOrDefault(s => s.EndsWith(key));
var correctedKey = book.Content.Html.Local.Select(s => s.Key).FirstOrDefault(s => s.EndsWith(key));
if (!string.IsNullOrEmpty(correctedKey))
{
key = correctedKey;
}
var stepsBack = CountParentDirectory(book.Content.NavigationHtmlFile?.FilePath); // FileName -> FilePath
if (mappings.TryGetValue(key, out _))
{
return key;
}
var modifiedKey = RemovePathSegments(key, stepsBack);
if (mappings.TryGetValue(modifiedKey, out _))
{
return modifiedKey;
}
return key;
}
public static string CoalesceKeyForAnyFile(EpubBookRef book, string key)
{
if (book.Content.AllFiles.ContainsKey(key)) return key;
if (book.Content.AllFiles.ContainsLocalFileRefWithKey(key)) return key;
var cleanedKey = CleanContentKeys(key);
if (book.Content.AllFiles.ContainsKey(cleanedKey)) return cleanedKey;
if (book.Content.AllFiles.ContainsLocalFileRefWithKey(cleanedKey)) return cleanedKey;
// TODO: Figure this out
// Fallback to searching for key (bad epub metadata)
var correctedKey = book.Content.AllFiles.Keys.SingleOrDefault(s => s.EndsWith(key));
if (!string.IsNullOrEmpty(correctedKey))
{
key = correctedKey;
}
// var correctedKey = book.Content.AllFiles.Keys.SingleOrDefault(s => s.EndsWith(key));
// if (!string.IsNullOrEmpty(correctedKey))
// {
// key = correctedKey;
// }
return key;
}
@ -796,42 +974,48 @@ public class BookService : IBookService
var navItems = await book.GetNavigationAsync();
var chaptersList = new List<BookChapterItem>();
foreach (var navigationItem in navItems)
if (navItems != null)
{
if (navigationItem.NestedItems.Count == 0)
foreach (var navigationItem in navItems)
{
CreateToCChapter(navigationItem, Array.Empty<BookChapterItem>(), chaptersList, mappings);
continue;
}
var nestedChapters = new List<BookChapterItem>();
foreach (var nestedChapter in navigationItem.NestedItems.Where(n => n.Link != null))
{
var key = CoalesceKey(book, mappings, nestedChapter.Link.ContentFileName);
if (mappings.ContainsKey(key))
if (navigationItem.NestedItems.Count == 0)
{
nestedChapters.Add(new BookChapterItem
{
Title = nestedChapter.Title,
Page = mappings[key],
Part = nestedChapter.Link.Anchor ?? string.Empty,
Children = new List<BookChapterItem>()
});
CreateToCChapter(book, navigationItem, Array.Empty<BookChapterItem>(), chaptersList, mappings);
continue;
}
}
CreateToCChapter(navigationItem, nestedChapters, chaptersList, mappings);
var nestedChapters = new List<BookChapterItem>();
foreach (var nestedChapter in navigationItem.NestedItems.Where(n => n.Link != null))
{
var key = CoalesceKey(book, mappings, nestedChapter.Link?.ContentFilePath);
if (mappings.TryGetValue(key, out var mapping))
{
nestedChapters.Add(new BookChapterItem
{
Title = nestedChapter.Title,
Page = mapping,
Part = nestedChapter.Link?.Anchor ?? string.Empty,
Children = new List<BookChapterItem>()
});
}
}
CreateToCChapter(book, navigationItem, nestedChapters, chaptersList, mappings);
}
}
if (chaptersList.Count != 0) return chaptersList;
// Generate from TOC from links (any point past this, Kavita is generating as a TOC doesn't exist)
var tocPage = book.Content.Html.Keys.FirstOrDefault(k => k.ToUpper().Contains("TOC"));
if (tocPage == null) return chaptersList;
var tocPage = book.Content.Html.Local.Select(s => s.Key).FirstOrDefault(k => k.Equals("TOC.XHTML", StringComparison.InvariantCultureIgnoreCase) ||
k.Equals("NAVIGATION.XHTML", StringComparison.InvariantCultureIgnoreCase));
if (string.IsNullOrEmpty(tocPage)) return chaptersList;
// Find all anchor tags, for each anchor we get inner text, to lower then title case on UI. Get href and generate page content
if (!book.Content.Html.TryGetLocalFileRefByKey(tocPage, out var file)) return chaptersList;
var content = await file.ReadContentAsync();
var doc = new HtmlDocument();
var content = await book.Content.Html[tocPage].ReadContentAsync();
doc.LoadHtml(content);
var anchors = doc.DocumentNode.SelectNodes("//a");
if (anchors == null) return chaptersList;
@ -860,6 +1044,38 @@ public class BookService : IBookService
return chaptersList;
}
private static int CountParentDirectory(string path)
{
const string pattern = @"\.\./";
var matches = Regex.Matches(path, pattern);
return matches.Count;
}
/// <summary>
/// Removes paths segments from the beginning of a path. Returns original path if any issues.
/// </summary>
/// <param name="path"></param>
/// <param name="segmentsToRemove"></param>
/// <returns></returns>
private static string RemovePathSegments(string path, int segmentsToRemove)
{
if (segmentsToRemove <= 0)
return path;
var startIndex = 0;
for (var i = 0; i < segmentsToRemove; i++)
{
var slashIndex = path.IndexOf('/', startIndex);
if (slashIndex == -1)
return path; // Not enough segments to remove
startIndex = slashIndex + 1;
}
return path.Substring(startIndex);
}
/// <summary>
/// This returns a single page within the epub book. All html will be rewritten to be scoped within our reader,
/// all css is scoped, etc.
@ -916,14 +1132,15 @@ public class BookService : IBookService
}
} catch (Exception ex)
{
// NOTE: We can log this to media analysis service
_logger.LogError(ex, "There was an issue reading one of the pages for {Book}", book.FilePath);
await _mediaErrorService.ReportMediaIssueAsync(book.FilePath, MediaErrorProducer.BookService,
"There was an issue reading one of the pages for", ex);
}
throw new KavitaException("Could not find the appropriate html for that page");
}
private static void CreateToCChapter(EpubNavigationItemRef navigationItem, IList<BookChapterItem> nestedChapters,
private static void CreateToCChapter(EpubBookRef book, EpubNavigationItemRef navigationItem, IList<BookChapterItem> nestedChapters,
ICollection<BookChapterItem> chaptersList, IReadOnlyDictionary<string, int> mappings)
{
if (navigationItem.Link == null)
@ -942,7 +1159,7 @@ public class BookService : IBookService
}
else
{
var groupKey = CleanContentKeys(navigationItem.Link.ContentFileName);
var groupKey = CoalesceKey(book, mappings, navigationItem.Link.ContentFilePath);
if (mappings.ContainsKey(groupKey))
{
chaptersList.Add(new BookChapterItem
@ -962,15 +1179,15 @@ public class BookService : IBookService
/// <param name="fileFilePath"></param>
/// <param name="fileName">Name of the new file.</param>
/// <param name="outputDirectory">Where to output the file, defaults to covers directory</param>
/// <param name="saveAsWebP">When saving the file, use WebP encoding instead of PNG</param>
/// <param name="encodeFormat">When saving the file, use encoding</param>
/// <returns></returns>
public string GetCoverImage(string fileFilePath, string fileName, string outputDirectory, bool saveAsWebP = false)
public string GetCoverImage(string fileFilePath, string fileName, string outputDirectory, EncodeFormat encodeFormat)
{
if (!IsValidFile(fileFilePath)) return string.Empty;
if (Parser.IsPdf(fileFilePath))
{
return GetPdfCoverImage(fileFilePath, fileName, outputDirectory, saveAsWebP);
return GetPdfCoverImage(fileFilePath, fileName, outputDirectory, encodeFormat);
}
using var epubBook = EpubReader.OpenBook(fileFilePath, BookReaderOptions);
@ -979,24 +1196,26 @@ public class BookService : IBookService
{
// Try to get the cover image from OPF file, if not set, try to parse it from all the files, then result to the first one.
var coverImageContent = epubBook.Content.Cover
?? epubBook.Content.Images.Values.FirstOrDefault(file => Parser.IsCoverImage(file.FileName))
?? epubBook.Content.Images.Values.FirstOrDefault();
?? epubBook.Content.Images.Local.FirstOrDefault(file => Parser.IsCoverImage(file.FilePath)) // FileName -> FilePath
?? epubBook.Content.Images.Local.FirstOrDefault();
if (coverImageContent == null) return string.Empty;
using var stream = coverImageContent.GetContentStream();
return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, saveAsWebP);
return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, encodeFormat);
}
catch (Exception ex)
{
_logger.LogWarning(ex, "[BookService] There was a critical error and prevented thumbnail generation on {BookFile}. Defaulting to no cover image", fileFilePath);
_mediaErrorService.ReportMediaIssue(fileFilePath, MediaErrorProducer.BookService,
"There was a critical error and prevented thumbnail generation", ex);
}
return string.Empty;
}
private string GetPdfCoverImage(string fileFilePath, string fileName, string outputDirectory, bool saveAsWebP)
private string GetPdfCoverImage(string fileFilePath, string fileName, string outputDirectory, EncodeFormat encodeFormat)
{
try
{
@ -1006,7 +1225,7 @@ public class BookService : IBookService
using var stream = StreamManager.GetStream("BookService.GetPdfPage");
GetPdfPage(docReader, 0, stream);
return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, saveAsWebP);
return _imageService.WriteCoverThumbnail(stream, fileName, outputDirectory, encodeFormat);
}
catch (Exception ex)
@ -1014,6 +1233,8 @@ public class BookService : IBookService
_logger.LogWarning(ex,
"[BookService] There was a critical error and prevented thumbnail generation on {BookFile}. Defaulting to no cover image",
fileFilePath);
_mediaErrorService.ReportMediaIssue(fileFilePath, MediaErrorProducer.BookService,
"There was a critical error and prevented thumbnail generation", ex);
}
return string.Empty;
@ -1046,18 +1267,19 @@ public class BookService : IBookService
}
// Remove comments from CSS
// body = CssComment().Replace(body, string.Empty);
//
// body = WhiteSpace1().Replace(body, "#");
// body = WhiteSpace2().Replace(body, string.Empty);
// body = WhiteSpace3().Replace(body, " ");
// body = WhiteSpace4().Replace(body, "$1");
body = Regex.Replace(body, @"/\*[\d\D]*?\*/", string.Empty, RegexOptions.None, Parser.RegexTimeout);
body = Regex.Replace(body, @"[a-zA-Z]+#", "#", RegexOptions.None, Parser.RegexTimeout);
body = Regex.Replace(body, @"[\n\r]+\s*", string.Empty, RegexOptions.None, Parser.RegexTimeout);
body = Regex.Replace(body, @"\s+", " ", RegexOptions.None, Parser.RegexTimeout);
body = Regex.Replace(body, @"\s?([:,;{}])\s?", "$1", RegexOptions.None, Parser.RegexTimeout);
// Handle <!-- which some books use (but shouldn't)
body = Regex.Replace(body, "<!--.*?-->", string.Empty, RegexOptions.None, Parser.RegexTimeout);
// Handle /* */
body = Regex.Replace(body, @"/\*.*?\*/", string.Empty, RegexOptions.None, Parser.RegexTimeout);
try
{
body = body.Replace(";}", "}");
@ -1067,7 +1289,6 @@ public class BookService : IBookService
//Swallow exception. Some css don't have style rules ending in ';'
}
//body = UnitPadding().Replace(body, "$1");
body = Regex.Replace(body, @"([\s:]0)(px|pt|%|em)", "$1", RegexOptions.None, Parser.RegexTimeout);
@ -1076,7 +1297,7 @@ public class BookService : IBookService
private void LogBookErrors(EpubBookRef book, EpubContentFileRef contentFileRef, HtmlDocument doc)
{
_logger.LogError("{FilePath} has an invalid html file (Page {PageName})", book.FilePath, contentFileRef.FileName);
_logger.LogError("{FilePath} has an invalid html file (Page {PageName})", book.FilePath, contentFileRef.Key);
foreach (var error in doc.ParseErrors)
{
_logger.LogError("Line {LineNumber}, Reason: {Reason}", error.Line, error.Reason);

223
API/Services/BookmarkService.cs
View file

@ -7,7 +7,6 @@ using API.Data;
using API.DTOs.Reader;
using API.Entities;
using API.Entities.Enums;
using API.SignalR;
using Hangfire;
using Microsoft.Extensions.Logging;
@ -19,9 +18,6 @@ public interface IBookmarkService
Task<bool> BookmarkPage(AppUser userWithBookmarks, BookmarkDto bookmarkDto, string imageToBookmark);
Task<bool> RemoveBookmarkPage(AppUser userWithBookmarks, BookmarkDto bookmarkDto);
Task<IEnumerable<string>> GetBookmarkFilesById(IEnumerable<int> bookmarkIds);
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
Task ConvertAllBookmarkToWebP();
Task ConvertAllCoverToWebP();
}
public class BookmarkService : IBookmarkService
@ -30,17 +26,15 @@ public class BookmarkService : IBookmarkService
private readonly ILogger<BookmarkService> _logger;
private readonly IUnitOfWork _unitOfWork;
private readonly IDirectoryService _directoryService;
private readonly IImageService _imageService;
private readonly IEventHub _eventHub;
private readonly IMediaConversionService _mediaConversionService;
public BookmarkService(ILogger<BookmarkService> logger, IUnitOfWork unitOfWork,
IDirectoryService directoryService, IImageService imageService, IEventHub eventHub)
IDirectoryService directoryService, IMediaConversionService mediaConversionService)
{
_logger = logger;
_unitOfWork = unitOfWork;
_directoryService = directoryService;
_imageService = imageService;
_eventHub = eventHub;
_mediaConversionService = mediaConversionService;
}
/// <summary>
@ -77,21 +71,25 @@ public class BookmarkService : IBookmarkService
/// This is a job that runs after a bookmark is saved
/// </summary>
/// <remarks>This must be public</remarks>
public async Task ConvertBookmarkToWebP(int bookmarkId)
public async Task ConvertBookmarkToEncoding(int bookmarkId)
{
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
var convertBookmarkToWebP =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertBookmarkToWebP;
var encodeFormat =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
if (!convertBookmarkToWebP) return;
if (encodeFormat == EncodeFormat.PNG)
{
_logger.LogError("Cannot convert media to PNG");
return;
}
// Validate the bookmark still exists
var bookmark = await _unitOfWork.UserRepository.GetBookmarkAsync(bookmarkId);
if (bookmark == null) return;
bookmark.FileName = await SaveAsWebP(bookmarkDirectory, bookmark.FileName,
BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId));
bookmark.FileName = await _mediaConversionService.SaveAsEncodingFormat(bookmarkDirectory, bookmark.FileName,
BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId), encodeFormat);
_unitOfWork.UserRepository.Update(bookmark);
await _unitOfWork.CommitAsync();
@ -137,10 +135,10 @@ public class BookmarkService : IBookmarkService
_unitOfWork.UserRepository.Add(bookmark);
await _unitOfWork.CommitAsync();
if (settings.ConvertBookmarkToWebP)
if (settings.EncodeMediaAs == EncodeFormat.WEBP)
{
// Enqueue a task to convert the bookmark to webP
BackgroundJob.Enqueue(() => ConvertBookmarkToWebP(bookmark.Id));
BackgroundJob.Enqueue(() => ConvertBookmarkToEncoding(bookmark.Id));
}
}
catch (Exception ex)
@ -192,198 +190,9 @@ public class BookmarkService : IBookmarkService
b.FileName)));
}
/// <summary>
/// This is a long-running job that will convert all bookmarks into WebP. Do not invoke anyway except via Hangfire.
/// </summary>
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
public async Task ConvertAllBookmarkToWebP()
{
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(0F, ProgressEventType.Started));
var bookmarks = (await _unitOfWork.UserRepository.GetAllBookmarksAsync())
.Where(b => !b.FileName.EndsWith(".webp")).ToList();
var count = 1F;
foreach (var bookmark in bookmarks)
{
bookmark.FileName = await SaveAsWebP(bookmarkDirectory, bookmark.FileName,
BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId));
_unitOfWork.UserRepository.Update(bookmark);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(count / bookmarks.Count, ProgressEventType.Started));
count++;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(1F, ProgressEventType.Ended));
_logger.LogInformation("[BookmarkService] Converted bookmarks to WebP");
}
/// <summary>
/// This is a long-running job that will convert all covers into WebP. Do not invoke anyway except via Hangfire.
/// </summary>
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
public async Task ConvertAllCoverToWebP()
{
_logger.LogInformation("[BookmarkService] Starting conversion of all covers to webp");
var coverDirectory = _directoryService.CoverImageDirectory;
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(0F, ProgressEventType.Started));
var chapterCovers = await _unitOfWork.ChapterRepository.GetAllChaptersWithNonWebPCovers();
var seriesCovers = await _unitOfWork.SeriesRepository.GetAllWithNonWebPCovers();
var readingListCovers = await _unitOfWork.ReadingListRepository.GetAllWithNonWebPCovers();
var libraryCovers = await _unitOfWork.LibraryRepository.GetAllWithNonWebPCovers();
var collectionCovers = await _unitOfWork.CollectionTagRepository.GetAllWithNonWebPCovers();
var totalCount = chapterCovers.Count + seriesCovers.Count + readingListCovers.Count +
libraryCovers.Count + collectionCovers.Count;
var count = 1F;
_logger.LogInformation("[BookmarkService] Starting conversion of chapters");
foreach (var chapter in chapterCovers)
{
if (string.IsNullOrEmpty(chapter.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, chapter.CoverImage, coverDirectory);
chapter.CoverImage = Path.GetFileName(newFile);
_unitOfWork.ChapterRepository.Update(chapter);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
_logger.LogInformation("[BookmarkService] Starting conversion of series");
foreach (var series in seriesCovers)
{
if (string.IsNullOrEmpty(series.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, series.CoverImage, coverDirectory);
series.CoverImage = Path.GetFileName(newFile);
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
_logger.LogInformation("[BookmarkService] Starting conversion of libraries");
foreach (var library in libraryCovers)
{
if (string.IsNullOrEmpty(library.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, library.CoverImage, coverDirectory);
library.CoverImage = Path.GetFileName(newFile);
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
_logger.LogInformation("[BookmarkService] Starting conversion of reading lists");
foreach (var readingList in readingListCovers)
{
if (string.IsNullOrEmpty(readingList.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, readingList.CoverImage, coverDirectory);
readingList.CoverImage = Path.GetFileName(newFile);
_unitOfWork.ReadingListRepository.Update(readingList);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
_logger.LogInformation("[BookmarkService] Starting conversion of collections");
foreach (var collection in collectionCovers)
{
if (string.IsNullOrEmpty(collection.CoverImage)) continue;
var newFile = await SaveAsWebP(coverDirectory, collection.CoverImage, coverDirectory);
collection.CoverImage = Path.GetFileName(newFile);
_unitOfWork.CollectionTagRepository.Update(collection);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Started));
count++;
}
// Now null out all series and volumes that aren't webp or custom
var nonCustomOrConvertedVolumeCovers = await _unitOfWork.VolumeRepository.GetAllWithNonWebPCovers();
foreach (var volume in nonCustomOrConvertedVolumeCovers)
{
if (string.IsNullOrEmpty(volume.CoverImage)) continue;
volume.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter
_unitOfWork.VolumeRepository.Update(volume);
await _unitOfWork.CommitAsync();
}
var nonCustomOrConvertedSeriesCovers = await _unitOfWork.SeriesRepository.GetAllWithNonWebPCovers(false);
foreach (var series in nonCustomOrConvertedSeriesCovers)
{
if (string.IsNullOrEmpty(series.CoverImage)) continue;
series.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(1F, ProgressEventType.Ended));
_logger.LogInformation("[BookmarkService] Converted covers to WebP");
}
/// <summary>
/// Converts an image file, deletes original and returns the new path back
/// </summary>
/// <param name="imageDirectory">Full Path to where files are stored</param>
/// <param name="filename">The file to convert</param>
/// <param name="targetFolder">Full path to where files should be stored or any stem</param>
/// <returns></returns>
public async Task<string> SaveAsWebP(string imageDirectory, string filename, string targetFolder)
{
// This must be Public as it's used in via Hangfire as a background task
var fullSourcePath = _directoryService.FileSystem.Path.Join(imageDirectory, filename);
var fullTargetDirectory = fullSourcePath.Replace(new FileInfo(filename).Name, string.Empty);
var newFilename = string.Empty;
_logger.LogDebug("Converting {Source} image into WebP at {Target}", fullSourcePath, fullTargetDirectory);
try
{
// Convert target file to webp then delete original target file and update bookmark
try
{
var targetFile = await _imageService.ConvertToWebP(fullSourcePath, fullTargetDirectory);
var targetName = new FileInfo(targetFile).Name;
newFilename = Path.Join(targetFolder, targetName);
_directoryService.DeleteFiles(new[] {fullSourcePath});
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not convert image {FilePath}", filename);
newFilename = filename;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not convert image to WebP");
}
return newFilename;
}
private static string BookmarkStem(int userId, int seriesId, int chapterId)
public static string BookmarkStem(int userId, int seriesId, int chapterId)
{
return Path.Join($"{userId}", $"{seriesId}", $"{chapterId}");
}

4
API/Services/DirectoryService.cs
View file

@ -22,6 +22,7 @@ public interface IDirectoryService
string TempDirectory { get; }
string ConfigDirectory { get; }
string SiteThemeDirectory { get; }
string FaviconDirectory { get; }
/// <summary>
/// Original BookmarkDirectory. Only used for resetting directory. Use <see cref="ServerSettingKey.BackupDirectory"/> for actual path.
/// </summary>
@ -75,6 +76,7 @@ public class DirectoryService : IDirectoryService
public string ConfigDirectory { get; }
public string BookmarkDirectory { get; }
public string SiteThemeDirectory { get; }
public string FaviconDirectory { get; }
private readonly ILogger<DirectoryService> _logger;
private const RegexOptions MatchOptions = RegexOptions.Compiled | RegexOptions.IgnoreCase;
@ -98,6 +100,7 @@ public class DirectoryService : IDirectoryService
ConfigDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config");
BookmarkDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "bookmarks");
SiteThemeDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "themes");
FaviconDirectory = FileSystem.Path.Join(FileSystem.Directory.GetCurrentDirectory(), "config", "favicons");
ExistOrCreate(SiteThemeDirectory);
ExistOrCreate(CoverImageDirectory);
@ -105,6 +108,7 @@ public class DirectoryService : IDirectoryService
ExistOrCreate(LogDirectory);
ExistOrCreate(TempDirectory);
ExistOrCreate(BookmarkDirectory);
ExistOrCreate(FaviconDirectory);
}
/// <summary>

199
API/Services/ImageService.cs
View file

@ -1,7 +1,16 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Constants;
using API.Entities.Enums;
using API.Extensions;
using EasyCaching.Core;
using Flurl;
using Flurl.Http;
using HtmlAgilityPack;
using Kavita.Common;
using Microsoft.Extensions.Logging;
using NetVips;
using Image = NetVips.Image;
@ -11,55 +20,58 @@ namespace API.Services;
public interface IImageService
{
void ExtractImages(string fileFilePath, string targetDirectory, int fileCount = 1);
string GetCoverImage(string path, string fileName, string outputDirectory, bool saveAsWebP = false);
string GetCoverImage(string path, string fileName, string outputDirectory, EncodeFormat encodeFormat);
/// <summary>
/// Creates a Thumbnail version of a base64 image
/// </summary>
/// <param name="encodedImage">base64 encoded image</param>
/// <param name="fileName"></param>
/// <param name="saveAsWebP">Convert and save as webp</param>
/// <param name="encodeFormat">Convert and save as encoding format</param>
/// <param name="thumbnailWidth">Width of thumbnail</param>
/// <returns>File name with extension of the file. This will always write to <see cref="DirectoryService.CoverImageDirectory"/></returns>
string CreateThumbnailFromBase64(string encodedImage, string fileName, bool saveAsWebP = false, int thumbnailWidth = 320);
string CreateThumbnailFromBase64(string encodedImage, string fileName, EncodeFormat encodeFormat, int thumbnailWidth = 320);
/// <summary>
/// Writes out a thumbnail by stream input
/// </summary>
/// <param name="stream"></param>
/// <param name="fileName"></param>
/// <param name="outputDirectory"></param>
/// <param name="saveAsWebP"></param>
/// <param name="encodeFormat"></param>
/// <returns></returns>
string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, bool saveAsWebP = false);
string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, EncodeFormat encodeFormat);
/// <summary>
/// Writes out a thumbnail by file path input
/// </summary>
/// <param name="sourceFile"></param>
/// <param name="fileName"></param>
/// <param name="outputDirectory"></param>
/// <param name="saveAsWebP"></param>
/// <param name="encodeFormat"></param>
/// <returns></returns>
string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, bool saveAsWebP = false);
string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, EncodeFormat encodeFormat);
/// <summary>
/// Converts the passed image to webP and outputs it in the same directory
/// Converts the passed image to encoding and outputs it in the same directory
/// </summary>
/// <param name="filePath">Full path to the image to convert</param>
/// <param name="outputPath">Where to output the file</param>
/// <returns>File of written webp image</returns>
Task<string> ConvertToWebP(string filePath, string outputPath);
/// <returns>File of written encoded image</returns>
Task<string> ConvertToEncodingFormat(string filePath, string outputPath, EncodeFormat encodeFormat);
Task<bool> IsImage(string filePath);
Task<string> DownloadFaviconAsync(string url, EncodeFormat encodeFormat);
}
public class ImageService : IImageService
{
public const string Name = "BookmarkService";
private readonly ILogger<ImageService> _logger;
private readonly IDirectoryService _directoryService;
private readonly IEasyCachingProviderFactory _cacheFactory;
public const string ChapterCoverImageRegex = @"v\d+_c\d+";
public const string SeriesCoverImageRegex = @"series\d+";
public const string CollectionTagCoverImageRegex = @"tag\d+";
public const string ReadingListCoverImageRegex = @"readinglist\d+";
/// <summary>
/// Width of the Thumbnail generation
/// </summary>
@ -69,10 +81,26 @@ public class ImageService : IImageService
/// </summary>
public const int LibraryThumbnailWidth = 32;
public ImageService(ILogger<ImageService> logger, IDirectoryService directoryService)
private static readonly string[] ValidIconRelations = {
"icon",
"apple-touch-icon",
"apple-touch-icon-precomposed",
"apple-touch-icon icon-precomposed" // ComicVine has it combined
};
/// <summary>
/// A mapping of urls that need to get the icon from another url, due to strangeness (like app.plex.tv loading a black icon)
/// </summary>
private static readonly IDictionary<string, string> FaviconUrlMapper = new Dictionary<string, string>
{
["https://app.plex.tv"] = "https://plex.tv"
};
public ImageService(ILogger<ImageService> logger, IDirectoryService directoryService, IEasyCachingProviderFactory cacheFactory)
{
_logger = logger;
_directoryService = directoryService;
_cacheFactory = cacheFactory;
}
public void ExtractImages(string? fileFilePath, string targetDirectory, int fileCount = 1)
@ -90,14 +118,14 @@ public class ImageService : IImageService
}
}
public string GetCoverImage(string path, string fileName, string outputDirectory, bool saveAsWebP = false)
public string GetCoverImage(string path, string fileName, string outputDirectory, EncodeFormat encodeFormat)
{
if (string.IsNullOrEmpty(path)) return string.Empty;
try
{
using var thumbnail = Image.Thumbnail(path, ThumbnailWidth);
var filename = fileName + (saveAsWebP ? ".webp" : ".png");
var filename = fileName + encodeFormat.GetExtension();
thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(outputDirectory, filename));
return filename;
}
@ -116,12 +144,12 @@ public class ImageService : IImageService
/// <param name="stream">Stream to write to disk. Ensure this is rewinded.</param>
/// <param name="fileName">filename to save as without extension</param>
/// <param name="outputDirectory">Where to output the file, defaults to covers directory</param>
/// <param name="saveAsWebP">Export the file as webP otherwise will default to png</param>
/// <param name="encodeFormat">Export the file as the passed encoding</param>
/// <returns>File name with extension of the file. This will always write to <see cref="DirectoryService.CoverImageDirectory"/></returns>
public string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, bool saveAsWebP = false)
public string WriteCoverThumbnail(Stream stream, string fileName, string outputDirectory, EncodeFormat encodeFormat)
{
using var thumbnail = Image.ThumbnailStream(stream, ThumbnailWidth);
var filename = fileName + (saveAsWebP ? ".webp" : ".png");
var filename = fileName + encodeFormat.GetExtension();
_directoryService.ExistOrCreate(outputDirectory);
try
{
@ -131,10 +159,10 @@ public class ImageService : IImageService
return filename;
}
public string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, bool saveAsWebP = false)
public string WriteCoverThumbnail(string sourceFile, string fileName, string outputDirectory, EncodeFormat encodeFormat)
{
using var thumbnail = Image.Thumbnail(sourceFile, ThumbnailWidth);
var filename = fileName + (saveAsWebP ? ".webp" : ".png");
var filename = fileName + encodeFormat.GetExtension();
_directoryService.ExistOrCreate(outputDirectory);
try
{
@ -144,11 +172,11 @@ public class ImageService : IImageService
return filename;
}
public Task<string> ConvertToWebP(string filePath, string outputPath)
public Task<string> ConvertToEncodingFormat(string filePath, string outputPath, EncodeFormat encodeFormat)
{
var file = _directoryService.FileSystem.FileInfo.New(filePath);
var fileName = file.Name.Replace(file.Extension, string.Empty);
var outputFile = Path.Join(outputPath, fileName + ".webp");
var outputFile = Path.Join(outputPath, fileName + encodeFormat.GetExtension());
using var sourceImage = Image.NewFromFile(filePath, false, Enums.Access.SequentialUnbuffered);
sourceImage.WriteToFile(outputFile);
@ -177,14 +205,133 @@ public class ImageService : IImageService
return false;
}
public async Task<string> DownloadFaviconAsync(string url, EncodeFormat encodeFormat)
{
// Parse the URL to get the domain (including subdomain)
var uri = new Uri(url);
var domain = uri.Host;
var baseUrl = uri.Scheme + "://" + uri.Host;
var provider = _cacheFactory.GetCachingProvider(EasyCacheProfiles.Favicon);
var res = await provider.GetAsync<string>(baseUrl);
if (res.HasValue)
{
_logger.LogInformation("Kavita has already tried to fetch from {BaseUrl} and failed. Skipping duplicate check", baseUrl);
throw new KavitaException($"Kavita has already tried to fetch from {baseUrl} and failed. Skipping duplicate check");
}
await provider.SetAsync(baseUrl, string.Empty, TimeSpan.FromDays(10));
if (FaviconUrlMapper.TryGetValue(baseUrl, out var value))
{
url = value;
}
var correctSizeLink = string.Empty;
try
{
var htmlContent = url.GetStringAsync().Result;
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(htmlContent);
var pngLinks = htmlDocument.DocumentNode.Descendants("link")
.Where(link => ValidIconRelations.Contains(link.GetAttributeValue("rel", string.Empty)))
.Select(link => link.GetAttributeValue("href", string.Empty))
.Where(href => href.Split("?")[0].EndsWith(".png", StringComparison.InvariantCultureIgnoreCase))
.ToList();
correctSizeLink = (pngLinks?.FirstOrDefault(pngLink => pngLink.Contains("32")) ?? pngLinks?.FirstOrDefault());
}
catch (Exception ex)
{
_logger.LogError(ex, "Error downloading favicon.png for {Domain}, will try fallback methods", domain);
}
try
{
correctSizeLink = FallbackToKavitaReaderFavicon(baseUrl);
if (string.IsNullOrEmpty(correctSizeLink))
{
throw new KavitaException($"Could not grab favicon from {baseUrl}");
}
var finalUrl = correctSizeLink;
// If starts with //, it's coming usually from an offsite cdn
if (correctSizeLink.StartsWith("//"))
{
finalUrl = "https:" + correctSizeLink;
}
else if (!correctSizeLink.StartsWith(uri.Scheme))
{
finalUrl = Url.Combine(baseUrl, correctSizeLink);
}
_logger.LogTrace("Fetching favicon from {Url}", finalUrl);
// Download the favicon.ico file using Flurl
var faviconStream = await finalUrl
.AllowHttpStatus("2xx,304")
.GetStreamAsync();
// Create the destination file path
using var image = Image.PngloadStream(faviconStream);
var filename = $"{domain}{encodeFormat.GetExtension()}";
switch (encodeFormat)
{
case EncodeFormat.PNG:
image.Pngsave(Path.Combine(_directoryService.FaviconDirectory, filename));
break;
case EncodeFormat.WEBP:
image.Webpsave(Path.Combine(_directoryService.FaviconDirectory, filename));
break;
case EncodeFormat.AVIF:
image.Heifsave(Path.Combine(_directoryService.FaviconDirectory, filename));
break;
default:
throw new ArgumentOutOfRangeException(nameof(encodeFormat), encodeFormat, null);
}
_logger.LogDebug("Favicon.png for {Domain} downloaded and saved successfully", domain);
return filename;
}catch (Exception ex)
{
_logger.LogError(ex, "Error downloading favicon.png for {Domain}", domain);
throw;
}
}
private static string FallbackToKavitaReaderFavicon(string baseUrl)
{
var correctSizeLink = string.Empty;
var allOverrides = "https://kavitareader.com/assets/favicons/urls.txt".GetStringAsync().Result;
if (!string.IsNullOrEmpty(allOverrides))
{
var cleanedBaseUrl = baseUrl.Replace("https://", string.Empty);
var externalFile = allOverrides
.Split("\n")
.FirstOrDefault(url =>
cleanedBaseUrl.Equals(url.Replace(".png", string.Empty)) ||
cleanedBaseUrl.Replace("www.", string.Empty).Equals(url.Replace(".png", string.Empty)
));
if (string.IsNullOrEmpty(externalFile))
{
throw new KavitaException($"Could not grab favicon from {baseUrl}");
}
correctSizeLink = "https://kavitareader.com/assets/favicons/" + externalFile;
}
return correctSizeLink;
}
/// <inheritdoc />
public string CreateThumbnailFromBase64(string encodedImage, string fileName, bool saveAsWebP = false, int thumbnailWidth = ThumbnailWidth)
public string CreateThumbnailFromBase64(string encodedImage, string fileName, EncodeFormat encodeFormat, int thumbnailWidth = ThumbnailWidth)
{
try
{
using var thumbnail = Image.ThumbnailBuffer(Convert.FromBase64String(encodedImage), thumbnailWidth);
fileName += (saveAsWebP ? ".webp" : ".png");
fileName += encodeFormat.GetExtension();
thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, fileName));
return fileName;
}
@ -244,6 +391,7 @@ public class ImageService : IImageService
/// <returns></returns>
public static string GetReadingListFormat(int readingListId)
{
// ReSharper disable once StringLiteralTypo
return $"readinglist{readingListId}";
}
@ -257,6 +405,11 @@ public class ImageService : IImageService
return $"thumbnail{chapterId}";
}
public static string GetWebLinkFormat(string url, EncodeFormat encodeFormat)
{
return $"{new Uri(url).Host}{encodeFormat.GetExtension()}";
}
public static string CreateMergedImage(List<string> coverImages, string dest)
{

312
API/Services/MediaConversionService.cs Normal file
View file

@ -0,0 +1,312 @@
using System;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
using API.Entities.Enums;
using API.Extensions;
using API.SignalR;
using Hangfire;
using Microsoft.Extensions.Logging;
namespace API.Services;
public interface IMediaConversionService
{
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
Task ConvertAllBookmarkToEncoding();
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
Task ConvertAllCoversToEncoding();
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
Task ConvertAllManagedMediaToEncodingFormat();
Task<string> SaveAsEncodingFormat(string imageDirectory, string filename, string targetFolder,
EncodeFormat encodeFormat);
}
public class MediaConversionService : IMediaConversionService
{
public const string Name = "MediaConversionService";
public static readonly string[] ConversionMethods = {"ConvertAllBookmarkToEncoding", "ConvertAllCoversToEncoding", "ConvertAllManagedMediaToEncodingFormat"};
private readonly IUnitOfWork _unitOfWork;
private readonly IImageService _imageService;
private readonly IEventHub _eventHub;
private readonly IDirectoryService _directoryService;
private readonly ILogger<MediaConversionService> _logger;
public MediaConversionService(IUnitOfWork unitOfWork, IImageService imageService, IEventHub eventHub,
IDirectoryService directoryService, ILogger<MediaConversionService> logger)
{
_unitOfWork = unitOfWork;
_imageService = imageService;
_eventHub = eventHub;
_directoryService = directoryService;
_logger = logger;
}
/// <summary>
/// Converts all Kavita managed media (bookmarks, covers, favicons, etc) to the saved target encoding.
/// Do not invoke anyway except via Hangfire.
/// </summary>
/// <remarks>This is a long-running job</remarks>
/// <returns></returns>
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
public async Task ConvertAllManagedMediaToEncodingFormat()
{
await ConvertAllBookmarkToEncoding();
await ConvertAllCoversToEncoding();
await CoverAllFaviconsToEncoding();
}
/// <summary>
/// This is a long-running job that will convert all bookmarks into a format that is not PNG. Do not invoke anyway except via Hangfire.
/// </summary>
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
public async Task ConvertAllBookmarkToEncoding()
{
var bookmarkDirectory =
(await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.BookmarkDirectory)).Value;
var encodeFormat =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
if (encodeFormat == EncodeFormat.PNG)
{
_logger.LogError("Cannot convert media to PNG");
return;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(0F, ProgressEventType.Started));
var bookmarks = (await _unitOfWork.UserRepository.GetAllBookmarksAsync())
.Where(b => !b.FileName.EndsWith(encodeFormat.GetExtension())).ToList();
var count = 1F;
foreach (var bookmark in bookmarks)
{
bookmark.FileName = await SaveAsEncodingFormat(bookmarkDirectory, bookmark.FileName,
BookmarkService.BookmarkStem(bookmark.AppUserId, bookmark.SeriesId, bookmark.ChapterId), encodeFormat);
_unitOfWork.UserRepository.Update(bookmark);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(count / bookmarks.Count, ProgressEventType.Updated));
count++;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(1F, ProgressEventType.Ended));
_logger.LogInformation("[MediaConversionService] Converted bookmarks to {Format}", encodeFormat);
}
/// <summary>
/// This is a long-running job that will convert all covers into WebP. Do not invoke anyway except via Hangfire.
/// </summary>
[DisableConcurrentExecution(timeoutInSeconds: 2 * 60 * 60), AutomaticRetry(Attempts = 0)]
public async Task ConvertAllCoversToEncoding()
{
var coverDirectory = _directoryService.CoverImageDirectory;
var encodeFormat =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
if (encodeFormat == EncodeFormat.PNG)
{
_logger.LogError("Cannot convert media to PNG");
return;
}
_logger.LogInformation("[MediaConversionService] Starting conversion of all covers to {Format}", encodeFormat);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(0F, ProgressEventType.Started));
var chapterCovers = await _unitOfWork.ChapterRepository.GetAllChaptersWithCoversInDifferentEncoding(encodeFormat);
var seriesCovers = await _unitOfWork.SeriesRepository.GetAllWithCoversInDifferentEncoding(encodeFormat);
var readingListCovers = await _unitOfWork.ReadingListRepository.GetAllWithCoversInDifferentEncoding(encodeFormat);
var libraryCovers = await _unitOfWork.LibraryRepository.GetAllWithCoversInDifferentEncoding(encodeFormat);
var collectionCovers = await _unitOfWork.CollectionTagRepository.GetAllWithCoversInDifferentEncoding(encodeFormat);
var totalCount = chapterCovers.Count + seriesCovers.Count + readingListCovers.Count +
libraryCovers.Count + collectionCovers.Count;
var count = 1F;
_logger.LogInformation("[MediaConversionService] Starting conversion of chapters");
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(0, ProgressEventType.Started));
foreach (var chapter in chapterCovers)
{
if (string.IsNullOrEmpty(chapter.CoverImage)) continue;
var newFile = await SaveAsEncodingFormat(coverDirectory, chapter.CoverImage, coverDirectory, encodeFormat);
chapter.CoverImage = Path.GetFileName(newFile);
_unitOfWork.ChapterRepository.Update(chapter);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated));
count++;
}
_logger.LogInformation("[MediaConversionService] Starting conversion of series");
foreach (var series in seriesCovers)
{
if (string.IsNullOrEmpty(series.CoverImage)) continue;
var newFile = await SaveAsEncodingFormat(coverDirectory, series.CoverImage, coverDirectory, encodeFormat);
series.CoverImage = Path.GetFileName(newFile);
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated));
count++;
}
_logger.LogInformation("[MediaConversionService] Starting conversion of libraries");
foreach (var library in libraryCovers)
{
if (string.IsNullOrEmpty(library.CoverImage)) continue;
var newFile = await SaveAsEncodingFormat(coverDirectory, library.CoverImage, coverDirectory, encodeFormat);
library.CoverImage = Path.GetFileName(newFile);
_unitOfWork.LibraryRepository.Update(library);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated));
count++;
}
_logger.LogInformation("[MediaConversionService] Starting conversion of reading lists");
foreach (var readingList in readingListCovers)
{
if (string.IsNullOrEmpty(readingList.CoverImage)) continue;
var newFile = await SaveAsEncodingFormat(coverDirectory, readingList.CoverImage, coverDirectory, encodeFormat);
readingList.CoverImage = Path.GetFileName(newFile);
_unitOfWork.ReadingListRepository.Update(readingList);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated));
count++;
}
_logger.LogInformation("[MediaConversionService] Starting conversion of collections");
foreach (var collection in collectionCovers)
{
if (string.IsNullOrEmpty(collection.CoverImage)) continue;
var newFile = await SaveAsEncodingFormat(coverDirectory, collection.CoverImage, coverDirectory, encodeFormat);
collection.CoverImage = Path.GetFileName(newFile);
_unitOfWork.CollectionTagRepository.Update(collection);
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(count / totalCount, ProgressEventType.Updated));
count++;
}
// Now null out all series and volumes that aren't webp or custom
var nonCustomOrConvertedVolumeCovers = await _unitOfWork.VolumeRepository.GetAllWithCoversInDifferentEncoding(encodeFormat);
foreach (var volume in nonCustomOrConvertedVolumeCovers)
{
if (string.IsNullOrEmpty(volume.CoverImage)) continue;
volume.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter
_unitOfWork.VolumeRepository.Update(volume);
await _unitOfWork.CommitAsync();
}
var nonCustomOrConvertedSeriesCovers = await _unitOfWork.SeriesRepository.GetAllWithCoversInDifferentEncoding(encodeFormat, false);
foreach (var series in nonCustomOrConvertedSeriesCovers)
{
if (string.IsNullOrEmpty(series.CoverImage)) continue;
series.CoverImage = null; // We null it out so when we call Refresh Metadata it will auto update from first chapter
_unitOfWork.SeriesRepository.Update(series);
await _unitOfWork.CommitAsync();
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertCoverProgressEvent(1F, ProgressEventType.Ended));
_logger.LogInformation("[MediaConversionService] Converted covers to {Format}", encodeFormat);
}
private async Task CoverAllFaviconsToEncoding()
{
var encodeFormat =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
if (encodeFormat == EncodeFormat.PNG)
{
_logger.LogError("Cannot convert media to PNG");
return;
}
_logger.LogInformation("[MediaConversionService] Starting conversion of favicons to {Format}", encodeFormat);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(0F, ProgressEventType.Started));
var pngFavicons = _directoryService.GetFiles(_directoryService.FaviconDirectory)
.Where(b => !b.EndsWith(encodeFormat.GetExtension())).
ToList();
var count = 1F;
foreach (var file in pngFavicons)
{
await SaveAsEncodingFormat(_directoryService.FaviconDirectory, _directoryService.FileSystem.FileInfo.New(file).Name, _directoryService.FaviconDirectory,
encodeFormat);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(count / pngFavicons.Count, ProgressEventType.Updated));
count++;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ConvertBookmarksProgressEvent(1F, ProgressEventType.Ended));
_logger.LogInformation("[MediaConversionService] Converted favicons to {Format}", encodeFormat);
}
/// <summary>
/// Converts an image file, deletes original and returns the new path back
/// </summary>
/// <param name="imageDirectory">Full Path to where files are stored</param>
/// <param name="filename">The file to convert</param>
/// <param name="targetFolder">Full path to where files should be stored or any stem</param>
/// <returns></returns>
public async Task<string> SaveAsEncodingFormat(string imageDirectory, string filename, string targetFolder, EncodeFormat encodeFormat)
{
// This must be Public as it's used in via Hangfire as a background task
var fullSourcePath = _directoryService.FileSystem.Path.Join(imageDirectory, filename);
var fullTargetDirectory = fullSourcePath.Replace(new FileInfo(filename).Name, string.Empty);
var newFilename = string.Empty;
_logger.LogDebug("Converting {Source} image into {Encoding} at {Target}", fullSourcePath, encodeFormat, fullTargetDirectory);
if (!File.Exists(fullSourcePath))
{
_logger.LogError("Requested to convert {File} but it doesn't exist", fullSourcePath);
return newFilename;
}
try
{
// Convert target file to format then delete original target file
try
{
var targetFile = await _imageService.ConvertToEncodingFormat(fullSourcePath, fullTargetDirectory, encodeFormat);
var targetName = new FileInfo(targetFile).Name;
newFilename = Path.Join(targetFolder, targetName);
_directoryService.DeleteFiles(new[] {fullSourcePath});
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not convert image {FilePath} to {Format}", filename, encodeFormat);
newFilename = filename;
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Could not convert image to {Format}", encodeFormat);
}
return newFilename;
}
}

67
API/Services/MediaErrorService.cs Normal file
View file

@ -0,0 +1,67 @@
using System;
using System.Threading.Tasks;
using API.Data;
using API.Helpers.Builders;
using Hangfire;
namespace API.Services;
public enum MediaErrorProducer
{
BookService = 0,
ArchiveService = 1
}
public interface IMediaErrorService
{
Task ReportMediaIssueAsync(string filename, MediaErrorProducer producer, string errorMessage, string details);
void ReportMediaIssue(string filename, MediaErrorProducer producer, string errorMessage, string details);
Task ReportMediaIssueAsync(string filename, MediaErrorProducer producer, string errorMessage, Exception ex);
void ReportMediaIssue(string filename, MediaErrorProducer producer, string errorMessage, Exception ex);
}
public class MediaErrorService : IMediaErrorService
{
private readonly IUnitOfWork _unitOfWork;
public MediaErrorService(IUnitOfWork unitOfWork)
{
_unitOfWork = unitOfWork;
}
public async Task ReportMediaIssueAsync(string filename, MediaErrorProducer producer, string errorMessage, Exception ex)
{
await ReportMediaIssueAsync(filename, producer, errorMessage, ex.Message);
}
public void ReportMediaIssue(string filename, MediaErrorProducer producer, string errorMessage, Exception ex)
{
// To avoid overhead on commits, do async. We don't need to wait.
BackgroundJob.Enqueue(() => ReportMediaIssueAsync(filename, producer, errorMessage, ex.Message));
}
public void ReportMediaIssue(string filename, MediaErrorProducer producer, string errorMessage, string details)
{
// To avoid overhead on commits, do async. We don't need to wait.
BackgroundJob.Enqueue(() => ReportMediaIssueAsync(filename, producer, errorMessage, details));
}
public async Task ReportMediaIssueAsync(string filename, MediaErrorProducer producer, string errorMessage, string details)
{
var error = new MediaErrorBuilder(filename)
.WithComment(errorMessage)
.WithDetails(details)
.Build();
if (await _unitOfWork.MediaErrorRepository.ExistsAsync(error))
{
return;
}
_unitOfWork.MediaErrorRepository.Attach(error);
await _unitOfWork.CommitAsync();
}
}

29
API/Services/MetadataService.cs
View file

@ -6,6 +6,7 @@ using System.Threading.Tasks;
using API.Comparators;
using API.Data;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers;
using API.SignalR;
@ -32,7 +33,7 @@ public interface IMetadataService
/// <param name="forceUpdate">Overrides any cache logic and forces execution</param>
Task GenerateCoversForSeries(int libraryId, int seriesId, bool forceUpdate = true);
Task GenerateCoversForSeries(Series series, bool convertToWebP, bool forceUpdate = false);
Task GenerateCoversForSeries(Series series, EncodeFormat encodeFormat, bool forceUpdate = false);
Task RemoveAbandonedMetadataKeys();
}
@ -63,8 +64,8 @@ public class MetadataService : IMetadataService
/// </summary>
/// <param name="chapter"></param>
/// <param name="forceUpdate">Force updating cover image even if underlying file has not been modified or chapter already has a cover image</param>
/// <param name="convertToWebPOnWrite">Convert image to WebP when extracting the cover</param>
private Task<bool> UpdateChapterCoverImage(Chapter chapter, bool forceUpdate, bool convertToWebPOnWrite)
/// <param name="encodeFormat">Convert image to Encoding Format when extracting the cover</param>
private Task<bool> UpdateChapterCoverImage(Chapter chapter, bool forceUpdate, EncodeFormat encodeFormat)
{
var firstFile = chapter.Files.MinBy(x => x.Chapter);
if (firstFile == null) return Task.FromResult(false);
@ -78,7 +79,7 @@ public class MetadataService : IMetadataService
_logger.LogDebug("[MetadataService] Generating cover image for {File}", firstFile.FilePath);
chapter.CoverImage = _readingItemService.GetCoverImage(firstFile.FilePath,
ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId), firstFile.Format, convertToWebPOnWrite);
ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId), firstFile.Format, encodeFormat);
_unitOfWork.ChapterRepository.Update(chapter);
_updateEvents.Add(MessageFactory.CoverUpdateEvent(chapter.Id, MessageFactoryEntityTypes.Chapter));
return Task.FromResult(true);
@ -141,8 +142,8 @@ public class MetadataService : IMetadataService
/// </summary>
/// <param name="series"></param>
/// <param name="forceUpdate"></param>
/// <param name="convertToWebP"></param>
private async Task ProcessSeriesCoverGen(Series series, bool forceUpdate, bool convertToWebP)
/// <param name="encodeFormat"></param>
private async Task ProcessSeriesCoverGen(Series series, bool forceUpdate, EncodeFormat encodeFormat)
{
_logger.LogDebug("[MetadataService] Processing cover image generation for series: {SeriesName}", series.OriginalName);
try
@ -155,7 +156,7 @@ public class MetadataService : IMetadataService
var index = 0;
foreach (var chapter in volume.Chapters)
{
var chapterUpdated = await UpdateChapterCoverImage(chapter, forceUpdate, convertToWebP);
var chapterUpdated = await UpdateChapterCoverImage(chapter, forceUpdate, encodeFormat);
// If cover was update, either the file has changed or first scan and we should force a metadata update
UpdateChapterLastModified(chapter, forceUpdate || chapterUpdated);
if (index == 0 && chapterUpdated)
@ -207,7 +208,7 @@ public class MetadataService : IMetadataService
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.CoverUpdateProgressEvent(library.Id, 0F, ProgressEventType.Started, $"Starting {library.Name}"));
var convertToWebP = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP;
var encodeFormat = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++)
{
@ -237,7 +238,7 @@ public class MetadataService : IMetadataService
try
{
await ProcessSeriesCoverGen(series, forceUpdate, convertToWebP);
await ProcessSeriesCoverGen(series, forceUpdate, encodeFormat);
}
catch (Exception ex)
{
@ -287,23 +288,23 @@ public class MetadataService : IMetadataService
return;
}
var convertToWebP = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP;
await GenerateCoversForSeries(series, convertToWebP, forceUpdate);
var encodeFormat = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
await GenerateCoversForSeries(series, encodeFormat, forceUpdate);
}
/// <summary>
/// Generate Cover for a Series. This is used by Scan Loop and should not be invoked directly via User Interaction.
/// </summary>
/// <param name="series">A full Series, with metadata, chapters, etc</param>
/// <param name="convertToWebP">When saving the file, use WebP encoding instead of PNG</param>
/// <param name="encodeFormat">When saving the file, what encoding should be used</param>
/// <param name="forceUpdate"></param>
public async Task GenerateCoversForSeries(Series series, bool convertToWebP, bool forceUpdate = false)
public async Task GenerateCoversForSeries(Series series, EncodeFormat encodeFormat, bool forceUpdate = false)
{
var sw = Stopwatch.StartNew();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.CoverUpdateProgressEvent(series.LibraryId, 0F, ProgressEventType.Started, series.Name));
await ProcessSeriesCoverGen(series, forceUpdate, convertToWebP);
await ProcessSeriesCoverGen(series, forceUpdate, encodeFormat);
if (_unitOfWork.HasChanges())

18
API/Services/ReaderService.cs
View file

@ -236,7 +236,6 @@ public class ReaderService : IReaderService
try
{
// TODO: Rewrite this code to just pull user object with progress for that particular appuserprogress, else create it
var userProgress =
await _unitOfWork.AppUserProgressRepository.GetUserProgressAsync(progressDto.ChapterId, userId);
@ -479,10 +478,9 @@ public class ReaderService : IReaderService
/// <returns></returns>
public async Task<ChapterDto> GetContinuePoint(int seriesId, int userId)
{
var progress = (await _unitOfWork.AppUserProgressRepository.GetUserProgressForSeriesAsync(seriesId, userId)).ToList();
var volumes = (await _unitOfWork.VolumeRepository.GetVolumesDtoAsync(seriesId, userId)).ToList();
if (progress.Count == 0)
if (!await _unitOfWork.AppUserProgressRepository.AnyUserProgressForSeriesAsync(seriesId, userId))
{
// I think i need a way to sort volumes last
return volumes.OrderBy(v => double.Parse(v.Number + string.Empty), _chapterSortComparer).First().Chapters
@ -503,7 +501,8 @@ public class ReaderService : IReaderService
if (currentlyReadingChapter != null) return currentlyReadingChapter;
// Order with volume 0 last so we prefer the natural order
return FindNextReadingChapter(volumes.OrderBy(v => v.Number, SortComparerZeroLast.Default).SelectMany(v => v.Chapters).ToList());
return FindNextReadingChapter(volumes.OrderBy(v => v.Number, SortComparerZeroLast.Default)
.SelectMany(v => v.Chapters.OrderBy(c => double.Parse(c.Number))).ToList());
}
private static ChapterDto FindNextReadingChapter(IList<ChapterDto> volumeChapters)
@ -524,13 +523,14 @@ public class ReaderService : IReaderService
return lastChapter;
}
// If the last chapter didn't fit, then we need the next chapter without any progress
var firstChapterWithoutProgress = volumeChapters.FirstOrDefault(c => c.PagesRead == 0);
// If the last chapter didn't fit, then we need the next chapter without full progress
var firstChapterWithoutProgress = volumeChapters.FirstOrDefault(c => c.PagesRead < c.Pages);
if (firstChapterWithoutProgress != null)
{
return firstChapterWithoutProgress;
}
// chaptersWithProgress are all read, then we need to get the next chapter that doesn't have progress
var lastIndexWithProgress = volumeChapters.IndexOf(lastChapter);
if (lastIndexWithProgress + 1 < volumeChapters.Count)
@ -667,15 +667,15 @@ public class ReaderService : IReaderService
_directoryService.FileSystem.Path.Join(_directoryService.TempDirectory, ImageService.GetThumbnailFormat(chapter.Id));
try
{
var saveAsWebp =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertBookmarkToWebP;
var encodeFormat =
(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs;
if (!Directory.Exists(outputDirectory))
{
var outputtedThumbnails = cachedImages
.Select((img, idx) =>
_directoryService.FileSystem.Path.Join(outputDirectory,
_imageService.WriteCoverThumbnail(img, $"{idx}", outputDirectory, saveAsWebp)))
_imageService.WriteCoverThumbnail(img, $"{idx}", outputDirectory, encodeFormat)))
.ToArray();
return CacheService.GetPageFromFiles(outputtedThumbnails, pageNum);
}

13
API/Services/ReadingItemService.cs
View file

@ -9,7 +9,7 @@ public interface IReadingItemService
{
ComicInfo? GetComicInfo(string filePath);
int GetNumberOfPages(string filePath, MangaFormat format);
string GetCoverImage(string filePath, string fileName, MangaFormat format, bool saveAsWebP);
string GetCoverImage(string filePath, string fileName, MangaFormat format, EncodeFormat encodeFormat);
void Extract(string fileFilePath, string targetDirectory, MangaFormat format, int imageCount = 1);
ParserInfo? ParseFile(string path, string rootPath, LibraryType type);
}
@ -89,6 +89,7 @@ public class ReadingItemService : IReadingItemService
}
// This is first time ComicInfo is called
info.ComicInfo = GetComicInfo(path);
if (info.ComicInfo == null) return info;
@ -160,7 +161,7 @@ public class ReadingItemService : IReadingItemService
}
}
public string GetCoverImage(string filePath, string fileName, MangaFormat format, bool saveAsWebP)
public string GetCoverImage(string filePath, string fileName, MangaFormat format, EncodeFormat encodeFormat)
{
if (string.IsNullOrEmpty(filePath) || string.IsNullOrEmpty(fileName))
{
@ -170,10 +171,10 @@ public class ReadingItemService : IReadingItemService
return format switch
{
MangaFormat.Epub => _bookService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, saveAsWebP),
MangaFormat.Archive => _archiveService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, saveAsWebP),
MangaFormat.Image => _imageService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, saveAsWebP),
MangaFormat.Pdf => _bookService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, saveAsWebP),
MangaFormat.Epub => _bookService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, encodeFormat),
MangaFormat.Archive => _archiveService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, encodeFormat),
MangaFormat.Image => _imageService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, encodeFormat),
MangaFormat.Pdf => _bookService.GetCoverImage(filePath, fileName, _directoryService.CoverImageDirectory, encodeFormat),
_ => string.Empty
};
}

40
API/Services/ReadingListService.cs
View file

@ -157,19 +157,19 @@ public class ReadingListService : IReadingListService
readingList.CoverImageLocked = dto.CoverImageLocked;
if (NumberHelper.IsValidMonth(dto.StartingMonth))
if (NumberHelper.IsValidMonth(dto.StartingMonth) || dto.StartingMonth == 0)
{
readingList.StartingMonth = dto.StartingMonth;
}
if (NumberHelper.IsValidYear(dto.StartingYear))
if (NumberHelper.IsValidYear(dto.StartingYear) || dto.StartingYear == 0)
{
readingList.StartingYear = dto.StartingYear;
}
if (NumberHelper.IsValidMonth(dto.EndingMonth))
if (NumberHelper.IsValidMonth(dto.EndingMonth) || dto.EndingMonth == 0)
{
readingList.EndingMonth = dto.EndingMonth;
}
if (NumberHelper.IsValidYear(dto.EndingYear))
if (NumberHelper.IsValidYear(dto.EndingYear) || dto.EndingYear == 0)
{
readingList.EndingYear = dto.EndingYear;
}
@ -336,7 +336,7 @@ public class ReadingListService : IReadingListService
// .Select(c => _directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, c)).ToList();
//
// var combinedFile = ImageService.CreateMergedImage(fullImages, _directoryService.FileSystem.Path.Join(_directoryService.TempDirectory, $"{readingListId}.png"));
// // webp needs to be handled
// // webp/avif needs to be handled
// return combinedFile;
}
@ -496,12 +496,13 @@ public class ReadingListService : IReadingListService
}
readingList.Items = items;
if (!_unitOfWork.HasChanges()) continue;
await CalculateReadingListAgeRating(readingList);
if (_unitOfWork.HasChanges())
{
await _unitOfWork.CommitAsync();
}
await CalculateStartAndEndDates(await _unitOfWork.ReadingListRepository.GetReadingListByTitleAsync(arcPair.Item1, user.Id, ReadingListIncludes.Items | ReadingListIncludes.ItemChapter));
await _unitOfWork.CommitAsync(); // TODO: See if we can avoid this extra commit by reworking bottom logic
await CalculateStartAndEndDates(await _unitOfWork.ReadingListRepository.GetReadingListByTitleAsync(arcPair.Item1,
user.Id, ReadingListIncludes.Items | ReadingListIncludes.ItemChapter));
await _unitOfWork.CommitAsync();
}
}
@ -512,23 +513,24 @@ public class ReadingListService : IReadingListService
var data = new List<Tuple<string, string>>();
if (string.IsNullOrEmpty(storyArc)) return data;
var arcs = storyArc.Split(",");
var arcNumbers = storyArcNumbers.Split(",");
var arcs = storyArc.Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
var arcNumbers = storyArcNumbers.Split(",", StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries);
if (arcNumbers.Count(s => !string.IsNullOrEmpty(s)) != arcs.Length)
{
_logger.LogWarning("There is a mismatch on StoryArc and StoryArcNumber for {FileName}. Def", filename);
_logger.LogWarning("There is a mismatch on StoryArc and StoryArcNumber for {FileName}", filename);
}
var maxPairs = Math.Min(arcs.Length, arcNumbers.Length);
var maxPairs = Math.Max(arcs.Length, arcNumbers.Length);
for (var i = 0; i < maxPairs; i++)
{
// When there is a mismatch on arcs and arc numbers, then we should default to a high number
if (string.IsNullOrEmpty(arcNumbers[i]) && !string.IsNullOrEmpty(arcs[i]))
var arcNumber = int.MaxValue.ToString();
if (arcNumbers.Length > i)
{
arcNumbers[i] = int.MaxValue.ToString();
arcNumber = arcNumbers[i];
}
if (string.IsNullOrEmpty(arcs[i]) || !int.TryParse(arcNumbers[i], out _)) continue;
data.Add(new Tuple<string, string>(arcs[i], arcNumbers[i]));
if (string.IsNullOrEmpty(arcs[i]) || !int.TryParse(arcNumber, out _)) continue;
data.Add(new Tuple<string, string>(arcs[i], arcNumber));
}
return data;

40
API/Services/SeriesService.cs
View file

@ -48,14 +48,25 @@ public class SeriesService : ISeriesService
/// <summary>
/// Returns the first chapter for a series to extract metadata from (ie Summary, etc)
/// </summary>
/// <param name="series"></param>
/// <param name="isBookLibrary"></param>
/// <param name="series">The full series with all volumes and chapters on it</param>
/// <returns></returns>
public static Chapter? GetFirstChapterForMetadata(Series series, bool isBookLibrary)
public static Chapter? GetFirstChapterForMetadata(Series series)
{
return series.Volumes.OrderBy(v => v.Number, ChapterSortComparer.Default)
var sortedVolumes = series.Volumes.OrderBy(v => v.Number, ChapterSortComparer.Default);
var minVolumeNumber = sortedVolumes
.Where(v => v.Number != 0)
.MinBy(v => v.Number);
var minChapter = series.Volumes
.SelectMany(v => v.Chapters.OrderBy(c => float.Parse(c.Number), ChapterSortComparer.Default))
.FirstOrDefault();
if (minVolumeNumber != null && minChapter != null && float.Parse(minChapter.Number) > minVolumeNumber.Number)
{
return minVolumeNumber.Chapters.MinBy(c => float.Parse(c.Number), ChapterSortComparer.Default);
}
return minChapter;
}
public async Task<bool> UpdateSeriesMetadata(UpdateSeriesMetadataDto updateSeriesMetadataDto)
@ -98,10 +109,10 @@ public class SeriesService : ISeriesService
}
// This shouldn't be needed post v0.5.3 release
if (string.IsNullOrEmpty(series.Metadata.Summary))
{
series.Metadata.Summary = string.Empty;
}
// if (string.IsNullOrEmpty(series.Metadata.Summary))
// {
// series.Metadata.Summary = string.Empty;
// }
if (string.IsNullOrEmpty(updateSeriesMetadataDto.SeriesMetadata.Summary))
{
@ -120,6 +131,19 @@ public class SeriesService : ISeriesService
series.Metadata.LanguageLocked = true;
}
if (string.IsNullOrEmpty(updateSeriesMetadataDto.SeriesMetadata?.WebLinks))
{
series.Metadata.WebLinks = string.Empty;
} else
{
series.Metadata.WebLinks = string.Join(",", updateSeriesMetadataDto.SeriesMetadata?.WebLinks
.Split(",")
.Where(s => !string.IsNullOrEmpty(s))
.Select(s => s.Trim())!
);
}
series.Metadata.CollectionTags ??= new List<CollectionTag>();
UpdateCollectionsList(updateSeriesMetadataDto.CollectionTags, series, allCollectionTags, (tag) =>
{

56
API/Services/TaskScheduler.cs
View file

@ -31,7 +31,7 @@ public interface ITaskScheduler
void CancelStatsTasks();
Task RunStatCollection();
void ScanSiteThemes();
Task CovertAllCoversToWebP();
Task CovertAllCoversToEncoding();
Task CleanupDbEntries();
}
@ -50,9 +50,9 @@ public class TaskScheduler : ITaskScheduler
private readonly IThemeService _themeService;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly IStatisticService _statisticService;
private readonly IBookmarkService _bookmarkService;
private readonly IMediaConversionService _mediaConversionService;
public static BackgroundJobServer Client => new BackgroundJobServer();
public static BackgroundJobServer Client => new ();
public const string ScanQueue = "scan";
public const string DefaultQueue = "default";
public const string RemoveFromWantToReadTaskId = "remove-from-want-to-read";
@ -68,12 +68,17 @@ public class TaskScheduler : ITaskScheduler
private static readonly Random Rnd = new Random();
private static readonly RecurringJobOptions RecurringJobOptions = new RecurringJobOptions()
{
TimeZone = TimeZoneInfo.Local
};
public TaskScheduler(ICacheService cacheService, ILogger<TaskScheduler> logger, IScannerService scannerService,
IUnitOfWork unitOfWork, IMetadataService metadataService, IBackupService backupService,
ICleanupService cleanupService, IStatsService statsService, IVersionUpdaterService versionUpdaterService,
IThemeService themeService, IWordCountAnalyzerService wordCountAnalyzerService, IStatisticService statisticService,
IBookmarkService bookmarkService)
IMediaConversionService mediaConversionService)
{
_cacheService = cacheService;
_logger = logger;
@ -87,7 +92,7 @@ public class TaskScheduler : ITaskScheduler
_themeService = themeService;
_wordCountAnalyzerService = wordCountAnalyzerService;
_statisticService = statisticService;
_bookmarkService = bookmarkService;
_mediaConversionService = mediaConversionService;
}
public async Task ScheduleTasks()
@ -100,28 +105,28 @@ public class TaskScheduler : ITaskScheduler
var scanLibrarySetting = setting;
_logger.LogDebug("Scheduling Scan Library Task for {Setting}", scanLibrarySetting);
RecurringJob.AddOrUpdate(ScanLibrariesTaskId, () => _scannerService.ScanLibraries(false),
() => CronConverter.ConvertToCronNotation(scanLibrarySetting), TimeZoneInfo.Local);
() => CronConverter.ConvertToCronNotation(scanLibrarySetting), RecurringJobOptions);
}
else
{
RecurringJob.AddOrUpdate(ScanLibrariesTaskId, () => ScanLibraries(false), Cron.Daily, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(ScanLibrariesTaskId, () => ScanLibraries(false), Cron.Daily, RecurringJobOptions);
}
setting = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.TaskBackup)).Value;
if (setting != null)
{
_logger.LogDebug("Scheduling Backup Task for {Setting}", setting);
RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), () => CronConverter.ConvertToCronNotation(setting), TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), () => CronConverter.ConvertToCronNotation(setting), RecurringJobOptions);
}
else
{
RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), Cron.Weekly, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(BackupTaskId, () => _backupService.BackupDatabase(), Cron.Weekly, RecurringJobOptions);
}
RecurringJob.AddOrUpdate(CleanupTaskId, () => _cleanupService.Cleanup(), Cron.Daily, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(CleanupDbTaskId, () => _cleanupService.CleanupDbEntries(), Cron.Daily, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(RemoveFromWantToReadTaskId, () => _cleanupService.CleanupWantToRead(), Cron.Daily, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(UpdateYearlyStatsTaskId, () => _statisticService.UpdateServerStatistics(), Cron.Monthly, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(CleanupTaskId, () => _cleanupService.Cleanup(), Cron.Daily, RecurringJobOptions);
RecurringJob.AddOrUpdate(CleanupDbTaskId, () => _cleanupService.CleanupDbEntries(), Cron.Daily, RecurringJobOptions);
RecurringJob.AddOrUpdate(RemoveFromWantToReadTaskId, () => _cleanupService.CleanupWantToRead(), Cron.Daily, RecurringJobOptions);
RecurringJob.AddOrUpdate(UpdateYearlyStatsTaskId, () => _statisticService.UpdateServerStatistics(), Cron.Monthly, RecurringJobOptions);
}
#region StatsTasks
@ -137,7 +142,7 @@ public class TaskScheduler : ITaskScheduler
}
_logger.LogDebug("Scheduling stat collection daily");
RecurringJob.AddOrUpdate(ReportStatsTaskId, () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), TimeZoneInfo.Local);
RecurringJob.AddOrUpdate(ReportStatsTaskId, () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), RecurringJobOptions);
}
public void AnalyzeFilesForLibrary(int libraryId, bool forceUpdate = false)
@ -182,10 +187,20 @@ public class TaskScheduler : ITaskScheduler
BackgroundJob.Enqueue(() => _themeService.Scan());
}
public async Task CovertAllCoversToWebP()
/// <summary>
/// Do not invoke this manually, always enqueue on a background thread
/// </summary>
public async Task CovertAllCoversToEncoding()
{
await _bookmarkService.ConvertAllCoverToWebP();
_logger.LogInformation("[BookmarkService] Queuing tasks to update Series and Volume references via Cover Refresh");
var defaultParams = Array.Empty<object>();
if (MediaConversionService.ConversionMethods.Any(method =>
HasAlreadyEnqueuedTask(MediaConversionService.Name, method, defaultParams, DefaultQueue, true)))
{
return;
}
await _mediaConversionService.ConvertAllManagedMediaToEncodingFormat();
_logger.LogInformation("Queuing tasks to update Series and Volume references via Cover Refresh");
var libraryIds = await _unitOfWork.LibraryRepository.GetLibrariesAsync();
foreach (var lib in libraryIds)
{
@ -200,8 +215,10 @@ public class TaskScheduler : ITaskScheduler
public void ScheduleUpdaterTasks()
{
_logger.LogInformation("Scheduling Auto-Update tasks");
// Schedule update check between noon and 6pm local time
RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local);
RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Daily(Rnd.Next(5, 23)), new RecurringJobOptions()
{
TimeZone = TimeZoneInfo.Local
});
}
public void ScanFolder(string folderPath, TimeSpan delay)
@ -399,6 +416,7 @@ public class TaskScheduler : ITaskScheduler
var scheduledJobs = JobStorage.Current.GetMonitoringApi().ScheduledJobs(0, int.MaxValue);
ret = scheduledJobs.Any(j =>
j.Value.Job != null &&
j.Value.Job.Method.DeclaringType != null && j.Value.Job.Args.SequenceEqual(args) &&
j.Value.Job.Method.Name.Equals(methodName) &&
j.Value.Job.Method.DeclaringType.Name.Equals(className));

7
API/Services/Tasks/BackupService.cs
View file

@ -120,6 +120,8 @@ public class BackupService : IBackupService
await SendProgress(0.75F, "Copying themes");
CopyThemesToBackupDirectory(tempDirectory);
await SendProgress(0.85F, "Copying favicons");
CopyFaviconsToBackupDirectory(tempDirectory);
try
{
@ -141,6 +143,11 @@ public class BackupService : IBackupService
_directoryService.CopyFilesToDirectory(files, _directoryService.FileSystem.Path.Join(tempDirectory, "logs"));
}
private void CopyFaviconsToBackupDirectory(string tempDirectory)
{
_directoryService.CopyDirectoryToDirectory(_directoryService.FaviconDirectory, tempDirectory);
}
private async Task CopyCoverImagesToBackupDirectory(string tempDirectory)
{
var outputTempDir = Path.Join(tempDirectory, "covers");

8
API/Services/Tasks/CleanupService.cs
View file

@ -58,14 +58,14 @@ public class CleanupService : ICleanupService
[AutomaticRetry(Attempts = 3, LogEvents = false, OnAttemptsExceeded = AttemptsExceededAction.Fail)]
public async Task Cleanup()
{
if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllCoverToWebP", Array.Empty<object>(),
if (TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllCoverToEncoding", Array.Empty<object>(),
TaskScheduler.DefaultQueue, true) ||
TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllBookmarkToWebP", Array.Empty<object>(),
TaskScheduler.HasAlreadyEnqueuedTask(BookmarkService.Name, "ConvertAllBookmarkToEncoding", Array.Empty<object>(),
TaskScheduler.DefaultQueue, true))
{
_logger.LogInformation("Cleanup put on hold as a conversion to WebP in progress");
_logger.LogInformation("Cleanup put on hold as a media conversion in progress");
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.ErrorEvent("Cleanup", "Cleanup put on hold as a conversion to WebP in progress"));
MessageFactory.ErrorEvent("Cleanup", "Cleanup put on hold as a media conversion in progress"));
return;
}

6
API/Services/Tasks/Metadata/WordCountAnalyzerService.cs
View file

@ -172,7 +172,7 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService
{
using var book = await EpubReader.OpenBookAsync(filePath, BookService.BookReaderOptions);
var totalPages = book.Content.Html.Values;
var totalPages = book.Content.Html.Local;
foreach (var bookPage in totalPages)
{
var progress = Math.Max(0F,
@ -238,10 +238,10 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService
}
private static async Task<int> GetWordCountFromHtml(EpubContentFileRef bookFile)
private static async Task<int> GetWordCountFromHtml(EpubLocalTextContentFileRef bookFile)
{
var doc = new HtmlDocument();
doc.LoadHtml(await bookFile.ReadContentAsTextAsync());
doc.LoadHtml(await bookFile.ReadContentAsync());
var textNodes = doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]");
if (textNodes == null) return 0;

25
API/Services/Tasks/Scanner/Parser/Parser.cs
View file

@ -14,7 +14,7 @@ public static class Parser
private const int RegexTimeoutMs = 5000000; // 500 ms
public static readonly TimeSpan RegexTimeout = TimeSpan.FromMilliseconds(500);
public const string ImageFileExtensions = @"^(\.png|\.jpeg|\.jpg|\.webp|\.gif)";
public const string ImageFileExtensions = @"^(\.png|\.jpeg|\.jpg|\.webp|\.gif|\.avif)";
public const string ArchiveFileExtensions = @"\.cbz|\.zip|\.rar|\.cbr|\.tar.gz|\.7zip|\.7z|\.cb7|\.cbt";
private const string BookFileExtensions = @"\.epub|\.pdf";
private const string XmlRegexExtensions = @"\.xml";
@ -321,13 +321,9 @@ public static class Parser
new Regex(
@"(?<Series>.*)( ?- ?)Ch\.\d+-?\d*",
MatchOptions, RegexTimeout),
// [BAA]_Darker_than_Black_Omake-1.zip
// [BAA]_Darker_than_Black_Omake-1, Bleach 001-002, Kodoja #001 (March 2016)
new Regex(
@"^(?!Vol)(?<Series>.*)(-)\d+-?\d*", // This catches a lot of stuff ^(?!Vol)(?<Series>.*)( |_)(\d+)
MatchOptions, RegexTimeout),
// Kodoja #001 (March 2016)
new Regex(
@"(?<Series>.*)(\s|_|-)#",
@"^(?!Vol)(?!Chapter)(?<Series>.+?)(-|_|\s|#)\d+(-\d+)?",
MatchOptions, RegexTimeout),
// Baketeriya ch01-05.zip, Akiiro Bousou Biyori - 01.jpg, Beelzebub_172_RHS.zip, Cynthia the Mission 29.rar, A Compendium of Ghosts - 031 - The Third Story_ Part 12 (Digital) (Cobalt001)
new Regex(
@ -1062,4 +1058,19 @@ public static class Parser
{
return string.IsNullOrEmpty(name) ? string.Empty : name.Replace('_', ' ');
}
public static string? ExtractFilename(string fileUrl)
{
var matches = Parser.CssImageUrlRegex.Matches(fileUrl);
foreach (Match match in matches)
{
if (!match.Success) continue;
// NOTE: This is failing for //localhost:5000/api/book/29919/book-resources?file=OPS/images/tick1.jpg
var importFile = match.Groups["Filename"].Value;
if (!importFile.Contains("?")) return importFile;
}
return null;
}
}

47
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -36,7 +36,7 @@ public interface IProcessSeries
void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false);
void UpdateChapters(Series series, Volume volume, IList<ParserInfo> parsedInfos, bool forceUpdate = false);
void AddOrUpdateFileForChapter(Chapter chapter, ParserInfo info, bool forceUpdate = false);
void UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? info);
void UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? comicInfo);
}
/// <summary>
@ -230,7 +230,7 @@ public class ProcessSeries : IProcessSeries
_logger.LogError(ex, "[ScannerService] There was an exception updating series for {SeriesName}", series.Name);
}
await _metadataService.GenerateCoversForSeries(series, (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).ConvertCoverToWebP);
await _metadataService.GenerateCoversForSeries(series, (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EncodeMediaAs);
EnqueuePostSeriesProcessTasks(series.LibraryId, series.Id);
}
@ -266,8 +266,7 @@ public class ProcessSeries : IProcessSeries
public void UpdateSeriesMetadata(Series series, Library library)
{
series.Metadata ??= new SeriesMetadataBuilder().Build();
var isBook = library.Type == LibraryType.Book;
var firstChapter = SeriesService.GetFirstChapterForMetadata(series, isBook);
var firstChapter = SeriesService.GetFirstChapterForMetadata(series);
var firstFile = firstChapter?.Files.FirstOrDefault();
if (firstFile == null) return;
@ -323,7 +322,7 @@ public class ProcessSeries : IProcessSeries
if (!string.IsNullOrEmpty(firstChapter?.SeriesGroup) && library.ManageCollections)
{
_logger.LogDebug("Collection tag(s) found for {SeriesName}, updating collections", series.Name);
foreach (var collection in firstChapter.SeriesGroup.Split(','))
foreach (var collection in firstChapter.SeriesGroup.Split(',', StringSplitOptions.TrimEntries | StringSplitOptions.RemoveEmptyEntries))
{
var normalizedName = Parser.Parser.Normalize(collection);
if (!_collectionTags.TryGetValue(normalizedName, out var tag))
@ -346,6 +345,8 @@ public class ProcessSeries : IProcessSeries
}
#region People
// Handle People
foreach (var chapter in chapters)
{
@ -490,6 +491,8 @@ public class ProcessSeries : IProcessSeries
}
});
#endregion
}
public void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
@ -657,19 +660,13 @@ public class ProcessSeries : IProcessSeries
}
}
public void UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? info)
public void UpdateChapterFromComicInfo(Chapter chapter, ComicInfo? comicInfo)
{
if (comicInfo == null) return;
var firstFile = chapter.Files.MinBy(x => x.Chapter);
if (firstFile == null ||
_cacheHelper.IsFileUnmodifiedSinceCreationOrLastScan(chapter, false, firstFile)) return;
var comicInfo = info;
if (info == null)
{
comicInfo = _readingItemService.GetComicInfo(firstFile.FilePath);
}
if (comicInfo == null) return;
_logger.LogTrace("[ScannerService] Read ComicInfo for {File}", firstFile.FilePath);
chapter.AgeRating = ComicInfo.ConvertAgeRatingToEnum(comicInfo.AgeRating);
@ -714,12 +711,24 @@ public class ProcessSeries : IProcessSeries
chapter.StoryArcNumber = comicInfo.StoryArcNumber;
}
if (comicInfo.AlternateCount > 0)
{
chapter.AlternateCount = comicInfo.AlternateCount;
}
if (!string.IsNullOrEmpty(comicInfo.Web))
{
chapter.WebLinks = string.Join(",", comicInfo.Web
.Split(",")
.Where(s => !string.IsNullOrEmpty(s))
.Select(s => s.Trim())
);
}
if (!string.IsNullOrEmpty(comicInfo.Isbn))
{
chapter.ISBN = comicInfo.Isbn;
}
if (comicInfo.Count > 0)
{
@ -807,11 +816,15 @@ public class ProcessSeries : IProcessSeries
private static IList<string> GetTagValues(string comicInfoTagSeparatedByComma)
{
// TODO: Move this to an extension and test it
if (!string.IsNullOrEmpty(comicInfoTagSeparatedByComma))
if (string.IsNullOrEmpty(comicInfoTagSeparatedByComma))
{
return comicInfoTagSeparatedByComma.Split(",").Select(s => s.Trim()).DistinctBy(Parser.Parser.Normalize).ToList();
return ImmutableList<string>.Empty;
}
return ImmutableList<string>.Empty;
return comicInfoTagSeparatedByComma.Split(",")
.Select(s => s.Trim())
.DistinctBy(Parser.Parser.Normalize)
.ToList();
}
/// <summary>

Some files were not shown because too many files have changed in this diff Show more