AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Koreader progress sync interface (#3025)

Browse source 
Co-authored-by: Joe Milazzo <josephmajora@gmail.com>
This commit is contained in:
Matthew DeAngelo 2024-10-26 07:09:00 -04:00 • committed by GitHub
parent 1a88dd4fc0
commit 180b49b8ea
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
45 changed files with 5848 additions and 8312 deletions
Download patch file Download diff file Expand all files Collapse all files

46
API/API.csproj
View file

@ -12,11 +12,13 @@
<LangVersion>latestmajor</LangVersion>
</PropertyGroup>
<!-- <Target Name="PostBuild" AfterTargets="Build" Condition=" '$(Configuration)' == 'Debug' ">-->
<!-- <Delete Files="../openapi.json" />-->
<!-- <Exec Command="swagger tofile &#45;&#45;output ../openapi.json bin/$(Configuration)/$(TargetFramework)/$(AssemblyName).dll v1" />-->
<!-- </Target>-->
<Target Name="PostBuild" AfterTargets="Build" Condition=" '$(Configuration)' == 'Debug' ">
<Delete Files="../openapi.json" />
<Exec Command="swagger tofile &#45;&#45;output ../openapi.json bin/$(Configuration)/$(TargetFramework)/$(AssemblyName).dll v1" />
</Target>
=======
>>>>>>> fc7f84f2bffadf9a0127d35fa01166c847b20dc0
<PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
<DebugSymbols>false</DebugSymbols>
<ApplicationIcon>../favicon.ico</ApplicationIcon>
@ -55,41 +57,41 @@
<ItemGroup>
<PackageReference Include="CsvHelper" Version="33.0.1" />
<PackageReference Include="MailKit" Version="4.7.1.1" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.8">
<PackageReference Include="MailKit" Version="4.8.0" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Design" Version="8.0.10">
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="12.0.1" />
<PackageReference Include="Docnet.Core" Version="2.6.0" />
<PackageReference Include="EasyCaching.InMemory" Version="1.9.2" />
<PackageReference Include="ExCSS" Version="4.2.5" />
<PackageReference Include="ExCSS" Version="4.3.0" />
<PackageReference Include="Flurl" Version="3.0.7" />
<PackageReference Include="Flurl.Http" Version="3.2.4" />
<PackageReference Include="Hangfire" Version="1.8.14" />
<PackageReference Include="Hangfire" Version="1.8.15" />
<PackageReference Include="Hangfire.InMemory" Version="1.0.0" />
<PackageReference Include="Hangfire.MaximumConcurrentExecutions" Version="1.1.0" />
<PackageReference Include="Hangfire.Storage.SQLite" Version="0.4.2" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.66" />
<PackageReference Include="HtmlAgilityPack" Version="1.11.69" />
<PackageReference Include="MarkdownDeep.NET.Core" Version="1.5.0.4" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.8.14" />
<PackageReference Include="Hangfire.AspNetCore" Version="1.8.15" />
<PackageReference Include="Microsoft.AspNetCore.SignalR" Version="1.1.0" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.8" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="8.0.8" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="8.0.8" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.8" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.0" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.JwtBearer" Version="8.0.10" />
<PackageReference Include="Microsoft.AspNetCore.Authentication.OpenIdConnect" Version="8.0.10" />
<PackageReference Include="Microsoft.AspNetCore.Identity.EntityFrameworkCore" Version="8.0.10" />
<PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="8.0.10" />
<PackageReference Include="Microsoft.Extensions.DependencyInjection" Version="8.0.1" />
<PackageReference Include="Microsoft.IO.RecyclableMemoryStream" Version="3.0.1" />
<PackageReference Include="MimeTypeMapOfficial" Version="1.0.17" />
<PackageReference Include="Nager.ArticleNumber" Version="1.0.7" />
<PackageReference Include="NetVips" Version="2.4.1" />
<PackageReference Include="NetVips" Version="2.4.2" />
<PackageReference Include="NetVips.Native" Version="8.15.3" />
<PackageReference Include="NReco.Logging.File" Version="1.2.1" />
<PackageReference Include="Serilog" Version="4.0.1" />
<PackageReference Include="Serilog.AspNetCore" Version="8.0.2" />
<PackageReference Include="Serilog" Version="4.1.0" />
<PackageReference Include="Serilog.AspNetCore" Version="8.0.3" />
<PackageReference Include="Serilog.Enrichers.Thread" Version="4.0.0" />
<PackageReference Include="Serilog.Extensions.Hosting" Version="8.0.0" />
<PackageReference Include="Serilog.Settings.Configuration" Version="8.0.2" />
<PackageReference Include="Serilog.Settings.Configuration" Version="8.0.4" />
<PackageReference Include="Serilog.Sinks.AspNetCore.SignalR" Version="0.4.0" />
<PackageReference Include="Serilog.Sinks.Console" Version="6.0.0" />
<PackageReference Include="Serilog.Sinks.File" Version="6.0.0" />
@ -100,11 +102,11 @@
<PrivateAssets>all</PrivateAssets>
<IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
</PackageReference>
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.7.3" />
<PackageReference Include="Swashbuckle.AspNetCore" Version="6.9.0" />
<PackageReference Include="Swashbuckle.AspNetCore.Filters" Version="8.0.2" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="8.0.2" />
<PackageReference Include="System.IdentityModel.Tokens.Jwt" Version="8.1.2" />
<PackageReference Include="System.IO.Abstractions" Version="21.0.29" />
<PackageReference Include="System.Drawing.Common" Version="8.0.8" />
<PackageReference Include="System.Drawing.Common" Version="8.0.10" />
<PackageReference Include="VersOne.Epub" Version="3.3.2" />
</ItemGroup>

99
API/Controllers/KoreaderController.cs Normal file
View file

@ -0,0 +1,99 @@
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using System;
using System.Threading.Tasks;
using API.Data;
using API.Data.Repositories;
using API.DTOs.Koreader;
using API.Entities;
using API.Services;
using Kavita.Common;
using Microsoft.AspNetCore.Identity;
using Microsoft.Extensions.Logging;
using static System.Net.WebRequestMethods;
namespace API.Controllers;
#nullable enable
/// <summary>
/// The endpoint to interface with Koreader's Progress Sync plugin.
/// </summary>
/// <remarks>
/// Koreader uses a different form of athentication. It stores the user name
/// and password in headers.
/// </remarks>
/// <see cref="https://github.com/koreader/koreader/blob/master/plugins/kosync.koplugin/KOSyncClient.lua"/>
[AllowAnonymous]
public class KoreaderController : BaseApiController
{
private readonly IUnitOfWork _unitOfWork;
private readonly ILocalizationService _localizationService;
private readonly IKoreaderService _koreaderService;
private readonly ILogger<KoreaderController> _logger;
public KoreaderController(IUnitOfWork unitOfWork, ILocalizationService localizationService,
IKoreaderService koreaderService, ILogger<KoreaderController> logger)
{
_unitOfWork = unitOfWork;
_localizationService = localizationService;
_koreaderService = koreaderService;
_logger = logger;
}
// We won't allow users to be created from Koreader. Rather, they
// must already have an account.
/*
[HttpPost("/users/create")]
public IActionResult CreateUser(CreateUserRequest request)
{
}
*/
[HttpGet("{apiKey}/users/auth")]
public async Task<IActionResult> Authenticate(string apiKey)
{
var userId = await GetUserId(apiKey);
var user = await _unitOfWork.UserRepository.GetUserByIdAsync(userId);
return Ok(new { username = user.UserName });
}
[HttpPut("{apiKey}/syncs/progress")]
public async Task<IActionResult> UpdateProgress(string apiKey, KoreaderBookDto request)
{
_logger.LogDebug("Koreader sync progress: {Progress}", request.Progress);
var userId = await GetUserId(apiKey);
await _koreaderService.SaveProgress(request, userId);
return Ok(new { document = request.Document, timestamp = DateTime.UtcNow });
}
[HttpGet("{apiKey}/syncs/progress/{ebookHash}")]
public async Task<IActionResult> GetProgress(string apiKey, string ebookHash)
{
var userId = await GetUserId(apiKey);
var response = await _koreaderService.GetProgress(ebookHash, userId);
_logger.LogDebug("Koreader response progress: {Progress}", response.Progress);
return Ok(response);
}
/// <summary>
/// Gets the user from the API key
/// </summary>
/// <returns>The user's Id</returns>
private async Task<int> GetUserId(string apiKey)
{
try
{
return await _unitOfWork.UserRepository.GetUserIdByApiKeyAsync(apiKey);
}
catch
{
throw new KavitaException(await _localizationService.Get("en", "user-doesnt-exist"));
}
}
}

33
API/DTOs/Koreader/KoreaderBookDto.cs Normal file
View file

@ -0,0 +1,33 @@
using API.DTOs.Progress;
namespace API.DTOs.Koreader;
/// <summary>
/// This is the interface for receiving and sending updates to Koreader. The only fields
/// that are actually used are the Document and Progress fields.
/// </summary>
public class KoreaderBookDto
{
/// <summary>
/// This is the Koreader hash of the book. It is used to identify the book.
/// </summary>
public string Document { get; set; }
/// <summary>
/// A randomly generated id from the koreader device. Only used to maintain the Koreader interface.
/// </summary>
public string Device_id { get; set; }
/// <summary>
/// The Koreader device name. Only used to maintain the Koreader interface.
/// </summary>
public string Device { get; set; }
/// <summary>
/// Percent progress of the book. Only used to maintain the Koreader interface.
/// </summary>
public float Percentage { get; set; }
/// <summary>
/// An XPath string read by Koreader to determine the location within the epub.
/// Essentially, it is Koreader's equivalent to ProgressDto.BookScrollId.
/// </summary>
/// <seealso cref="ProgressDto.BookScrollId"/>
public string Progress { get; set; }
}

3148
API/Data/Migrations/20240914041512_KoreaderHash.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

28
API/Data/Migrations/20240914041512_KoreaderHash.cs Normal file
View file

@ -0,0 +1,28 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
/// <inheritdoc />
public partial class KoreaderHash : Migration
{
/// <inheritdoc />
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<string>(
name: "KoreaderHash",
table: "MangaFile",
type: "TEXT",
nullable: true);
}
/// <inheritdoc />
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "KoreaderHash",
table: "MangaFile");
}
}
}

3
API/Data/Migrations/DataContextModelSnapshot.cs
View file

@ -1174,6 +1174,9 @@ namespace API.Data.Migrations
b.Property<int>("Format")
.HasColumnType("INTEGER");
b.Property<string>("KoreaderHash")
.HasColumnType("TEXT");
b.Property<DateTime>("LastFileAnalysis")
.HasColumnType("TEXT");

10
API/Data/Repositories/MangaFileRepository.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using API.Entities;
@ -10,6 +11,7 @@ public interface IMangaFileRepository
{
void Update(MangaFile file);
Task<IList<MangaFile>> GetAllWithMissingExtension();
Task<MangaFile> GetByKoreaderHash(string hash);
}
public class MangaFileRepository : IMangaFileRepository
@ -32,4 +34,10 @@ public class MangaFileRepository : IMangaFileRepository
.Where(f => string.IsNullOrEmpty(f.Extension))
.ToListAsync();
}
public Task<MangaFile> GetByKoreaderHash(string hash)
{
return _context.MangaFile
.FirstOrDefaultAsync(f => f.KoreaderHash == hash.ToUpper());
}
}

7
API/Entities/MangaFile.cs
View file

@ -1,4 +1,4 @@

using System;
using System.IO;
using API.Entities.Enums;
@ -21,6 +21,11 @@ public class MangaFile : IEntityDate
/// </summary>
public required string FilePath { get; set; }
/// <summary>
/// A hash of the document using Koreader's unique hashing algorithm
/// <remark> KoreaderHash is only available for epub types </remark>
/// </summary>
public string? KoreaderHash { get; set; }
/// <summary>
/// Number of pages for the given file
/// </summary>
public int Pages { get; set; }

3
API/Extensions/ApplicationServiceExtensions.cs
View file

@ -1,4 +1,4 @@
using System.IO.Abstractions;
using System.IO.Abstractions;
using API.Constants;
using API.Data;
using API.Helpers;
@ -54,6 +54,7 @@ public static class ApplicationServiceExtensions
services.AddScoped<IMediaConversionService, MediaConversionService>();
services.AddScoped<IRecommendationService, RecommendationService>();
services.AddScoped<IStreamService, StreamService>();
services.AddScoped<IKoreaderService, KoreaderService>();
services.AddScoped<IScannerService, ScannerService>();
services.AddScoped<IProcessSeries, ProcessSeries>();

14
API/Helpers/Builders/MangaFileBuilder.cs
View file

@ -1,4 +1,4 @@
using System;
using System;
using System.IO;
using API.Entities;
using API.Entities.Enums;
@ -20,7 +20,7 @@ public class MangaFileBuilder : IEntityBuilder<MangaFile>
Pages = pages,
LastModified = File.GetLastWriteTime(filePath),
LastModifiedUtc = File.GetLastWriteTimeUtc(filePath),
FileName = Parser.RemoveExtensionIfSupported(filePath)
FileName = Parser.RemoveExtensionIfSupported(filePath),
};
}
@ -60,4 +60,14 @@ public class MangaFileBuilder : IEntityBuilder<MangaFile>
_mangaFile.Id = Math.Max(id, 0);
return this;
}
public MangaFileBuilder WithHash()
{
if (_mangaFile.Format == MangaFormat.Epub)
{
_mangaFile.KoreaderHash = KoreaderHelper.HashContents(_mangaFile.FilePath);
}
return this;
}
}

107
API/Helpers/KoreaderHelper.cs Normal file
View file

@ -0,0 +1,107 @@
using API.DTOs.Koreader;
using API.DTOs.Progress;
using API.Services;
using System;
using System.IO;
using System.Linq;
using System.Security.Cryptography;
using System.Text;
using System.Text.RegularExpressions;
namespace API.Helpers;
public static class KoreaderHelper
{
/// <summary>
/// Hashes the document according to a custom Koreader hashing algorithm.
/// Look at the util.partialMD5 method in the attached link.
/// </summary>
/// <remarks>The hashing algorithm is relatively quick as it only hashes ~10,000 bytes for the biggest of files.</remarks>
/// <see href="https://github.com/koreader/koreader/blob/master/frontend/util.lua#L1040"/>
/// <param name="filePath">The path to the file to hash</param>
public static string HashContents(string filePath)
{
if (string.IsNullOrEmpty(filePath))
{
return null;
}
using var file = File.OpenRead(filePath);
var step = 1024;
var size = 1024;
MD5 md5 = MD5.Create();
byte[] buffer = new byte[size];
for (var i = -1; i < 10; i++)
{
file.Position = step << 2 * i;
var bytesRead = file.Read(buffer, 0, size);
if (bytesRead > 0)
{
md5.TransformBlock(buffer, 0, bytesRead, buffer, 0);
}
else
{
break;
}
}
file.Close();
md5.TransformFinalBlock(new byte[0], 0, 0);
return BitConverter.ToString(md5.Hash).Replace("-", string.Empty).ToUpper();
}
/// <summary>
/// Koreader can identitfy documents based on contents or title.
/// For now we only support by contents.
/// </summary>
public static string HashTitle(string filePath)
{
var fileName = Path.GetFileName(filePath);
var fileNameBytes = Encoding.ASCII.GetBytes(fileName);
var bytes = MD5.HashData(fileNameBytes);
return BitConverter.ToString(bytes).Replace("-", string.Empty);
}
public static void UpdateProgressDto(string koreaderPosition, ProgressDto progress)
{
var path = koreaderPosition.Split('/');
if (path.Length < 6)
{
return;
}
var docNumber = path[2].Replace("DocFragment[", string.Empty).Replace("]", string.Empty);
progress.PageNum = Int32.Parse(docNumber) - 1;
var lastTag = path[5].ToUpper();
if (lastTag == "A")
{
progress.BookScrollId = null;
}
else
{
// The format that Kavita accpets as a progress string. It tells Kavita where Koreader last left off.
progress.BookScrollId = $"//html[1]/BODY/APP-ROOT[1]/DIV[1]/DIV[1]/DIV[1]/APP-BOOK-READER[1]/DIV[1]/DIV[2]/DIV[1]/DIV[1]/DIV[1]/{lastTag}";
}
}
public static string GetKoreaderPosition(ProgressDto progressDto)
{
string lastTag;
var koreaderPageNumber = progressDto.PageNum + 1;
if (string.IsNullOrEmpty(progressDto.BookScrollId))
{
lastTag = "a";
}
else
{
lastTag = progressDto.BookScrollId.Split('/').Last().ToLower();
}
// The format that Koreader accepts as a progress string. It tells Koreader where Kavita last left off.
return $"/body/DocFragment[{koreaderPageNumber}]/body/div/{lastTag}";
}
}

60
API/Services/KoreaderService.cs Normal file
View file

@ -0,0 +1,60 @@
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Koreader;
using API.Helpers;
using Microsoft.Extensions.Logging;
namespace API.Services;
#nullable enable
public interface IKoreaderService
{
Task SaveProgress(KoreaderBookDto koreaderBookDto, int userId);
Task<KoreaderBookDto> GetProgress(string bookHash, int userId);
}
public class KoreaderService : IKoreaderService
{
private IReaderService _readerService;
private IUnitOfWork _unitOfWork;
private ILogger<KoreaderService> _logger;
public KoreaderService(IReaderService readerService, IUnitOfWork unitOfWork,
ILogger<KoreaderService> logger)
{
_readerService = readerService;
_unitOfWork = unitOfWork;
_logger = logger;
}
public async Task SaveProgress(KoreaderBookDto koreaderBookDto, int userId)
{
var file = await _unitOfWork.MangaFileRepository.GetByKoreaderHash(koreaderBookDto.Document);
var userProgressDto = await _unitOfWork.AppUserProgressRepository.GetUserProgressDtoAsync(file.ChapterId, userId);
_logger.LogInformation("Saving Koreader progress to Kavita: {KoreaderProgress}", koreaderBookDto.Progress);
KoreaderHelper.UpdateProgressDto(koreaderBookDto.Progress, userProgressDto);
await _readerService.SaveReadingProgress(userProgressDto, userId);
await _unitOfWork.CommitAsync();
}
public async Task<KoreaderBookDto> GetProgress(string bookHash, int userId)
{
var file = await _unitOfWork.MangaFileRepository.GetByKoreaderHash(bookHash);
var progressDto = await _unitOfWork.AppUserProgressRepository.GetUserProgressDtoAsync(file.ChapterId, userId);
_logger.LogInformation("Transmitting Kavita progress to Koreader: {KoreaderProgress}", progressDto.BookScrollId);
var koreaderProgress = KoreaderHelper.GetKoreaderPosition(progressDto);
var settingsDto = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
return new KoreaderBookDto
{
Document = bookHash,
Device_id = settingsDto.InstallId,
Device = "Kavita",
Progress = koreaderProgress,
Percentage = progressDto.PageNum / (float) file.Pages
};
}
}

3
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

@ -1,4 +1,4 @@
using System;
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
@ -816,6 +816,7 @@ public class ProcessSeries : IProcessSeries
var file = new MangaFileBuilder(info.FullFilePath, info.Format, _readingItemService.GetNumberOfPages(info.FullFilePath, info.Format))
.WithExtension(fileInfo.Extension)
.WithBytes(fileInfo.Length)
.WithHash()
.Build();
chapter.Files.Add(file);
}