More Fixes (#1993)
* Strip just isbn: from epub isbns and log when it's back (books) * Tweaked to allow invalid GTINs but only valid ISBN 10/13s will be saved to Kavita. * Fixed a bug with parsing series from a filename that is just a chapter range and no chapter/volume keywords. * Show the media issue count before you open accordion * Added a inpage filter for Media issues * Cleanup styles * Fixed up some code in epub isbn parsing when it's null * Encode filenames when downloading so that non english characters can be passed properly to UI. * Added support to parse ComicInfo's with Empty Tags. * Reset development settings. * Tweaked the code in generating reading lists to avoid extra work when not needed. * Fix comicvine's favicon * Fixed up a unit test * Tweaked the favicon code to ignore icons that have query parameters * More favicon work. Expanded ability to grab icons a bit. Added in ability to not keep requesting favicons when we failed to parse already. * Added a note for later * Fixed stats server url * Added more debugging * Fixed unit tests
This commit is contained in:
Joe Milazzo
GitHub
parent
cd8fca993b
commit
25703d6fe0
27 changed files with 171 additions and 75 deletions
|
|
@ -56,6 +56,7 @@
|
|||
<PackageReference Include="AutoMapper.Extensions.Microsoft.DependencyInjection" Version="12.0.1" />
|
||||
<PackageReference Include="CsvHelper" Version="30.0.1" />
|
||||
<PackageReference Include="Docnet.Core" Version="2.4.0-alpha.4" />
|
||||
<PackageReference Include="EasyCaching.InMemory" Version="1.9.0" />
|
||||
<PackageReference Include="ExCSS" Version="4.1.0" />
|
||||
<PackageReference Include="Flurl" Version="3.0.7" />
|
||||
<PackageReference Include="Flurl.Http" Version="3.2.4" />
|
||||
|
|
|
|||
|
|
@ -117,7 +117,7 @@ public class DownloadController : BaseApiController
|
|||
private ActionResult GetFirstFileDownload(IEnumerable<MangaFile> files)
|
||||
{
|
||||
var (zipFile, contentType, fileDownloadName) = _downloadService.GetFirstFileDownload(files);
|
||||
return PhysicalFile(zipFile, contentType, fileDownloadName, true);
|
||||
return PhysicalFile(zipFile, contentType, System.Web.HttpUtility.UrlEncode(fileDownloadName), true);
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -163,7 +163,7 @@ public class DownloadController : BaseApiController
|
|||
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
|
||||
MessageFactory.DownloadProgressEvent(User.GetUsername(),
|
||||
Path.GetFileNameWithoutExtension(downloadName), 1F, "ended"));
|
||||
return PhysicalFile(filePath, DefaultContentType, downloadName, true);
|
||||
return PhysicalFile(filePath, DefaultContentType, System.Web.HttpUtility.UrlEncode(downloadName), true);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
|
|
@ -220,7 +220,7 @@ public class DownloadController : BaseApiController
|
|||
MessageFactory.DownloadProgressEvent(username, Path.GetFileNameWithoutExtension(filename), 1F));
|
||||
|
||||
|
||||
return PhysicalFile(filePath, DefaultContentType, filename, true);
|
||||
return PhysicalFile(filePath, DefaultContentType, System.Web.HttpUtility.UrlEncode(filename), true);
|
||||
}
|
||||
|
||||
}
|
||||
|
|
|
|||
|
|
@ -150,7 +150,8 @@ public class ServerController : BaseApiController
|
|||
try
|
||||
{
|
||||
var zipPath = _archiveService.CreateZipForDownload(files, "logs");
|
||||
return PhysicalFile(zipPath, "application/zip", Path.GetFileName(zipPath), true);
|
||||
return PhysicalFile(zipPath, "application/zip",
|
||||
System.Web.HttpUtility.UrlEncode(Path.GetFileName(zipPath)), true);
|
||||
}
|
||||
catch (KavitaException ex)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -214,7 +214,14 @@ public class SettingsController : BaseApiController
|
|||
? $"{path}/"
|
||||
: path;
|
||||
setting.Value = path;
|
||||
Configuration.BaseUrl = updateSettingsDto.BaseUrl;
|
||||
try
|
||||
{
|
||||
Configuration.BaseUrl = updateSettingsDto.BaseUrl;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Could not set base url. Give this exception to majora2007");
|
||||
}
|
||||
_unitOfWork.SettingsRepository.Update(setting);
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -33,6 +33,9 @@ public class UsersController : BaseApiController
|
|||
var user = await _unitOfWork.UserRepository.GetUserByUsernameAsync(username);
|
||||
_unitOfWork.UserRepository.Delete(user);
|
||||
|
||||
//(TODO: After updating a role or removing a user, delete their token)
|
||||
// await _userManager.RemoveAuthenticationTokenAsync(user, TokenOptions.DefaultProvider, RefreshTokenName);
|
||||
|
||||
if (await _unitOfWork.CommitAsync()) return Ok();
|
||||
|
||||
return BadRequest("Could not delete the user.");
|
||||
|
|
|
|||
|
|
@ -153,7 +153,7 @@ public class ComicInfo
|
|||
info.CoverArtist = Services.Tasks.Scanner.Parser.Parser.CleanAuthor(info.CoverArtist);
|
||||
|
||||
// We need to convert GTIN to ISBN
|
||||
if (!string.IsNullOrEmpty(info.GTIN) && ArticleNumberHelper.IsValidGtin(info.GTIN))
|
||||
if (!string.IsNullOrEmpty(info.GTIN))
|
||||
{
|
||||
// This is likely a valid ISBN
|
||||
if (info.GTIN[0] == '0')
|
||||
|
|
@ -163,9 +163,10 @@ public class ComicInfo
|
|||
{
|
||||
info.Isbn = potentialISBN;
|
||||
}
|
||||
|
||||
} else if (ArticleNumberHelper.IsValidIsbn10(info.GTIN) || ArticleNumberHelper.IsValidIsbn13(info.GTIN))
|
||||
{
|
||||
info.Isbn = info.GTIN;
|
||||
}
|
||||
|
||||
}
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -22,9 +22,7 @@ public static class ApplicationServiceExtensions
|
|||
services.AddAutoMapper(typeof(AutoMapperProfiles).Assembly);
|
||||
|
||||
services.AddScoped<IUnitOfWork, UnitOfWork>();
|
||||
services.AddScoped<IDirectoryService, DirectoryService>();
|
||||
services.AddScoped<ITokenService, TokenService>();
|
||||
services.AddScoped<IFileSystem, FileSystem>();
|
||||
services.AddScoped<IFileService, FileService>();
|
||||
services.AddScoped<ICacheHelper, CacheHelper>();
|
||||
|
||||
|
|
@ -35,7 +33,6 @@ public static class ApplicationServiceExtensions
|
|||
services.AddScoped<IBackupService, BackupService>();
|
||||
services.AddScoped<ICleanupService, CleanupService>();
|
||||
services.AddScoped<IBookService, BookService>();
|
||||
services.AddScoped<IImageService, ImageService>();
|
||||
services.AddScoped<IVersionUpdaterService, VersionUpdaterService>();
|
||||
services.AddScoped<IDownloadService, DownloadService>();
|
||||
services.AddScoped<IReaderService, ReaderService>();
|
||||
|
|
@ -59,11 +56,20 @@ public static class ApplicationServiceExtensions
|
|||
services.AddScoped<ITachiyomiService, TachiyomiService>();
|
||||
services.AddScoped<ICollectionTagService, CollectionTagService>();
|
||||
|
||||
services.AddScoped<IPresenceTracker, PresenceTracker>();
|
||||
services.AddScoped<IFileSystem, FileSystem>();
|
||||
services.AddScoped<IDirectoryService, DirectoryService>();
|
||||
services.AddScoped<IEventHub, EventHub>();
|
||||
services.AddScoped<IPresenceTracker, PresenceTracker>();
|
||||
|
||||
services.AddScoped<IImageService, ImageService>();
|
||||
|
||||
services.AddSqLite(env);
|
||||
services.AddSignalR(opt => opt.EnableDetailedErrors = true);
|
||||
|
||||
services.AddEasyCaching(options =>
|
||||
{
|
||||
options.UseInMemory("favicon");
|
||||
});
|
||||
}
|
||||
|
||||
private static void AddSqLite(this IServiceCollection services, IHostEnvironment env)
|
||||
|
|
|
|||
|
|
@ -4,6 +4,7 @@ using System.Diagnostics;
|
|||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Xml.Linq;
|
||||
using System.Xml.Serialization;
|
||||
using API.Archive;
|
||||
using API.Data.Metadata;
|
||||
|
|
@ -372,10 +373,7 @@ public class ArchiveService : IArchiveService
|
|||
if (entry != null)
|
||||
{
|
||||
using var stream = entry.Open();
|
||||
var serializer = new XmlSerializer(typeof(ComicInfo));
|
||||
var info = (ComicInfo?) serializer.Deserialize(stream);
|
||||
ComicInfo.CleanComicInfo(info);
|
||||
return info;
|
||||
return Deserialize(stream);
|
||||
}
|
||||
|
||||
break;
|
||||
|
|
@ -390,9 +388,7 @@ public class ArchiveService : IArchiveService
|
|||
if (entry != null)
|
||||
{
|
||||
using var stream = entry.OpenEntryStream();
|
||||
var serializer = new XmlSerializer(typeof(ComicInfo));
|
||||
var info = (ComicInfo?) serializer.Deserialize(stream);
|
||||
ComicInfo.CleanComicInfo(info);
|
||||
var info = Deserialize(stream);
|
||||
return info;
|
||||
}
|
||||
|
||||
|
|
@ -418,6 +414,28 @@ public class ArchiveService : IArchiveService
|
|||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Strips out empty tags before deserializing
|
||||
/// </summary>
|
||||
/// <param name="stream"></param>
|
||||
/// <returns></returns>
|
||||
private static ComicInfo? Deserialize(Stream stream)
|
||||
{
|
||||
var comicInfoXml = XDocument.Load(stream);
|
||||
comicInfoXml.Descendants()
|
||||
.Where(e => e.IsEmpty || string.IsNullOrWhiteSpace(e.Value))
|
||||
.Remove();
|
||||
|
||||
var serializer = new XmlSerializer(typeof(ComicInfo));
|
||||
using var reader = comicInfoXml.Root?.CreateReader();
|
||||
if (reader == null) return null;
|
||||
|
||||
var info = (ComicInfo?) serializer.Deserialize(reader);
|
||||
ComicInfo.CleanComicInfo(info);
|
||||
return info;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void ExtractArchiveEntities(IEnumerable<IArchiveEntry> entries, string extractPath)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -437,8 +437,13 @@ public class BookService : IBookService
|
|||
|
||||
foreach (var identifier in epubBook.Schema.Package.Metadata.Identifiers.Where(id => id.Scheme.Equals("ISBN")))
|
||||
{
|
||||
var isbn = identifier.Identifier.Replace("urn:isbn:", string.Empty);
|
||||
if (!ArticleNumberHelper.IsValidIsbn10(isbn) && !ArticleNumberHelper.IsValidIsbn13(isbn)) continue;
|
||||
if (string.IsNullOrEmpty(identifier.Identifier)) continue;
|
||||
var isbn = identifier.Identifier.Replace("urn:isbn:", string.Empty).Replace("isbn:", string.Empty);
|
||||
if (!ArticleNumberHelper.IsValidIsbn10(isbn) && !ArticleNumberHelper.IsValidIsbn13(isbn))
|
||||
{
|
||||
_logger.LogDebug("[BookService] {File} has invalid ISBN number", filePath);
|
||||
continue;
|
||||
}
|
||||
info.Isbn = isbn;
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using API.Entities.Enums;
|
||||
using API.Extensions;
|
||||
using EasyCaching.Core;
|
||||
using Flurl;
|
||||
using Flurl.Http;
|
||||
using HtmlAgilityPack;
|
||||
|
|
@ -63,11 +65,13 @@ public class ImageService : IImageService
|
|||
public const string Name = "BookmarkService";
|
||||
private readonly ILogger<ImageService> _logger;
|
||||
private readonly IDirectoryService _directoryService;
|
||||
private readonly IEasyCachingProviderFactory _cacheFactory;
|
||||
public const string ChapterCoverImageRegex = @"v\d+_c\d+";
|
||||
public const string SeriesCoverImageRegex = @"series\d+";
|
||||
public const string CollectionTagCoverImageRegex = @"tag\d+";
|
||||
public const string ReadingListCoverImageRegex = @"readinglist\d+";
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Width of the Thumbnail generation
|
||||
/// </summary>
|
||||
|
|
@ -80,7 +84,8 @@ public class ImageService : IImageService
|
|||
private static readonly string[] ValidIconRelations = {
|
||||
"icon",
|
||||
"apple-touch-icon",
|
||||
"apple-touch-icon-precomposed"
|
||||
"apple-touch-icon-precomposed",
|
||||
"apple-touch-icon icon-precomposed" // ComicVine has it combined
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -91,10 +96,11 @@ public class ImageService : IImageService
|
|||
["https://app.plex.tv"] = "https://plex.tv"
|
||||
};
|
||||
|
||||
public ImageService(ILogger<ImageService> logger, IDirectoryService directoryService)
|
||||
public ImageService(ILogger<ImageService> logger, IDirectoryService directoryService, IEasyCachingProviderFactory cacheFactory)
|
||||
{
|
||||
_logger = logger;
|
||||
_directoryService = directoryService;
|
||||
_cacheFactory = cacheFactory;
|
||||
}
|
||||
|
||||
public void ExtractImages(string? fileFilePath, string targetDirectory, int fileCount = 1)
|
||||
|
|
@ -207,6 +213,15 @@ public class ImageService : IImageService
|
|||
var baseUrl = uri.Scheme + "://" + uri.Host;
|
||||
|
||||
|
||||
var provider = _cacheFactory.GetCachingProvider("favicon");
|
||||
var res = await provider.GetAsync<string>(baseUrl);
|
||||
if (res.HasValue)
|
||||
{
|
||||
_logger.LogInformation("Kavita has already tried to fetch from {BaseUrl} and failed. Skipping duplicate check", baseUrl);
|
||||
throw new KavitaException($"Kavita has already tried to fetch from {baseUrl} and failed. Skipping duplicate check");
|
||||
}
|
||||
|
||||
await provider.SetAsync(baseUrl, string.Empty, TimeSpan.FromDays(10));
|
||||
if (FaviconUrlMapper.TryGetValue(baseUrl, out var value))
|
||||
{
|
||||
url = value;
|
||||
|
|
@ -220,7 +235,7 @@ public class ImageService : IImageService
|
|||
var pngLinks = htmlDocument.DocumentNode.Descendants("link")
|
||||
.Where(link => ValidIconRelations.Contains(link.GetAttributeValue("rel", string.Empty)))
|
||||
.Select(link => link.GetAttributeValue("href", string.Empty))
|
||||
.Where(href => href.EndsWith(".png") || href.EndsWith(".PNG"))
|
||||
.Where(href => href.Split("?")[0].EndsWith(".png", StringComparison.InvariantCultureIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
if (pngLinks == null)
|
||||
|
|
@ -234,7 +249,13 @@ public class ImageService : IImageService
|
|||
}
|
||||
|
||||
var finalUrl = correctSizeLink;
|
||||
if (!correctSizeLink.StartsWith(uri.Scheme))
|
||||
|
||||
// If starts with //, it's coming usually from an offsite cdn
|
||||
if (correctSizeLink.StartsWith("//"))
|
||||
{
|
||||
finalUrl = Url.Combine("https:", correctSizeLink);
|
||||
}
|
||||
else if (!correctSizeLink.StartsWith(uri.Scheme))
|
||||
{
|
||||
finalUrl = Url.Combine(baseUrl, correctSizeLink);
|
||||
}
|
||||
|
|
@ -269,7 +290,7 @@ public class ImageService : IImageService
|
|||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error downloading favicon.png for ${Domain}", domain);
|
||||
_logger.LogError(ex, "Error downloading favicon.png for {Domain}", domain);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -496,12 +496,13 @@ public class ReadingListService : IReadingListService
|
|||
}
|
||||
|
||||
readingList.Items = items;
|
||||
|
||||
if (!_unitOfWork.HasChanges()) continue;
|
||||
|
||||
await CalculateReadingListAgeRating(readingList);
|
||||
if (_unitOfWork.HasChanges())
|
||||
{
|
||||
await _unitOfWork.CommitAsync();
|
||||
}
|
||||
await CalculateStartAndEndDates(await _unitOfWork.ReadingListRepository.GetReadingListByTitleAsync(arcPair.Item1, user.Id, ReadingListIncludes.Items | ReadingListIncludes.ItemChapter));
|
||||
await _unitOfWork.CommitAsync(); // TODO: See if we can avoid this extra commit by reworking bottom logic
|
||||
await CalculateStartAndEndDates(await _unitOfWork.ReadingListRepository.GetReadingListByTitleAsync(arcPair.Item1,
|
||||
user.Id, ReadingListIncludes.Items | ReadingListIncludes.ItemChapter));
|
||||
await _unitOfWork.CommitAsync();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -321,13 +321,9 @@ public static class Parser
|
|||
new Regex(
|
||||
@"(?<Series>.*)( ?- ?)Ch\.\d+-?\d*",
|
||||
MatchOptions, RegexTimeout),
|
||||
// [BAA]_Darker_than_Black_Omake-1.zip
|
||||
// [BAA]_Darker_than_Black_Omake-1, Bleach 001-002, Kodoja #001 (March 2016)
|
||||
new Regex(
|
||||
@"^(?!Vol)(?<Series>.*)(-)\d+-?\d*", // This catches a lot of stuff ^(?!Vol)(?<Series>.*)( |_)(\d+)
|
||||
MatchOptions, RegexTimeout),
|
||||
// Kodoja #001 (March 2016)
|
||||
new Regex(
|
||||
@"(?<Series>.*)(\s|_|-)#",
|
||||
@"^(?!Vol)(?!Chapter)(?<Series>.+?)(-|_|\s|#)\d+(-\d+)?",
|
||||
MatchOptions, RegexTimeout),
|
||||
// Baketeriya ch01-05.zip, Akiiro Bousou Biyori - 01.jpg, Beelzebub_172_RHS.zip, Cynthia the Mission 29.rar, A Compendium of Ghosts - 031 - The Third Story_ Part 12 (Digital) (Cobalt001)
|
||||
new Regex(
|
||||
|
|
|
|||
|
|
@ -346,6 +346,8 @@ public class ProcessSeries : IProcessSeries
|
|||
}
|
||||
|
||||
|
||||
#region People
|
||||
|
||||
// Handle People
|
||||
foreach (var chapter in chapters)
|
||||
{
|
||||
|
|
@ -490,6 +492,8 @@ public class ProcessSeries : IProcessSeries
|
|||
}
|
||||
});
|
||||
|
||||
#endregion
|
||||
|
||||
}
|
||||
|
||||
public void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ public class StatsService : IStatsService
|
|||
private readonly IUnitOfWork _unitOfWork;
|
||||
private readonly DataContext _context;
|
||||
private readonly IStatisticService _statisticService;
|
||||
private const string ApiUrl = "http://localhost:5003";
|
||||
private const string ApiUrl = "https://stats.kavitareader.com";
|
||||
|
||||
public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork, DataContext context, IStatisticService statisticService)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -90,12 +90,12 @@ public class TokenService : ITokenService
|
|||
Token = await CreateToken(user),
|
||||
RefreshToken = await CreateRefreshToken(user)
|
||||
};
|
||||
} catch (SecurityTokenExpiredException)
|
||||
} catch (SecurityTokenExpiredException ex)
|
||||
{
|
||||
// Handle expired token
|
||||
return null;
|
||||
}
|
||||
catch (Exception)
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Handle other exceptions
|
||||
return null;
|
||||
|
|
|
|||
|
|
@ -2,5 +2,5 @@
|
|||
"TokenKey": "super secret unguessable key that is longer because we require it",
|
||||
"Port": 5000,
|
||||
"IpAddresses": "",
|
||||
"BaseUrl": "/joe/"
|
||||
}
|
||||
"BaseUrl": "/"
|
||||
}
|
||||
Loading…
Add table
Add a link
Reference in a new issue