More Fixes (#1993)
* Strip just isbn: from epub isbns and log when it's back (books) * Tweaked to allow invalid GTINs but only valid ISBN 10/13s will be saved to Kavita. * Fixed a bug with parsing series from a filename that is just a chapter range and no chapter/volume keywords. * Show the media issue count before you open accordion * Added a inpage filter for Media issues * Cleanup styles * Fixed up some code in epub isbn parsing when it's null * Encode filenames when downloading so that non english characters can be passed properly to UI. * Added support to parse ComicInfo's with Empty Tags. * Reset development settings. * Tweaked the code in generating reading lists to avoid extra work when not needed. * Fix comicvine's favicon * Fixed up a unit test * Tweaked the favicon code to ignore icons that have query parameters * More favicon work. Expanded ability to grab icons a bit. Added in ability to not keep requesting favicons when we failed to parse already. * Added a note for later * Fixed stats server url * Added more debugging * Fixed unit tests
This commit is contained in:
Joe Milazzo
GitHub
parent
cd8fca993b
commit
25703d6fe0
27 changed files with 171 additions and 75 deletions
|
|
@ -4,6 +4,7 @@ using System.Diagnostics;
|
|||
using System.IO;
|
||||
using System.IO.Compression;
|
||||
using System.Linq;
|
||||
using System.Xml.Linq;
|
||||
using System.Xml.Serialization;
|
||||
using API.Archive;
|
||||
using API.Data.Metadata;
|
||||
|
|
@ -372,10 +373,7 @@ public class ArchiveService : IArchiveService
|
|||
if (entry != null)
|
||||
{
|
||||
using var stream = entry.Open();
|
||||
var serializer = new XmlSerializer(typeof(ComicInfo));
|
||||
var info = (ComicInfo?) serializer.Deserialize(stream);
|
||||
ComicInfo.CleanComicInfo(info);
|
||||
return info;
|
||||
return Deserialize(stream);
|
||||
}
|
||||
|
||||
break;
|
||||
|
|
@ -390,9 +388,7 @@ public class ArchiveService : IArchiveService
|
|||
if (entry != null)
|
||||
{
|
||||
using var stream = entry.OpenEntryStream();
|
||||
var serializer = new XmlSerializer(typeof(ComicInfo));
|
||||
var info = (ComicInfo?) serializer.Deserialize(stream);
|
||||
ComicInfo.CleanComicInfo(info);
|
||||
var info = Deserialize(stream);
|
||||
return info;
|
||||
}
|
||||
|
||||
|
|
@ -418,6 +414,28 @@ public class ArchiveService : IArchiveService
|
|||
return null;
|
||||
}
|
||||
|
||||
/// <summary>
|
||||
/// Strips out empty tags before deserializing
|
||||
/// </summary>
|
||||
/// <param name="stream"></param>
|
||||
/// <returns></returns>
|
||||
private static ComicInfo? Deserialize(Stream stream)
|
||||
{
|
||||
var comicInfoXml = XDocument.Load(stream);
|
||||
comicInfoXml.Descendants()
|
||||
.Where(e => e.IsEmpty || string.IsNullOrWhiteSpace(e.Value))
|
||||
.Remove();
|
||||
|
||||
var serializer = new XmlSerializer(typeof(ComicInfo));
|
||||
using var reader = comicInfoXml.Root?.CreateReader();
|
||||
if (reader == null) return null;
|
||||
|
||||
var info = (ComicInfo?) serializer.Deserialize(reader);
|
||||
ComicInfo.CleanComicInfo(info);
|
||||
return info;
|
||||
|
||||
}
|
||||
|
||||
|
||||
private void ExtractArchiveEntities(IEnumerable<IArchiveEntry> entries, string extractPath)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -437,8 +437,13 @@ public class BookService : IBookService
|
|||
|
||||
foreach (var identifier in epubBook.Schema.Package.Metadata.Identifiers.Where(id => id.Scheme.Equals("ISBN")))
|
||||
{
|
||||
var isbn = identifier.Identifier.Replace("urn:isbn:", string.Empty);
|
||||
if (!ArticleNumberHelper.IsValidIsbn10(isbn) && !ArticleNumberHelper.IsValidIsbn13(isbn)) continue;
|
||||
if (string.IsNullOrEmpty(identifier.Identifier)) continue;
|
||||
var isbn = identifier.Identifier.Replace("urn:isbn:", string.Empty).Replace("isbn:", string.Empty);
|
||||
if (!ArticleNumberHelper.IsValidIsbn10(isbn) && !ArticleNumberHelper.IsValidIsbn13(isbn))
|
||||
{
|
||||
_logger.LogDebug("[BookService] {File} has invalid ISBN number", filePath);
|
||||
continue;
|
||||
}
|
||||
info.Isbn = isbn;
|
||||
break;
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,10 +1,12 @@
|
|||
using System;
|
||||
using System.Collections.Concurrent;
|
||||
using System.Collections.Generic;
|
||||
using System.IO;
|
||||
using System.Linq;
|
||||
using System.Threading.Tasks;
|
||||
using API.Entities.Enums;
|
||||
using API.Extensions;
|
||||
using EasyCaching.Core;
|
||||
using Flurl;
|
||||
using Flurl.Http;
|
||||
using HtmlAgilityPack;
|
||||
|
|
@ -63,11 +65,13 @@ public class ImageService : IImageService
|
|||
public const string Name = "BookmarkService";
|
||||
private readonly ILogger<ImageService> _logger;
|
||||
private readonly IDirectoryService _directoryService;
|
||||
private readonly IEasyCachingProviderFactory _cacheFactory;
|
||||
public const string ChapterCoverImageRegex = @"v\d+_c\d+";
|
||||
public const string SeriesCoverImageRegex = @"series\d+";
|
||||
public const string CollectionTagCoverImageRegex = @"tag\d+";
|
||||
public const string ReadingListCoverImageRegex = @"readinglist\d+";
|
||||
|
||||
|
||||
/// <summary>
|
||||
/// Width of the Thumbnail generation
|
||||
/// </summary>
|
||||
|
|
@ -80,7 +84,8 @@ public class ImageService : IImageService
|
|||
private static readonly string[] ValidIconRelations = {
|
||||
"icon",
|
||||
"apple-touch-icon",
|
||||
"apple-touch-icon-precomposed"
|
||||
"apple-touch-icon-precomposed",
|
||||
"apple-touch-icon icon-precomposed" // ComicVine has it combined
|
||||
};
|
||||
|
||||
/// <summary>
|
||||
|
|
@ -91,10 +96,11 @@ public class ImageService : IImageService
|
|||
["https://app.plex.tv"] = "https://plex.tv"
|
||||
};
|
||||
|
||||
public ImageService(ILogger<ImageService> logger, IDirectoryService directoryService)
|
||||
public ImageService(ILogger<ImageService> logger, IDirectoryService directoryService, IEasyCachingProviderFactory cacheFactory)
|
||||
{
|
||||
_logger = logger;
|
||||
_directoryService = directoryService;
|
||||
_cacheFactory = cacheFactory;
|
||||
}
|
||||
|
||||
public void ExtractImages(string? fileFilePath, string targetDirectory, int fileCount = 1)
|
||||
|
|
@ -207,6 +213,15 @@ public class ImageService : IImageService
|
|||
var baseUrl = uri.Scheme + "://" + uri.Host;
|
||||
|
||||
|
||||
var provider = _cacheFactory.GetCachingProvider("favicon");
|
||||
var res = await provider.GetAsync<string>(baseUrl);
|
||||
if (res.HasValue)
|
||||
{
|
||||
_logger.LogInformation("Kavita has already tried to fetch from {BaseUrl} and failed. Skipping duplicate check", baseUrl);
|
||||
throw new KavitaException($"Kavita has already tried to fetch from {baseUrl} and failed. Skipping duplicate check");
|
||||
}
|
||||
|
||||
await provider.SetAsync(baseUrl, string.Empty, TimeSpan.FromDays(10));
|
||||
if (FaviconUrlMapper.TryGetValue(baseUrl, out var value))
|
||||
{
|
||||
url = value;
|
||||
|
|
@ -220,7 +235,7 @@ public class ImageService : IImageService
|
|||
var pngLinks = htmlDocument.DocumentNode.Descendants("link")
|
||||
.Where(link => ValidIconRelations.Contains(link.GetAttributeValue("rel", string.Empty)))
|
||||
.Select(link => link.GetAttributeValue("href", string.Empty))
|
||||
.Where(href => href.EndsWith(".png") || href.EndsWith(".PNG"))
|
||||
.Where(href => href.Split("?")[0].EndsWith(".png", StringComparison.InvariantCultureIgnoreCase))
|
||||
.ToList();
|
||||
|
||||
if (pngLinks == null)
|
||||
|
|
@ -234,7 +249,13 @@ public class ImageService : IImageService
|
|||
}
|
||||
|
||||
var finalUrl = correctSizeLink;
|
||||
if (!correctSizeLink.StartsWith(uri.Scheme))
|
||||
|
||||
// If starts with //, it's coming usually from an offsite cdn
|
||||
if (correctSizeLink.StartsWith("//"))
|
||||
{
|
||||
finalUrl = Url.Combine("https:", correctSizeLink);
|
||||
}
|
||||
else if (!correctSizeLink.StartsWith(uri.Scheme))
|
||||
{
|
||||
finalUrl = Url.Combine(baseUrl, correctSizeLink);
|
||||
}
|
||||
|
|
@ -269,7 +290,7 @@ public class ImageService : IImageService
|
|||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
_logger.LogError(ex, "Error downloading favicon.png for ${Domain}", domain);
|
||||
_logger.LogError(ex, "Error downloading favicon.png for {Domain}", domain);
|
||||
throw;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -496,12 +496,13 @@ public class ReadingListService : IReadingListService
|
|||
}
|
||||
|
||||
readingList.Items = items;
|
||||
|
||||
if (!_unitOfWork.HasChanges()) continue;
|
||||
|
||||
await CalculateReadingListAgeRating(readingList);
|
||||
if (_unitOfWork.HasChanges())
|
||||
{
|
||||
await _unitOfWork.CommitAsync();
|
||||
}
|
||||
await CalculateStartAndEndDates(await _unitOfWork.ReadingListRepository.GetReadingListByTitleAsync(arcPair.Item1, user.Id, ReadingListIncludes.Items | ReadingListIncludes.ItemChapter));
|
||||
await _unitOfWork.CommitAsync(); // TODO: See if we can avoid this extra commit by reworking bottom logic
|
||||
await CalculateStartAndEndDates(await _unitOfWork.ReadingListRepository.GetReadingListByTitleAsync(arcPair.Item1,
|
||||
user.Id, ReadingListIncludes.Items | ReadingListIncludes.ItemChapter));
|
||||
await _unitOfWork.CommitAsync();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -321,13 +321,9 @@ public static class Parser
|
|||
new Regex(
|
||||
@"(?<Series>.*)( ?- ?)Ch\.\d+-?\d*",
|
||||
MatchOptions, RegexTimeout),
|
||||
// [BAA]_Darker_than_Black_Omake-1.zip
|
||||
// [BAA]_Darker_than_Black_Omake-1, Bleach 001-002, Kodoja #001 (March 2016)
|
||||
new Regex(
|
||||
@"^(?!Vol)(?<Series>.*)(-)\d+-?\d*", // This catches a lot of stuff ^(?!Vol)(?<Series>.*)( |_)(\d+)
|
||||
MatchOptions, RegexTimeout),
|
||||
// Kodoja #001 (March 2016)
|
||||
new Regex(
|
||||
@"(?<Series>.*)(\s|_|-)#",
|
||||
@"^(?!Vol)(?!Chapter)(?<Series>.+?)(-|_|\s|#)\d+(-\d+)?",
|
||||
MatchOptions, RegexTimeout),
|
||||
// Baketeriya ch01-05.zip, Akiiro Bousou Biyori - 01.jpg, Beelzebub_172_RHS.zip, Cynthia the Mission 29.rar, A Compendium of Ghosts - 031 - The Third Story_ Part 12 (Digital) (Cobalt001)
|
||||
new Regex(
|
||||
|
|
|
|||
|
|
@ -346,6 +346,8 @@ public class ProcessSeries : IProcessSeries
|
|||
}
|
||||
|
||||
|
||||
#region People
|
||||
|
||||
// Handle People
|
||||
foreach (var chapter in chapters)
|
||||
{
|
||||
|
|
@ -490,6 +492,8 @@ public class ProcessSeries : IProcessSeries
|
|||
}
|
||||
});
|
||||
|
||||
#endregion
|
||||
|
||||
}
|
||||
|
||||
public void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos, bool forceUpdate = false)
|
||||
|
|
|
|||
|
|
@ -34,7 +34,7 @@ public class StatsService : IStatsService
|
|||
private readonly IUnitOfWork _unitOfWork;
|
||||
private readonly DataContext _context;
|
||||
private readonly IStatisticService _statisticService;
|
||||
private const string ApiUrl = "http://localhost:5003";
|
||||
private const string ApiUrl = "https://stats.kavitareader.com";
|
||||
|
||||
public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork, DataContext context, IStatisticService statisticService)
|
||||
{
|
||||
|
|
|
|||
|
|
@ -90,12 +90,12 @@ public class TokenService : ITokenService
|
|||
Token = await CreateToken(user),
|
||||
RefreshToken = await CreateRefreshToken(user)
|
||||
};
|
||||
} catch (SecurityTokenExpiredException)
|
||||
} catch (SecurityTokenExpiredException ex)
|
||||
{
|
||||
// Handle expired token
|
||||
return null;
|
||||
}
|
||||
catch (Exception)
|
||||
catch (Exception ex)
|
||||
{
|
||||
// Handle other exceptions
|
||||
return null;
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue