AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Performance, Scan Loop, Specials, and cleanup (#150)

Browse source 
* More cases for parsing regex

* Fixed a bug where chapter cover images weren't being updated due to a missed not.

* Removed a piece of code that was needed for upgrading, since all beta users agreed to wipe db.

* Fixed InProgress to properly respect order and show more recent activity first. Issue is with IEntityDate LastModified not updating in DataContext.

* Updated dependencies to lastest stable.

* LastModified on Volumes wasn't updating, validated it does update when data is changed.

* Rewrote a check to avoid a small heap object warning.

* Ensure UpdateSeries checks all libraries for unique name.

* Took care of some todos, removed unused imports, on dev go ahead and schedule reoocuring jobs since LiteDB caused the locking issue.

* No Tracking when we aren't using entities.

* Added code to remove abandoned progress rows after a chapter gets deleted.

* RefreshMetadata uses one large query rather than many trips to DB for updating metadata. Significantly faster.

* Fixed a bug where UpdateSeries would always complain about a unique name even when we weren't updating name.

* Files that are linked to a series but can't parse out Vol/Chapter information are properly grouped like other Specials.

* Refresh metadata on UI should call the task directly

* Fixed a bug on updating series to make sure we don't complain if we aren't trying to update the name to an existing name.

* Fixed #142 - Library cards should be sorted.

* Refactored the name of some variables to be more agnostic to comics.

* Implemented ScanLibrary but abandoning it.

* Code Cleanup & removing ScanSeries code.

* Some more tests and new Comparators for natural sorting.

* Fixed #137 - When performing I/O on archives, ignore __MACOSX folders completely.

* Fixed #137 - When performing I/O on archives, ignore __MACOSX folders completely.

* All entities that will show under specials tab should be marked special, rather than just what has a special keyword.

* Don't let specials generate cover images

* Don't let specials generate cover images

* SearchResults should send LocalizedName back since we are searching against it.

* Added some tests around macosx folders found from my actual server.

* Put extra notes about a case where duplicates come about, logger will now tell user about this issue.

* Missed a build issue somehow...

* Some code smells
This commit is contained in:
Joseph Milazzo 2021-04-05 08:37:45 -05:00 • committed by GitHub
parent 7790cf31fd
commit d3c14863d6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
39 changed files with 401 additions and 184 deletions
Download patch file Download diff file Expand all files Collapse all files

47
API/Services/ArchiveService.cs
View file

@ -10,6 +10,7 @@ using API.Extensions;
using API.Interfaces.Services;
using API.Services.Tasks;
using Microsoft.Extensions.Logging;
using Microsoft.IO;
using SharpCompress.Archives;
using SharpCompress.Common;
using Image = NetVips.Image;
@ -22,7 +23,8 @@ namespace API.Services
public class ArchiveService : IArchiveService
{
private readonly ILogger<ArchiveService> _logger;
private const int ThumbnailWidth = 320; // 153w x 230h TODO: Look into optimizing the images to be smaller
private const int ThumbnailWidth = 320; // 153w x 230h
private static readonly RecyclableMemoryStreamManager _streamManager = new RecyclableMemoryStreamManager();
public ArchiveService(ILogger<ArchiveService> logger)
{
@ -74,13 +76,15 @@ namespace API.Services
{
_logger.LogDebug("Using default compression handling");
using ZipArchive archive = ZipFile.OpenRead(archivePath);
return archive.Entries.Count(e => Parser.Parser.IsImage(e.FullName));
return archive.Entries.Count(e => !e.FullName.Contains("__MACOSX") && Parser.Parser.IsImage(e.FullName));
}
case ArchiveLibrary.SharpCompress:
{
_logger.LogDebug("Using SharpCompress compression handling");
using var archive = ArchiveFactory.Open(archivePath);
return archive.Entries.Count(entry => !entry.IsDirectory && Parser.Parser.IsImage(entry.Key));
return archive.Entries.Count(entry => !entry.IsDirectory &&
!(Path.GetDirectoryName(entry.Key) ?? string.Empty).Contains("__MACOSX")
&& Parser.Parser.IsImage(entry.Key));
}
case ArchiveLibrary.NotSupported:
_logger.LogError("[GetNumberOfPagesFromArchive] This archive cannot be read: {ArchivePath}. Defaulting to 0 pages", archivePath);
@ -117,8 +121,8 @@ namespace API.Services
{
_logger.LogDebug("Using default compression handling");
using var archive = ZipFile.OpenRead(archivePath);
var folder = archive.Entries.SingleOrDefault(x => Path.GetFileNameWithoutExtension(x.Name).ToLower() == "folder");
var entries = archive.Entries.Where(x => Path.HasExtension(x.FullName) && Parser.Parser.IsImage(x.FullName)).OrderBy(x => x.FullName).ToList();
var folder = archive.Entries.SingleOrDefault(x => !x.FullName.Contains("__MACOSX") && Path.GetFileNameWithoutExtension(x.Name).ToLower() == "folder");
var entries = archive.Entries.Where(x => Path.HasExtension(x.FullName) && !x.FullName.Contains("__MACOSX") && Parser.Parser.IsImage(x.FullName)).OrderBy(x => x.FullName).ToList();
var entry = folder ?? entries[0];
return createThumbnail ? CreateThumbnail(entry) : ConvertEntryToByteArray(entry);
@ -127,7 +131,9 @@ namespace API.Services
{
_logger.LogDebug("Using SharpCompress compression handling");
using var archive = ArchiveFactory.Open(archivePath);
return FindCoverImage(archive.Entries.Where(entry => !entry.IsDirectory && Parser.Parser.IsImage(entry.Key)), createThumbnail);
return FindCoverImage(archive.Entries.Where(entry => !entry.IsDirectory
&& !(Path.GetDirectoryName(entry.Key) ?? string.Empty).Contains("__MACOSX")
&& Parser.Parser.IsImage(entry.Key)), createThumbnail);
}
case ArchiveLibrary.NotSupported:
_logger.LogError("[GetCoverImage] This archive cannot be read: {ArchivePath}. Defaulting to no cover image", archivePath);
@ -152,10 +158,11 @@ namespace API.Services
{
if (Path.GetFileNameWithoutExtension(entry.Key).ToLower() == "folder")
{
using var ms = new MemoryStream();
using var ms = _streamManager.GetStream();
entry.WriteTo(ms);
ms.Position = 0;
return createThumbnail ? CreateThumbnail(ms.ToArray(), Path.GetExtension(entry.Key)) : ms.ToArray();
var data = ms.ToArray();
return createThumbnail ? CreateThumbnail(data, Path.GetExtension(entry.Key)) : data;
}
}
@ -163,7 +170,7 @@ namespace API.Services
{
var entry = images.OrderBy(e => e.Key).FirstOrDefault();
if (entry == null) return Array.Empty<byte>();
using var ms = new MemoryStream();
using var ms = _streamManager.GetStream();
entry.WriteTo(ms);
ms.Position = 0;
var data = ms.ToArray();
@ -176,11 +183,9 @@ namespace API.Services
private static byte[] ConvertEntryToByteArray(ZipArchiveEntry entry)
{
using var stream = entry.Open();
using var ms = new MemoryStream();
using var ms = _streamManager.GetStream();
stream.CopyTo(ms);
var data = ms.ToArray();
return data;
return ms.ToArray();
}
/// <summary>
@ -194,7 +199,7 @@ namespace API.Services
// Sometimes ZipArchive will list the directory and others it will just keep it in the FullName
return archive.Entries.Count > 0 &&
!Path.HasExtension(archive.Entries.ElementAt(0).FullName) ||
archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar));
archive.Entries.Any(e => e.FullName.Contains(Path.AltDirectorySeparatorChar) && !e.FullName.Contains("__MACOSX"));
}
private byte[] CreateThumbnail(byte[] entry, string formatExtension = ".jpg")
@ -211,7 +216,7 @@ namespace API.Services
}
catch (Exception ex)
{
_logger.LogError(ex, "[CreateThumbnail] There was a critical error and prevented thumbnail generation. Defaulting to no cover image");
_logger.LogError(ex, "[CreateThumbnail] There was a critical error and prevented thumbnail generation. Defaulting to no cover image. Format Extension {Extension}", formatExtension);
}
return Array.Empty<byte>();
@ -263,7 +268,7 @@ namespace API.Services
{
if (Path.GetFileNameWithoutExtension(entry.Key).ToLower().EndsWith("comicinfo") && Parser.Parser.IsXml(entry.Key))
{
using var ms = new MemoryStream();
using var ms = _streamManager.GetStream();
entry.WriteTo(ms);
ms.Position = 0;
@ -295,7 +300,7 @@ namespace API.Services
{
_logger.LogDebug("Using default compression handling");
using var archive = ZipFile.OpenRead(archivePath);
var entry = archive.Entries.SingleOrDefault(x => Path.GetFileNameWithoutExtension(x.Name).ToLower() == "comicinfo" && Parser.Parser.IsXml(x.FullName));
var entry = archive.Entries.SingleOrDefault(x => !x.FullName.Contains("__MACOSX") && Path.GetFileNameWithoutExtension(x.Name).ToLower() == "comicinfo" && Parser.Parser.IsXml(x.FullName));
if (entry != null)
{
using var stream = entry.Open();
@ -308,7 +313,9 @@ namespace API.Services
{
_logger.LogDebug("Using SharpCompress compression handling");
using var archive = ArchiveFactory.Open(archivePath);
info = FindComicInfoXml(archive.Entries.Where(entry => !entry.IsDirectory && Parser.Parser.IsXml(entry.Key)));
info = FindComicInfoXml(archive.Entries.Where(entry => !entry.IsDirectory
&& !(Path.GetDirectoryName(entry.Key) ?? string.Empty).Contains("__MACOSX")
&& Parser.Parser.IsXml(entry.Key)));
break;
}
case ArchiveLibrary.NotSupported:
@ -392,7 +399,9 @@ namespace API.Services
{
_logger.LogDebug("Using SharpCompress compression handling");
using var archive = ArchiveFactory.Open(archivePath);
ExtractArchiveEntities(archive.Entries.Where(entry => !entry.IsDirectory && Parser.Parser.IsImage(entry.Key)), extractPath);
ExtractArchiveEntities(archive.Entries.Where(entry => !entry.IsDirectory
&& !(Path.GetDirectoryName(entry.Key) ?? string.Empty).Contains("__MACOSX")
&& Parser.Parser.IsImage(entry.Key)), extractPath);
break;
}
case ArchiveLibrary.NotSupported: