AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged develop in

Browse source
This commit is contained in:
Joseph Milazzo 2025-04-26 16:17:05 -05:00
commit d12a79892f
1443 changed files with 215765 additions and 44113 deletions
Download patch file Download diff file Expand all files Collapse all files

26
API/Services/Tasks/BackupService.cs
View file

@ -45,8 +45,6 @@ public class BackupService : IBackupService
_backupFiles = new List<string>()
{
"appsettings.json",
"Hangfire.db", // This is not used atm
"Hangfire-log.db", // This is not used atm
"kavita.db",
"kavita.db-shm", // This wont always be there
"kavita.db-wal" // This wont always be there
@ -106,22 +104,29 @@ public class BackupService : IBackupService
_directoryService.ExistOrCreate(tempDirectory);
_directoryService.ClearDirectory(tempDirectory);
await SendProgress(0.1F, "Copying config files");
_directoryService.CopyFilesToDirectory(
_backupFiles.Select(file => _directoryService.FileSystem.Path.Join(_directoryService.ConfigDirectory, file)).ToList(), tempDirectory);
_backupFiles.Select(file => _directoryService.FileSystem.Path.Join(_directoryService.ConfigDirectory, file)), tempDirectory);
// Copy any csv's as those are used for manual migrations
_directoryService.CopyFilesToDirectory(
_directoryService.GetFilesWithCertainExtensions(_directoryService.ConfigDirectory, @"\.csv"), tempDirectory);
await SendProgress(0.2F, "Copying logs");
CopyLogsToBackupDirectory(tempDirectory);
await SendProgress(0.25F, "Copying cover images");
await CopyCoverImagesToBackupDirectory(tempDirectory);
await SendProgress(0.5F, "Copying bookmarks");
await SendProgress(0.35F, "Copying templates images");
CopyTemplatesToBackupDirectory(tempDirectory);
await SendProgress(0.5F, "Copying bookmarks");
await CopyBookmarksToBackupDirectory(tempDirectory);
await SendProgress(0.75F, "Copying themes");
CopyThemesToBackupDirectory(tempDirectory);
await SendProgress(0.85F, "Copying favicons");
CopyFaviconsToBackupDirectory(tempDirectory);
@ -150,6 +155,11 @@ public class BackupService : IBackupService
_directoryService.CopyDirectoryToDirectory(_directoryService.FaviconDirectory, _directoryService.FileSystem.Path.Join(tempDirectory, "favicons"));
}
private void CopyTemplatesToBackupDirectory(string tempDirectory)
{
_directoryService.CopyDirectoryToDirectory(_directoryService.TemplateDirectory, _directoryService.FileSystem.Path.Join(tempDirectory, "templates"));
}
private async Task CopyCoverImagesToBackupDirectory(string tempDirectory)
{
var outputTempDir = Path.Join(tempDirectory, "covers");
@ -169,6 +179,10 @@ public class BackupService : IBackupService
_directoryService.CopyFilesToDirectory(
chapterImages.Select(s => _directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, s)), outputTempDir);
var volumeImages = await _unitOfWork.VolumeRepository.GetCoverImagesForLockedVolumesAsync();
_directoryService.CopyFilesToDirectory(
volumeImages.Select(s => _directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, s)), outputTempDir);
var libraryImages = await _unitOfWork.LibraryRepository.GetAllCoverImagesAsync();
_directoryService.CopyFilesToDirectory(
libraryImages.Select(s => _directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, s)), outputTempDir);

139
API/Services/Tasks/CleanupService.cs
View file

@ -8,8 +8,10 @@ using API.DTOs.Filtering;
using API.Entities;
using API.Entities.Enums;
using API.Helpers;
using API.Services.Tasks.Scanner.Parser;
using API.SignalR;
using Hangfire;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Services.Tasks;
@ -20,6 +22,7 @@ public interface ICleanupService
Task Cleanup();
Task CleanupDbEntries();
void CleanupCacheAndTempDirectories();
void CleanupCacheDirectory();
Task DeleteSeriesCoverImages();
Task DeleteChapterCoverImages();
Task DeleteTagCoverImages();
@ -32,6 +35,11 @@ public interface ICleanupService
/// </summary>
/// <returns></returns>
Task CleanupWantToRead();
Task ConsolidateProgress();
Task CleanupMediaErrors();
}
/// <summary>
/// Cleans up after operations on reoccurring basis
@ -73,13 +81,23 @@ public class CleanupService : ICleanupService
_logger.LogInformation("Starting Cleanup");
await SendProgress(0F, "Starting cleanup");
_logger.LogInformation("Cleaning temp directory");
_directoryService.ClearDirectory(_directoryService.TempDirectory);
await SendProgress(0.1F, "Cleaning temp directory");
CleanupCacheAndTempDirectories();
await SendProgress(0.25F, "Cleaning old database backups");
_logger.LogInformation("Cleaning old database backups");
await CleanupBackups();
await SendProgress(0.35F, "Consolidating Progress Events");
await ConsolidateProgress();
await SendProgress(0.4F, "Consolidating Media Errors");
await CleanupMediaErrors();
await SendProgress(0.50F, "Cleaning deleted cover images");
_logger.LogInformation("Cleaning deleted cover images");
await DeleteSeriesCoverImages();
@ -106,7 +124,7 @@ public class CleanupService : ICleanupService
await _unitOfWork.AppUserProgressRepository.CleanupAbandonedChapters();
await _unitOfWork.PersonRepository.RemoveAllPeopleNoLongerAssociated();
await _unitOfWork.GenreRepository.RemoveAllGenreNoLongerAssociated();
await _unitOfWork.CollectionTagRepository.RemoveTagsWithoutSeries();
await _unitOfWork.CollectionTagRepository.RemoveCollectionsWithoutSeries();
await _unitOfWork.ReadingListRepository.RemoveReadingListsWithoutSeries();
}
@ -178,6 +196,23 @@ public class CleanupService : ICleanupService
_logger.LogInformation("Cache and temp directory purged");
}
public void CleanupCacheDirectory()
{
_logger.LogInformation("Performing cleanup of Cache directories");
_directoryService.ExistOrCreate(_directoryService.CacheDirectory);
try
{
_directoryService.ClearDirectory(_directoryService.CacheDirectory);
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an issue deleting one or more folders/files during cleanup");
}
_logger.LogInformation("Cache directory purged");
}
/// <summary>
/// Removes Database backups older than configured total backups. If all backups are older than total backups days, only the latest is kept.
/// </summary>
@ -208,6 +243,108 @@ public class CleanupService : ICleanupService
_logger.LogInformation("Finished cleanup of Database backups at {Time}", DateTime.Now);
}
/// <summary>
/// Find any progress events that have duplicate, find the highest page read event, then copy over information from that and delete others, to leave one.
/// </summary>
public async Task ConsolidateProgress()
{
_logger.LogInformation("Consolidating Progress Events");
// AppUserProgress
var allProgress = await _unitOfWork.AppUserProgressRepository.GetAllProgress();
// Group by the unique identifiers that would make a progress entry unique
var duplicateGroups = allProgress
.GroupBy(p => new
{
p.AppUserId,
p.ChapterId,
})
.Where(g => g.Count() > 1);
foreach (var group in duplicateGroups)
{
// Find the entry with the highest pages read
var highestProgress = group
.OrderByDescending(p => p.PagesRead)
.ThenByDescending(p => p.LastModifiedUtc)
.First();
// Get the duplicate entries to remove (all except the highest progress)
var duplicatesToRemove = group
.Where(p => p.Id != highestProgress.Id)
.ToList();
// Copy over any non-null BookScrollId if the highest progress entry doesn't have one
if (string.IsNullOrEmpty(highestProgress.BookScrollId))
{
var firstValidScrollId = duplicatesToRemove
.FirstOrDefault(p => !string.IsNullOrEmpty(p.BookScrollId))
?.BookScrollId;
if (firstValidScrollId != null)
{
highestProgress.BookScrollId = firstValidScrollId;
highestProgress.MarkModified();
}
}
// Remove the duplicates
foreach (var duplicate in duplicatesToRemove)
{
_unitOfWork.AppUserProgressRepository.Remove(duplicate);
}
}
// Save changes
await _unitOfWork.CommitAsync();
}
/// <summary>
/// Scans through Media Error and removes any entries that have been fixed and are within the DB (proper files where wordcount/pagecount > 0)
/// </summary>
public async Task CleanupMediaErrors()
{
try
{
List<string> errorStrings = ["This archive cannot be read or not supported", "File format not supported"];
var mediaErrors = await _unitOfWork.MediaErrorRepository.GetAllErrorsAsync(errorStrings);
_logger.LogInformation("Beginning consolidation of {Count} Media Errors", mediaErrors.Count);
var pathToErrorMap = mediaErrors
.GroupBy(me => Parser.NormalizePath(me.FilePath))
.ToDictionary(
group => group.Key,
group => group.ToList() // The same file can be duplicated (rare issue when network drives die out midscan)
);
var normalizedPaths = pathToErrorMap.Keys.ToList();
// Find all files that are valid
var validFiles = await _unitOfWork.DataContext.MangaFile
.Where(f => normalizedPaths.Contains(f.FilePath) && f.Pages > 0)
.Select(f => f.FilePath)
.ToListAsync();
var removalCount = 0;
foreach (var validFilePath in validFiles)
{
if (!pathToErrorMap.TryGetValue(validFilePath, out var mediaError)) continue;
_unitOfWork.MediaErrorRepository.Remove(mediaError);
removalCount++;
}
await _unitOfWork.CommitAsync();
_logger.LogInformation("Finished consolidation of {Count} Media Errors, Removed: {RemovalCount}",
mediaErrors.Count, removalCount);
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an exception consolidating media errors");
}
}
public async Task CleanupLogs()
{
_logger.LogInformation("Performing cleanup of logs directory");

643
API/Services/Tasks/Metadata/CoverDbService.cs Normal file
View file

@ -0,0 +1,643 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
using API.Constants;
using API.Data;
using API.Data.Repositories;
using API.Entities;
using API.Entities.Enums;
using API.Entities.Person;
using API.Extensions;
using API.SignalR;
using EasyCaching.Core;
using Flurl;
using Flurl.Http;
using HtmlAgilityPack;
using Kavita.Common;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using NetVips;
namespace API.Services.Tasks.Metadata;
#nullable enable
public interface ICoverDbService
{
Task<string> DownloadFaviconAsync(string url, EncodeFormat encodeFormat);
Task<string> DownloadPublisherImageAsync(string publisherName, EncodeFormat encodeFormat);
Task<string?> DownloadPersonImageAsync(Person person, EncodeFormat encodeFormat);
Task<string?> DownloadPersonImageAsync(Person person, EncodeFormat encodeFormat, string url);
Task SetPersonCoverByUrl(Person person, string url, bool fromBase64 = true, bool checkNoImagePlaceholder = false);
Task SetSeriesCoverByUrl(Series series, string url, bool fromBase64 = true, bool chooseBetterImage = false);
Task SetChapterCoverByUrl(Chapter chapter, string url, bool fromBase64 = true, bool chooseBetterImage = false);
}
public class CoverDbService : ICoverDbService
{
private readonly ILogger<CoverDbService> _logger;
private readonly IDirectoryService _directoryService;
private readonly IEasyCachingProviderFactory _cacheFactory;
private readonly IHostEnvironment _env;
private readonly IImageService _imageService;
private readonly IUnitOfWork _unitOfWork;
private readonly IEventHub _eventHub;
private const string NewHost = "https://www.kavitareader.com/CoversDB/";
private static readonly string[] ValidIconRelations = {
"icon",
"apple-touch-icon",
"apple-touch-icon-precomposed",
"apple-touch-icon icon-precomposed" // ComicVine has it combined
};
/// <summary>
/// A mapping of urls that need to get the icon from another url, due to strangeness (like app.plex.tv loading a black icon)
/// </summary>
private static readonly Dictionary<string, string> FaviconUrlMapper = new()
{
["https://app.plex.tv"] = "https://plex.tv"
};
/// <summary>
/// Cache of the publisher/favicon list
/// </summary>
private static readonly TimeSpan CacheDuration = TimeSpan.FromDays(1);
public CoverDbService(ILogger<CoverDbService> logger, IDirectoryService directoryService,
IEasyCachingProviderFactory cacheFactory, IHostEnvironment env, IImageService imageService,
IUnitOfWork unitOfWork, IEventHub eventHub)
{
_logger = logger;
_directoryService = directoryService;
_cacheFactory = cacheFactory;
_env = env;
_imageService = imageService;
_unitOfWork = unitOfWork;
_eventHub = eventHub;
}
public async Task<string> DownloadFaviconAsync(string url, EncodeFormat encodeFormat)
{
// Parse the URL to get the domain (including subdomain)
var uri = new Uri(url);
var domain = uri.Host.Replace(Environment.NewLine, string.Empty);
var baseUrl = uri.Scheme + "://" + uri.Host;
var provider = _cacheFactory.GetCachingProvider(EasyCacheProfiles.Favicon);
var res = await provider.GetAsync<string>(baseUrl);
if (res.HasValue)
{
var sanitizedBaseUrl = baseUrl.Sanitize();
_logger.LogInformation("Kavita has already tried to fetch from {BaseUrl} and failed. Skipping duplicate check", sanitizedBaseUrl);
throw new KavitaException($"Kavita has already tried to fetch from {sanitizedBaseUrl} and failed. Skipping duplicate check");
}
await provider.SetAsync(baseUrl, string.Empty, TimeSpan.FromDays(10));
if (FaviconUrlMapper.TryGetValue(baseUrl, out var value))
{
url = value;
}
var correctSizeLink = string.Empty;
try
{
var htmlContent = url.GetStringAsync().Result;
var htmlDocument = new HtmlDocument();
htmlDocument.LoadHtml(htmlContent);
var pngLinks = htmlDocument.DocumentNode.Descendants("link")
.Where(link => ValidIconRelations.Contains(link.GetAttributeValue("rel", string.Empty)))
.Select(link => link.GetAttributeValue("href", string.Empty))
.Where(href => href.Split("?")[0].EndsWith(".png", StringComparison.InvariantCultureIgnoreCase))
.ToList();
correctSizeLink = (pngLinks?.Find(pngLink => pngLink.Contains("32")) ?? pngLinks?.FirstOrDefault());
}
catch (Exception ex)
{
_logger.LogError(ex, "Error downloading favicon.png for {Domain}, will try fallback methods", domain);
}
try
{
if (string.IsNullOrEmpty(correctSizeLink))
{
correctSizeLink = await FallbackToKavitaReaderFavicon(baseUrl);
}
if (string.IsNullOrEmpty(correctSizeLink))
{
throw new KavitaException($"Could not grab favicon from {baseUrl}");
}
var finalUrl = correctSizeLink;
// If starts with //, it's coming usually from an offsite cdn
if (correctSizeLink.StartsWith("//"))
{
finalUrl = "https:" + correctSizeLink;
}
else if (!correctSizeLink.StartsWith(uri.Scheme))
{
finalUrl = Url.Combine(baseUrl, correctSizeLink);
}
_logger.LogTrace("Fetching favicon from {Url}", finalUrl);
// Download the favicon.ico file using Flurl
var faviconStream = await finalUrl
.AllowHttpStatus("2xx,304")
.GetStreamAsync();
// Create the destination file path
using var image = Image.PngloadStream(faviconStream);
var filename = ImageService.GetWebLinkFormat(baseUrl, encodeFormat);
switch (encodeFormat)
{
case EncodeFormat.PNG:
image.Pngsave(Path.Combine(_directoryService.FaviconDirectory, filename));
break;
case EncodeFormat.WEBP:
image.Webpsave(Path.Combine(_directoryService.FaviconDirectory, filename));
break;
case EncodeFormat.AVIF:
image.Heifsave(Path.Combine(_directoryService.FaviconDirectory, filename));
break;
default:
throw new ArgumentOutOfRangeException(nameof(encodeFormat), encodeFormat, null);
}
_logger.LogDebug("Favicon for {Domain} downloaded and saved successfully", domain);
return filename;
} catch (Exception ex)
{
_logger.LogError(ex, "Error downloading favicon for {Domain}", domain);
throw;
}
}
public async Task<string> DownloadPublisherImageAsync(string publisherName, EncodeFormat encodeFormat)
{
try
{
var publisherLink = await FallbackToKavitaReaderPublisher(publisherName);
if (string.IsNullOrEmpty(publisherLink))
{
throw new KavitaException($"Could not grab publisher image for {publisherName}");
}
_logger.LogTrace("Fetching publisher image from {Url}", publisherLink.Sanitize());
// Download the publisher file using Flurl
var publisherStream = await publisherLink
.AllowHttpStatus("2xx,304")
.GetStreamAsync();
// Create the destination file path
using var image = Image.NewFromStream(publisherStream);
var filename = ImageService.GetPublisherFormat(publisherName, encodeFormat);
switch (encodeFormat)
{
case EncodeFormat.PNG:
image.Pngsave(Path.Combine(_directoryService.PublisherDirectory, filename));
break;
case EncodeFormat.WEBP:
image.Webpsave(Path.Combine(_directoryService.PublisherDirectory, filename));
break;
case EncodeFormat.AVIF:
image.Heifsave(Path.Combine(_directoryService.PublisherDirectory, filename));
break;
default:
throw new ArgumentOutOfRangeException(nameof(encodeFormat), encodeFormat, null);
}
_logger.LogDebug("Publisher image for {PublisherName} downloaded and saved successfully", publisherName.Sanitize());
return filename;
} catch (Exception ex)
{
_logger.LogError(ex, "Error downloading image for {PublisherName}", publisherName.Sanitize());
throw;
}
}
/// <summary>
/// Attempts to download the Person image from CoverDB while matching against metadata within the Person
/// </summary>
/// <param name="person"></param>
/// <param name="encodeFormat"></param>
/// <returns>Person image (in correct directory) or null if not found/error</returns>
public async Task<string?> DownloadPersonImageAsync(Person person, EncodeFormat encodeFormat)
{
try
{
var personImageLink = await GetCoverPersonImagePath(person);
if (string.IsNullOrEmpty(personImageLink))
{
throw new KavitaException($"Could not grab person image for {person.Name}");
}
return await DownloadPersonImageAsync(person, encodeFormat, personImageLink);
} catch (Exception ex)
{
_logger.LogError(ex, "Error downloading image for {PersonName}", person.Name);
}
return null;
}
/// <summary>
/// Attempts to download the Person cover image from a Url
/// </summary>
/// <param name="person"></param>
/// <param name="encodeFormat"></param>
/// <param name="url"></param>
/// <returns></returns>
/// <exception cref="KavitaException"></exception>
/// <exception cref="ArgumentOutOfRangeException"></exception>
public async Task<string?> DownloadPersonImageAsync(Person person, EncodeFormat encodeFormat, string url)
{
try
{
var personImageLink = await GetCoverPersonImagePath(person);
if (string.IsNullOrEmpty(personImageLink))
{
throw new KavitaException($"Could not grab person image for {person.Name}");
}
var filename = await DownloadImageFromUrl(ImageService.GetPersonFormat(person.Id), encodeFormat, personImageLink);
_logger.LogDebug("Person image for {PersonName} downloaded and saved successfully", person.Name);
return filename;
} catch (Exception ex)
{
_logger.LogError(ex, "Error downloading image for {PersonName}", person.Name);
}
return null;
}
private async Task<string> DownloadImageFromUrl(string filenameWithoutExtension, EncodeFormat encodeFormat, string url)
{
// Create the destination file path
var filename = filenameWithoutExtension + encodeFormat.GetExtension();
var targetFile = Path.Combine(_directoryService.CoverImageDirectory, filename);
// Ensure if file exists, we delete to overwrite
_logger.LogTrace("Fetching person image from {Url}", url.Sanitize());
// Download the file using Flurl
var personStream = await url
.AllowHttpStatus("2xx,304")
.GetStreamAsync();
using var image = Image.NewFromStream(personStream);
switch (encodeFormat)
{
case EncodeFormat.PNG:
image.Pngsave(targetFile);
break;
case EncodeFormat.WEBP:
image.Webpsave(targetFile);
break;
case EncodeFormat.AVIF:
image.Heifsave(targetFile);
break;
default:
throw new ArgumentOutOfRangeException(nameof(encodeFormat), encodeFormat, null);
}
return filename;
}
private async Task<string> GetCoverPersonImagePath(Person person)
{
var tempFile = Path.Join(_directoryService.LongTermCacheDirectory, "people.yml");
// Check if the file already exists and skip download in Development environment
if (File.Exists(tempFile))
{
if (_env.IsDevelopment())
{
_logger.LogInformation("Using existing people.yml file in Development environment");
}
else
{
// Remove file if not in Development and file is older than 7 days
if (File.GetLastWriteTime(tempFile) < DateTime.Now.AddDays(-7))
{
File.Delete(tempFile);
}
}
}
// Download the file if it doesn't exist or was deleted due to age
if (!File.Exists(tempFile))
{
var masterPeopleFile = await $"{NewHost}people/people.yml"
.DownloadFileAsync(_directoryService.LongTermCacheDirectory);
if (!File.Exists(tempFile) || string.IsNullOrEmpty(masterPeopleFile))
{
_logger.LogError("Could not download people.yml from Github");
return null;
}
}
var coverDbRepository = new CoverDbRepository(tempFile);
var coverAuthor = coverDbRepository.FindBestAuthorMatch(person);
if (coverAuthor == null || string.IsNullOrEmpty(coverAuthor.ImagePath))
{
throw new KavitaException($"Could not grab person image for {person.Name}");
}
return $"{NewHost}{coverAuthor.ImagePath}";
}
private async Task<string> FallbackToKavitaReaderFavicon(string baseUrl)
{
const string urlsFileName = "publishers.txt";
var correctSizeLink = string.Empty;
var allOverrides = await GetCachedData(urlsFileName) ??
await $"{NewHost}favicons/{urlsFileName}".GetStringAsync();
// Cache immediately
await CacheDataAsync(urlsFileName, allOverrides);
if (!string.IsNullOrEmpty(allOverrides))
{
var cleanedBaseUrl = baseUrl.Replace("https://", string.Empty);
var externalFile = allOverrides
.Split("\n")
.FirstOrDefault(url =>
cleanedBaseUrl.Equals(url.Replace(".png", string.Empty)) ||
cleanedBaseUrl.Replace("www.", string.Empty).Equals(url.Replace(".png", string.Empty)
));
if (string.IsNullOrEmpty(externalFile))
{
throw new KavitaException($"Could not grab favicon from {baseUrl.Sanitize()}");
}
correctSizeLink = $"{NewHost}favicons/" + externalFile;
}
return correctSizeLink;
}
private async Task<string> FallbackToKavitaReaderPublisher(string publisherName)
{
const string publisherFileName = "publishers.txt";
var externalLink = string.Empty;
var allOverrides = await GetCachedData(publisherFileName) ??
await $"{NewHost}publishers/{publisherFileName}".GetStringAsync();
// Cache immediately
await CacheDataAsync(publisherFileName, allOverrides);
if (!string.IsNullOrEmpty(allOverrides))
{
var externalFile = allOverrides
.Split("\n")
.Select(publisherLine =>
{
var tokens = publisherLine.Split("|");
if (tokens.Length != 2) return null;
var aliases = tokens[0];
// Multiple publisher aliases are separated by #
if (aliases.Split("#").Any(name => name.ToLowerInvariant().Trim().Equals(publisherName.ToLowerInvariant().Trim())))
{
return tokens[1];
}
return null;
})
.FirstOrDefault(url => !string.IsNullOrEmpty(url));
if (string.IsNullOrEmpty(externalFile))
{
throw new KavitaException($"Could not grab publisher image for {publisherName}");
}
externalLink = $"{NewHost}publishers/" + externalFile;
}
return externalLink;
}
private async Task CacheDataAsync(string fileName, string? content)
{
if (content == null) return;
try
{
var filePath = _directoryService.FileSystem.Path.Join(_directoryService.LongTermCacheDirectory, fileName);
await File.WriteAllTextAsync(filePath, content);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to cache {FileName}", fileName);
}
}
private async Task<string?> GetCachedData(string cacheFile)
{
// Form the full file path:
var filePath = _directoryService.FileSystem.Path.Join(_directoryService.LongTermCacheDirectory, cacheFile);
if (!File.Exists(filePath)) return null;
var fileInfo = new FileInfo(filePath);
if (DateTime.UtcNow - fileInfo.LastWriteTimeUtc <= CacheDuration)
{
return await File.ReadAllTextAsync(filePath);
}
return null;
}
/// <summary>
///
/// </summary>
/// <param name="person"></param>
/// <param name="url"></param>
/// <param name="fromBase64"></param>
/// <param name="checkNoImagePlaceholder">Will check against all known null image placeholders to avoid writing it</param>
public async Task SetPersonCoverByUrl(Person person, string url, bool fromBase64 = true, bool checkNoImagePlaceholder = false)
{
// TODO: Refactor checkNoImagePlaceholder bool to an action that evaluates how to process Image
if (!string.IsNullOrEmpty(url))
{
var filePath = await CreateThumbnail(url, $"{ImageService.GetPersonFormat(person.Id)}", fromBase64);
// Additional check to see if downloaded image is similar and we have a higher resolution
if (checkNoImagePlaceholder)
{
var matchRating = Path.Join(_directoryService.AssetsDirectory, "anilist-no-image-placeholder.jpg").GetSimilarity(Path.Join(_directoryService.CoverImageDirectory, filePath))!;
if (matchRating >= 0.9f)
{
if (string.IsNullOrEmpty(person.CoverImage))
{
filePath = null;
}
else
{
filePath = Path.GetFileName(Path.Join(_directoryService.CoverImageDirectory, person.CoverImage));
}
}
}
if (!string.IsNullOrEmpty(filePath))
{
person.CoverImage = filePath;
person.CoverImageLocked = true;
_imageService.UpdateColorScape(person);
_unitOfWork.PersonRepository.Update(person);
}
}
else
{
person.CoverImage = string.Empty;
person.CoverImageLocked = false;
_imageService.UpdateColorScape(person);
_unitOfWork.PersonRepository.Update(person);
}
if (_unitOfWork.HasChanges())
{
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.CoverUpdate,
MessageFactory.CoverUpdateEvent(person.Id, MessageFactoryEntityTypes.Person), false);
}
}
/// <summary>
/// Sets the series cover by url
/// </summary>
/// <param name="series"></param>
/// <param name="url"></param>
/// <param name="fromBase64"></param>
/// <param name="chooseBetterImage">If images are similar, will choose the higher quality image</param>
public async Task SetSeriesCoverByUrl(Series series, string url, bool fromBase64 = true, bool chooseBetterImage = false)
{
if (!string.IsNullOrEmpty(url))
{
var filePath = await CreateThumbnail(url, $"{ImageService.GetSeriesFormat(series.Id)}", fromBase64);
if (!string.IsNullOrEmpty(filePath))
{
// Additional check to see if downloaded image is similar and we have a higher resolution
if (chooseBetterImage && !string.IsNullOrEmpty(series.CoverImage))
{
try
{
var betterImage = Path.Join(_directoryService.CoverImageDirectory, series.CoverImage)
.GetBetterImage(Path.Join(_directoryService.CoverImageDirectory, filePath))!;
filePath = Path.GetFileName(betterImage);
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an issue trying to choose a better cover image for Series: {SeriesName} ({SeriesId})", series.Name, series.Id);
}
}
series.CoverImage = filePath;
series.CoverImageLocked = true;
if (series.CoverImage == null)
{
_logger.LogDebug("[SeriesCoverImageBug] Setting Series Cover Image to null");
}
_imageService.UpdateColorScape(series);
_unitOfWork.SeriesRepository.Update(series);
}
}
else
{
series.CoverImage = null;
series.CoverImageLocked = false;
if (series.CoverImage == null)
{
_logger.LogDebug("[SeriesCoverImageBug] Setting Series Cover Image to null");
}
_imageService.UpdateColorScape(series);
_unitOfWork.SeriesRepository.Update(series);
}
if (_unitOfWork.HasChanges())
{
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.CoverUpdate,
MessageFactory.CoverUpdateEvent(series.Id, MessageFactoryEntityTypes.Series), false);
}
}
public async Task SetChapterCoverByUrl(Chapter chapter, string url, bool fromBase64 = true, bool chooseBetterImage = false)
{
if (!string.IsNullOrEmpty(url))
{
var filePath = await CreateThumbnail(url, $"{ImageService.GetChapterFormat(chapter.Id, chapter.VolumeId)}", fromBase64);
if (!string.IsNullOrEmpty(filePath))
{
// Additional check to see if downloaded image is similar and we have a higher resolution
if (chooseBetterImage && !string.IsNullOrEmpty(chapter.CoverImage))
{
try
{
var betterImage = Path.Join(_directoryService.CoverImageDirectory, chapter.CoverImage)
.GetBetterImage(Path.Join(_directoryService.CoverImageDirectory, filePath))!;
filePath = Path.GetFileName(betterImage);
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an issue trying to choose a better cover image for Chapter: {FileName} ({ChapterId})", chapter.Range, chapter.Id);
}
}
chapter.CoverImage = filePath;
chapter.CoverImageLocked = true;
_imageService.UpdateColorScape(chapter);
_unitOfWork.ChapterRepository.Update(chapter);
}
}
else
{
chapter.CoverImage = null;
chapter.CoverImageLocked = false;
_imageService.UpdateColorScape(chapter);
_unitOfWork.ChapterRepository.Update(chapter);
}
if (_unitOfWork.HasChanges())
{
await _unitOfWork.CommitAsync();
await _eventHub.SendMessageAsync(MessageFactory.CoverUpdate,
MessageFactory.CoverUpdateEvent(chapter.Id, MessageFactoryEntityTypes.Chapter), false);
}
}
private async Task<string> CreateThumbnail(string url, string filename, bool fromBase64 = true)
{
var settings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
var encodeFormat = settings.EncodeMediaAs;
var coverImageSize = settings.CoverImageSize;
if (fromBase64)
{
return _imageService.CreateThumbnailFromBase64(url,
filename, encodeFormat, coverImageSize.GetDimensions().Width);
}
return await DownloadImageFromUrl(filename, encodeFormat, url);
}
}

29
API/Services/Tasks/Metadata/WordCountAnalyzerService.cs
View file

@ -33,17 +33,19 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService
private readonly IEventHub _eventHub;
private readonly ICacheHelper _cacheHelper;
private readonly IReaderService _readerService;
private readonly IMediaErrorService _mediaErrorService;
private const int AverageCharactersPerWord = 5;
public WordCountAnalyzerService(ILogger<WordCountAnalyzerService> logger, IUnitOfWork unitOfWork, IEventHub eventHub,
ICacheHelper cacheHelper, IReaderService readerService)
ICacheHelper cacheHelper, IReaderService readerService, IMediaErrorService mediaErrorService)
{
_logger = logger;
_unitOfWork = unitOfWork;
_eventHub = eventHub;
_cacheHelper = cacheHelper;
_readerService = readerService;
_mediaErrorService = mediaErrorService;
}
@ -177,7 +179,7 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService
var pageCounter = 1;
try
{
using var book = await EpubReader.OpenBookAsync(filePath, BookService.BookReaderOptions);
using var book = await EpubReader.OpenBookAsync(filePath, BookService.LenientBookReaderOptions);
var totalPages = book.Content.Html.Local;
foreach (var bookPage in totalPages)
@ -188,7 +190,7 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.WordCountAnalyzerProgressEvent(series.LibraryId, progress,
ProgressEventType.Updated, useFileName ? filePath : series.Name));
sum += await GetWordCountFromHtml(bookPage);
sum += await GetWordCountFromHtml(bookPage, filePath);
pageCounter++;
}
@ -215,6 +217,7 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService
chapter.MinHoursToRead = est.MinHours;
chapter.MaxHoursToRead = est.MaxHours;
chapter.AvgHoursToRead = est.AvgHours;
foreach (var file in chapter.Files)
{
UpdateFileAnalysis(file);
@ -245,13 +248,23 @@ public class WordCountAnalyzerService : IWordCountAnalyzerService
}
private static async Task<int> GetWordCountFromHtml(EpubLocalTextContentFileRef bookFile)
private async Task<int> GetWordCountFromHtml(EpubLocalTextContentFileRef bookFile, string filePath)
{
var doc = new HtmlDocument();
doc.LoadHtml(await bookFile.ReadContentAsync());
try
{
var doc = new HtmlDocument();
doc.LoadHtml(await bookFile.ReadContentAsync());
var textNodes = doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]");
return textNodes?.Sum(node => node.InnerText.Count(char.IsLetter)) / AverageCharactersPerWord ?? 0;
var textNodes = doc.DocumentNode.SelectNodes("//body//text()[not(parent::script)]");
return textNodes?.Sum(node => node.InnerText.Count(char.IsLetter)) / AverageCharactersPerWord ?? 0;
}
catch (EpubContentException ex)
{
_logger.LogError(ex, "Error when counting words in epub {EpubPath}", filePath);
await _mediaErrorService.ReportMediaIssueAsync(filePath, MediaErrorProducer.BookService,
$"Invalid Epub Metadata, {bookFile.FilePath} does not exist", ex.Message);
return 0;
}
}
}

50
API/Services/Tasks/Scanner/LibraryWatcher.cs
View file

@ -56,9 +56,9 @@ public class LibraryWatcher : ILibraryWatcher
/// <summary>
/// Counts within a time frame how many times the buffer became full. Is used to reschedule LibraryWatcher to start monitoring much later rather than instantly
/// </summary>
private int _bufferFullCounter;
private int _restartCounter;
private DateTime _lastErrorTime = DateTime.MinValue;
private static int _bufferFullCounter;
private static int _restartCounter;
private static DateTime _lastErrorTime = DateTime.MinValue;
/// <summary>
/// Used to lock buffer Full Counter
/// </summary>
@ -148,15 +148,30 @@ public class LibraryWatcher : ILibraryWatcher
private void OnChanged(object sender, FileSystemEventArgs e)
{
_logger.LogDebug("[LibraryWatcher] Changed: {FullPath}, {Name}, {ChangeType}", e.FullPath, e.Name, e.ChangeType);
_logger.LogTrace("[LibraryWatcher] Changed: {FullPath}, {Name}, {ChangeType}", e.FullPath, e.Name, e.ChangeType);
if (e.ChangeType != WatcherChangeTypes.Changed) return;
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name))));
var isDirectoryChange = string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name));
if (TaskScheduler.HasAlreadyEnqueuedTask("LibraryWatcher", "ProcessChange", [e.FullPath, isDirectoryChange],
checkRunningJobs: true))
{
return;
}
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, isDirectoryChange));
}
private void OnCreated(object sender, FileSystemEventArgs e)
{
_logger.LogDebug("[LibraryWatcher] Created: {FullPath}, {Name}", e.FullPath, e.Name);
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, !_directoryService.FileSystem.File.Exists(e.Name)));
_logger.LogTrace("[LibraryWatcher] Created: {FullPath}, {Name}", e.FullPath, e.Name);
var isDirectoryChange = !_directoryService.FileSystem.File.Exists(e.Name);
if (TaskScheduler.HasAlreadyEnqueuedTask("LibraryWatcher", "ProcessChange", [e.FullPath, isDirectoryChange],
checkRunningJobs: true))
{
return;
}
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, isDirectoryChange));
}
/// <summary>
@ -167,7 +182,12 @@ public class LibraryWatcher : ILibraryWatcher
private void OnDeleted(object sender, FileSystemEventArgs e) {
var isDirectory = string.IsNullOrEmpty(_directoryService.FileSystem.Path.GetExtension(e.Name));
if (!isDirectory) return;
_logger.LogDebug("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name);
_logger.LogTrace("[LibraryWatcher] Deleted: {FullPath}, {Name}", e.FullPath, e.Name);
if (TaskScheduler.HasAlreadyEnqueuedTask("LibraryWatcher", "ProcessChange", [e.FullPath, true],
checkRunningJobs: true))
{
return;
}
BackgroundJob.Enqueue(() => ProcessChange(e.FullPath, true));
}
@ -258,21 +278,23 @@ public class LibraryWatcher : ILibraryWatcher
_logger.LogTrace("Folder path: {FolderPath}", fullPath);
if (string.IsNullOrEmpty(fullPath))
{
_logger.LogTrace("[LibraryWatcher] Change from {FilePath} could not find root level folder, ignoring change", filePath);
_logger.LogInformation("[LibraryWatcher] Change from {FilePath} could not find root level folder, ignoring change", filePath);
return;
}
_taskScheduler.ScanFolder(fullPath, _queueWaitTime);
_taskScheduler.ScanFolder(fullPath, filePath, _queueWaitTime);
}
catch (Exception ex)
{
_logger.LogError(ex, "[LibraryWatcher] An error occured when processing a watch event");
}
_logger.LogDebug("[LibraryWatcher] ProcessChange completed in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds);
_logger.LogTrace("[LibraryWatcher] ProcessChange completed in {ElapsedMilliseconds}ms", sw.ElapsedMilliseconds);
}
private string GetFolder(string filePath, IEnumerable<string> libraryFolders)
{
// TODO: I can optimize this to avoid a library scan and instead do a Series Scan by finding the series that has a lowestFolderPath higher or equal to the filePath
var parentDirectory = _directoryService.GetParentDirectoryName(filePath);
_logger.LogTrace("[LibraryWatcher] Parent Directory: {ParentDirectory}", parentDirectory);
if (string.IsNullOrEmpty(parentDirectory)) return string.Empty;
@ -285,10 +307,10 @@ public class LibraryWatcher : ILibraryWatcher
var rootFolder = _directoryService.GetFoldersTillRoot(libraryFolder, filePath).ToList();
_logger.LogTrace("[LibraryWatcher] Root Folders: {RootFolders}", rootFolder);
if (!rootFolder.Any()) return string.Empty;
if (rootFolder.Count == 0) return string.Empty;
// Select the first folder and join with library folder, this should give us the folder to scan.
return Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder[rootFolder.Count - 1]));
return Parser.Parser.NormalizePath(_directoryService.FileSystem.Path.Join(libraryFolder, rootFolder[rootFolder.Count - 1]));
}
@ -296,7 +318,7 @@ public class LibraryWatcher : ILibraryWatcher
/// This is called via Hangfire to decrement the counter. Must work around a lock
/// </summary>
// ReSharper disable once MemberCanBePrivate.Global
public void UpdateLastBufferOverflow()
public static void UpdateLastBufferOverflow()
{
lock (Lock)
{

846
API/Services/Tasks/Scanner/ParseScannedFiles.cs
View file

@ -1,6 +1,8 @@
using System;
using System.Collections.Concurrent;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Threading.Tasks;
@ -9,6 +11,7 @@ using API.Entities.Enums;
using API.Extensions;
using API.Services.Tasks.Scanner.Parser;
using API.SignalR;
using ExCSS;
using Kavita.Common.Helpers;
using Microsoft.Extensions.Logging;
@ -29,11 +32,59 @@ public class ParsedSeries
/// Format of the Series
/// </summary>
public required MangaFormat Format { get; init; }
/// <summary>
/// Has this Series changed or not aka do we need to process it or not.
/// </summary>
public bool HasChanged { get; set; }
}
public class ScanResult
{
/// <summary>
/// A list of files in the Folder. Empty if HasChanged = false
/// </summary>
public IList<string> Files { get; set; }
/// <summary>
/// A nested folder from Library Root (at any level)
/// </summary>
public string Folder { get; set; }
/// <summary>
/// The library root
/// </summary>
public string LibraryRoot { get; set; }
/// <summary>
/// Was the Folder scanned or not. If not modified since last scan, this will be false and Files empty
/// </summary>
public bool HasChanged { get; set; }
/// <summary>
/// Set in Stage 2: Parsed Info from the Files
/// </summary>
public IList<ParserInfo> ParserInfos { get; set; }
}
/// <summary>
/// The final product of ParseScannedFiles. This has all the processed parserInfo and is ready for tracking/processing into entities
/// </summary>
public class ScannedSeriesResult
{
/// <summary>
/// Was the Folder scanned or not. If not modified since last scan, this will be false and indicates that upstream should count this as skipped
/// </summary>
public bool HasChanged { get; set; }
/// <summary>
/// The Parsed Series information used for tracking
/// </summary>
public ParsedSeries ParsedSeries { get; set; }
/// <summary>
/// Parsed files
/// </summary>
public IList<ParserInfo> ParsedInfos { get; set; }
}
public class SeriesModified
{
public required string FolderPath { get; set; }
public required string? FolderPath { get; set; }
public required string? LowestFolderPath { get; set; }
public required string SeriesName { get; set; }
public DateTime LastScanned { get; set; }
public MangaFormat Format { get; set; }
@ -68,119 +119,282 @@ public class ParseScannedFiles
_eventHub = eventHub;
}
/// <summary>
/// This will Scan all files in a folder path. For each folder within the folderPath, FolderAction will be invoked for all files contained
/// </summary>
/// <param name="scanDirectoryByDirectory">Scan directory by directory and for each, call folderAction</param>
/// <param name="seriesPaths">A dictionary mapping a normalized path to a list of <see cref="SeriesModified"/> to help scanner skip I/O</param>
/// <param name="folderPath">A library folder or series folder</param>
/// <param name="folderAction">A callback async Task to be called once all files for each folder path are found</param>
/// <param name="forceCheck">If we should bypass any folder last write time checks on the scan and force I/O</param>
public async Task ProcessFiles(string folderPath, bool scanDirectoryByDirectory,
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<IList<string>, string,Task> folderAction, Library library, bool forceCheck = false)
public async Task<IList<ScanResult>> ScanFiles(string folderPath, bool scanDirectoryByDirectory,
IDictionary<string, IList<SeriesModified>> seriesPaths, Library library, bool forceCheck = false)
{
string normalizedPath;
var fileExtensions = string.Join("|", library.LibraryFileTypes.Select(l => l.FileTypeGroup.GetRegex()));
// If there are no library file types, skip scanning entirely
if (string.IsNullOrWhiteSpace(fileExtensions))
{
return ArraySegment<ScanResult>.Empty;
}
var matcher = BuildMatcher(library);
var result = new List<ScanResult>();
// Not to self: this whole thing can be parallelized because we don't deal with any DB or global state
if (scanDirectoryByDirectory)
{
// This is used in library scan, so we should check first for a ignore file and use that here as well
var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(folderPath, DirectoryService.KavitaIgnoreFile);
var matcher = _directoryService.CreateMatcherFromFile(potentialIgnoreFile);
if (matcher != null)
{
_logger.LogWarning(".kavitaignore found! Ignore files is deprecated in favor of Library Settings. Please update and remove file at {Path}", potentialIgnoreFile);
}
if (library.LibraryExcludePatterns.Count != 0)
{
matcher ??= new GlobMatcher();
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
{
matcher.AddExclude(pattern.Pattern);
}
}
var directories = _directoryService.GetDirectories(folderPath, matcher).ToList();
foreach (var directory in directories)
{
normalizedPath = Parser.Parser.NormalizePath(directory);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
{
await folderAction(new List<string>(), directory);
}
else
{
// For a scan, this is doing everything in the directory loop before the folder Action is called...which leads to no progress indication
await folderAction(_directoryService.ScanFiles(directory, fileExtensions, matcher), directory);
}
}
return;
return await ScanDirectories(folderPath, seriesPaths, library, forceCheck, matcher, result, fileExtensions);
}
normalizedPath = Parser.Parser.NormalizePath(folderPath);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedPath, forceCheck))
{
await folderAction(new List<string>(), folderPath);
return;
}
// We need to calculate all folders till library root and see if any kavitaignores
var seriesMatcher = BuildIgnoreFromLibraryRoot(folderPath, seriesPaths);
await folderAction(_directoryService.ScanFiles(folderPath, fileExtensions, seriesMatcher), folderPath);
return await ScanSingleDirectory(folderPath, seriesPaths, library, forceCheck, result, fileExtensions, matcher);
}
/// <summary>
/// Used in ScanSeries, which enters at a lower level folder and hence needs a .kavitaignore from higher (up to root) to be built before
/// the scan takes place.
/// </summary>
/// <param name="folderPath"></param>
/// <param name="seriesPaths"></param>
/// <returns>A GlobMatter. Empty if not applicable</returns>
private GlobMatcher BuildIgnoreFromLibraryRoot(string folderPath, IDictionary<string, IList<SeriesModified>> seriesPaths)
private async Task<IList<ScanResult>> ScanDirectories(string folderPath, IDictionary<string, IList<SeriesModified>> seriesPaths,
Library library, bool forceCheck, GlobMatcher matcher, List<ScanResult> result, string fileExtensions)
{
var seriesMatcher = new GlobMatcher();
try
{
var roots = seriesPaths[folderPath][0].LibraryRoots.Select(Parser.Parser.NormalizePath).ToList();
var libraryFolder = roots.SingleOrDefault(folderPath.Contains);
var allDirectories = _directoryService.GetAllDirectories(folderPath, matcher)
.Select(Parser.Parser.NormalizePath)
.OrderByDescending(d => d.Length)
.ToList();
if (string.IsNullOrEmpty(libraryFolder) || !Directory.Exists(folderPath))
var processedDirs = new HashSet<string>();
_logger.LogDebug("[ScannerService] Step 1.C Found {DirectoryCount} directories to process for {FolderPath}", allDirectories.Count, folderPath);
foreach (var directory in allDirectories)
{
// Don't process any folders where we've already scanned everything below
if (processedDirs.Any(d => d.StartsWith(directory + Path.AltDirectorySeparatorChar) || d.Equals(directory)))
{
return seriesMatcher;
var hasChanged = !HasSeriesFolderNotChangedSinceLastScan(library, seriesPaths, directory, forceCheck);
// Skip this directory as we've already processed a parent unless there are loose files at that directory
// and they have changes
CheckSurfaceFiles(result, directory, folderPath, fileExtensions, matcher, hasChanged);
continue;
}
var allParents = _directoryService.GetFoldersTillRoot(libraryFolder, folderPath);
var path = libraryFolder;
// Apply the library root level kavitaignore
var potentialIgnoreFile = _directoryService.FileSystem.Path.Join(path, DirectoryService.KavitaIgnoreFile);
seriesMatcher.Merge(_directoryService.CreateMatcherFromFile(potentialIgnoreFile));
// Then apply kavitaignores for each folder down to where the series folder is
foreach (var folderPart in allParents.Reverse())
// Skip directories ending with "Specials", let the parent handle it
if (directory.EndsWith("Specials", StringComparison.OrdinalIgnoreCase))
{
path = Parser.Parser.NormalizePath(Path.Join(libraryFolder, folderPart));
potentialIgnoreFile = _directoryService.FileSystem.Path.Join(path, DirectoryService.KavitaIgnoreFile);
seriesMatcher.Merge(_directoryService.CreateMatcherFromFile(potentialIgnoreFile));
// Log or handle that we are skipping this directory
_logger.LogDebug("Skipping {Directory} as it ends with 'Specials'", directory);
continue;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(directory, library.Name, ProgressEventType.Updated));
if (HasSeriesFolderNotChangedSinceLastScan(library, seriesPaths, directory, forceCheck))
{
HandleUnchangedFolder(result, folderPath, directory);
}
else
{
PerformFullScan(result, directory, folderPath, fileExtensions, matcher);
}
processedDirs.Add(directory);
}
return result;
}
/// <summary>
/// Checks against all folder paths on file if the last scanned is >= the directory's last write time, down to the second
/// </summary>
/// <param name="library"></param>
/// <param name="seriesPaths"></param>
/// <param name="directory">This should be normalized</param>
/// <param name="forceCheck"></param>
/// <returns></returns>
private bool HasSeriesFolderNotChangedSinceLastScan(Library library, IDictionary<string, IList<SeriesModified>> seriesPaths, string directory, bool forceCheck)
{
// Reverting code from: https://github.com/Kareadita/Kavita/pull/3619/files#diff-0625df477047ab9d8e97a900201f2f29b2dc0599ba58eb75cfbbd073a9f3c72f
// This is to be able to release hotfix and tackle this in appropriate time
// With the bottom-up approach, this can report a false positive where a nested folder will get scanned even though a parent is the series
// This can't really be avoided. This is more likely to happen on Image chapter folder library layouts.
if (forceCheck || !seriesPaths.TryGetValue(directory, out var seriesList))
{
return false;
}
// if (forceCheck)
// {
// return false;
// }
// TryGetSeriesList falls back to parent folders to match to seriesList
// var seriesList = TryGetSeriesList(library, seriesPaths, directory);
// if (seriesList == null)
// {
// return false;
// }
foreach (var series in seriesList)
{
var lastWriteTime = _directoryService.GetLastWriteTime(series.LowestFolderPath!).Truncate(TimeSpan.TicksPerSecond);
var seriesLastScanned = series.LastScanned.Truncate(TimeSpan.TicksPerSecond);
if (seriesLastScanned < lastWriteTime)
{
return false;
}
}
catch (Exception ex)
return true;
}
private IList<SeriesModified>? TryGetSeriesList(Library library, IDictionary<string, IList<SeriesModified>> seriesPaths, string directory)
{
if (seriesPaths.Count == 0)
{
_logger.LogError(ex,
"[ScannerService] There was an error trying to find and apply .kavitaignores above the Series Folder. Scanning without them present");
return null;
}
return seriesMatcher;
if (string.IsNullOrEmpty(directory))
{
return null;
}
if (library.Folders.Any(fp => fp.Path.Equals(directory)))
{
return null;
}
if (seriesPaths.TryGetValue(directory, out var seriesList))
{
return seriesList;
}
return TryGetSeriesList(library, seriesPaths, _directoryService.GetParentDirectoryName(directory));
}
/// <summary>
/// Handles directories that haven't changed since the last scan.
/// </summary>
private void HandleUnchangedFolder(List<ScanResult> result, string folderPath, string directory)
{
if (result.Exists(r => r.Folder == directory))
{
_logger.LogDebug("[ProcessFiles] Skipping adding {Directory} as it's already added, this indicates a bad layout issue", directory);
}
else
{
_logger.LogDebug("[ProcessFiles] Skipping {Directory} as it hasn't changed since last scan", directory);
result.Add(CreateScanResult(directory, folderPath, false, ArraySegment<string>.Empty));
}
}
/// <summary>
/// Performs a full scan of the directory and adds it to the result.
/// </summary>
private void PerformFullScan(List<ScanResult> result, string directory, string folderPath, string fileExtensions, GlobMatcher matcher)
{
_logger.LogDebug("[ProcessFiles] Performing full scan on {Directory}", directory);
var files = _directoryService.ScanFiles(directory, fileExtensions, matcher);
if (files.Count == 0)
{
_logger.LogDebug("[ProcessFiles] Empty directory: {Directory}. Keeping empty will cause Kavita to scan this each time", directory);
}
result.Add(CreateScanResult(directory, folderPath, true, files));
}
/// <summary>
/// Performs a full scan of the directory and adds it to the result.
/// </summary>
private void CheckSurfaceFiles(List<ScanResult> result, string directory, string folderPath, string fileExtensions, GlobMatcher matcher, bool hasChanged)
{
var files = _directoryService.ScanFiles(directory, fileExtensions, matcher, SearchOption.TopDirectoryOnly);
if (files.Count == 0)
{
return;
}
// Revert of https://github.com/Kareadita/Kavita/pull/3629/files#diff-0625df477047ab9d8e97a900201f2f29b2dc0599ba58eb75cfbbd073a9f3c72f
// for Hotfix v0.8.5.x
result.Add(CreateScanResult(directory, folderPath, true, files));
}
/// <summary>
/// Scans a single directory and processes the scan result.
/// </summary>
private async Task<IList<ScanResult>> ScanSingleDirectory(string folderPath, IDictionary<string, IList<SeriesModified>> seriesPaths, Library library, bool forceCheck, List<ScanResult> result,
string fileExtensions, GlobMatcher matcher)
{
var normalizedPath = Parser.Parser.NormalizePath(folderPath);
var libraryRoot =
library.Folders.FirstOrDefault(f =>
normalizedPath.Contains(Parser.Parser.NormalizePath(f.Path)))?.Path ??
folderPath;
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(normalizedPath, library.Name, ProgressEventType.Updated));
if (HasSeriesFolderNotChangedSinceLastScan(library, seriesPaths, normalizedPath, forceCheck))
{
result.Add(CreateScanResult(folderPath, libraryRoot, false, ArraySegment<string>.Empty));
}
else
{
result.Add(CreateScanResult(folderPath, libraryRoot, true,
_directoryService.ScanFiles(folderPath, fileExtensions, matcher)));
}
return result;
}
private static GlobMatcher BuildMatcher(Library library)
{
var matcher = new GlobMatcher();
foreach (var pattern in library.LibraryExcludePatterns.Where(p => !string.IsNullOrEmpty(p.Pattern)))
{
matcher.AddExclude(pattern.Pattern);
}
return matcher;
}
private static ScanResult CreateScanResult(string folderPath, string libraryRoot, bool hasChanged,
IList<string> files)
{
return new ScanResult()
{
Files = files,
Folder = Parser.Parser.NormalizePath(folderPath),
LibraryRoot = libraryRoot,
HasChanged = hasChanged
};
}
/// <summary>
/// Processes scanResults to track all series across the combined results.
/// Ensures series are correctly grouped even if they span multiple folders.
/// </summary>
/// <param name="scanResults">A collection of scan results</param>
/// <param name="scannedSeries">A concurrent dictionary to store the tracked series</param>
private void TrackSeriesAcrossScanResults(IList<ScanResult> scanResults, ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries)
{
// Flatten all ParserInfos from scanResults
var allInfos = scanResults.SelectMany(sr => sr.ParserInfos).ToList();
// Iterate through each ParserInfo and track the series
foreach (var info in allInfos)
{
if (info == null) continue;
try
{
TrackSeries(scannedSeries, info);
}
catch (Exception ex)
{
_logger.LogError(ex, "[ScannerService] Exception occurred during tracking {FilePath}. Skipping this file", info?.FullFilePath);
}
}
}
/// <summary>
/// Attempts to either add a new instance of a show mapping to the _scannedSeries bag or adds to an existing.
/// Attempts to either add a new instance of a series mapping to the _scannedSeries bag or adds to an existing.
/// This will check if the name matches an existing series name (multiple fields) <see cref="MergeName"/>
/// </summary>
/// <param name="scannedSeries">A localized list of a series' parsed infos</param>
@ -192,6 +406,8 @@ public class ParseScannedFiles
// Check if normalized info.Series already exists and if so, update info to use that name instead
info.Series = MergeName(scannedSeries, info);
// BUG: This will fail for Solo Leveling & Solo Leveling (Manga)
var normalizedSeries = info.Series.ToNormalized();
var normalizedSortSeries = info.SeriesSort.ToNormalized();
var normalizedLocalizedSeries = info.LocalizedSeries.ToNormalized();
@ -209,7 +425,7 @@ public class ParseScannedFiles
NormalizedName = normalizedSeries
};
scannedSeries.AddOrUpdate(existingKey, new List<ParserInfo>() {info}, (_, oldValue) =>
scannedSeries.AddOrUpdate(existingKey, [info], (_, oldValue) =>
{
oldValue ??= new List<ParserInfo>();
if (!oldValue.Contains(info))
@ -222,13 +438,13 @@ public class ParseScannedFiles
}
catch (Exception ex)
{
_logger.LogCritical(ex, "[ScannerService] {SeriesName} matches against multiple series in the parsed series. This indicates a critical kavita issue. Key will be skipped", info.Series);
_logger.LogCritical("[ScannerService] {SeriesName} matches against multiple series in the parsed series. This indicates a critical kavita issue. Key will be skipped", info.Series);
foreach (var seriesKey in scannedSeries.Keys.Where(ps =>
ps.Format == info.Format && (ps.NormalizedName.Equals(normalizedSeries)
|| ps.NormalizedName.Equals(normalizedLocalizedSeries)
|| ps.NormalizedName.Equals(normalizedSortSeries))))
{
_logger.LogCritical("[ScannerService] Matches: {SeriesName} matches on {SeriesKey}", info.Series, seriesKey.Name);
_logger.LogCritical("[ScannerService] Matches: '{SeriesName}' matches on '{SeriesKey}'", info.Series, seriesKey.Name);
}
}
}
@ -267,11 +483,12 @@ public class ParseScannedFiles
}
catch (Exception ex)
{
_logger.LogCritical(ex, "[ScannerService] Multiple series detected for {SeriesName} ({File})! This is critical to fix! There should only be 1", info.Series, info.FullFilePath);
_logger.LogCritical("[ScannerService] Multiple series detected for {SeriesName} ({File})! This is critical to fix! There should only be 1", info.Series, info.FullFilePath);
var values = scannedSeries.Where(p =>
(p.Key.NormalizedName.ToNormalized() == normalizedSeries ||
p.Key.NormalizedName.ToNormalized() == normalizedLocalSeries) &&
p.Key.Format == info.Format);
foreach (var pair in values)
{
_logger.LogCritical("[ScannerService] Duplicate Series in DB matches with {SeriesName}: {DuplicateName}", info.Series, pair.Key.Name);
@ -282,7 +499,6 @@ public class ParseScannedFiles
return info.Series;
}
/// <summary>
/// This will process series by folder groups. This is used solely by ScanSeries
/// </summary>
@ -290,107 +506,135 @@ public class ParseScannedFiles
/// <param name="folders"></param>
/// <param name="isLibraryScan">If true, does a directory scan first (resulting in folders being tackled in parallel), else does an immediate scan files</param>
/// <param name="seriesPaths">A map of Series names -> existing folder paths to handle skipping folders</param>
/// <param name="processSeriesInfos">Action which returns if the folder was skipped and the infos from said folder</param>
/// <param name="forceCheck">Defaults to false</param>
/// <returns></returns>
public async Task ScanLibrariesForSeries(Library library,
IEnumerable<string> folders, bool isLibraryScan,
IDictionary<string, IList<SeriesModified>> seriesPaths, Func<Tuple<bool, IList<ParserInfo>>, Task>? processSeriesInfos, bool forceCheck = false)
public async Task<IList<ScannedSeriesResult>> ScanLibrariesForSeries(Library library,
IList<string> folders, bool isLibraryScan,
IDictionary<string, IList<SeriesModified>> seriesPaths, bool forceCheck = false)
{
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Starting", library.Name, ProgressEventType.Started));
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent("File Scan Starting", library.Name, ProgressEventType.Started));
foreach (var folderPath in folders)
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 1.A: Process {FolderCount} folders", library.Name, folders.Count);
var processedScannedSeries = new ConcurrentBag<ScannedSeriesResult>();
foreach (var folder in folders)
{
try
{
await ProcessFiles(folderPath, isLibraryScan, seriesPaths, ProcessFolder, library, forceCheck);
await ScanAndParseFolder(folder, library, isLibraryScan, seriesPaths, processedScannedSeries, forceCheck);
}
catch (ArgumentException ex)
{
_logger.LogError(ex, "[ScannerService] The directory '{FolderPath}' does not exist", folderPath);
_logger.LogError(ex, "[ScannerService] The directory '{FolderPath}' does not exist", folder);
}
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.FileScanProgressEvent("File Scan Done", library.Name, ProgressEventType.Ended));
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent("File Scan Done", library.Name, ProgressEventType.Ended));
async Task ProcessFolder(IList<string> files, string folder)
{
var normalizedFolder = Parser.Parser.NormalizePath(folder);
if (HasSeriesFolderNotChangedSinceLastScan(seriesPaths, normalizedFolder, forceCheck))
{
var parsedInfos = seriesPaths[normalizedFolder].Select(fp => new ParserInfo()
{
Series = fp.SeriesName,
Format = fp.Format,
}).ToList();
if (processSeriesInfos != null)
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(true, parsedInfos));
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed since last scan", folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent("Skipped " + normalizedFolder, library.Name, ProgressEventType.Updated));
return;
}
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, folder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{files.Count} files in {folder}", library.Name, ProgressEventType.Updated));
if (files.Count == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty or is no longer in this location", folder);
return;
}
var scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
var infos = files
.Select(file => _readingItemService.ParseFile(file, folder, library.Type))
.Where(info => info != null)
.ToList();
MergeLocalizedSeriesWithSeries(infos);
foreach (var info in infos)
{
try
{
TrackSeries(scannedSeries, info);
}
catch (Exception ex)
{
_logger.LogError(ex,
"[ScannerService] There was an exception that occurred during tracking {FilePath}. Skipping this file",
info?.FullFilePath);
}
}
foreach (var series in scannedSeries.Keys)
{
if (scannedSeries[series].Count > 0 && processSeriesInfos != null)
{
await processSeriesInfos.Invoke(new Tuple<bool, IList<ParserInfo>>(false, scannedSeries[series]));
}
}
}
return processedScannedSeries.ToList();
}
/// <summary>
/// Checks against all folder paths on file if the last scanned is >= the directory's last write down to the second
/// Helper method to scan and parse a folder
/// </summary>
/// <param name="folderPath"></param>
/// <param name="library"></param>
/// <param name="isLibraryScan"></param>
/// <param name="seriesPaths"></param>
/// <param name="normalizedFolder"></param>
/// <param name="processedScannedSeries"></param>
/// <param name="forceCheck"></param>
/// <returns></returns>
private bool HasSeriesFolderNotChangedSinceLastScan(IDictionary<string, IList<SeriesModified>> seriesPaths, string normalizedFolder, bool forceCheck = false)
private async Task ScanAndParseFolder(string folderPath, Library library,
bool isLibraryScan, IDictionary<string, IList<SeriesModified>> seriesPaths,
ConcurrentBag<ScannedSeriesResult> processedScannedSeries, bool forceCheck)
{
if (forceCheck) return false;
_logger.LogDebug("\t[ScannerService] Library {LibraryName} Step 1.B: Scan files in {Folder}", library.Name, folderPath);
var scanResults = await ScanFiles(folderPath, isLibraryScan, seriesPaths, library, forceCheck);
return seriesPaths.ContainsKey(normalizedFolder) && seriesPaths[normalizedFolder].All(f => f.LastScanned.Truncate(TimeSpan.TicksPerSecond) >=
_directoryService.GetLastWriteTime(normalizedFolder).Truncate(TimeSpan.TicksPerSecond));
// Aggregate the scanned series across all scanResults
var scannedSeries = new ConcurrentDictionary<ParsedSeries, List<ParserInfo>>();
_logger.LogDebug("\t[ScannerService] Library {LibraryName} Step 1.C: Process files in {Folder}", library.Name, folderPath);
foreach (var scanResult in scanResults)
{
await ParseFiles(scanResult, seriesPaths, library);
}
_logger.LogDebug("\t[ScannerService] Library {LibraryName} Step 1.D: Merge any localized series with series {Folder}", library.Name, folderPath);
scanResults = MergeLocalizedSeriesAcrossScanResults(scanResults);
_logger.LogDebug("\t[ScannerService] Library {LibraryName} Step 1.E: Group all parsed data into logical Series", library.Name);
TrackSeriesAcrossScanResults(scanResults, scannedSeries);
// Now transform and add to processedScannedSeries AFTER everything is processed
_logger.LogDebug("\t[ScannerService] Library {LibraryName} Step 1.F: Generate Sort Order for Series and Finalize", library.Name);
GenerateProcessedScannedSeries(scannedSeries, scanResults, processedScannedSeries);
}
/// <summary>
/// Checks if there are any ParserInfos that have a Series that matches the LocalizedSeries field in any other info. If so,
/// rewrites the infos with series name instead of the localized name, so they stack.
/// Processes and generates the final results for processedScannedSeries after updating sort order.
/// </summary>
/// <param name="scannedSeries">A concurrent dictionary of tracked series and their parsed infos</param>
/// <param name="scanResults">List of all scan results, used to determine if any series has changed</param>
/// <param name="processedScannedSeries">A thread-safe concurrent bag of processed series results</param>
private void GenerateProcessedScannedSeries(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, IList<ScanResult> scanResults, ConcurrentBag<ScannedSeriesResult> processedScannedSeries)
{
// First, update the sort order for all series
UpdateSeriesSortOrder(scannedSeries);
// Now, generate the final processed scanned series results
CreateFinalSeriesResults(scannedSeries, scanResults, processedScannedSeries);
}
/// <summary>
/// Updates the sort order for all series in the scannedSeries dictionary.
/// </summary>
/// <param name="scannedSeries">A concurrent dictionary of tracked series and their parsed infos</param>
private void UpdateSeriesSortOrder(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries)
{
foreach (var series in scannedSeries.Keys)
{
if (scannedSeries[series].Count <= 0) continue;
try
{
UpdateSortOrder(scannedSeries, series); // Call to method that updates sort order
}
catch (Exception ex)
{
_logger.LogError(ex, "[ScannerService] Issue occurred while setting IssueOrder for series {SeriesName}", series.Name);
}
}
}
/// <summary>
/// Generates the final processed scanned series results after processing the sort order.
/// </summary>
/// <param name="scannedSeries">A concurrent dictionary of tracked series and their parsed infos</param>
/// <param name="scanResults">List of all scan results, used to determine if any series has changed</param>
/// <param name="processedScannedSeries">The list where processed results will be added</param>
private static void CreateFinalSeriesResults(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries,
IList<ScanResult> scanResults, ConcurrentBag<ScannedSeriesResult> processedScannedSeries)
{
foreach (var series in scannedSeries.Keys)
{
if (scannedSeries[series].Count <= 0) continue;
processedScannedSeries.Add(new ScannedSeriesResult
{
HasChanged = scanResults.Any(sr => sr.HasChanged), // Combine HasChanged flag across all scanResults
ParsedSeries = series,
ParsedInfos = scannedSeries[series]
});
}
}
/// <summary>
/// Merges localized series with the series field across all scan results.
/// Combines ParserInfos from all scanResults and processes them collectively
/// to ensure consistent series names.
/// </summary>
/// <example>
/// Accel World v01.cbz has Series "Accel World" and Localized Series "World of Acceleration"
@ -398,47 +642,263 @@ public class ParseScannedFiles
/// After running this code, we'd have:
/// World of Acceleration v02.cbz having Series "Accel World" and Localized Series of "World of Acceleration"
/// </example>
/// <param name="infos">A collection of ParserInfos</param>
private void MergeLocalizedSeriesWithSeries(IReadOnlyCollection<ParserInfo?> infos)
/// <param name="scanResults">A collection of scan results</param>
/// <returns>A new list of scan results with merged series</returns>
private IList<ScanResult> MergeLocalizedSeriesAcrossScanResults(IList<ScanResult> scanResults)
{
var hasLocalizedSeries = infos.Any(i => !string.IsNullOrEmpty(i.LocalizedSeries));
if (!hasLocalizedSeries) return;
// Flatten all ParserInfos across scanResults
var allInfos = scanResults.SelectMany(sr => sr.ParserInfos).ToList();
var localizedSeries = infos
.Where(i => !i.IsSpecial)
// Filter relevant infos (non-special and with localized series)
var relevantInfos = GetRelevantInfos(allInfos);
if (relevantInfos.Count == 0) return scanResults;
// Get distinct localized series and process each one
var distinctLocalizedSeries = relevantInfos
.Select(i => i.LocalizedSeries)
.Distinct()
.FirstOrDefault(i => !string.IsNullOrEmpty(i));
if (string.IsNullOrEmpty(localizedSeries)) return;
.ToList();
// NOTE: If we have multiple series in a folder with a localized title, then this will fail. It will group into one series. User needs to fix this themselves.
string? nonLocalizedSeries;
// Normalize this as many of the cases is a capitalization difference
var nonLocalizedSeriesFound = infos
.Where(i => !i.IsSpecial)
.Select(i => i.Series).DistinctBy(Parser.Parser.Normalize).ToList();
if (nonLocalizedSeriesFound.Count == 1)
foreach (var localizedSeries in distinctLocalizedSeries)
{
nonLocalizedSeries = nonLocalizedSeriesFound[0];
if (string.IsNullOrEmpty(localizedSeries)) continue;
// Process the localized series for merging
ProcessLocalizedSeries(scanResults, allInfos, relevantInfos, localizedSeries);
}
// Remove or clear any scan results that now have no ParserInfos after merging
return scanResults.Where(sr => sr.ParserInfos.Count > 0).ToList();
}
private static List<ParserInfo> GetRelevantInfos(List<ParserInfo> allInfos)
{
return allInfos
.Where(i => !i.IsSpecial && !string.IsNullOrEmpty(i.LocalizedSeries))
.GroupBy(i => i.Format)
.SelectMany(g => g.ToList())
.ToList();
}
private void ProcessLocalizedSeries(IList<ScanResult> scanResults, List<ParserInfo> allInfos, List<ParserInfo> relevantInfos, string localizedSeries)
{
var seriesForLocalized = GetSeriesForLocalized(relevantInfos, localizedSeries);
if (seriesForLocalized.Count == 0) return;
var nonLocalizedSeries = GetNonLocalizedSeries(seriesForLocalized, localizedSeries);
if (nonLocalizedSeries == null) return;
// Remap and update relevant ParserInfos
RemapSeries(scanResults, allInfos, localizedSeries, nonLocalizedSeries);
}
private static List<string> GetSeriesForLocalized(List<ParserInfo> relevantInfos, string localizedSeries)
{
return relevantInfos
.Where(i => i.LocalizedSeries == localizedSeries)
.DistinctBy(r => r.Series)
.Select(r => r.Series)
.ToList();
}
private string? GetNonLocalizedSeries(List<string> seriesForLocalized, string localizedSeries)
{
switch (seriesForLocalized.Count)
{
case 1:
return seriesForLocalized[0];
case <= 2:
return seriesForLocalized.FirstOrDefault(s => !s.Equals(Parser.Parser.Normalize(localizedSeries)));
default:
_logger.LogError(
"[ScannerService] Multiple series detected across scan results that contain localized series. " +
"This will cause them to group incorrectly. Please separate series into their own dedicated folder: {LocalizedSeries}",
string.Join(", ", seriesForLocalized)
);
return null;
}
}
private static void RemapSeries(IList<ScanResult> scanResults, List<ParserInfo> allInfos, string localizedSeries, string nonLocalizedSeries)
{
// If the series names are identical, no remapping is needed (rare but valid)
if (localizedSeries.ToNormalized().Equals(nonLocalizedSeries.ToNormalized()))
{
return;
}
// Find all infos that need to be remapped from the localized series to the non-localized series
var normalizedLocalizedSeries = localizedSeries.ToNormalized();
var seriesToBeRemapped = allInfos.Where(i => i.Series.ToNormalized().Equals(normalizedLocalizedSeries)).ToList();
foreach (var infoNeedingMapping in seriesToBeRemapped)
{
infoNeedingMapping.Series = nonLocalizedSeries;
// Find the scan result containing the localized info
var localizedScanResult = scanResults.FirstOrDefault(sr => sr.ParserInfos.Contains(infoNeedingMapping));
if (localizedScanResult == null) continue;
// Remove the localized series from this scan result
localizedScanResult.ParserInfos.Remove(infoNeedingMapping);
// Find the scan result that should be merged with
var nonLocalizedScanResult = scanResults.FirstOrDefault(sr => sr.ParserInfos.Any(pi => pi.Series == nonLocalizedSeries));
if (nonLocalizedScanResult == null) continue;
// Add the remapped info to the non-localized scan result
nonLocalizedScanResult.ParserInfos.Add(infoNeedingMapping);
// Assign the higher folder path (i.e., the one closer to the root)
//nonLocalizedScanResult.Folder = DirectoryService.GetDeepestCommonPath(localizedScanResult.Folder, nonLocalizedScanResult.Folder);
}
}
/// <summary>
/// For a given ScanResult, sets the ParserInfos on the result
/// </summary>
/// <param name="result"></param>
/// <param name="seriesPaths"></param>
/// <param name="library"></param>
private async Task ParseFiles(ScanResult result, IDictionary<string, IList<SeriesModified>> seriesPaths, Library library)
{
var normalizedFolder = Parser.Parser.NormalizePath(result.Folder);
// If folder hasn't changed, generate fake ParserInfos
if (!result.HasChanged)
{
result.ParserInfos = seriesPaths[normalizedFolder]
.Select(fp => new ParserInfo { Series = fp.SeriesName, Format = fp.Format })
.ToList();
// // We are certain TryGetSeriesList will return a valid result here, if the series wasn't present yet. It will have been changed.
// result.ParserInfos = TryGetSeriesList(library, seriesPaths, normalizedFolder)!
// .Select(fp => new ParserInfo { Series = fp.SeriesName, Format = fp.Format })
// .ToList();
_logger.LogDebug("[ScannerService] Skipped File Scan for {Folder} as it hasn't changed", normalizedFolder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"Skipped {normalizedFolder}", library.Name, ProgressEventType.Updated));
return;
}
var files = result.Files;
var fileCount = files.Count;
if (fileCount == 0)
{
_logger.LogInformation("[ScannerService] {Folder} is empty or has no matching file types", normalizedFolder);
result.ParserInfos = ArraySegment<ParserInfo>.Empty;
return;
}
_logger.LogDebug("[ScannerService] Found {Count} files for {Folder}", files.Count, normalizedFolder);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent($"{fileCount} files in {normalizedFolder}", library.Name, ProgressEventType.Updated));
// Parse files into ParserInfos
if (fileCount < 100)
{
// Process files sequentially
result.ParserInfos = files
.Select(file => _readingItemService.ParseFile(file, normalizedFolder, result.LibraryRoot, library.Type))
.Where(info => info != null)
.ToList()!;
}
else
{
// There can be a case where there are multiple series in a folder that causes merging.
if (nonLocalizedSeriesFound.Count > 2)
{
_logger.LogError("[ScannerService] There are multiple series within one folder that contain localized series. This will cause them to group incorrectly. Please separate series into their own dedicated folder or ensure there is only 2 potential series (localized and series): {LocalizedSeries}", string.Join(", ", nonLocalizedSeriesFound));
}
nonLocalizedSeries = nonLocalizedSeriesFound.Find(s => !s.Equals(localizedSeries));
// Process files in parallel
var tasks = files.Select(file => Task.Run(() =>
_readingItemService.ParseFile(file, normalizedFolder, result.LibraryRoot, library.Type)));
var infos = await Task.WhenAll(tasks);
result.ParserInfos = infos.Where(info => info != null).ToList()!;
}
}
if (nonLocalizedSeries == null) return;
var normalizedNonLocalizedSeries = nonLocalizedSeries.ToNormalized();
foreach (var infoNeedingMapping in infos.Where(i =>
!i.Series.ToNormalized().Equals(normalizedNonLocalizedSeries)))
private static void UpdateSortOrder(ConcurrentDictionary<ParsedSeries, List<ParserInfo>> scannedSeries, ParsedSeries series)
{
// Set the Sort order per Volume
var volumes = scannedSeries[series].GroupBy(info => info.Volumes);
foreach (var volume in volumes)
{
infoNeedingMapping.Series = nonLocalizedSeries;
infoNeedingMapping.LocalizedSeries = localizedSeries;
var infos = scannedSeries[series].Where(info => info.Volumes == volume.Key).ToList();
IList<ParserInfo> chapters;
var specialTreatment = infos.TrueForAll(info => info.IsSpecial);
var hasAnySpMarker = infos.Exists(info => info.SpecialIndex > 0);
var counter = 0f;
// Handle specials with SpecialIndex
if (specialTreatment && hasAnySpMarker)
{
chapters = infos
.OrderBy(info => info.SpecialIndex)
.ToList();
foreach (var chapter in chapters)
{
chapter.IssueOrder = counter;
counter++;
}
continue;
}
// Handle specials without SpecialIndex (natural order)
if (specialTreatment)
{
chapters = infos
.OrderByNatural(info => Parser.Parser.RemoveExtensionIfSupported(info.Filename)!)
.ToList();
foreach (var chapter in chapters)
{
chapter.IssueOrder = counter;
counter++;
}
continue;
}
// Ensure chapters are sorted numerically when possible, otherwise push unparseable to the end
chapters = infos
.OrderBy(info => float.TryParse(info.Chapters, NumberStyles.Any, CultureInfo.InvariantCulture, out var val) ? val : float.MaxValue)
.ToList();
counter = 0f;
var prevIssue = string.Empty;
foreach (var chapter in chapters)
{
// Use MinNumber in case there is a range, as otherwise sort order will cause it to be processed last
var chapterNum =
$"{Parser.Parser.MinNumberFromRange(chapter.Chapters).ToString(CultureInfo.InvariantCulture)}";
if (float.TryParse(chapterNum, NumberStyles.Any, CultureInfo.InvariantCulture, out var parsedChapter))
{
// Parsed successfully, use the numeric value
counter = parsedChapter;
chapter.IssueOrder = counter;
// Increment for next chapter (unless the next has a similar value, then add 0.1)
if (!string.IsNullOrEmpty(prevIssue) && float.TryParse(prevIssue, NumberStyles.Any, CultureInfo.InvariantCulture, out var prevIssueFloat) && parsedChapter.Is(prevIssueFloat))
{
counter += 0.1f; // bump if same value as the previous issue
}
prevIssue = $"{parsedChapter.ToString(CultureInfo.InvariantCulture)}";
}
else
{
// Unparsed chapters: use the current counter and bump for the next
if (!string.IsNullOrEmpty(prevIssue) && prevIssue == counter.ToString(CultureInfo.InvariantCulture))
{
counter += 0.1f; // bump if same value as the previous issue
}
chapter.IssueOrder = counter;
counter++;
prevIssue = chapter.Chapters;
}
}
}
}
}

130
API/Services/Tasks/Scanner/Parser/BasicParser.cs Normal file
View file

@ -0,0 +1,130 @@
using System;
using System.IO;
using API.Data.Metadata;
using API.Entities.Enums;
namespace API.Services.Tasks.Scanner.Parser;
#nullable enable
/// <summary>
/// This is the basic parser for handling Manga/Comic/Book libraries. This was previously DefaultParser before splitting each parser
/// into their own classes.
/// </summary>
public class BasicParser(IDirectoryService directoryService, IDefaultParser imageParser) : DefaultParser(directoryService)
{
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null)
{
var fileName = directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this.
if (type != LibraryType.Image && Parser.IsCoverImage(directoryService.FileSystem.Path.GetFileName(filePath))) return null;
if (Parser.IsImage(filePath))
{
return imageParser.Parse(filePath, rootPath, libraryRoot, LibraryType.Image, comicInfo);
}
var ret = new ParserInfo()
{
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Parser.RemoveExtensionIfSupported(fileName)!,
FullFilePath = Parser.NormalizePath(filePath),
Series = Parser.ParseSeries(fileName, type),
ComicInfo = comicInfo,
Chapters = Parser.ParseChapter(fileName, type),
Volumes = Parser.ParseVolume(fileName, type),
};
if (ret.Series == string.Empty || Parser.IsImage(filePath))
{
// Try to parse information out of each folder all the way to rootPath
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
var edition = Parser.ParseEdition(fileName);
if (!string.IsNullOrEmpty(edition))
{
ret.Series = Parser.CleanTitle(ret.Series.Replace(edition, string.Empty), type is LibraryType.Comic);
ret.Edition = edition;
}
var isSpecial = Parser.IsSpecial(fileName, type);
// We must ensure that we can only parse a special out. As some files will have v20 c171-180+Omake and that
// could cause a problem as Omake is a special term, but there is valid volume/chapter information.
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.LooseLeafVolume && isSpecial)
{
ret.IsSpecial = true;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret); // NOTE: This can cause some complications, we should try to be a bit less aggressive to fallback to folder
}
// If we are a special with marker, we need to ensure we use the correct series name. we can do this by falling back to Folder name
if (Parser.HasSpecialMarker(fileName))
{
ret.IsSpecial = true;
ret.SpecialIndex = Parser.ParseSpecialIndex(fileName);
ret.Chapters = Parser.DefaultChapter;
ret.Volumes = Parser.SpecialVolume;
// NOTE: This uses rootPath. LibraryRoot works better for manga, but it's not always that way.
// It might be worth writing some logic if the file is a special, to take the folder above the Specials/
// if present
var tempRootPath = rootPath;
if (rootPath.EndsWith("Specials") || rootPath.EndsWith("Specials/"))
{
tempRootPath = rootPath.Replace("Specials", string.Empty).TrimEnd('/');
}
// Check if the folder the file exists in is Specials/ and if so, take the parent directory as series (cleaned)
var fileDirectory = Path.GetDirectoryName(filePath);
if (!string.IsNullOrEmpty(fileDirectory) &&
(fileDirectory.EndsWith("Specials", StringComparison.OrdinalIgnoreCase) ||
fileDirectory.EndsWith("Specials/", StringComparison.OrdinalIgnoreCase)))
{
ret.Series = Parser.CleanTitle(Directory.GetParent(fileDirectory)?.Name ?? string.Empty);
}
else
{
ParseFromFallbackFolders(filePath, tempRootPath, type, ref ret);
}
ret.Title = Parser.CleanSpecialTitle(fileName);
}
if (string.IsNullOrEmpty(ret.Series))
{
ret.Series = Parser.CleanTitle(fileName, type is LibraryType.Comic);
}
// Pdfs may have .pdf in the series name, remove that
if (Parser.IsPdf(filePath) && ret.Series.ToLower().EndsWith(".pdf"))
{
ret.Series = ret.Series.Substring(0, ret.Series.Length - ".pdf".Length);
}
// Patch in other information from ComicInfo
UpdateFromComicInfo(ret);
if (ret.Volumes == Parser.LooseLeafVolume && ret.Chapters == Parser.DefaultChapter)
{
ret.IsSpecial = true;
}
// v0.8.x: Introducing a change where Specials will go in a separate Volume with a reserved number
if (ret.IsSpecial)
{
ret.Volumes = Parser.SpecialVolume;
}
return ret.Series == string.Empty ? null : ret;
}
/// <summary>
/// Applicable for everything but ComicVine and Image library types
/// </summary>
/// <param name="filePath"></param>
/// <param name="type"></param>
/// <returns></returns>
public override bool IsApplicable(string filePath, LibraryType type)
{
return type != LibraryType.ComicVine && type != LibraryType.Image;
}
}

62
API/Services/Tasks/Scanner/Parser/BookParser.cs Normal file
View file

@ -0,0 +1,62 @@
using API.Data.Metadata;
using API.Entities.Enums;
namespace API.Services.Tasks.Scanner.Parser;
public class BookParser(IDirectoryService directoryService, IBookService bookService, BasicParser basicParser) : DefaultParser(directoryService)
{
public override ParserInfo Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo comicInfo = null)
{
var info = bookService.ParseInfo(filePath);
if (info == null) return null;
info.ComicInfo = comicInfo;
// We need a special piece of code to override the Series IF there is a special marker in the filename for epub files
if (info.IsSpecial && info.Volumes is "0" or "0.0" && info.ComicInfo.Series != info.Series)
{
info.Series = info.ComicInfo.Series;
}
// This catches when original library type is Manga/Comic and when parsing with non
if (Parser.ParseVolume(info.Series, type) != Parser.LooseLeafVolume)
{
var hasVolumeInTitle = !Parser.ParseVolume(info.Title, type)
.Equals(Parser.LooseLeafVolume);
var hasVolumeInSeries = !Parser.ParseVolume(info.Series, type)
.Equals(Parser.LooseLeafVolume);
if (string.IsNullOrEmpty(info.ComicInfo?.Volume) && hasVolumeInTitle && (hasVolumeInSeries || string.IsNullOrEmpty(info.Series)))
{
// NOTE: I'm not sure the comment is true. I've never seen this triggered
// This is likely a light novel for which we can set series from parsed title
info.Series = Parser.ParseSeries(info.Title, type);
info.Volumes = Parser.ParseVolume(info.Title, type);
}
else
{
var info2 = basicParser.Parse(filePath, rootPath, libraryRoot, LibraryType.Book, comicInfo);
info.Merge(info2);
if (hasVolumeInSeries && info2 != null && Parser.ParseVolume(info2.Series, type)
.Equals(Parser.LooseLeafVolume))
{
// Override the Series name so it groups appropriately
info.Series = info2.Series;
}
}
}
return string.IsNullOrEmpty(info.Series) ? null : info;
}
/// <summary>
/// Only applicable for Epub files
/// </summary>
/// <param name="filePath"></param>
/// <param name="type"></param>
/// <returns></returns>
public override bool IsApplicable(string filePath, LibraryType type)
{
return Parser.IsEpub(filePath);
}
}

134
API/Services/Tasks/Scanner/Parser/ComicVineParser.cs Normal file
View file

@ -0,0 +1,134 @@
using System.IO;
using System.Linq;
using API.Data.Metadata;
using API.Entities.Enums;
namespace API.Services.Tasks.Scanner.Parser;
#nullable enable
/// <summary>
/// Responsible for Parsing ComicVine Comics.
/// </summary>
/// <param name="directoryService"></param>
public class ComicVineParser(IDirectoryService directoryService) : DefaultParser(directoryService)
{
/// <summary>
/// This Parser generates Series name to be defined as Series + first Issue Volume, so "Batman (2020)".
/// </summary>
/// <param name="filePath"></param>
/// <param name="rootPath"></param>
/// <param name="type"></param>
/// <returns></returns>
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null)
{
if (type != LibraryType.ComicVine) return null;
var fileName = directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// Mylar often outputs cover.jpg, ignore it by default
if (string.IsNullOrEmpty(fileName) || Parser.IsCoverImage(directoryService.FileSystem.Path.GetFileName(filePath))) return null;
var directoryName = directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
var info = new ParserInfo()
{
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Parser.RemoveExtensionIfSupported(fileName)!,
FullFilePath = Parser.NormalizePath(filePath),
Series = string.Empty,
ComicInfo = comicInfo,
Chapters = Parser.ParseChapter(fileName, type),
Volumes = Parser.ParseVolume(fileName, type)
};
// See if we can formulate the name from the ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo?.Series) && !string.IsNullOrEmpty(info.ComicInfo?.Volume))
{
info.Series = $"{info.ComicInfo.Series} ({info.ComicInfo.Volume})";
}
if (string.IsNullOrEmpty(info.Series))
{
// Check if we need to fallback to the Folder name AND that the folder matches the format "Series (Year)"
var directories = directoryService.GetFoldersTillRoot(rootPath, filePath).ToList();
if (directories.Count > 0)
{
foreach (var directory in directories)
{
if (!Parser.IsSeriesAndYear(directory)) continue;
info.Series = directory;
info.Volumes = Parser.ParseYearFromSeries(directory);
break;
}
// When there was at least one directory and we failed to parse the series, this is the final fallback
if (string.IsNullOrEmpty(info.Series))
{
info.Series = Parser.CleanTitle(directories[0], true);
}
}
else
{
if (Parser.IsSeriesAndYear(directoryName))
{
info.Series = directoryName;
info.Volumes = Parser.ParseYearFromSeries(directoryName);
}
}
}
// Check if this is a Special/Annual
info.IsSpecial = Parser.IsSpecial(info.Filename, type) || Parser.IsSpecial(info.ComicInfo?.Format, type);
// Patch in other information from ComicInfo
UpdateFromComicInfo(info);
if (string.IsNullOrEmpty(info.Series))
{
info.Series = Parser.CleanTitle(directoryName, true);
}
return string.IsNullOrEmpty(info.Series) ? null : info;
}
/// <summary>
/// Only applicable for ComicVine library type
/// </summary>
/// <param name="filePath"></param>
/// <param name="type"></param>
/// <returns></returns>
public override bool IsApplicable(string filePath, LibraryType type)
{
return type == LibraryType.ComicVine;
}
private new static void UpdateFromComicInfo(ParserInfo info)
{
if (info.ComicInfo == null) return;
if (!string.IsNullOrEmpty(info.ComicInfo.Volume))
{
info.Volumes = info.ComicInfo.Volume;
}
if (string.IsNullOrEmpty(info.LocalizedSeries) && !string.IsNullOrEmpty(info.ComicInfo.LocalizedSeries))
{
info.LocalizedSeries = info.ComicInfo.LocalizedSeries.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
{
info.Chapters = info.ComicInfo.Number;
if (info.IsSpecial && Parser.DefaultChapter != info.Chapters)
{
info.IsSpecial = false;
info.Volumes = $"{Parser.SpecialVolumeNumber}";
}
}
// Patch is SeriesSort from ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo.TitleSort))
{
info.SeriesSort = info.ComicInfo.TitleSort.Trim();
}
}
}

266
API/Services/Tasks/Scanner/Parser/DefaultParser.cs
View file

@ -1,5 +1,6 @@
using System.IO;
using System.Linq;
using API.Data.Metadata;
using API.Entities.Enums;
namespace API.Services.Tasks.Scanner.Parser;
@ -7,213 +8,26 @@ namespace API.Services.Tasks.Scanner.Parser;
public interface IDefaultParser
{
ParserInfo? Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga);
ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null);
void ParseFromFallbackFolders(string filePath, string rootPath, LibraryType type, ref ParserInfo ret);
bool IsApplicable(string filePath, LibraryType type);
}
/// <summary>
/// This is an implementation of the Parser that is the basis for everything
/// </summary>
public class DefaultParser : IDefaultParser
public abstract class DefaultParser(IDirectoryService directoryService) : IDefaultParser
{
private readonly IDirectoryService _directoryService;
public DefaultParser(IDirectoryService directoryService)
{
_directoryService = directoryService;
}
/// <summary>
/// Parses information out of a file path. Will fallback to using directory name if Series couldn't be parsed
/// Parses information out of a file path. Can fallback to using directory name if Series couldn't be parsed
/// from filename.
/// </summary>
/// <param name="filePath"></param>
/// <param name="rootPath">Root folder</param>
/// <param name="type">Defaults to Manga. Allows different Regex to be used for parsing.</param>
/// <param name="type">Allows different Regex to be used for parsing.</param>
/// <returns><see cref="ParserInfo"/> or null if Series was empty</returns>
public ParserInfo? Parse(string filePath, string rootPath, LibraryType type = LibraryType.Manga)
{
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
// We can now remove this as there is the ability to turn off images for non-image libraries
// TODO: Potential Bug: This will return null, but on Image libraries, if all images, we would want to include this.
if (type != LibraryType.Image && Parser.IsCoverImage(_directoryService.FileSystem.Path.GetFileName(filePath))) return null;
var ret = new ParserInfo()
{
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Path.GetFileNameWithoutExtension(fileName),
FullFilePath = filePath,
Series = string.Empty
};
// If library type is Image or this is not a cover image in a non-image library, then use dedicated parsing mechanism
if (type == LibraryType.Image || Parser.IsImage(filePath))
{
// TODO: We can move this up one level (out of DefaultParser - If we do different Parsers)
return ParseImage(filePath, rootPath, ret);
}
if (type == LibraryType.Magazine)
{
return ParseMagazine(filePath, rootPath, ret);
}
// This will be called if the epub is already parsed once then we call and merge the information, if the
if (Parser.IsEpub(filePath))
{
ret.Chapters = Parser.ParseChapter(fileName);
ret.Series = Parser.ParseSeries(fileName);
ret.Volumes = Parser.ParseVolume(fileName);
}
else
{
ret.Chapters = type == LibraryType.Comic
? Parser.ParseComicChapter(fileName)
: Parser.ParseChapter(fileName);
ret.Series = type == LibraryType.Comic ? Parser.ParseComicSeries(fileName) : Parser.ParseSeries(fileName);
ret.Volumes = type == LibraryType.Comic ? Parser.ParseComicVolume(fileName) : Parser.ParseVolume(fileName);
}
if (ret.Series == string.Empty || Parser.IsImage(filePath))
{
// Try to parse information out of each folder all the way to rootPath
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
var edition = Parser.ParseEdition(fileName);
if (!string.IsNullOrEmpty(edition))
{
ret.Series = Parser.CleanTitle(ret.Series.Replace(edition, string.Empty), type is LibraryType.Comic);
ret.Edition = edition;
}
var isSpecial = type == LibraryType.Comic ? Parser.IsComicSpecial(fileName) : Parser.IsMangaSpecial(fileName);
// We must ensure that we can only parse a special out. As some files will have v20 c171-180+Omake and that
// could cause a problem as Omake is a special term, but there is valid volume/chapter information.
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.DefaultVolume && isSpecial)
{
ret.IsSpecial = true;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret); // NOTE: This can cause some complications, we should try to be a bit less aggressive to fallback to folder
}
// If we are a special with marker, we need to ensure we use the correct series name. we can do this by falling back to Folder name
if (Parser.HasSpecialMarker(fileName))
{
ret.IsSpecial = true;
ret.Chapters = Parser.DefaultChapter;
ret.Volumes = Parser.DefaultVolume;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
if (string.IsNullOrEmpty(ret.Series))
{
ret.Series = Parser.CleanTitle(fileName, type is LibraryType.Comic);
}
// Pdfs may have .pdf in the series name, remove that
if (Parser.IsPdf(filePath) && ret.Series.ToLower().EndsWith(".pdf"))
{
ret.Series = ret.Series.Substring(0, ret.Series.Length - ".pdf".Length);
}
return ret.Series == string.Empty ? null : ret;
}
private ParserInfo ParseMagazine(string filePath, string rootPath, ParserInfo ret)
{
// Try to parse Series from the filename
var libraryPath = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Parent?.FullName ?? rootPath;
var fileName = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
ret.Series = Parser.ParseMagazineSeries(fileName);
ret.Volumes = Parser.ParseMagazineVolume(fileName);
ret.Chapters = Parser.ParseMagazineChapter(fileName);
if (string.IsNullOrEmpty(ret.Series) || (string.IsNullOrEmpty(ret.Chapters) && string.IsNullOrEmpty(ret.Volumes)))
{
// Fallback to the parent folder. We can also likely grab Volume (year) from here
var folders = _directoryService.GetFoldersTillRoot(libraryPath, filePath).ToList();
// Usually the LAST folder is the Series and everything up to can have Volume
if (string.IsNullOrEmpty(ret.Series))
{
ret.Series = Parser.CleanTitle(folders[^1]);
}
var hasGeoCode = !string.IsNullOrEmpty(Parser.ParseGeoCode(ret.Series));
foreach (var folder in folders[..^1])
{
if (ret.Volumes == Parser.DefaultVolume)
{
var vol = Parser.ParseYear(folder);
if (!string.IsNullOrEmpty(vol) && vol != folder)
{
ret.Volumes = vol;
}
}
// If folder has a language code in it, then we add that to the Series (Wired (UK))
if (!hasGeoCode)
{
var geoCode = Parser.ParseGeoCode(folder);
if (!string.IsNullOrEmpty(geoCode))
{
ret.Series = $"{ret.Series} ({geoCode})";
hasGeoCode = true;
}
}
}
}
return ret;
}
private ParserInfo ParseImage(string filePath, string rootPath, ParserInfo ret)
{
ret.Volumes = Parser.DefaultVolume;
ret.Chapters = Parser.DefaultChapter;
var directoryName = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
ret.Series = directoryName;
ParseFromFallbackFolders(filePath, rootPath, LibraryType.Image, ref ret);
if (IsEmptyOrDefault(ret.Volumes, ret.Chapters))
{
ret.IsSpecial = true;
}
else
{
var parsedVolume = Parser.ParseVolume(ret.Filename);
var parsedChapter = Parser.ParseChapter(ret.Filename);
if (IsEmptyOrDefault(ret.Volumes, string.Empty) && !parsedVolume.Equals(Parser.DefaultVolume))
{
ret.Volumes = parsedVolume;
}
if (IsEmptyOrDefault(string.Empty, ret.Chapters) && !parsedChapter.Equals(Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}
}
// Override the series name, as fallback folders needs it to try and parse folder name
if (string.IsNullOrEmpty(ret.Series) || ret.Series.Equals(directoryName))
{
ret.Series = Parser.CleanTitle(directoryName, replaceSpecials: false);
}
return ret;
}
private static bool IsEmptyOrDefault(string volumes, string chapters)
{
return (string.IsNullOrEmpty(chapters) || chapters == Parser.DefaultChapter) &&
(string.IsNullOrEmpty(volumes) || volumes == Parser.DefaultVolume);
}
public abstract ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null);
/// <summary>
/// Fills out <see cref="ParserInfo"/> by trying to parse volume, chapters, and series from folders
@ -224,14 +38,14 @@ public class DefaultParser : IDefaultParser
/// <param name="ret">Expects a non-null ParserInfo which this method will populate</param>
public void ParseFromFallbackFolders(string filePath, string rootPath, LibraryType type, ref ParserInfo ret)
{
var fallbackFolders = _directoryService.GetFoldersTillRoot(rootPath, filePath)
.Where(f => !Parser.IsMangaSpecial(f))
var fallbackFolders = directoryService.GetFoldersTillRoot(rootPath, filePath)
.Where(f => !Parser.IsSpecial(f, type))
.ToList();
if (fallbackFolders.Count == 0)
{
var rootFolderName = _directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
var series = Parser.ParseSeries(rootFolderName);
var rootFolderName = directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
var series = Parser.ParseSeries(rootFolderName, type);
if (string.IsNullOrEmpty(series))
{
@ -250,16 +64,18 @@ public class DefaultParser : IDefaultParser
{
var folder = fallbackFolders[i];
var parsedVolume = type is LibraryType.Manga ? Parser.ParseVolume(folder) : Parser.ParseComicVolume(folder);
var parsedChapter = type is LibraryType.Manga ? Parser.ParseChapter(folder) : Parser.ParseComicChapter(folder);
var parsedVolume = Parser.ParseVolume(folder, type);
var parsedChapter = Parser.ParseChapter(folder, type);
if (!parsedVolume.Equals(Parser.DefaultVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
if (!parsedVolume.Equals(Parser.LooseLeafVolume) || !parsedChapter.Equals(Parser.DefaultChapter))
{
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.DefaultVolume)) && !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.DefaultVolume))
if ((string.IsNullOrEmpty(ret.Volumes) || ret.Volumes.Equals(Parser.LooseLeafVolume))
&& !string.IsNullOrEmpty(parsedVolume) && !parsedVolume.Equals(Parser.LooseLeafVolume))
{
ret.Volumes = parsedVolume;
}
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter)) && !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter))
if ((string.IsNullOrEmpty(ret.Chapters) || ret.Chapters.Equals(Parser.DefaultChapter))
&& !string.IsNullOrEmpty(parsedChapter) && !parsedChapter.Equals(Parser.DefaultChapter))
{
ret.Chapters = parsedChapter;
}
@ -268,7 +84,7 @@ public class DefaultParser : IDefaultParser
// Generally users group in series folders. Let's try to parse series from the top folder
if (!folder.Equals(ret.Series) && i == fallbackFolders.Count - 1)
{
var series = Parser.ParseSeries(folder);
var series = Parser.ParseSeries(folder, type);
if (string.IsNullOrEmpty(series))
{
@ -284,4 +100,48 @@ public class DefaultParser : IDefaultParser
}
}
}
protected static void UpdateFromComicInfo(ParserInfo info)
{
if (info.ComicInfo == null) return;
if (!string.IsNullOrEmpty(info.ComicInfo.Volume))
{
info.Volumes = info.ComicInfo.Volume;
}
if (!string.IsNullOrEmpty(info.ComicInfo.Number))
{
info.Chapters = info.ComicInfo.Number;
}
if (!string.IsNullOrEmpty(info.ComicInfo.Series))
{
info.Series = info.ComicInfo.Series.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.LocalizedSeries))
{
info.LocalizedSeries = info.ComicInfo.LocalizedSeries.Trim();
}
if (!string.IsNullOrEmpty(info.ComicInfo.Format) && Parser.HasComicInfoSpecial(info.ComicInfo.Format))
{
info.IsSpecial = true;
info.Chapters = Parser.DefaultChapter;
info.Volumes = Parser.SpecialVolume;
}
// Patch is SeriesSort from ComicInfo
if (!string.IsNullOrEmpty(info.ComicInfo.SeriesSort))
{
info.SeriesSort = info.ComicInfo.SeriesSort.Trim();
}
}
public abstract bool IsApplicable(string filePath, LibraryType type);
protected static bool IsEmptyOrDefault(string volumes, string chapters)
{
return (string.IsNullOrEmpty(chapters) || chapters == Parser.DefaultChapter) &&
(string.IsNullOrEmpty(volumes) || volumes == Parser.LooseLeafVolume);
}
}

55
API/Services/Tasks/Scanner/Parser/ImageParser.cs Normal file
View file

@ -0,0 +1,55 @@
using System.IO;
using API.Data.Metadata;
using API.Entities.Enums;
namespace API.Services.Tasks.Scanner.Parser;
#nullable enable
public class ImageParser(IDirectoryService directoryService) : DefaultParser(directoryService)
{
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo? comicInfo = null)
{
if (!IsApplicable(filePath, type)) return null;
var directoryName = directoryService.FileSystem.DirectoryInfo.New(rootPath).Name;
var fileName = directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
var ret = new ParserInfo
{
Series = directoryName,
Volumes = Parser.LooseLeafVolume,
Chapters = Parser.DefaultChapter,
ComicInfo = comicInfo,
Format = MangaFormat.Image,
Filename = Path.GetFileName(filePath),
FullFilePath = Parser.NormalizePath(filePath),
Title = fileName,
};
ParseFromFallbackFolders(filePath, libraryRoot, LibraryType.Image, ref ret);
if (IsEmptyOrDefault(ret.Volumes, ret.Chapters))
{
ret.IsSpecial = true;
ret.Volumes = Parser.SpecialVolume;
}
// Override the series name, as fallback folders needs it to try and parse folder name
if (string.IsNullOrEmpty(ret.Series) || ret.Series.Equals(directoryName))
{
ret.Series = Parser.CleanTitle(directoryName);
}
return string.IsNullOrEmpty(ret.Series) ? null : ret;
}
/// <summary>
/// Only applicable for Image files and Image library type
/// </summary>
/// <param name="filePath"></param>
/// <param name="type"></param>
/// <returns></returns>
public override bool IsApplicable(string filePath, LibraryType type)
{
return type == LibraryType.Image && Parser.IsImage(filePath);
}
}

84
API/Services/Tasks/Scanner/Parser/MagazineParser.cs Normal file
View file

@ -0,0 +1,84 @@
using System.IO;
using System.Linq;
using API.Data.Metadata;
using API.Entities.Enums;
namespace API.Services.Tasks.Scanner.Parser;
public class MagazineParser(IDirectoryService directoryService) : DefaultParser(directoryService)
{
public override ParserInfo? Parse(string filePath, string rootPath, string libraryRoot, LibraryType type,
ComicInfo? comicInfo = null)
{
if (!IsApplicable(filePath, type)) return null;
var ret = new ParserInfo
{
Volumes = Parser.LooseLeafVolume,
Chapters = Parser.DefaultChapter,
ComicInfo = comicInfo,
Format = MangaFormat.Image,
Filename = Path.GetFileName(filePath),
FullFilePath = Parser.NormalizePath(filePath),
Series = string.Empty,
};
// Try to parse Series from the filename
var libraryPath = directoryService.FileSystem.DirectoryInfo.New(rootPath).Parent?.FullName ?? rootPath;
var fileName = directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
ret.Series = Parser.ParseMagazineSeries(fileName);
ret.Volumes = Parser.ParseMagazineVolume(fileName);
ret.Chapters = Parser.ParseMagazineChapter(fileName);
if (string.IsNullOrEmpty(ret.Series) || (string.IsNullOrEmpty(ret.Chapters) && string.IsNullOrEmpty(ret.Volumes)))
{
// Fallback to the parent folder. We can also likely grab Volume (year) from here
var folders = directoryService.GetFoldersTillRoot(libraryPath, filePath).ToList();
// Usually the LAST folder is the Series and everything up to can have Volume
if (string.IsNullOrEmpty(ret.Series))
{
ret.Series = Parser.CleanTitle(folders[^1]);
}
var hasGeoCode = !string.IsNullOrEmpty(Parser.ParseGeoCode(ret.Series));
foreach (var folder in folders[..^1])
{
if (ret.Volumes == Parser.LooseLeafVolume)
{
var vol = Parser.ParseYear(folder); // TODO: This might be better as YearFromSeries
if (!string.IsNullOrEmpty(vol) && vol != folder)
{
ret.Volumes = vol;
}
}
// If folder has a language code in it, then we add that to the Series (Wired (UK))
if (!hasGeoCode)
{
var geoCode = Parser.ParseGeoCode(folder);
if (!string.IsNullOrEmpty(geoCode))
{
ret.Series = $"{ret.Series} ({geoCode})";
hasGeoCode = true;
}
}
}
}
return ret;
}
/// <summary>
/// Only applicable for Image files and Image library type
/// </summary>
/// <param name="filePath"></param>
/// <param name="type"></param>
/// <returns></returns>
public override bool IsApplicable(string filePath, LibraryType type)
{
return type == LibraryType.Magazine && Parser.IsPdf(filePath);
}
}

660
API/Services/Tasks/Scanner/Parser/Parser.cs
View file

@ -13,11 +13,20 @@ namespace API.Services.Tasks.Scanner.Parser;
public static partial class Parser
{
public const string DefaultChapter = "0";
public const string DefaultVolume = "0";
// NOTE: If you change this, don't forget to change in the UI (see Series Detail)
public const string DefaultChapter = "-100000"; // -2147483648
public const string LooseLeafVolume = "-100000";
public const int DefaultChapterNumber = -100_000;
public const int LooseLeafVolumeNumber = -100_000;
/// <summary>
/// The Volume Number of Specials to reside in
/// </summary>
public const int SpecialVolumeNumber = 100_000;
public const string SpecialVolume = "100000";
public static readonly TimeSpan RegexTimeout = TimeSpan.FromMilliseconds(500);
public const string ImageFileExtensions = @"^(\.png|\.jpeg|\.jpg|\.webp|\.gif|\.avif)"; // Don't forget to update CoverChooser
public const string ImageFileExtensions = @"(\.png|\.jpeg|\.jpg|\.webp|\.gif|\.avif)"; // Don't forget to update CoverChooser
public const string ArchiveFileExtensions = @"\.cbz|\.zip|\.rar|\.cbr|\.tar.gz|\.7zip|\.7z|\.cb7|\.cbt";
public const string EpubFileExtension = @"\.epub";
public const string PdfFileExtension = @"\.pdf";
@ -36,30 +45,26 @@ public static partial class Parser
"One Shot", "One-Shot", "Prologue", "TPB", "Trade Paper Back", "Omnibus", "Compendium", "Absolute", "Graphic Novel",
"GN", "FCBD", "Giant Size");
private static readonly char[] LeadingZeroesTrimChars = new[] { '0' };
private static readonly char[] LeadingZeroesTrimChars = ['0'];
private static readonly char[] SpacesAndSeparators = { '\0', '\t', '\r', ' ', '-', ','};
private static readonly char[] SpacesAndSeparators = ['\0', '\t', '\r', ' ', '-', ','];
private const string Number = @"\d+(\.\d)?";
private const string NumberRange = Number + @"(-" + Number + @")?";
/// <summary>
/// non greedy matching of a string where parenthesis are balanced
/// non-greedy matching of a string where parenthesis are balanced
/// </summary>
public const string BalancedParen = @"(?:[^()]|(?<open>\()|(?<-open>\)))*?(?(open)(?!))";
/// <summary>
/// non greedy matching of a string where square brackets are balanced
/// non-greedy matching of a string where square brackets are balanced
/// </summary>
public const string BalancedBracket = @"(?:[^\[\]]|(?<open>\[)|(?<-open>\]))*?(?(open)(?!))";
/// <summary>
/// Matches [Complete], release tags like [kmts] but not [ Complete ] or [kmts ]
/// </summary>
private const string TagsInBrackets = $@"\[(?!\s){BalancedBracket}(?<!\s)\]";
/// <summary>
/// Common regex patterns present in both Comics and Mangas
/// </summary>
private const string CommonSpecial = @"Specials?|One[- ]?Shot|Extra(?:\sChapter)?(?=\s)|Art Collection|Side Stories|Bonus";
[GeneratedRegex(@"^\d+$")]
private static partial Regex IsNumberRegex();
@ -68,48 +73,138 @@ public static partial class Parser
/// Matches against font-family css syntax. Does not match if url import has data: starting, as that is binary data
/// </summary>
/// <remarks>See here for some examples https://developer.mozilla.org/en-US/docs/Web/CSS/@font-face</remarks>
public static readonly Regex FontSrcUrlRegex = new Regex(@"(?<Start>(?:src:\s?)?(?:url|local)\((?!data:)" + "(?:[\"']?)" + @"(?!data:))"
+ "(?<Filename>(?!data:)[^\"']+?)" + "(?<End>[\"']?" + @"\);?)",
public static readonly Regex FontSrcUrlRegex = new(@"(?<Start>(?:src:\s?)?(?:url|local)\((?!data:)" + "(?:[\"']?)" + @"(?!data:))"
+ "(?<Filename>(?!data:)[^\"']+?)" + "(?<End>[\"']?" + @"\);?)",
MatchOptions, RegexTimeout);
/// <summary>
/// https://developer.mozilla.org/en-US/docs/Web/CSS/@import
/// </summary>
public static readonly Regex CssImportUrlRegex = new Regex("(@import\\s([\"|']|url\\([\"|']))(?<Filename>[^'\"]+)([\"|']\\)?);",
public static readonly Regex CssImportUrlRegex = new("(@import\\s([\"|']|url\\([\"|']))(?<Filename>[^'\"]+)([\"|']\\)?);",
MatchOptions | RegexOptions.Multiline, RegexTimeout);
/// <summary>
/// Misc css image references, like background-image: url(), border-image, or list-style-image
/// </summary>
/// Original prepend: (background|border|list-style)-image:\s?)?
public static readonly Regex CssImageUrlRegex = new Regex(@"(url\((?!data:).(?!data:))" + "(?<Filename>(?!data:)[^\"']*)" + @"(.\))",
public static readonly Regex CssImageUrlRegex = new(@"(url\((?!data:).(?!data:))" + "(?<Filename>(?!data:)[^\"']*)" + @"(.\))",
MatchOptions, RegexTimeout);
private static readonly Regex ImageRegex = new Regex(ImageFileExtensions,
private static readonly Regex ImageRegex = new(ImageFileExtensions,
MatchOptions, RegexTimeout);
private static readonly Regex ArchiveFileRegex = new Regex(ArchiveFileExtensions,
private static readonly Regex ArchiveFileRegex = new(ArchiveFileExtensions,
MatchOptions, RegexTimeout);
private static readonly Regex ComicInfoArchiveRegex = new Regex(@"\.cbz|\.cbr|\.cb7|\.cbt",
private static readonly Regex ComicInfoArchiveRegex = new(@"\.cbz|\.cbr|\.cb7|\.cbt",
MatchOptions, RegexTimeout);
private static readonly Regex XmlRegex = new Regex(XmlRegexExtensions,
private static readonly Regex XmlRegex = new(XmlRegexExtensions,
MatchOptions, RegexTimeout);
private static readonly Regex BookFileRegex = new Regex(BookFileExtensions,
private static readonly Regex BookFileRegex = new(BookFileExtensions,
MatchOptions, RegexTimeout);
private static readonly Regex CoverImageRegex = new Regex(@"(?<![[a-z]\d])(?:!?)(?<!back)(?<!back_)(?<!back-)(cover|folder)(?![\w\d])",
private static readonly Regex CoverImageRegex = new(@"(?<!back[\s_-])(?<!\(back )(?<!back)(?:^|[^a-zA-Z0-9])(!?cover|folder)(?![a-zA-Z0-9]|s\b)",
MatchOptions, RegexTimeout);
private static readonly Regex NormalizeRegex = new Regex(@"[^\p{L}0-9\+!]",
/// <summary>
/// Normalize everything within Kavita. Some characters don't fall under Unicode, like full-width characters and need to be
/// added on a case-by-case basis.
/// </summary>
private static readonly Regex NormalizeRegex = new(@"[^\p{L}0-9\+!＊！＋]",
MatchOptions, RegexTimeout);
/// <summary>
/// Supports Batman (2020) or Batman (2)
/// </summary>
private static readonly Regex SeriesAndYearRegex = new(@"^\D+\s\((?<Year>\d+)\)$",
MatchOptions, RegexTimeout);
/// <summary>
/// Recognizes the Special token only
/// </summary>
private static readonly Regex SpecialTokenRegex = new Regex(@"SP\d+",
private static readonly Regex SpecialTokenRegex = new(@"SP\d+",
MatchOptions, RegexTimeout);
#region Manga
private static readonly Regex[] MangaSeriesRegex = new[]
{
private static readonly Regex[] MangaVolumeRegex =
[
// Thai Volume: เล่ม n -> Volume n
new Regex(
@"(เล่ม|เล่มที่)(\s)?(\.?)(\s|_)?(?<Volume>\d+(\-\d+)?(\.\d+)?)",
MatchOptions, RegexTimeout),
// Dance in the Vampire Bund v16-17
new Regex(
@"(?<Series>.*)(\b|_)v(?<Volume>\d+-?\d+)( |_)",
MatchOptions, RegexTimeout),
// Nagasarete Airantou - Vol. 30 Ch. 187.5 - Vol.31 Omake
new Regex(
@"^(?<Series>.+?)(\s*Chapter\s*\d+)?(\s|_|\-\s)+(Vol(ume)?\.?(\s|_)?)(?<Volume>\d+(\.\d+)?)(.+?|$)",
MatchOptions, RegexTimeout),
// Historys Strongest Disciple Kenichi_v11_c90-98.zip or Dance in the Vampire Bund v16-17
new Regex(
@"(?<Series>.*)(\b|_)(?!\[)v(?<Volume>" + NumberRange + @")(?!\])",
MatchOptions, RegexTimeout),
// Kodomo no Jikan vol. 10, [dmntsf.net] One Piece - Digital Colored Comics Vol. 20.5-21.5 Ch. 177
new Regex(
@"(?<Series>.*)(\b|_)(vol\.? ?)(?<Volume>\d+(\.\d)?(-\d+)?(\.\d)?)",
MatchOptions, RegexTimeout),
// Killing Bites Vol. 0001 Ch. 0001 - Galactica Scanlations (gb)
new Regex(
@"(vol\.? ?)(?<Volume>\d+(\.\d)?)",
MatchOptions, RegexTimeout),
// Tonikaku Cawaii [Volume 11].cbz
new Regex(
@"(volume )(?<Volume>\d+(\.\d)?)",
MatchOptions, RegexTimeout),
// Tower Of God S01 014 (CBT) (digital).cbz
new Regex(
@"(?<Series>.*)(\b|_|)(S(?<Volume>\d+))",
MatchOptions, RegexTimeout),
// vol_001-1.cbz for MangaPy default naming convention
new Regex(
@"(vol_)(?<Volume>\d+(\.\d)?)",
MatchOptions, RegexTimeout),
// Chinese Volume: 第n卷 -> Volume n, 第n册 -> Volume n, 幽游白书完全版 第03卷 天下 or 阿衰online 第1册
new Regex(
@"第(?<Volume>\d+)(卷|册)",
MatchOptions, RegexTimeout),
// Chinese Volume: 卷n -> Volume n, 册n -> Volume n
new Regex(
@"(卷|册)(?<Volume>\d+)",
MatchOptions, RegexTimeout),
// Korean Volume: 제n화|권|회|장 -> Volume n, n화|권|회|장 -> Volume n, 63권#200.zip -> Volume 63 (no chapter, #200 is just files inside)
new Regex(
@"제?(?<Volume>\d+(\.\d+)?)(권|회|화|장)",
MatchOptions, RegexTimeout),
// Korean Season: 시즌n -> Season n,
new Regex(
@"시즌(?<Volume>\d+\-?\d+)",
MatchOptions, RegexTimeout),
// Korean Season: 시즌n -> Season n, n시즌 -> season n
new Regex(
@"(?<Volume>\d+(\-|~)?\d+?)시즌",
MatchOptions, RegexTimeout),
// Korean Season: 시즌n -> Season n, n시즌 -> season n
new Regex(
@"시즌(?<Volume>\d+(\-|~)?\d+?)",
MatchOptions, RegexTimeout),
// Japanese Volume: n巻 -> Volume n
new Regex(
@"(?<Volume>\d+(?:(\-)\d+)?)巻",
MatchOptions, RegexTimeout),
// Russian Volume: Том n -> Volume n, Тома n -> Volume
new Regex(
@"Том(а?)(\.?)(\s|_)?(?<Volume>\d+(?:(\-)\d+)?)",
MatchOptions, RegexTimeout),
// Russian Volume: n Том -> Volume n
new Regex(
@"(\s|_)?(?<Volume>\d+(?:(\-)\d+)?)(\s|_)Том(а?)",
MatchOptions, RegexTimeout)
];
private static readonly Regex[] MangaSeriesRegex =
[
// Thai Volume: เล่ม n -> Volume n
new Regex(
@"(?<Series>.+?)(เล่ม|เล่มที่)(\s)?(\.?)(\s|_)?(?<Volume>\d+(\-\d+)?(\.\d+)?)",
MatchOptions, RegexTimeout),
// Russian Volume: Том n -> Volume n, Тома n -> Volume
new Regex(
@"(?<Series>.+?)Том(а?)(\.?)(\s|_)?(?<Volume>\d+(?:(\-)\d+)?)",
@ -139,7 +234,7 @@ public static partial class Parser
// [SugoiSugoi]_NEEDLESS_Vol.2_-_Disk_The_Informant_5_[ENG].rar, Yuusha Ga Shinda! - Vol.tbd Chapter 27.001 V2 Infection ①.cbz,
// Nagasarete Airantou - Vol. 30 Ch. 187.5 - Vol.30 Omake
new Regex(
@"^(?<Series>.+?)(\s*Chapter\s*\d+)?(\s|_|\-\s)+Vol(ume)?\.?(\d+|tbd|\s\d).+?",
@"^(?<Series>.+?)(?:\s*|_|\-\s*)+(?:Ch(?:apter|\.|)\s*\d+(?:\.\d+)?(?:\s*|_|\-\s*)+)?Vol(?:ume|\.|)\s*(?:\d+|tbd)(?:\s|_|\-\s*).+",
MatchOptions, RegexTimeout),
// Ichiban_Ushiro_no_Daimaou_v04_ch34_[VISCANS].zip, VanDread-v01-c01.zip
new Regex(
@ -148,7 +243,7 @@ public static partial class Parser
RegexTimeout),
// Gokukoku no Brynhildr - c001-008 (v01) [TrinityBAKumA], Black Bullet - v4 c17 [batoto]
new Regex(
@"(?<Series>.*)( - )(?:v|vo|c|chapters)\d",
@"(?<Series>.+?)( - )(?:v|vo|c|chapters)\d",
MatchOptions, RegexTimeout),
// Kedouin Makoto - Corpse Party Musume, Chapter 19 [Dametrans].zip
new Regex(
@ -175,7 +270,7 @@ public static partial class Parser
RegexTimeout),
//Knights of Sidonia c000 (S2 LE BD Omake - BLAME!) [Habanero Scans]
new Regex(
@"(?<Series>.*)(\bc\d+\b)",
@"(?<Series>.*?)(?<!\()\bc\d+\b",
MatchOptions, RegexTimeout),
//Tonikaku Cawaii [Volume 11], Darling in the FranXX - Volume 01.cbz
new Regex(
@ -278,163 +373,16 @@ public static partial class Parser
// Japanese Volume: n巻 -> Volume n
new Regex(
@"(?<Series>.+?)第(?<Volume>\d+(?:(\-)\d+)?)巻",
MatchOptions, RegexTimeout),
MatchOptions, RegexTimeout)
};
private static readonly Regex[] MangaVolumeRegex = new[]
{
// Dance in the Vampire Bund v16-17
new Regex(
@"(?<Series>.*)(\b|_)v(?<Volume>\d+-?\d+)( |_)",
MatchOptions, RegexTimeout),
// Nagasarete Airantou - Vol. 30 Ch. 187.5 - Vol.31 Omake
new Regex(
@"^(?<Series>.+?)(\s*Chapter\s*\d+)?(\s|_|\-\s)+(Vol(ume)?\.?(\s|_)?)(?<Volume>\d+(\.\d+)?)(.+?|$)",
MatchOptions, RegexTimeout),
// Historys Strongest Disciple Kenichi_v11_c90-98.zip or Dance in the Vampire Bund v16-17
new Regex(
@"(?<Series>.*)(\b|_)(?!\[)v(?<Volume>" + NumberRange + @")(?!\])",
MatchOptions, RegexTimeout),
// Kodomo no Jikan vol. 10, [dmntsf.net] One Piece - Digital Colored Comics Vol. 20.5-21.5 Ch. 177
new Regex(
@"(?<Series>.*)(\b|_)(vol\.? ?)(?<Volume>\d+(\.\d)?(-\d+)?(\.\d)?)",
MatchOptions, RegexTimeout),
// Killing Bites Vol. 0001 Ch. 0001 - Galactica Scanlations (gb)
new Regex(
@"(vol\.? ?)(?<Volume>\d+(\.\d)?)",
MatchOptions, RegexTimeout),
// Tonikaku Cawaii [Volume 11].cbz
new Regex(
@"(volume )(?<Volume>\d+(\.\d)?)",
MatchOptions, RegexTimeout),
// Tower Of God S01 014 (CBT) (digital).cbz
new Regex(
@"(?<Series>.*)(\b|_|)(S(?<Volume>\d+))",
MatchOptions, RegexTimeout),
// vol_001-1.cbz for MangaPy default naming convention
new Regex(
@"(vol_)(?<Volume>\d+(\.\d)?)",
MatchOptions, RegexTimeout),
];
// Chinese Volume: 第n卷 -> Volume n, 第n册 -> Volume n, 幽游白书完全版 第03卷 天下 or 阿衰online 第1册
private static readonly Regex[] ComicSeriesRegex =
[
// Thai Volume: เล่ม n -> Volume n
new Regex(
@"第(?<Volume>\d+)(卷|册)",
@"(?<Series>.+?)(เล่ม|เล่มที่)(\s)?(\.?)(\s|_)?(?<Volume>\d+(\-\d+)?(\.\d+)?)",
MatchOptions, RegexTimeout),
// Chinese Volume: 卷n -> Volume n, 册n -> Volume n
new Regex(
@"(卷|册)(?<Volume>\d+)",
MatchOptions, RegexTimeout),
// Korean Volume: 제n화|권|회|장 -> Volume n, n화|권|회|장 -> Volume n, 63권#200.zip -> Volume 63 (no chapter, #200 is just files inside)
new Regex(
@"제?(?<Volume>\d+(\.\d)?)(권|회|화|장)",
MatchOptions, RegexTimeout),
// Korean Season: 시즌n -> Season n,
new Regex(
@"시즌(?<Volume>\d+\-?\d+)",
MatchOptions, RegexTimeout),
// Korean Season: 시즌n -> Season n, n시즌 -> season n
new Regex(
@"(?<Volume>\d+(\-|~)?\d+?)시즌",
MatchOptions, RegexTimeout),
// Korean Season: 시즌n -> Season n, n시즌 -> season n
new Regex(
@"시즌(?<Volume>\d+(\-|~)?\d+?)",
MatchOptions, RegexTimeout),
// Japanese Volume: n巻 -> Volume n
new Regex(
@"(?<Volume>\d+(?:(\-)\d+)?)巻",
MatchOptions, RegexTimeout),
// Russian Volume: Том n -> Volume n, Тома n -> Volume
new Regex(
@"Том(а?)(\.?)(\s|_)?(?<Volume>\d+(?:(\-)\d+)?)",
MatchOptions, RegexTimeout),
// Russian Volume: n Том -> Volume n
new Regex(
@"(\s|_)?(?<Volume>\d+(?:(\-)\d+)?)(\s|_)Том(а?)",
MatchOptions, RegexTimeout),
};
private static readonly Regex[] MangaChapterRegex = new[]
{
// Historys Strongest Disciple Kenichi_v11_c90-98.zip, ...c90.5-100.5
new Regex(
@"(\b|_)(c|ch)(\.?\s?)(?<Chapter>(\d+(\.\d)?)(-c?\d+(\.\d)?)?)",
MatchOptions, RegexTimeout),
// [Suihei Kiki]_Kasumi_Otoko_no_Ko_[Taruby]_v1.1.zip
new Regex(
@"v\d+\.(\s|_)(?<Chapter>\d+(?:.\d+|-\d+)?)",
MatchOptions, RegexTimeout),
// Umineko no Naku Koro ni - Episode 3 - Banquet of the Golden Witch #02.cbz (Rare case, if causes issue remove)
new Regex(
@"^(?<Series>.*)(?: |_)#(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// Green Worldz - Chapter 027, Kimi no Koto ga Daidaidaidaidaisuki na 100-nin no Kanojo Chapter 11-10
new Regex(
@"^(?!Vol)(?<Series>.*)\s?(?<!vol\. )\sChapter\s(?<Chapter>\d+(?:\.?[\d-]+)?)",
MatchOptions, RegexTimeout),
// Russian Chapter: Главы n -> Chapter n
new Regex(
@"(Глава|глава|Главы|Глава)(\.?)(\s|_)?(?<Chapter>\d+(?:.\d+|-\d+)?)",
MatchOptions, RegexTimeout),
// Hinowa ga CRUSH! 018 (2019) (Digital) (LuCaZ).cbz, Hinowa ga CRUSH! 018.5 (2019) (Digital) (LuCaZ).cbz
new Regex(
@"^(?<Series>.+?)(?<!Vol)(?<!Vol.)(?<!Volume)\s(\d\s)?(?<Chapter>\d+(?:\.\d+|-\d+)?)(?:\s\(\d{4}\))?(\b|_|-)",
MatchOptions, RegexTimeout),
// Tower Of God S01 014 (CBT) (digital).cbz
new Regex(
@"(?<Series>.*)\sS(?<Volume>\d+)\s(?<Chapter>\d+(?:.\d+|-\d+)?)",
MatchOptions, RegexTimeout),
// Beelzebub_01_[Noodles].zip, Beelzebub_153b_RHS.zip
new Regex(
@"^((?!v|vo|vol|Volume).)*(\s|_)(?<Chapter>\.?\d+(?:.\d+|-\d+)?)(?<Part>b)?(\s|_|\[|\()",
MatchOptions, RegexTimeout),
// Yumekui-Merry_DKThias_Chapter21.zip
new Regex(
@"Chapter(?<Chapter>\d+(-\d+)?)", //(?:.\d+|-\d+)?
MatchOptions, RegexTimeout),
// [Hidoi]_Amaenaideyo_MS_vol01_chp02.rar
new Regex(
@"(?<Series>.*)(\s|_)(vol\d+)?(\s|_)Chp\.? ?(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// Vol 1 Chapter 2
new Regex(
@"(?<Volume>((vol|volume|v))?(\s|_)?\.?\d+)(\s|_)(Chp|Chapter)\.?(\s|_)?(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// Chinese Chapter: 第n话 -> Chapter n, 【TFO汉化&Petit汉化】迷你偶像漫画第25话
new Regex(
@"第(?<Chapter>\d+)话",
MatchOptions, RegexTimeout),
// Korean Chapter: 제n화 -> Chapter n, 가디언즈 오브 갤럭시 죽음의 보석.E0008.7화#44
new Regex(
@"제?(?<Chapter>\d+\.?\d+)(회|화|장)",
MatchOptions, RegexTimeout),
// Korean Chapter: 第10話 -> Chapter n, [ハレム]ナナとカオル ～高校生のSMごっこ～　第1話
new Regex(
@"第?(?<Chapter>\d+(?:\.\d+|-\d+)?)話",
MatchOptions, RegexTimeout),
// Russian Chapter: n Главa -> Chapter n
new Regex(
@"(?!Том)(?<!Том\.)\s\d+(\s|_)?(?<Chapter>\d+(?:\.\d+|-\d+)?)(\s|_)(Глава|глава|Главы|Глава)",
MatchOptions, RegexTimeout),
};
private static readonly Regex MangaEditionRegex = new Regex(
// Tenjo Tenge {Full Contact Edition} v01 (2011) (Digital) (ASTC).cbz
// To Love Ru v01 Uncensored (Ch.001-007)
@"\b(?:Omnibus(?:\s?Edition)?|Uncensored)\b",
MatchOptions, RegexTimeout
);
private static readonly Regex MangaSpecialRegex = new Regex(
// All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle.
$@"\b(?:{CommonSpecial}|Omake)\b",
MatchOptions, RegexTimeout
);
#endregion
#region Comic
private static readonly Regex[] ComicSeriesRegex = new[]
{
// Russian Volume: Том n -> Volume n, Тома n -> Volume
new Regex(
@"(?<Series>.+?)Том(а?)(\.?)(\s|_)?(?<Volume>\d+(?:(\-)\d+)?)",
@ -518,11 +466,15 @@ public static partial class Parser
// MUST BE LAST: Batman & Daredevil - King of New York
new Regex(
@"^(?<Series>.*)",
MatchOptions, RegexTimeout),
};
MatchOptions, RegexTimeout)
];
private static readonly Regex[] ComicVolumeRegex = new[]
{
private static readonly Regex[] ComicVolumeRegex =
[
// Thai Volume: เล่ม n -> Volume n
new Regex(
@"(เล่ม|เล่มที่)(\s)?(\.?)(\s|_)?(?<Volume>\d+(\-\d+)?(\.\d+)?)",
MatchOptions, RegexTimeout),
// Teen Titans v1 001 (1966-02) (digital) (OkC.O.M.P.U.T.O.-Novus)
new Regex(
@"^(?<Series>.+?)(?: |_)(t|v)(?<Volume>" + NumberRange + @")",
@ -554,11 +506,15 @@ public static partial class Parser
// Russian Volume: n Том -> Volume n
new Regex(
@"(\s|_)?(?<Volume>\d+(?:(\-)\d+)?)(\s|_)Том(а?)",
MatchOptions, RegexTimeout),
};
MatchOptions, RegexTimeout)
];
private static readonly Regex[] ComicChapterRegex = new[]
{
private static readonly Regex[] ComicChapterRegex =
[
// Thai Volume: บทที่ n -> Chapter n, ตอนที่ n -> Chapter n
new Regex(
@"(บทที่|ตอนที่)(\s)?(\.?)(\s|_)?(?<Chapter>\d+(\-\d+)?(\.\d+)?)",
MatchOptions, RegexTimeout),
// Batman & Wildcat (1 of 3)
new Regex(
@"(?<Series>.*(\d{4})?)( |_)(?:\((?<Chapter>\d+) of \d+)",
@ -619,22 +575,101 @@ public static partial class Parser
// spawn-123, spawn-chapter-123 (from https://github.com/Girbons/comics-downloader)
new Regex(
@"^(?<Series>.+?)-(chapter-)?(?<Chapter>\d+)",
MatchOptions, RegexTimeout)
];
private static readonly Regex[] MangaChapterRegex =
[
// Thai Chapter: บทที่ n -> Chapter n, ตอนที่ n -> Chapter n, เล่ม n -> Volume n, เล่มที่ n -> Volume n
new Regex(
@"(?<Volume>((เล่ม|เล่มที่))?(\s|_)?\.?\d+)(\s|_)(บทที่|ตอนที่)\.?(\s|_)?(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// Historys Strongest Disciple Kenichi_v11_c90-98.zip, ...c90.5-100.5
new Regex(
@"(\b|_)(c|ch)(\.?\s?)(?<Chapter>(\d+(\.\d)?)(-c?\d+(\.\d)?)?)",
MatchOptions, RegexTimeout),
// [Suihei Kiki]_Kasumi_Otoko_no_Ko_[Taruby]_v1.1.zip
new Regex(
@"v\d+\.(\s|_)(?<Chapter>\d+(?:.\d+|-\d+)?)",
MatchOptions, RegexTimeout),
// Umineko no Naku Koro ni - Episode 3 - Banquet of the Golden Witch #02.cbz (Rare case, if causes issue remove)
new Regex(
@"^(?<Series>.*)(?: |_)#(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// Green Worldz - Chapter 027, Kimi no Koto ga Daidaidaidaidaisuki na 100-nin no Kanojo Chapter 11-10
new Regex(
@"^(?!Vol)(?<Series>.*)\s?(?<!vol\. )\sChapter\s(?<Chapter>\d+(?:\.?[\d-]+)?)",
MatchOptions, RegexTimeout),
// Russian Chapter: Главы n -> Chapter n
new Regex(
@"(Глава|глава|Главы|Глава)(\.?)(\s|_)?(?<Chapter>\d+(?:.\d+|-\d+)?)",
MatchOptions, RegexTimeout),
};
private static readonly Regex ComicSpecialRegex = new Regex(
// All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle.
$@"\b(?:{CommonSpecial}|\d.+?(\W|-|^)Annual|Annual(\W|-|$)|Book \d.+?|Compendium(\W|-|$|\s.+?)|Omnibus(\W|-|$|\s.+?)|FCBD \d.+?|Absolute(\W|-|$|\s.+?)|Preview(\W|-|$|\s.+?)|Hors[ -]S[ée]rie|TPB|HS|THS)\b",
// Hinowa ga CRUSH! 018 (2019) (Digital) (LuCaZ).cbz, Hinowa ga CRUSH! 018.5 (2019) (Digital) (LuCaZ).cbz
new Regex(
@"^(?<Series>.+?)(?<!Vol)(?<!Vol.)(?<!Volume)\s(\d\s)?(?<Chapter>\d+(?:\.\d+|-\d+)?)(?:\s\(\d{4}\))?(\b|_|-)",
MatchOptions, RegexTimeout),
// Tower Of God S01 014 (CBT) (digital).cbz
new Regex(
@"(?<Series>.*)\sS(?<Volume>\d+)\s(?<Chapter>\d+(?:.\d+|-\d+)?)",
MatchOptions, RegexTimeout),
// Beelzebub_01_[Noodles].zip, Beelzebub_153b_RHS.zip
new Regex(
@"^((?!v|vo|vol|Volume).)*(\s|_)(?<Chapter>\.?\d+(?:.\d+|-\d+)?)(?<Part>b)?(\s|_|\[|\()",
MatchOptions, RegexTimeout),
// Yumekui-Merry_DKThias_Chapter21.zip
new Regex(
@"Chapter(?<Chapter>\d+(-\d+)?)", //(?:.\d+|-\d+)?
MatchOptions, RegexTimeout),
// [Hidoi]_Amaenaideyo_MS_vol01_chp02.rar
new Regex(
@"(?<Series>.*)(\s|_)(vol\d+)?(\s|_)Chp\.? ?(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// Vol 1 Chapter 2
new Regex(
@"(?<Volume>((vol|volume|v))?(\s|_)?\.?\d+)(\s|_)(Chp|Chapter)\.?(\s|_)?(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// Chinese Chapter: 第n话 -> Chapter n, 【TFO汉化&Petit汉化】迷你偶像漫画第25话
new Regex(
@"第(?<Chapter>\d+)话",
MatchOptions, RegexTimeout),
// Korean Chapter: 제n화 -> Chapter n, 가디언즈 오브 갤럭시 죽음의 보석.E0008.7화#44
new Regex(
@"제?(?<Chapter>\d+\.?\d+)(회|화|장)",
MatchOptions, RegexTimeout),
// Korean Chapter: 第10話 -> Chapter n, [ハレム]ナナとカオル ～高校生のSMごっこ～　第1話
new Regex(
@"第?(?<Chapter>\d+(?:\.\d+|-\d+)?)話",
MatchOptions, RegexTimeout),
// Russian Chapter: n Главa -> Chapter n
new Regex(
@"(?!Том)(?<!Том\.)\s\d+(\s|_)?(?<Chapter>\d+(?:\.\d+|-\d+)?)(\s|_)(Глава|глава|Главы|Глава)",
MatchOptions, RegexTimeout)
];
private static readonly Regex MangaEditionRegex = new Regex(
// Tenjo Tenge {Full Contact Edition} v01 (2011) (Digital) (ASTC).cbz
// To Love Ru v01 Uncensored (Ch.001-007)
@"\b(?:Omnibus(?:\s?Edition)?|Uncensored)\b",
MatchOptions, RegexTimeout
);
private static readonly Regex EuropeanComicRegex = new Regex(
// All Keywords, does not account for checking if contains volume/chapter identification. Parser.Parse() will handle.
@"\b(?:Bd[-\s]Fr)\b",
// Matches anything between balanced parenthesis, tags between brackets, {} and {Complete}
private static readonly Regex CleanupRegex = new Regex(
$@"(?:\({BalancedParen}\)|{TagsInBrackets}|\{{\}}|\{{Complete\}})",
MatchOptions, RegexTimeout
);
#endregion
// If SP\d+ is in the filename, we force treat it as a special regardless if volume or chapter might have been found.
private static readonly Regex SpecialMarkerRegex = new Regex(
@"SP\d+",
MatchOptions, RegexTimeout
);
private static readonly Regex EmptySpaceRegex = new Regex(
@"\s{2,}",
MatchOptions, RegexTimeout
);
#region Magazine
@ -692,7 +727,7 @@ public static partial class Parser
MatchOptions, RegexTimeout),
};
private static readonly Regex YearRegex = new Regex(
private static readonly Regex YearRegex = new(
@"(\b|\s|_)[1-9]{1}\d{3}(\b|\s|_)",
MatchOptions, RegexTimeout
);
@ -700,24 +735,6 @@ public static partial class Parser
#endregion
// Matches anything between balanced parenthesis, tags between brackets, {} and {Complete}
private static readonly Regex CleanupRegex = new Regex(
$@"(?:\({BalancedParen}\)|{TagsInBrackets}|\{{\}}|\{{Complete\}})",
MatchOptions, RegexTimeout
);
// If SP\d+ is in the filename, we force treat it as a special regardless if volume or chapter might have been found.
private static readonly Regex SpecialMarkerRegex = new Regex(
@"SP\d+",
MatchOptions, RegexTimeout
);
private static readonly Regex EmptySpaceRegex = new Regex(
@"\s{2,}",
MatchOptions, RegexTimeout
);
public static MangaFormat ParseFormat(string filePath)
{
@ -740,24 +757,25 @@ public static partial class Parser
/// </summary>
/// <param name="filePath"></param>
/// <returns></returns>
public static bool HasSpecialMarker(string filePath)
public static bool HasSpecialMarker(string? filePath)
{
if (string.IsNullOrEmpty(filePath)) return false;
return SpecialMarkerRegex.IsMatch(filePath);
}
public static bool IsMangaSpecial(string filePath)
public static int ParseSpecialIndex(string filePath)
{
filePath = ReplaceUnderscores(filePath);
return MangaSpecialRegex.IsMatch(filePath);
var match = SpecialMarkerRegex.Match(filePath).Value.Replace("SP", string.Empty);
if (string.IsNullOrEmpty(match)) return 0;
return int.Parse(match);
}
public static bool IsComicSpecial(string filePath)
public static bool IsSpecial(string? filePath, LibraryType type)
{
filePath = ReplaceUnderscores(filePath);
return ComicSpecialRegex.IsMatch(filePath);
return HasSpecialMarker(filePath);
}
public static string ParseSeries(string filename)
private static string ParseMangaSeries(string filename)
{
foreach (var regex in MangaSeriesRegex)
{
@ -765,7 +783,11 @@ public static partial class Parser
var group = matches
.Select(match => match.Groups["Series"])
.FirstOrDefault(group => group.Success && group != Match.Empty);
if (group != null) return CleanTitle(group.Value);
if (group != null)
{
return CleanTitle(group.Value);
}
}
return string.Empty;
@ -798,7 +820,7 @@ public static partial class Parser
return string.Empty;
}
public static string ParseVolume(string filename)
public static string ParseMangaVolume(string filename)
{
foreach (var regex in MangaVolumeRegex)
{
@ -813,7 +835,7 @@ public static partial class Parser
}
}
return DefaultVolume;
return LooseLeafVolume;
}
public static string ParseComicVolume(string filename)
@ -831,9 +853,10 @@ public static partial class Parser
}
}
return DefaultVolume;
return LooseLeafVolume;
}
public static string ParseMagazineVolume(string filename)
{
foreach (var regex in MagazineVolumeRegex)
@ -848,7 +871,7 @@ public static partial class Parser
}
}
return DefaultVolume;
return LooseLeafVolume;
}
private static string[] CreateCountryCodes()
@ -934,11 +957,6 @@ public static partial class Parser
return null;
}
public static string? ParseYear(string? value)
{
if (string.IsNullOrEmpty(value)) return value;
return YearRegex.Match(value).Value;
}
private static string FormatValue(string value, bool hasPart)
{
@ -949,6 +967,7 @@ public static partial class Parser
var tokens = value.Split("-");
var from = RemoveLeadingZeroes(tokens[0]);
if (tokens.Length != 2) return from;
// Occasionally users will use c01-c02 instead of c01-02, clean any leftover c
@ -960,7 +979,49 @@ public static partial class Parser
return $"{from}-{to}";
}
public static string ParseChapter(string filename)
public static string ParseSeries(string filename, LibraryType type)
{
return type switch
{
LibraryType.Manga => ParseMangaSeries(filename),
LibraryType.Comic => ParseComicSeries(filename),
LibraryType.Book => ParseMangaSeries(filename),
LibraryType.Image => ParseMangaSeries(filename),
LibraryType.LightNovel => ParseMangaSeries(filename),
LibraryType.ComicVine => ParseComicSeries(filename),
_ => string.Empty
};
}
public static string ParseVolume(string filename, LibraryType type)
{
return type switch
{
LibraryType.Manga => ParseMangaVolume(filename),
LibraryType.Comic => ParseComicVolume(filename),
LibraryType.Book => ParseMangaVolume(filename),
LibraryType.Image => ParseMangaVolume(filename),
LibraryType.LightNovel => ParseMangaVolume(filename),
LibraryType.ComicVine => ParseComicVolume(filename),
_ => LooseLeafVolume
};
}
public static string ParseChapter(string filename, LibraryType type)
{
return type switch
{
LibraryType.Manga => ParseMangaChapter(filename),
LibraryType.Comic => ParseComicChapter(filename),
LibraryType.Book => ParseMangaChapter(filename),
LibraryType.Image => ParseMangaChapter(filename),
LibraryType.LightNovel => ParseMangaChapter(filename),
LibraryType.ComicVine => ParseComicChapter(filename),
_ => DefaultChapter
};
}
private static string ParseMangaChapter(string filename)
{
foreach (var regex in MangaChapterRegex)
{
@ -989,7 +1050,7 @@ public static partial class Parser
return $"{value}.5";
}
public static string ParseComicChapter(string filename)
private static string ParseComicChapter(string filename)
{
foreach (var regex in ComicChapterRegex)
{
@ -1016,22 +1077,6 @@ public static partial class Parser
return title;
}
private static string RemoveMangaSpecialTags(string title)
{
return MangaSpecialRegex.Replace(title, string.Empty);
}
private static string RemoveEuropeanTags(string title)
{
return EuropeanComicRegex.Replace(title, string.Empty);
}
private static string RemoveComicSpecialTags(string title)
{
return ComicSpecialRegex.Replace(title, string.Empty);
}
/// <summary>
/// Translates _ -> spaces, trims front and back of string, removes release groups
@ -1043,27 +1088,13 @@ public static partial class Parser
/// <param name="isComic"></param>
/// <returns></returns>
public static string CleanTitle(string title, bool isComic = false, bool replaceSpecials = true)
public static string CleanTitle(string title, bool isComic = false)
{
title = ReplaceUnderscores(title);
title = RemoveEditionTagHolders(title);
if (replaceSpecials)
{
if (isComic)
{
title = RemoveComicSpecialTags(title);
title = RemoveEuropeanTags(title);
}
else
{
title = RemoveMangaSpecialTags(title);
}
}
title = title.Trim(SpacesAndSeparators);
title = EmptySpaceRegex.Replace(title, " ");
@ -1131,35 +1162,52 @@ public static partial class Parser
{
try
{
if (!Regex.IsMatch(range, @"^[\d\-.]+$", MatchOptions, RegexTimeout))
// Check if the range string is not null or empty
if (string.IsNullOrEmpty(range) || !Regex.IsMatch(range, @"^[\d\-.]+$", MatchOptions, RegexTimeout))
{
return (float) 0.0;
return 0.0f;
}
var tokens = range.Replace("_", string.Empty).Split("-");
return tokens.Min(t => t.AsFloat());
// Check if there is a range or not
if (NumberRangeRegex().IsMatch(range))
{
var tokens = range.Replace("_", string.Empty).Split("-", StringSplitOptions.RemoveEmptyEntries);
return tokens.Min(t => t.AsFloat());
}
return range.AsFloat();
}
catch
catch (Exception)
{
return (float) 0.0;
return 0.0f;
}
}
public static float MaxNumberFromRange(string range)
{
try
{
if (!Regex.IsMatch(range, @"^[\d\-.]+$", MatchOptions, RegexTimeout))
// Check if the range string is not null or empty
if (string.IsNullOrEmpty(range) || !Regex.IsMatch(range, @"^[\d\-.]+$", MatchOptions, RegexTimeout))
{
return (float) 0.0;
return 0.0f;
}
var tokens = range.Replace("_", string.Empty).Split("-");
return tokens.Max(t => t.AsFloat());
// Check if there is a range or not
if (NumberRangeRegex().IsMatch(range))
{
var tokens = range.Replace("_", string.Empty).Split("-", StringSplitOptions.RemoveEmptyEntries);
return tokens.Max(t => t.AsFloat());
}
return range.AsFloat();
}
catch
catch (Exception)
{
return (float) 0.0;
return 0.0f;
}
}
@ -1177,11 +1225,6 @@ public static partial class Parser
{
if (string.IsNullOrEmpty(name)) return name;
var cleaned = SpecialTokenRegex.Replace(name.Replace('_', ' '), string.Empty).Trim();
var lastIndex = cleaned.LastIndexOf('.');
if (lastIndex > 0)
{
cleaned = cleaned.Substring(0, cleaned.LastIndexOf('.')).Trim();
}
return string.IsNullOrEmpty(cleaned) ? name : cleaned;
}
@ -1199,7 +1242,7 @@ public static partial class Parser
}
/// <summary>
/// Validates that a Path doesn't start with certain blacklisted folders, like __MACOSX, @Recently-Snapshot, etc and that if a full path, the filename
/// Validates that a Path doesn't start with certain blacklisted folders, like __MACOSX, @Recently-Snapshot, etc. and that if a full path, the filename
/// doesn't start with ._, which is a metadata file on MACOSX.
/// </summary>
/// <param name="path"></param>
@ -1209,6 +1252,7 @@ public static partial class Parser
return path.Contains("__MACOSX") || path.StartsWith("@Recently-Snapshot") || path.StartsWith("@recycle")
|| path.StartsWith("._") || Path.GetFileName(path).StartsWith("._") || path.Contains(".qpkg")
|| path.StartsWith("#recycle")
|| path.Contains(".yacreaderlibrary")
|| path.Contains(".caltrash");
}
@ -1281,10 +1325,52 @@ public static partial class Parser
// NOTE: This is failing for //localhost:5000/api/book/29919/book-resources?file=OPS/images/tick1.jpg
var importFile = match.Groups["Filename"].Value;
if (!importFile.Contains("?")) return importFile;
if (!importFile.Contains('?')) return importFile;
}
return null;
}
/// <summary>
/// If the name matches exactly Series (Volume digits)
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public static bool IsSeriesAndYear(string? name)
{
return !string.IsNullOrEmpty(name) && SeriesAndYearRegex.IsMatch(name);
}
/// <summary>
/// Extracts year from Series (Year)
/// </summary>
/// <param name="name"></param>
/// <returns></returns>
public static string ParseYearFromSeries(string? name)
{
if (string.IsNullOrEmpty(name)) return string.Empty;
var match = SeriesAndYearRegex.Match(name);
return !match.Success ? string.Empty : match.Groups["Year"].Value;
}
public static string ParseYear(string? value)
{
return string.IsNullOrEmpty(value) ? string.Empty : YearRegex.Match(value).Value;
}
public static string? RemoveExtensionIfSupported(string? filename)
{
if (string.IsNullOrEmpty(filename)) return filename;
if (SupportedExtensionsRegex().IsMatch(filename))
{
return SupportedExtensionsRegex().Replace(filename, string.Empty);
}
return filename;
}
[GeneratedRegex(SupportedExtensions)]
private static partial Regex SupportedExtensionsRegex();
[GeneratedRegex(@"\d-{1}\d")]
private static partial Regex NumberRangeRegex();
}

14
API/Services/Tasks/Scanner/Parser/ParserInfo.cs
View file

@ -60,6 +60,10 @@ public class ParserInfo
/// If the file contains no volume/chapter information or contains Special Keywords <see cref="Parser.MangaSpecialRegex"/>
/// </summary>
public bool IsSpecial { get; set; }
/// <summary>
/// If the file has a Special Marker explicitly, this will contain the index
/// </summary>
public int SpecialIndex { get; set; } = 0;
/// <summary>
/// Used for specials or books, stores what the UI should show.
@ -67,13 +71,19 @@ public class ParserInfo
/// </summary>
public string Title { get; set; } = string.Empty;
/// <summary>
/// This can be filled in from ComicInfo.xml during scanning. Will update the SortOrder field on <see cref="Entities.Chapter"/>.
/// Falls back to Parsed Chapter number
/// </summary>
public float IssueOrder { get; set; }
/// <summary>
/// If the ParserInfo has the IsSpecial tag or both volumes and chapters are default aka 0
/// </summary>
/// <returns></returns>
public bool IsSpecialInfo()
{
return (IsSpecial || (Volumes == Parser.DefaultVolume && Chapters == Parser.DefaultChapter));
return (IsSpecial || (Volumes == Parser.LooseLeafVolume && Chapters == Parser.DefaultChapter));
}
/// <summary>
@ -91,7 +101,7 @@ public class ParserInfo
{
if (info2 == null) return;
Chapters = string.IsNullOrEmpty(Chapters) || Chapters == Parser.DefaultChapter ? info2.Chapters: Chapters;
Volumes = string.IsNullOrEmpty(Volumes) || Volumes == Parser.DefaultVolume ? info2.Volumes : Volumes;
Volumes = string.IsNullOrEmpty(Volumes) || Volumes == Parser.LooseLeafVolume ? info2.Volumes : Volumes;
Edition = string.IsNullOrEmpty(Edition) ? info2.Edition : Edition;
Title = string.IsNullOrEmpty(Title) ? info2.Title : Title;
Series = string.IsNullOrEmpty(Series) ? info2.Series : Series;

130
API/Services/Tasks/Scanner/Parser/PdfParser.cs Normal file
View file

@ -0,0 +1,130 @@
using System.IO;
using API.Data.Metadata;
using API.Entities.Enums;
namespace API.Services.Tasks.Scanner.Parser;
public class PdfParser(IDirectoryService directoryService) : DefaultParser(directoryService)
{
public override ParserInfo Parse(string filePath, string rootPath, string libraryRoot, LibraryType type, ComicInfo comicInfo = null)
{
var fileName = directoryService.FileSystem.Path.GetFileNameWithoutExtension(filePath);
var ret = new ParserInfo
{
Filename = Path.GetFileName(filePath),
Format = Parser.ParseFormat(filePath),
Title = Parser.RemoveExtensionIfSupported(fileName)!,
FullFilePath = Parser.NormalizePath(filePath),
Series = string.Empty,
ComicInfo = comicInfo,
Chapters = Parser.ParseChapter(fileName, type)
};
if (type == LibraryType.Book)
{
ret.Chapters = Parser.DefaultChapter;
}
ret.Series = Parser.ParseSeries(fileName, type);
ret.Volumes = Parser.ParseVolume(fileName, type);
if (ret.Series == string.Empty)
{
// Try to parse information out of each folder all the way to rootPath
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
var edition = Parser.ParseEdition(fileName);
if (!string.IsNullOrEmpty(edition))
{
ret.Series = Parser.CleanTitle(ret.Series.Replace(edition, string.Empty), type is LibraryType.Comic);
ret.Edition = edition;
}
var isSpecial = Parser.IsSpecial(fileName, type);
// We must ensure that we can only parse a special out. As some files will have v20 c171-180+Omake and that
// could cause a problem as Omake is a special term, but there is valid volume/chapter information.
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.LooseLeafVolume && isSpecial)
{
ret.IsSpecial = true;
// NOTE: This can cause some complications, we should try to be a bit less aggressive to fallback to folder
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
// If we are a special with marker, we need to ensure we use the correct series name. we can do this by falling back to Folder name
if (Parser.HasSpecialMarker(fileName))
{
ret.IsSpecial = true;
ret.SpecialIndex = Parser.ParseSpecialIndex(fileName);
ret.Chapters = Parser.DefaultChapter;
ret.Volumes = Parser.SpecialVolume;
var tempRootPath = rootPath;
if (rootPath.EndsWith("Specials") || rootPath.EndsWith("Specials/"))
{
tempRootPath = rootPath.Replace("Specials", string.Empty).TrimEnd('/');
}
ParseFromFallbackFolders(filePath, tempRootPath, type, ref ret);
}
// Patch in other information from ComicInfo
UpdateFromComicInfo(ret);
if (comicInfo != null && !string.IsNullOrEmpty(comicInfo.Title))
{
ret.Title = comicInfo.Title.Trim();
}
if (ret.Chapters == Parser.DefaultChapter && ret.Volumes == Parser.LooseLeafVolume && type == LibraryType.Book)
{
ret.IsSpecial = true;
ret.Chapters = Parser.DefaultChapter;
ret.Volumes = Parser.SpecialVolume;
ParseFromFallbackFolders(filePath, rootPath, type, ref ret);
}
if (type == LibraryType.Book && comicInfo != null)
{
// For books, fall back to the Title for Series.
if (!string.IsNullOrEmpty(comicInfo.Series))
{
ret.Series = comicInfo.Series.Trim();
}
else if (!string.IsNullOrEmpty(comicInfo.Title))
{
ret.Series = comicInfo.Title.Trim();
}
}
if (string.IsNullOrEmpty(ret.Series))
{
ret.Series = Parser.CleanTitle(fileName, type is LibraryType.Comic);
}
// Pdfs may have .pdf in the series name, remove that
if (Parser.IsPdf(filePath) && ret.Series.ToLower().EndsWith(".pdf"))
{
ret.Series = ret.Series.Substring(0, ret.Series.Length - ".pdf".Length);
}
// v0.8.x: Introducing a change where Specials will go in a separate Volume with a reserved number
if (ret.IsSpecial)
{
ret.Volumes = $"{Parser.SpecialVolumeNumber}";
}
return string.IsNullOrEmpty(ret.Series) ? null : ret;
}
/// <summary>
/// Only applicable for PDF files
/// </summary>
/// <param name="filePath"></param>
/// <param name="type"></param>
/// <returns></returns>
public override bool IsApplicable(string filePath, LibraryType type)
{
return Parser.IsPdf(filePath);
}
}

1170
API/Services/Tasks/Scanner/ProcessSeries.cs
View file

File diff suppressed because it is too large Load diff

465
API/Services/Tasks/ScannerService.cs
View file

@ -12,6 +12,7 @@ using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers;
using API.Helpers.Builders;
using API.Services.Tasks.Metadata;
using API.Services.Tasks.Scanner;
using API.Services.Tasks.Scanner.Parser;
@ -33,7 +34,7 @@ public interface IScannerService
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task ScanLibrary(int libraryId, bool forceUpdate = false);
Task ScanLibrary(int libraryId, bool forceUpdate = false, bool isSingleScan = true);
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
@ -45,7 +46,7 @@ public interface IScannerService
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true);
Task ScanFolder(string folder);
Task ScanFolder(string folder, string originalPath);
Task AnalyzeFiles();
}
@ -76,6 +77,7 @@ public enum ScanCancelReason
public class ScannerService : IScannerService
{
public const string Name = "ScannerService";
private const int Timeout = 60 * 60 * 60; // 2.5 days
private readonly IUnitOfWork _unitOfWork;
private readonly ILogger<ScannerService> _logger;
private readonly IMetadataService _metadataService;
@ -86,8 +88,6 @@ public class ScannerService : IScannerService
private readonly IProcessSeries _processSeries;
private readonly IWordCountAnalyzerService _wordCountAnalyzerService;
private readonly SemaphoreSlim _seriesProcessingSemaphore = new SemaphoreSlim(1, 1);
public ScannerService(IUnitOfWork unitOfWork, ILogger<ScannerService> logger,
IMetadataService metadataService, ICacheService cacheService, IEventHub eventHub,
IDirectoryService directoryService, IReadingItemService readingItemService,
@ -137,41 +137,47 @@ public class ScannerService : IScannerService
/// Given a generic folder path, will invoke a Series scan or Library scan.
/// </summary>
/// <remarks>This will Schedule the job to run 1 minute in the future to allow for any close-by duplicate requests to be dropped</remarks>
/// <param name="folder"></param>
public async Task ScanFolder(string folder)
/// <param name="folder">Normalized folder</param>
/// <param name="originalPath">If invoked from LibraryWatcher, this maybe a nested folder and can allow for optimization</param>
public async Task ScanFolder(string folder, string originalPath)
{
Series? series = null;
try
{
series = await _unitOfWork.SeriesRepository.GetSeriesByFolderPath(folder, SeriesIncludes.Library);
series = await _unitOfWork.SeriesRepository.GetSeriesThatContainsLowestFolderPath(originalPath,
SeriesIncludes.Library) ??
await _unitOfWork.SeriesRepository.GetSeriesByFolderPath(originalPath, SeriesIncludes.Library) ??
await _unitOfWork.SeriesRepository.GetSeriesByFolderPath(folder, SeriesIncludes.Library);
}
catch (InvalidOperationException ex)
{
if (ex.Message.Equals("Sequence contains more than one element."))
{
_logger.LogCritical("[ScannerService] Multiple series map to this folder. Library scan will be used for ScanFolder");
_logger.LogCritical(ex, "[ScannerService] Multiple series map to this folder or folder is at library root. Library scan will be used for ScanFolder");
}
}
// TODO: Figure out why we have the library type restriction here
if (series != null && (series.Library.Type != LibraryType.Book || series.Library.Type != LibraryType.LightNovel))
if (series != null)
{
if (TaskScheduler.HasScanTaskRunningForSeries(series.Id))
{
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder);
_logger.LogTrace("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this series. Dropping request", folder);
return;
}
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder}, Series matched to folder and ScanSeries enqueued for 1 minute", folder);
BackgroundJob.Schedule(() => ScanSeries(series.Id, true), TimeSpan.FromMinutes(1));
return;
}
// This is basically rework of what's already done in Library Watcher but is needed if invoked via API
var parentDirectory = _directoryService.GetParentDirectoryName(folder);
if (string.IsNullOrEmpty(parentDirectory)) return;
var libraries = (await _unitOfWork.LibraryRepository.GetLibraryDtosAsync()).ToList();
var libraryFolders = libraries.SelectMany(l => l.Folders);
var libraryFolder = libraryFolders.Select(Scanner.Parser.Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory));
var libraryFolder = libraryFolders.Select(Parser.NormalizePath).FirstOrDefault(f => f.Contains(parentDirectory));
if (string.IsNullOrEmpty(libraryFolder)) return;
var library = libraries.Find(l => l.Folders.Select(Parser.NormalizePath).Contains(libraryFolder));
@ -180,10 +186,10 @@ public class ScannerService : IScannerService
{
if (TaskScheduler.HasScanTaskRunningForLibrary(library.Id))
{
_logger.LogInformation("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
_logger.LogTrace("[ScannerService] Scan folder invoked for {Folder} but a task is already queued for this library. Dropping request", folder);
return;
}
BackgroundJob.Schedule(() => ScanLibrary(library.Id, false), TimeSpan.FromMinutes(1));
BackgroundJob.Schedule(() => ScanLibrary(library.Id, false, true), TimeSpan.FromMinutes(1));
}
}
@ -193,28 +199,42 @@ public class ScannerService : IScannerService
/// <param name="seriesId"></param>
/// <param name="bypassFolderOptimizationChecks">Not Used. Scan series will always force</param>
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(Timeout)]
[AutomaticRetry(Attempts = 200, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanSeries(int seriesId, bool bypassFolderOptimizationChecks = true)
{
if (TaskScheduler.HasAlreadyEnqueuedTask(Name, "ScanSeries", [seriesId, bypassFolderOptimizationChecks], TaskScheduler.ScanQueue))
{
_logger.LogInformation("[ScannerService] Scan series invoked but a task is already running/enqueued. Dropping request");
return;
}
var sw = Stopwatch.StartNew();
var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId);
var series = await _unitOfWork.SeriesRepository.GetFullSeriesForSeriesIdAsync(seriesId);
if (series == null) return; // This can occur when UI deletes a series but doesn't update and user re-requests update
var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var existingChapterIdsToClean = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(series.LibraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
if (library == null) return;
var libraryPaths = library.Folders.Select(f => f.Path).ToList();
if (await ShouldScanSeries(seriesId, library, libraryPaths, series, true) != ScanCancelReason.NoCancel)
{
BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(series.LibraryId, seriesId, false));
BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(series.LibraryId, seriesId, false, false));
BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(library.Id, seriesId, bypassFolderOptimizationChecks));
return;
}
var folderPath = series.FolderPath;
// TODO: We need to refactor this to handle the path changes better
var folderPath = series.LowestFolderPath ?? series.FolderPath;
if (string.IsNullOrEmpty(folderPath) || !_directoryService.Exists(folderPath))
{
// We don't care if it's multiple due to new scan loop enforcing all in one root directory
var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(libraryPaths, files.Select(f => f.FilePath).ToList());
var files = await _unitOfWork.SeriesRepository.GetFilesForSeries(seriesId);
var seriesDirs = _directoryService.FindHighestDirectoriesFromFiles(libraryPaths,
files.Select(f => f.FilePath).ToList());
if (seriesDirs.Keys.Count == 0)
{
_logger.LogCritical("Scan Series has files spread outside a main series folder. Defaulting to library folder (this is expensive)");
@ -240,29 +260,24 @@ public class ScannerService : IScannerService
return;
}
// If the series path doesn't exist anymore, it was either moved or renamed. We need to essentially delete it
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name));
await _processSeries.Prime();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Started, series.Name, 1));
_logger.LogInformation("Beginning file scan on {SeriesName}", series.Name);
var scanElapsedTime = await ScanFiles(library, new []{ folderPath }, false, TrackFiles, true);
_logger.LogInformation("ScanFiles for {Series} took {Time}", series.Name, scanElapsedTime);
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
var (scanElapsedTime, parsedSeries) = await ScanFiles(library, [folderPath],
false, true);
_logger.LogInformation("ScanFiles for {Series} took {Time} milliseconds", series.Name, scanElapsedTime);
// Remove any parsedSeries keys that don't belong to our series. This can occur when users store 2 series in the same folder
RemoveParsedInfosNotForSeries(parsedSeries, series);
// If nothing was found, first validate any of the files still exist. If they don't then we have a deletion and can skip the rest of the logic flow
if (parsedSeries.Count == 0)
{
// If nothing was found, first validate any of the files still exist. If they don't then we have a deletion and can skip the rest of the logic flow
if (parsedSeries.Count == 0)
{
var seriesFiles = (await _unitOfWork.SeriesRepository.GetFilesForSeries(series.Id));
if (!string.IsNullOrEmpty(series.FolderPath) && !seriesFiles.Where(f => f.FilePath.Contains(series.FolderPath)).Any(m => File.Exists(m.FilePath)))
if (!string.IsNullOrEmpty(series.FolderPath) &&
!seriesFiles.Where(f => f.FilePath.Contains(series.FolderPath)).Any(m => File.Exists(m.FilePath)))
{
try
{
@ -287,44 +302,57 @@ public class ScannerService : IScannerService
await _unitOfWork.RollbackAsync();
return;
}
// At this point, parsedSeries will have at least one key and we can perform the update. If it still doesn't, just return and don't do anything
if (parsedSeries.Count == 0) return;
}
}
// At this point, parsedSeries will have at least one key then we can perform the update. If it still doesn't, just return and don't do anything
// Don't allow any processing on files that aren't part of this series
var toProcess = parsedSeries.Keys.Where(key =>
key.NormalizedName.Equals(series.NormalizedName) ||
key.NormalizedName.Equals(series.OriginalName?.ToNormalized()))
.ToList();
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name));
var seriesLeftToProcess = toProcess.Count;
foreach (var pSeries in toProcess)
{
// Process Series
var seriesProcessStopWatch = Stopwatch.StartNew();
await _processSeries.ProcessSeriesAsync(parsedSeries[pSeries], library, seriesLeftToProcess, bypassFolderOptimizationChecks);
_logger.LogTrace("[TIME] Kavita took {Time} ms to process {SeriesName}", seriesProcessStopWatch.ElapsedMilliseconds, parsedSeries[pSeries][0].Series);
seriesLeftToProcess--;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, series.Name, 0));
// Tell UI that this series is done
await _eventHub.SendMessageAsync(MessageFactory.ScanSeries,
MessageFactory.ScanSeriesEvent(library.Id, seriesId, series.Name));
await _metadataService.RemoveAbandonedMetadataKeys();
//BackgroundJob.Enqueue(() => _metadataService.GenerateCoversForSeries(series.LibraryId, seriesId, false));
//BackgroundJob.Enqueue(() => _wordCountAnalyzerService.ScanSeries(library.Id, seriesId, false));
BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(chapterIds));
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return;
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(existingChapterIdsToClean));
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.CacheDirectory));
}
private static Dictionary<ParsedSeries, IList<ParserInfo>> TrackFoundSeriesAndFiles(IList<ScannedSeriesResult> seenSeries)
{
// Why does this only grab things that have changed?
var parsedSeries = new Dictionary<ParsedSeries, IList<ParserInfo>>();
foreach (var series in seenSeries.Where(s => s.ParsedInfos.Count > 0)) // && s.HasChanged
{
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return;
var parsedFiles = series.ParsedInfos;
series.ParsedSeries.HasChanged = series.HasChanged;
var foundParsedSeries = new ParsedSeries()
if (series.HasChanged)
{
Name = parsedFiles[0].Series,
NormalizedName = parsedFiles[0].Series.ToNormalized(),
Format = parsedFiles[0].Format
};
// For Scan Series, we need to filter out anything that isn't our Series
if (!foundParsedSeries.NormalizedName.Equals(series.NormalizedName) && !foundParsedSeries.NormalizedName.Equals(series.OriginalName?.ToNormalized()))
{
return;
parsedSeries.Add(series.ParsedSeries, parsedFiles);
}
else
{
parsedSeries.Add(series.ParsedSeries, []);
}
await _processSeries.ProcessSeriesAsync(parsedFiles, library, bypassFolderOptimizationChecks);
parsedSeries.Add(foundParsedSeries, parsedFiles);
}
return parsedSeries;
}
private async Task<ScanCancelReason> ShouldScanSeries(int seriesId, Library library, IList<string> libraryPaths, Series series, bool bypassFolderChecks = false)
@ -416,7 +444,7 @@ public class ScannerService : IScannerService
// Check if any of the folder roots are not available (ie disconnected from network, etc) and fail if any of them are
if (folders.Any(f => !_directoryService.IsDriveMounted(f)))
{
_logger.LogCritical("Some of the root folders for library ({LibraryName} are not accessible. Please check that drives are connected and rescan. Scan will be aborted", libraryName);
_logger.LogCritical("[ScannerService] Some of the root folders for library ({LibraryName} are not accessible. Please check that drives are connected and rescan. Scan will be aborted", libraryName);
await _eventHub.SendMessageAsync(MessageFactory.Error,
MessageFactory.ErrorEvent("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted",
@ -430,14 +458,14 @@ public class ScannerService : IScannerService
if (folders.Any(f => _directoryService.IsDirectoryEmpty(f)))
{
// That way logging and UI informing is all in one place with full context
_logger.LogError("Some of the root folders for the library are empty. " +
_logger.LogError("[ScannerService] Some of the root folders for the library are empty. " +
"Either your mount has been disconnected or you are trying to delete all series in the library. " +
"Scan has be aborted. " +
"Scan has been aborted. " +
"Check that your mount is connected or change the library's root folder and rescan");
await _eventHub.SendMessageAsync(MessageFactory.Error, MessageFactory.ErrorEvent( $"Some of the root folders for the library, {libraryName}, are empty.",
"Either your mount has been disconnected or you are trying to delete all series in the library. " +
"Scan has be aborted. " +
"Scan has been aborted. " +
"Check that your mount is connected or change the library's root folder and rescan"));
return false;
@ -447,16 +475,25 @@ public class ScannerService : IScannerService
}
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
[DisableConcurrentExecution(Timeout)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibraries(bool forceUpdate = false)
{
_logger.LogInformation("Starting Scan of All Libraries");
_logger.LogInformation("[ScannerService] Starting Scan of All Libraries, Forced: {Forced}", forceUpdate);
foreach (var lib in await _unitOfWork.LibraryRepository.GetLibrariesAsync())
{
await ScanLibrary(lib.Id, forceUpdate);
// BUG: This will trigger the first N libraries to scan over and over if there is always an interruption later in the chain
if (TaskScheduler.HasScanTaskRunningForLibrary(lib.Id))
{
// We don't need to send SignalR event as this is a background job that user doesn't need insight into
_logger.LogInformation("[ScannerService] Scan library invoked via nightly scan job but a task is already running for {LibraryName}. Rescheduling for 4 hours", lib.Name);
await Task.Delay(TimeSpan.FromHours(4));
}
await ScanLibrary(lib.Id, forceUpdate, true);
}
_logger.LogInformation("Scan of All Libraries Finished");
_logger.LogInformation("[ScannerService] Scan of All Libraries Finished");
}
@ -467,13 +504,16 @@ public class ScannerService : IScannerService
/// </summary>
/// <param name="libraryId"></param>
/// <param name="forceUpdate">Defaults to false</param>
/// <param name="isSingleScan">Defaults to true. Is this a standalone invocation or is it in a loop?</param>
[Queue(TaskScheduler.ScanQueue)]
[DisableConcurrentExecution(60 * 60 * 60)]
[DisableConcurrentExecution(Timeout)]
[AutomaticRetry(Attempts = 3, OnAttemptsExceeded = AttemptsExceededAction.Delete)]
public async Task ScanLibrary(int libraryId, bool forceUpdate = false)
public async Task ScanLibrary(int libraryId, bool forceUpdate = false, bool isSingleScan = true)
{
var sw = Stopwatch.StartNew();
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId,
LibraryIncludes.Folders | LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns);
var libraryFolderPaths = library!.Folders.Select(fp => fp.Path).ToList();
if (!await CheckMounts(library.Name, libraryFolderPaths)) return;
@ -485,77 +525,39 @@ public class ScannerService : IScannerService
var shouldUseLibraryScan = !(await _unitOfWork.LibraryRepository.DoAnySeriesFoldersMatch(libraryFolderPaths));
if (!shouldUseLibraryScan)
{
_logger.LogError("Library {LibraryName} consists of one or more Series folders, using series scan", library.Name);
_logger.LogError("[ScannerService] Library {LibraryName} consists of one or more Series folders as a library root, using series scan", library.Name);
}
var totalFiles = 0;
var seenSeries = new List<ParsedSeries>();
await _processSeries.Prime();
//var processTasks = new List<Func<Task>>();
var scanElapsedTime = await ScanFiles(library, libraryFolderPaths, shouldUseLibraryScan, TrackFiles, forceUpdate);
// NOTE: This runs sync after every file is scanned
// foreach (var task in processTasks)
// {
// await task();
// }
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(string.Empty, library.Name, ProgressEventType.Ended));
_logger.LogInformation("[ScannerService] Finished file scan in {ScanAndUpdateTime} milliseconds. Updating database", scanElapsedTime);
var time = DateTime.Now;
foreach (var folderPath in library.Folders)
{
folderPath.UpdateLastScanned(time);
}
library.UpdateLastScanned(time);
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 1: Scan & Parse Files", library.Name);
var (scanElapsedTime, parsedSeries) = await ScanFiles(library, libraryFolderPaths,
shouldUseLibraryScan, forceUpdate);
// We need to remove any keys where there is no actual parser info
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 2: Process and Update Database", library.Name);
var totalFiles = await ProcessParsedSeries(forceUpdate, parsedSeries, library, scanElapsedTime);
UpdateLastScanned(library);
_unitOfWork.LibraryRepository.Update(library);
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 3: Save Library", library.Name);
if (await _unitOfWork.CommitAsync())
{
if (totalFiles == 0)
{
_logger.LogInformation(
"[ScannerService] Finished library scan of {ParsedSeriesCount} series in {ElapsedScanTime} milliseconds for {LibraryName}. There were no changes",
seenSeries.Count, sw.ElapsedMilliseconds, library.Name);
parsedSeries.Count, sw.ElapsedMilliseconds, library.Name);
}
else
{
_logger.LogInformation(
"[ScannerService] Finished library scan of {TotalFiles} files and {ParsedSeriesCount} series in {ElapsedScanTime} milliseconds for {LibraryName}",
totalFiles, seenSeries.Count, sw.ElapsedMilliseconds, library.Name);
totalFiles, parsedSeries.Count, sw.ElapsedMilliseconds, library.Name);
}
try
{
// Could I delete anything in a Library's Series where the LastScan date is before scanStart?
// NOTE: This implementation is expensive
_logger.LogDebug("[ScannerService] Removing Series that were not found during the scan");
var removedSeries = await _unitOfWork.SeriesRepository.RemoveSeriesNotInList(seenSeries, library.Id);
_logger.LogDebug("[ScannerService] Found {Count} series that needs to be removed: {SeriesList}",
removedSeries.Count, removedSeries.Select(s => s.Name));
_logger.LogDebug("[ScannerService] Removing Series that were not found during the scan - complete");
await _unitOfWork.CommitAsync();
foreach (var s in removedSeries)
{
await _eventHub.SendMessageAsync(MessageFactory.SeriesRemoved,
MessageFactory.SeriesRemovedEvent(s.Id, s.Name, s.LibraryId), false);
}
}
catch (Exception ex)
{
_logger.LogCritical(ex, "[ScannerService] There was an issue deleting series for cleanup. Please check logs and rescan");
}
_logger.LogDebug("[ScannerService] Library {LibraryName} Step 5: Remove Deleted Series", library.Name);
await RemoveSeriesNotFound(parsedSeries, library);
}
else
{
@ -563,70 +565,205 @@ public class ScannerService : IScannerService
"[ScannerService] There was a critical error that resulted in a failed scan. Please check logs and rescan");
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress, MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, string.Empty));
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.LibraryScanProgressEvent(library.Name, ProgressEventType.Ended, string.Empty));
await _metadataService.RemoveAbandonedMetadataKeys();
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.TempDirectory));
return;
BackgroundJob.Enqueue(() => _directoryService.ClearDirectory(_directoryService.CacheDirectory));
}
// Responsible for transforming parsedInfo into an actual ParsedSeries then calling the actual processing of the series
async Task TrackFiles(Tuple<bool, IList<ParserInfo>> parsedInfo)
private async Task RemoveSeriesNotFound(Dictionary<ParsedSeries, IList<ParserInfo>> parsedSeries, Library library)
{
try
{
var skippedScan = parsedInfo.Item1;
var parsedFiles = parsedInfo.Item2;
if (parsedFiles.Count == 0) return;
_logger.LogDebug("[ScannerService] Removing series that were not found during the scan");
var foundParsedSeries = new ParsedSeries()
{
Name = parsedFiles[0].Series,
NormalizedName = Parser.Normalize(parsedFiles[0].Series),
Format = parsedFiles[0].Format,
};
var removedSeries = await _unitOfWork.SeriesRepository.RemoveSeriesNotInList(parsedSeries.Keys.ToList(), library.Id);
_logger.LogDebug("[ScannerService] Found {Count} series to remove: {SeriesList}",
removedSeries.Count, string.Join(", ", removedSeries.Select(s => s.Name)));
if (skippedScan)
// Commit the changes
await _unitOfWork.CommitAsync();
// Notify for each removed series
foreach (var series in removedSeries)
{
seenSeries.AddRange(parsedFiles.Select(pf => new ParsedSeries()
{
Name = pf.Series,
NormalizedName = Parser.Normalize(pf.Series),
Format = pf.Format
}));
return;
await _eventHub.SendMessageAsync(
MessageFactory.SeriesRemoved,
MessageFactory.SeriesRemovedEvent(series.Id, series.Name, series.LibraryId),
false
);
}
totalFiles += parsedFiles.Count;
seenSeries.Add(foundParsedSeries);
await _seriesProcessingSemaphore.WaitAsync();
try
{
await _processSeries.ProcessSeriesAsync(parsedFiles, library, forceUpdate);
}
finally
{
_seriesProcessingSemaphore.Release();
}
_logger.LogDebug("[ScannerService] Series removal process completed");
}
catch (Exception ex)
{
_logger.LogCritical(ex, "[ScannerService] Error during series cleanup. Please check logs and rescan");
}
}
private async Task<long> ScanFiles(Library library, IEnumerable<string> dirs,
bool isLibraryScan, Func<Tuple<bool, IList<ParserInfo>>, Task>? processSeriesInfos = null, bool forceChecks = false)
private async Task<int> ProcessParsedSeries(bool forceUpdate, Dictionary<ParsedSeries, IList<ParserInfo>> parsedSeries, Library library, long scanElapsedTime)
{
// Iterate over the dictionary and remove only the ParserInfos that don't need processing
var toProcess = new Dictionary<ParsedSeries, IList<ParserInfo>>();
var scanSw = Stopwatch.StartNew();
foreach (var series in parsedSeries)
{
if (!series.Key.HasChanged)
{
_logger.LogDebug("{Series} hasn't changed", series.Key.Name);
continue;
}
// Filter out ParserInfos where FullFilePath is empty (i.e., folder not modified)
var validInfos = series.Value.Where(info => !string.IsNullOrEmpty(info.Filename)).ToList();
if (validInfos.Count != 0)
{
toProcess[series.Key] = validInfos;
}
}
if (toProcess.Count > 0)
{
// For all Genres in the ParserInfos, do a bulk check against the DB on what is not in the DB and create them
// This will ensure all Genres are pre-created and allow our Genre lookup (and Priming) to be much simpler. It will be slower, but more consistent.
var allGenres = toProcess
.SelectMany(s => s.Value
.SelectMany(p => p.ComicInfo?.Genre?
.Split(",", StringSplitOptions.RemoveEmptyEntries) // Split on comma and remove empty entries
.Select(g => g.Trim()) // Trim each genre
.Where(g => !string.IsNullOrWhiteSpace(g)) // Ensure no null/empty genres
?? [])); // Handle null Genre or ComicInfo safely
await CreateAllGenresAsync(allGenres.Distinct().ToList());
var allTags = toProcess
.SelectMany(s => s.Value
.SelectMany(p => p.ComicInfo?.Tags?
.Split(",", StringSplitOptions.RemoveEmptyEntries) // Split on comma and remove empty entries
.Select(g => g.Trim()) // Trim each genre
.Where(g => !string.IsNullOrWhiteSpace(g)) // Ensure no null/empty genres
?? [])); // Handle null Tag or ComicInfo safely
await CreateAllTagsAsync(allTags.Distinct().ToList());
}
var totalFiles = 0;
var seriesLeftToProcess = toProcess.Count;
_logger.LogInformation("[ScannerService] Found {SeriesCount} Series that need processing in {Time} ms", toProcess.Count, scanSw.ElapsedMilliseconds + scanElapsedTime);
foreach (var pSeries in toProcess)
{
totalFiles += pSeries.Value.Count;
var seriesProcessStopWatch = Stopwatch.StartNew();
await _processSeries.ProcessSeriesAsync(pSeries.Value, library, seriesLeftToProcess, forceUpdate);
_logger.LogTrace("[TIME] Kavita took {Time} ms to process {SeriesName}", seriesProcessStopWatch.ElapsedMilliseconds, pSeries.Value[0].Series);
seriesLeftToProcess--;
}
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.FileScanProgressEvent(string.Empty, library.Name, ProgressEventType.Ended));
_logger.LogInformation("[ScannerService] Finished file scan in {ScanAndUpdateTime} milliseconds. Updating database", scanElapsedTime);
return totalFiles;
}
private static void UpdateLastScanned(Library library)
{
var time = DateTime.Now;
foreach (var folderPath in library.Folders)
{
folderPath.UpdateLastScanned(time);
}
library.UpdateLastScanned(time);
}
private async Task<Tuple<long, Dictionary<ParsedSeries, IList<ParserInfo>>>> ScanFiles(Library library, IList<string> dirs,
bool isLibraryScan, bool forceChecks = false)
{
var scanner = new ParseScannedFiles(_logger, _directoryService, _readingItemService, _eventHub);
var scanWatch = Stopwatch.StartNew();
await scanner.ScanLibrariesForSeries(library, dirs,
isLibraryScan, await _unitOfWork.SeriesRepository.GetFolderPathMap(library.Id), processSeriesInfos, forceChecks);
var processedSeries = await scanner.ScanLibrariesForSeries(library, dirs,
isLibraryScan, await _unitOfWork.SeriesRepository.GetFolderPathMap(library.Id), forceChecks);
var scanElapsedTime = scanWatch.ElapsedMilliseconds;
return scanElapsedTime;
var parsedSeries = TrackFoundSeriesAndFiles(processedSeries);
return Tuple.Create(scanElapsedTime, parsedSeries);
}
public static IEnumerable<Series> FindSeriesNotOnDisk(IEnumerable<Series> existingSeries, Dictionary<ParsedSeries, IList<ParserInfo>> parsedSeries)
/// <summary>
/// Given a list of all Genres, generates new Genre entries for any that do not exist.
/// Does not delete anything, that will be handled by nightly task
/// </summary>
/// <param name="genres"></param>
private async Task CreateAllGenresAsync(ICollection<string> genres)
{
return existingSeries.Where(es => !ParserInfoHelpers.SeriesHasMatchingParserInfoFormat(es, parsedSeries));
_logger.LogInformation("[ScannerService] Attempting to pre-save all Genres");
try
{
// Pass the non-normalized genres directly to the repository
var nonExistingGenres = await _unitOfWork.GenreRepository.GetAllGenresNotInListAsync(genres);
// Create and attach new genres using the non-normalized names
foreach (var genre in nonExistingGenres)
{
var newGenre = new GenreBuilder(genre).Build();
_unitOfWork.GenreRepository.Attach(newGenre);
}
// Commit changes
if (nonExistingGenres.Count > 0)
{
await _unitOfWork.CommitAsync();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[ScannerService] There was an unknown issue when pre-saving all Genres");
}
}
/// <summary>
/// Given a list of all Tags, generates new Tag entries for any that do not exist.
/// Does not delete anything, that will be handled by nightly task
/// </summary>
/// <param name="tags"></param>
private async Task CreateAllTagsAsync(ICollection<string> tags)
{
_logger.LogInformation("[ScannerService] Attempting to pre-save all Tags");
try
{
// Pass the non-normalized tags directly to the repository
var nonExistingTags = await _unitOfWork.TagRepository.GetAllTagsNotInListAsync(tags);
// Create and attach new genres using the non-normalized names
foreach (var tag in nonExistingTags)
{
var newTag = new TagBuilder(tag).Build();
_unitOfWork.TagRepository.Attach(newTag);
}
// Commit changes
if (nonExistingTags.Count > 0)
{
await _unitOfWork.CommitAsync();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "[ScannerService] There was an unknown issue when pre-saving all Tags");
}
}
}

499
API/Services/Tasks/SiteThemeService.cs
View file

@ -1,35 +1,108 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text.Json.Serialization;
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Theme;
using API.Entities;
using API.Entities.Enums.Theme;
using API.Extensions;
using API.Services.Tasks.Scanner.Parser;
using API.SignalR;
using Flurl.Http;
using HtmlAgilityPack;
using Kavita.Common;
using Microsoft.AspNetCore.Authorization;
using Kavita.Common.EnvironmentInfo;
using MarkdownDeep;
using Microsoft.Extensions.Caching.Memory;
using Microsoft.Extensions.Logging;
using Newtonsoft.Json;
namespace API.Services.Tasks;
#nullable enable
internal class GitHubContent
{
[JsonProperty("name")]
public string Name { get; set; }
[JsonProperty("path")]
public string Path { get; set; }
[JsonProperty("type")]
public string Type { get; set; }
[JsonPropertyName("download_url")]
[JsonProperty("download_url")]
public string DownloadUrl { get; set; }
[JsonProperty("sha")]
public string Sha { get; set; }
}
/// <summary>
/// The readme of the Theme repo
/// </summary>
internal class ThemeMetadata
{
public string Author { get; set; }
public string AuthorUrl { get; set; }
public string Description { get; set; }
public Version LastCompatible { get; set; }
}
public interface IThemeService
{
Task<string> GetContent(int themeId);
Task Scan();
Task UpdateDefault(int themeId);
/// <summary>
/// Browse theme repo for themes to download
/// </summary>
/// <returns></returns>
Task<List<DownloadableSiteThemeDto>> GetDownloadableThemes();
Task<SiteTheme> DownloadRepoTheme(DownloadableSiteThemeDto dto);
Task DeleteTheme(int siteThemeId);
Task<SiteTheme> CreateThemeFromFile(string tempFile, string username);
Task SyncThemes();
}
public class ThemeService : IThemeService
{
private readonly IDirectoryService _directoryService;
private readonly IUnitOfWork _unitOfWork;
private readonly IEventHub _eventHub;
private readonly IFileService _fileService;
private readonly ILogger<ThemeService> _logger;
private readonly Markdown _markdown = new();
private readonly IMemoryCache _cache;
private readonly MemoryCacheEntryOptions _cacheOptions;
public ThemeService(IDirectoryService directoryService, IUnitOfWork unitOfWork, IEventHub eventHub)
private const string GithubBaseUrl = "https://api.github.com";
/// <summary>
/// Used for refreshing metadata around themes
/// </summary>
private const string GithubReadme = "https://raw.githubusercontent.com/Kareadita/Themes/main/README.md";
public ThemeService(IDirectoryService directoryService, IUnitOfWork unitOfWork,
IEventHub eventHub, IFileService fileService, ILogger<ThemeService> logger, IMemoryCache cache)
{
_directoryService = directoryService;
_unitOfWork = unitOfWork;
_eventHub = eventHub;
_fileService = fileService;
_logger = logger;
_cache = cache;
_cacheOptions = new MemoryCacheEntryOptions()
.SetSize(1)
.SetAbsoluteExpiration(TimeSpan.FromMinutes(30));
}
/// <summary>
@ -39,8 +112,7 @@ public class ThemeService : IThemeService
/// <returns></returns>
public async Task<string> GetContent(int themeId)
{
var theme = await _unitOfWork.SiteThemeRepository.GetThemeDto(themeId);
if (theme == null) throw new KavitaException("theme-doesnt-exist");
var theme = await _unitOfWork.SiteThemeRepository.GetThemeDto(themeId) ?? throw new KavitaException("theme-doesnt-exist");
var themeFile = _directoryService.FileSystem.Path.Join(_directoryService.SiteThemeDirectory, theme.FileName);
if (string.IsNullOrEmpty(themeFile) || !_directoryService.FileSystem.File.Exists(themeFile))
throw new KavitaException("theme-doesnt-exist");
@ -48,78 +120,366 @@ public class ThemeService : IThemeService
return await _directoryService.FileSystem.File.ReadAllTextAsync(themeFile);
}
/// <summary>
/// Scans the site theme directory for custom css files and updates what the system has on store
/// </summary>
public async Task Scan()
public async Task<List<DownloadableSiteThemeDto>> GetDownloadableThemes()
{
_directoryService.ExistOrCreate(_directoryService.SiteThemeDirectory);
var reservedNames = Seed.DefaultThemes.Select(t => t.NormalizedName).ToList();
var themeFiles = _directoryService
.GetFilesWithExtension(Scanner.Parser.Parser.NormalizePath(_directoryService.SiteThemeDirectory), @"\.css")
.Where(name => !reservedNames.Contains(name.ToNormalized()) && !name.Contains(" "))
.ToList();
const string cacheKey = "browse";
// Avoid a duplicate Dark issue some users faced during migration
var existingThemes = (await _unitOfWork.SiteThemeRepository.GetThemeDtos())
.GroupBy(k => k.Name)
.ToDictionary(g => g.Key, g => g.First());
var allThemes = (await _unitOfWork.SiteThemeRepository.GetThemes()).ToList();
// First remove any files from allThemes that are User Defined and not on disk
var userThemes = allThemes.Where(t => t.Provider == ThemeProvider.User).ToList();
foreach (var userTheme in userThemes)
if (_cache.TryGetValue(cacheKey, out List<DownloadableSiteThemeDto>? themes) && themes != null)
{
var filepath = Scanner.Parser.Parser.NormalizePath(
_directoryService.FileSystem.Path.Join(_directoryService.SiteThemeDirectory, userTheme.FileName));
if (_directoryService.FileSystem.File.Exists(filepath)) continue;
// I need to do the removal different. I need to update all user preferences to use DefaultTheme
allThemes.Remove(userTheme);
await RemoveTheme(userTheme);
}
// Add new custom themes
var allThemeNames = allThemes.Select(t => t.NormalizedName).ToList();
foreach (var themeFile in themeFiles)
{
var themeName =
_directoryService.FileSystem.Path.GetFileNameWithoutExtension(themeFile).ToNormalized();
if (allThemeNames.Contains(themeName)) continue;
_unitOfWork.SiteThemeRepository.Add(new SiteTheme()
foreach (var t in themes)
{
Name = _directoryService.FileSystem.Path.GetFileNameWithoutExtension(themeFile),
NormalizedName = themeName,
FileName = _directoryService.FileSystem.Path.GetFileName(themeFile),
Provider = ThemeProvider.User,
IsDefault = false,
});
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.SiteThemeProgressEvent(_directoryService.FileSystem.Path.GetFileName(themeFile), themeName,
ProgressEventType.Updated));
t.AlreadyDownloaded = existingThemes.ContainsKey(t.Name);
}
return themes;
}
// Fetch contents of the Native Themes directory
var themesContents = await GetDirectoryContent("Native%20Themes");
if (_unitOfWork.HasChanges())
{
await _unitOfWork.CommitAsync();
}
// Filter out directories
var themeDirectories = themesContents.Where(c => c.Type == "dir").ToList();
// if there are no default themes, reselect Dark as default
var postSaveThemes = (await _unitOfWork.SiteThemeRepository.GetThemes()).ToList();
if (!postSaveThemes.Exists(t => t.IsDefault))
// Get the Readme and augment the theme data
var themeMetadata = await GetReadme();
var themeDtos = new List<DownloadableSiteThemeDto>();
foreach (var themeDir in themeDirectories)
{
var defaultThemeName = Seed.DefaultThemes.Single(t => t.IsDefault).NormalizedName;
var theme = postSaveThemes.SingleOrDefault(t => t.NormalizedName == defaultThemeName);
if (theme != null)
var themeName = themeDir.Name.Trim();
// Fetch contents of the theme directory
var themeContents = await GetDirectoryContent(themeDir.Path);
// Find css and preview files
var cssFile = themeContents.FirstOrDefault(c => c.Name.EndsWith(".css"));
var previewUrls = GetPreviewUrls(themeContents);
if (cssFile == null) continue;
var cssUrl = cssFile.DownloadUrl;
var dto = new DownloadableSiteThemeDto()
{
theme.IsDefault = true;
_unitOfWork.SiteThemeRepository.Update(theme);
await _unitOfWork.CommitAsync();
Name = themeName,
CssUrl = cssUrl,
CssFile = cssFile.Name,
PreviewUrls = previewUrls,
Sha = cssFile.Sha,
Path = themeDir.Path,
};
if (themeMetadata.TryGetValue(themeName, out var metadata))
{
dto.Author = metadata.Author;
dto.LastCompatibleVersion = metadata.LastCompatible.ToString();
dto.IsCompatible = BuildInfo.Version <= metadata.LastCompatible;
dto.AlreadyDownloaded = existingThemes.ContainsKey(themeName);
dto.Description = metadata.Description;
}
themeDtos.Add(dto);
}
_cache.Set(cacheKey, themeDtos, _cacheOptions);
return themeDtos;
}
private static List<string> GetPreviewUrls(IEnumerable<GitHubContent> themeContents)
{
return themeContents
.Where(c => Parser.IsImage(c.Name) )
.Select(p => p.DownloadUrl)
.ToList();
}
private static async Task<IList<GitHubContent>> GetDirectoryContent(string path)
{
var json = await $"{GithubBaseUrl}/repos/Kareadita/Themes/contents/{path}"
.WithHeader("Accept", "application/vnd.github+json")
.WithHeader("User-Agent", "Kavita")
.GetStringAsync();
return string.IsNullOrEmpty(json) ? [] : JsonConvert.DeserializeObject<List<GitHubContent>>(json);
}
/// <summary>
/// Returns a map of all Native Themes names mapped to their metadata
/// </summary>
/// <returns></returns>
private async Task<IDictionary<string, ThemeMetadata>> GetReadme()
{
// Try and delete a Readme file if it already exists
var existingReadmeFile = _directoryService.FileSystem.Path.Join(_directoryService.TempDirectory, "README.md");
if (_directoryService.FileSystem.File.Exists(existingReadmeFile))
{
_directoryService.DeleteFiles([existingReadmeFile]);
}
var tempDownloadFile = await GithubReadme.DownloadFileAsync(_directoryService.TempDirectory);
// Read file into Markdown
var htmlContent = _markdown.Transform(await _directoryService.FileSystem.File.ReadAllTextAsync(tempDownloadFile));
var htmlDoc = new HtmlDocument();
htmlDoc.LoadHtml(htmlContent);
// Find the table of Native Themes
var tableContent = htmlDoc.DocumentNode
.SelectSingleNode("//h2[contains(text(),'Native Themes')]/following-sibling::p").InnerText;
// Initialize dictionary to store theme metadata
var themes = new Dictionary<string, ThemeMetadata>();
// Split the table content by rows
var rows = tableContent.Split("\r\n").Select(row => row.Trim()).Where(row => !string.IsNullOrWhiteSpace(row)).ToList();
// Parse each row in the Native Themes table
foreach (var row in rows.Skip(2))
{
var cells = row.Split('|').Skip(1).Select(cell => cell.Trim()).ToList();
// Extract information from each cell
var themeName = cells[0];
var authorName = cells[1];
var description = cells[2];
var compatibility = Version.Parse(cells[3]);
// Create ThemeMetadata object
var themeMetadata = new ThemeMetadata
{
Author = authorName,
Description = description,
LastCompatible = compatibility
};
// Add theme metadata to dictionary
themes.Add(themeName, themeMetadata);
}
return themes;
}
private async Task<string> DownloadSiteTheme(DownloadableSiteThemeDto dto)
{
if (string.IsNullOrEmpty(dto.Sha))
{
throw new ArgumentException("SHA cannot be null or empty for already downloaded themes.");
}
_directoryService.ExistOrCreate(_directoryService.SiteThemeDirectory);
var existingTempFile = _directoryService.FileSystem.Path.Join(_directoryService.SiteThemeDirectory,
_directoryService.FileSystem.FileInfo.New(dto.CssUrl).Name);
_directoryService.DeleteFiles([existingTempFile]);
var tempDownloadFile = await dto.CssUrl.DownloadFileAsync(_directoryService.TempDirectory);
// Validate the hash on the downloaded file
// if (!_fileService.ValidateSha(tempDownloadFile, dto.Sha))
// {
// throw new KavitaException("Cannot download theme, hash does not match");
// }
_directoryService.CopyFileToDirectory(tempDownloadFile, _directoryService.SiteThemeDirectory);
var finalLocation = _directoryService.FileSystem.Path.Join(_directoryService.SiteThemeDirectory, dto.CssFile);
return finalLocation;
}
public async Task<SiteTheme> DownloadRepoTheme(DownloadableSiteThemeDto dto)
{
// Validate we don't have a collision with existing or existing doesn't already exist
var existingThemes = _directoryService.ScanFiles(_directoryService.SiteThemeDirectory, string.Empty);
if (existingThemes.Any(f => Path.GetFileName(f) == dto.CssFile))
{
// This can happen if you delete then immediately download (to refresh). We should just delete the old file and download. Users can always rollback their version with github directly
_directoryService.DeleteFiles(existingThemes.Where(f => Path.GetFileName(f) == dto.CssFile));
}
var finalLocation = await DownloadSiteTheme(dto);
// Create a new entry and note that this is downloaded
var theme = new SiteTheme()
{
Name = dto.Name,
NormalizedName = dto.Name.ToNormalized(),
FileName = _directoryService.FileSystem.Path.GetFileName(finalLocation),
Provider = ThemeProvider.Custom,
IsDefault = false,
GitHubPath = dto.Path,
Description = dto.Description,
PreviewUrls = string.Join('|', dto.PreviewUrls),
Author = dto.Author,
ShaHash = dto.Sha,
CompatibleVersion = dto.LastCompatibleVersion,
};
_unitOfWork.SiteThemeRepository.Add(theme);
await _unitOfWork.CommitAsync();
// Inform about the new theme
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.SiteThemeProgressEvent("", "", ProgressEventType.Ended));
MessageFactory.SiteThemeProgressEvent(_directoryService.FileSystem.Path.GetFileName(theme.FileName), theme.Name,
ProgressEventType.Ended));
return theme;
}
public async Task SyncThemes()
{
var themes = await _unitOfWork.SiteThemeRepository.GetThemes();
var themeMetadata = await GetReadme();
foreach (var theme in themes)
{
await SyncTheme(theme, themeMetadata);
}
_logger.LogInformation("Sync Themes complete");
}
/// <summary>
/// If the Theme is from the Theme repo, see if there is a new version that is compatible
/// </summary>
/// <param name="theme"></param>
/// <param name="themeMetadata">The Readme information</param>
private async Task SyncTheme(SiteTheme? theme, IDictionary<string, ThemeMetadata> themeMetadata)
{
// Given a theme, first validate that it is applicable
if (theme == null || theme.Provider == ThemeProvider.System || string.IsNullOrEmpty(theme.GitHubPath))
{
_logger.LogInformation("Cannot Sync {ThemeName} as it is not valid", theme?.Name);
return;
}
if (new Version(theme.CompatibleVersion) > BuildInfo.Version)
{
_logger.LogDebug("{ThemeName} theme supports a more up-to-date version ({Version}) of Kavita. Please update", theme.Name, theme.CompatibleVersion);
return;
}
var themeContents = await GetDirectoryContent(theme.GitHubPath);
var cssFile = themeContents.FirstOrDefault(c => c.Name.EndsWith(".css"));
if (cssFile == null) return;
// Update any metadata
if (themeMetadata.TryGetValue(theme.Name, out var metadata))
{
theme.Description = metadata.Description;
theme.Author = metadata.Author;
theme.CompatibleVersion = metadata.LastCompatible.ToString();
theme.PreviewUrls = string.Join('|', GetPreviewUrls(themeContents));
}
var hasUpdated = cssFile.Sha != theme.ShaHash;
if (hasUpdated)
{
_logger.LogDebug("Theme {ThemeName} is out of date, updating", theme.Name);
var tempLocation = _directoryService.FileSystem.Path.Join(_directoryService.TempDirectory, theme.FileName);
_directoryService.DeleteFiles([tempLocation]);
var location = await cssFile.DownloadUrl.DownloadFileAsync(_directoryService.TempDirectory);
if (_directoryService.FileSystem.File.Exists(location))
{
_directoryService.CopyFileToDirectory(location, _directoryService.SiteThemeDirectory);
_logger.LogInformation("Updated Theme on disk for {ThemeName}", theme.Name);
}
}
await _unitOfWork.CommitAsync();
if (hasUpdated)
{
await _eventHub.SendMessageAsync(MessageFactory.SiteThemeUpdated,
MessageFactory.SiteThemeUpdatedEvent(theme.Name));
}
// Send an update to refresh metadata around the themes
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.SiteThemeProgressEvent(_directoryService.FileSystem.Path.GetFileName(theme.FileName), theme.Name,
ProgressEventType.Ended));
_logger.LogInformation("Theme Sync complete");
}
/// <summary>
/// Deletes a SiteTheme. The CSS file will be moved to temp/ to allow user to recover data
/// </summary>
/// <param name="siteThemeId"></param>
public async Task DeleteTheme(int siteThemeId)
{
// Validate no one else is using this theme
var inUse = await _unitOfWork.SiteThemeRepository.IsThemeInUse(siteThemeId);
if (inUse)
{
throw new KavitaException("errors.delete-theme-in-use");
}
var siteTheme = await _unitOfWork.SiteThemeRepository.GetTheme(siteThemeId);
if (siteTheme == null) return;
await RemoveTheme(siteTheme);
}
/// <summary>
/// This assumes a file is already in temp directory and will be used for
/// </summary>
/// <param name="tempFile"></param>
/// <returns></returns>
public async Task<SiteTheme> CreateThemeFromFile(string tempFile, string username)
{
if (!_directoryService.FileSystem.File.Exists(tempFile))
{
_logger.LogInformation("Unable to create theme from manual upload as file not in temp");
throw new KavitaException("errors.theme-manual-upload");
}
var filename = _directoryService.FileSystem.FileInfo.New(tempFile).Name;
var themeName = Path.GetFileNameWithoutExtension(filename);
if (await _unitOfWork.SiteThemeRepository.GetThemeDtoByName(themeName) != null)
{
throw new KavitaException("errors.theme-already-in-use");
}
_directoryService.CopyFileToDirectory(tempFile, _directoryService.SiteThemeDirectory);
var finalLocation = _directoryService.FileSystem.Path.Join(_directoryService.SiteThemeDirectory, filename);
// Create a new entry and note that this is downloaded
var theme = new SiteTheme()
{
Name = Path.GetFileNameWithoutExtension(filename),
NormalizedName = themeName.ToNormalized(),
FileName = _directoryService.FileSystem.Path.GetFileName(finalLocation),
Provider = ThemeProvider.Custom,
IsDefault = false,
Description = $"Manually uploaded via UI by {username}",
PreviewUrls = string.Empty,
Author = username,
};
_unitOfWork.SiteThemeRepository.Add(theme);
await _unitOfWork.CommitAsync();
// Inform about the new theme
await _eventHub.SendMessageAsync(MessageFactory.NotificationProgress,
MessageFactory.SiteThemeProgressEvent(_directoryService.FileSystem.Path.GetFileName(theme.FileName), theme.Name,
ProgressEventType.Ended));
return theme;
}
@ -130,6 +490,7 @@ public class ThemeService : IThemeService
/// <param name="theme"></param>
private async Task RemoveTheme(SiteTheme theme)
{
_logger.LogInformation("Removing {ThemeName}. File can be found in temp/ until nightly cleanup", theme.Name);
var prefs = await _unitOfWork.UserRepository.GetAllPreferencesByThemeAsync(theme.Id);
var defaultTheme = await _unitOfWork.SiteThemeRepository.GetDefaultTheme();
foreach (var pref in prefs)
@ -137,6 +498,20 @@ public class ThemeService : IThemeService
pref.Theme = defaultTheme;
_unitOfWork.UserRepository.Update(pref);
}
try
{
// Copy the theme file to temp for nightly removal (to give user time to reclaim if made a mistake)
var existingLocation =
_directoryService.FileSystem.Path.Join(_directoryService.SiteThemeDirectory, theme.FileName);
var newLocation =
_directoryService.FileSystem.Path.Join(_directoryService.TempDirectory, theme.FileName);
_directoryService.CopyFileToDirectory(existingLocation, newLocation);
_directoryService.DeleteFiles([existingLocation]);
}
catch (Exception) { /* Swallow */ }
_unitOfWork.SiteThemeRepository.Remove(theme);
await _unitOfWork.CommitAsync();
}

366
API/Services/Tasks/StatsService.cs
View file

@ -1,20 +1,29 @@
using System;
using System.Collections.Generic;
using System.IO;
using System.Diagnostics;
using System.Globalization;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Threading.Tasks;
using API.Data;
using API.Data.Misc;
using API.Data.Repositories;
using API.DTOs.Stats;
using API.DTOs.Stats.V3;
using API.Entities;
using API.Entities.Enums;
using API.Entities.Enums.UserPreferences;
using API.Extensions;
using API.Services.Plus;
using API.Services.Tasks.Scanner.Parser;
using Flurl.Http;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Kavita.Common.Helpers;
using Microsoft.AspNetCore.Http;
using Microsoft.AspNetCore.Identity;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
namespace API.Services.Tasks;
@ -24,7 +33,6 @@ namespace API.Services.Tasks;
public interface IStatsService
{
Task Send();
Task<ServerInfoDto> GetServerInfo();
Task<ServerInfoSlimDto> GetServerInfoSlim();
Task SendCancellation();
}
@ -36,23 +44,33 @@ public class StatsService : IStatsService
private readonly ILogger<StatsService> _logger;
private readonly IUnitOfWork _unitOfWork;
private readonly DataContext _context;
private readonly IStatisticService _statisticService;
private const string ApiUrl = "https://stats.kavitareader.com";
private readonly ILicenseService _licenseService;
private readonly UserManager<AppUser> _userManager;
private readonly IEmailService _emailService;
private readonly ICacheService _cacheService;
private readonly string _apiUrl = "";
private const string ApiKey = "MsnvA2DfQqxSK5jh"; // It's not important this is public, just a way to keep bots from hitting the API willy-nilly
public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork, DataContext context, IStatisticService statisticService)
public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork, DataContext context,
ILicenseService licenseService, UserManager<AppUser> userManager, IEmailService emailService,
ICacheService cacheService, IHostEnvironment environment)
{
_logger = logger;
_unitOfWork = unitOfWork;
_context = context;
_statisticService = statisticService;
_licenseService = licenseService;
_userManager = userManager;
_emailService = emailService;
_cacheService = cacheService;
FlurlHttp.ConfigureClient(ApiUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
FlurlConfiguration.ConfigureClientForUrl(Configuration.StatsApiUrl);
_apiUrl = environment.IsDevelopment() ? "http://localhost:5001" : Configuration.StatsApiUrl;
}
/// <summary>
/// Due to all instances firing this at the same time, we can DDOS our server. This task when fired will schedule the task to be run
/// randomly over a 6 hour spread
/// randomly over a six-hour spread
/// </summary>
public async Task Send()
{
@ -71,24 +89,22 @@ public class StatsService : IStatsService
// ReSharper disable once MemberCanBePrivate.Global
public async Task SendData()
{
var data = await GetServerInfo();
var sw = Stopwatch.StartNew();
var data = await GetStatV3Payload();
_logger.LogDebug("Collecting stats took {Time} ms", sw.ElapsedMilliseconds);
sw.Stop();
await SendDataToStatsServer(data);
}
private async Task SendDataToStatsServer(ServerInfoDto data)
private async Task SendDataToStatsServer(ServerInfoV3Dto data)
{
var responseContent = string.Empty;
try
{
var response = await (ApiUrl + "/api/v2/stats")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(30))
var response = await (_apiUrl + "/api/v3/stats")
.WithBasicHeaders(ApiKey)
.PostJsonAsync(data);
if (response.StatusCode != StatusCodes.Status200OK)
@ -112,67 +128,6 @@ public class StatsService : IStatsService
}
}
public async Task<ServerInfoDto> GetServerInfo()
{
var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
var serverInfo = new ServerInfoDto
{
InstallId = serverSettings.InstallId,
Os = RuntimeInformation.OSDescription,
KavitaVersion = serverSettings.InstallVersion,
DotnetVersion = Environment.Version.ToString(),
IsDocker = OsInfo.IsDocker,
NumOfCores = Math.Max(Environment.ProcessorCount, 1),
UsersWithEmulateComicBook = await _context.AppUserPreferences.CountAsync(p => p.EmulateBook),
TotalReadingHours = await _statisticService.TimeSpentReadingForUsersAsync(ArraySegment<int>.Empty, ArraySegment<int>.Empty),
PercentOfLibrariesWithFolderWatchingEnabled = await GetPercentageOfLibrariesWithFolderWatchingEnabled(),
PercentOfLibrariesIncludedInRecommended = await GetPercentageOfLibrariesIncludedInRecommended(),
PercentOfLibrariesIncludedInDashboard = await GetPercentageOfLibrariesIncludedInDashboard(),
PercentOfLibrariesIncludedInSearch = await GetPercentageOfLibrariesIncludedInSearch(),
HasBookmarks = (await _unitOfWork.UserRepository.GetAllBookmarksAsync()).Any(),
NumberOfLibraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).Count(),
NumberOfCollections = (await _unitOfWork.CollectionTagRepository.GetAllTagsAsync()).Count(),
NumberOfReadingLists = await _unitOfWork.ReadingListRepository.Count(),
OPDSEnabled = serverSettings.EnableOpds,
NumberOfUsers = (await _unitOfWork.UserRepository.GetAllUsersAsync()).Count(),
TotalFiles = await _unitOfWork.LibraryRepository.GetTotalFiles(),
TotalGenres = await _unitOfWork.GenreRepository.GetCountAsync(),
TotalPeople = await _unitOfWork.PersonRepository.GetCountAsync(),
UsingSeriesRelationships = await GetIfUsingSeriesRelationship(),
EncodeMediaAs = serverSettings.EncodeMediaAs,
MaxSeriesInALibrary = await MaxSeriesInAnyLibrary(),
MaxVolumesInASeries = await MaxVolumesInASeries(),
MaxChaptersInASeries = await MaxChaptersInASeries(),
MangaReaderBackgroundColors = await AllMangaReaderBackgroundColors(),
MangaReaderPageSplittingModes = await AllMangaReaderPageSplitting(),
MangaReaderLayoutModes = await AllMangaReaderLayoutModes(),
FileFormats = AllFormats(),
UsingRestrictedProfiles = await GetUsingRestrictedProfiles(),
LastReadTime = await _unitOfWork.AppUserProgressRepository.GetLatestProgress()
};
var usersWithPref = (await _unitOfWork.UserRepository.GetAllUsersAsync(AppUserIncludes.UserPreferences)).ToList();
serverInfo.UsersOnCardLayout =
usersWithPref.Count(u => u.UserPreferences.GlobalPageLayoutMode == PageLayoutMode.Cards);
serverInfo.UsersOnListLayout =
usersWithPref.Count(u => u.UserPreferences.GlobalPageLayoutMode == PageLayoutMode.List);
var firstAdminUser = (await _unitOfWork.UserRepository.GetAdminUsersAsync()).FirstOrDefault();
if (firstAdminUser != null)
{
var firstAdminUserPref = (await _unitOfWork.UserRepository.GetPreferencesAsync(firstAdminUser.UserName!));
var activeTheme = firstAdminUserPref?.Theme ?? Seed.DefaultThemes.First(t => t.IsDefault);
serverInfo.ActiveSiteTheme = activeTheme.Name;
if (firstAdminUserPref != null) serverInfo.MangaReaderMode = firstAdminUserPref.ReaderMode;
}
return serverInfo;
}
public async Task<ServerInfoSlimDto> GetServerInfoSlim()
{
@ -181,7 +136,9 @@ public class StatsService : IStatsService
{
InstallId = serverSettings.InstallId,
KavitaVersion = serverSettings.InstallVersion,
IsDocker = OsInfo.IsDocker
IsDocker = OsInfo.IsDocker,
FirstInstallDate = serverSettings.FirstInstallDate,
FirstInstallVersion = serverSettings.FirstInstallVersion
};
}
@ -194,12 +151,8 @@ public class StatsService : IStatsService
try
{
var response = await (ApiUrl + "/api/v2/stats/opt-out?installId=" + installId)
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
var response = await (_apiUrl + "/api/v2/stats/opt-out?installId=" + installId)
.WithBasicHeaders(ApiKey)
.WithTimeout(TimeSpan.FromSeconds(30))
.PostAsync();
@ -218,42 +171,32 @@ public class StatsService : IStatsService
}
}
private async Task<float> GetPercentageOfLibrariesWithFolderWatchingEnabled()
private static async Task<long> PingStatsApi()
{
var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();
if (libraries.Count == 0) return 0.0f;
return libraries.Count(l => l.FolderWatching) / (1.0f * libraries.Count);
}
try
{
var sw = Stopwatch.StartNew();
var response = await (Configuration.StatsApiUrl + "/api/health/")
.WithBasicHeaders(ApiKey)
.GetAsync();
private async Task<float> GetPercentageOfLibrariesIncludedInRecommended()
{
var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();
if (libraries.Count == 0) return 0.0f;
return libraries.Count(l => l.IncludeInRecommended) / (1.0f * libraries.Count);
}
if (response.StatusCode == StatusCodes.Status200OK)
{
sw.Stop();
return sw.ElapsedMilliseconds;
}
}
catch (Exception)
{
/* Swallow */
}
private async Task<float> GetPercentageOfLibrariesIncludedInDashboard()
{
var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();
if (libraries.Count == 0) return 0.0f;
return libraries.Count(l => l.IncludeInDashboard) / (1.0f * libraries.Count);
}
private async Task<float> GetPercentageOfLibrariesIncludedInSearch()
{
var libraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync()).ToList();
if (libraries.Count == 0) return 0.0f;
return libraries.Count(l => l.IncludeInSearch) / (1.0f * libraries.Count);
}
private Task<bool> GetIfUsingSeriesRelationship()
{
return _context.SeriesRelation.AnyAsync();
return 0;
}
private async Task<int> MaxSeriesInAnyLibrary()
{
// If first time flow, just return 0
// If first time flow, return 0
if (!await _context.Series.AnyAsync()) return 0;
return await _context.Series
.Select(s => _context.Library.Where(l => l.Id == s.LibraryId).SelectMany(l => l.Series!).Count())
@ -279,50 +222,191 @@ public class StatsService : IStatsService
{
// If first time flow, just return 0
if (!await _context.Chapter.AnyAsync()) return 0;
return await _context.Series
.AsNoTracking()
.AsSplitQuery()
.MaxAsync(s => s.Volumes!
.Where(v => v.MinNumber == 0)
.Where(v => v.MinNumber == Parser.LooseLeafVolumeNumber)
.SelectMany(v => v.Chapters!)
.Count());
}
private async Task<IEnumerable<string>> AllMangaReaderBackgroundColors()
private async Task<ServerInfoV3Dto> GetStatV3Payload()
{
return await _context.AppUserPreferences.Select(p => p.BackgroundColor).Distinct().ToListAsync();
}
var serverSettings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
var mediaSettings = await _unitOfWork.SettingsRepository.GetMetadataSettings();
var dto = new ServerInfoV3Dto()
{
InstallId = serverSettings.InstallId,
KavitaVersion = serverSettings.InstallVersion,
InitialKavitaVersion = serverSettings.FirstInstallVersion,
InitialInstallDate = (DateTime)serverSettings.FirstInstallDate!,
IsDocker = OsInfo.IsDocker,
Os = RuntimeInformation.OSDescription,
NumOfCores = Math.Max(Environment.ProcessorCount, 1),
DotnetVersion = Environment.Version.ToString(),
OpdsEnabled = serverSettings.EnableOpds,
EncodeMediaAs = serverSettings.EncodeMediaAs,
MatchedMetadataEnabled = mediaSettings.Enabled
};
private async Task<IEnumerable<PageSplitOption>> AllMangaReaderPageSplitting()
{
return await _context.AppUserPreferences.Select(p => p.PageSplitOption).Distinct().ToListAsync();
}
dto.OsLocale = CultureInfo.CurrentCulture.EnglishName;
dto.LastReadTime = await _unitOfWork.AppUserProgressRepository.GetLatestProgress();
dto.MaxSeriesInALibrary = await MaxSeriesInAnyLibrary();
dto.MaxVolumesInASeries = await MaxVolumesInASeries();
dto.MaxChaptersInASeries = await MaxChaptersInASeries();
dto.TotalFiles = await _context.MangaFile.CountAsync();
dto.TotalGenres = await _context.Genre.CountAsync();
dto.TotalPeople = await _context.Person.CountAsync();
dto.TotalSeries = await _context.Series.CountAsync();
dto.TotalLibraries = await _context.Library.CountAsync();
dto.NumberOfCollections = await _context.AppUserCollection.CountAsync();
dto.NumberOfReadingLists = await _context.ReadingList.CountAsync();
try
{
var license = (await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.LicenseKey)).Value;
dto.ActiveKavitaPlusSubscription = await _licenseService.HasActiveSubscription(license);
}
catch (Exception)
{
dto.ActiveKavitaPlusSubscription = false;
}
private async Task<IEnumerable<LayoutMode>> AllMangaReaderLayoutModes()
{
return await _context.AppUserPreferences.Select(p => p.LayoutMode).Distinct().ToListAsync();
}
// Find a random cbz/zip file and open it for reading
await OpenRandomFile(dto);
dto.TimeToPingKavitaStatsApi = await PingStatsApi();
private IEnumerable<FileFormatDto> AllFormats()
{
#region Relationships
var results = _context.MangaFile
.AsNoTracking()
.AsEnumerable()
.Select(m => new FileFormatDto()
dto.Relationships = await _context.SeriesRelation
.GroupBy(sr => sr.RelationKind)
.Select(g => new RelationshipStatV3
{
Format = m.Format,
Extension = m.Extension
Relationship = g.Key,
Count = g.Count()
})
.DistinctBy(f => f.Extension)
.ToList();
.ToListAsync();
return results;
#endregion
#region Libraries
var allLibraries = (await _unitOfWork.LibraryRepository.GetLibrariesAsync(LibraryIncludes.Folders |
LibraryIncludes.FileTypes | LibraryIncludes.ExcludePatterns | LibraryIncludes.AppUser)).ToList();
dto.Libraries ??= [];
foreach (var library in allLibraries)
{
var libDto = new LibraryStatV3();
libDto.IncludeInDashboard = library.IncludeInDashboard;
libDto.IncludeInSearch = library.IncludeInSearch;
libDto.LastScanned = library.LastScanned;
libDto.NumberOfFolders = library.Folders.Count;
libDto.FileTypes = library.LibraryFileTypes.Select(s => s.FileTypeGroup).Distinct().ToList();
libDto.UsingExcludePatterns = library.LibraryExcludePatterns.Any(p => !string.IsNullOrEmpty(p.Pattern));
libDto.UsingFolderWatching = library.FolderWatching;
libDto.CreateCollectionsFromMetadata = library.ManageCollections;
libDto.CreateReadingListsFromMetadata = library.ManageReadingLists;
libDto.LibraryType = library.Type;
dto.Libraries.Add(libDto);
}
#endregion
#region Users
// Create a dictionary mapping user IDs to the libraries they have access to
var userLibraryAccess = allLibraries
.SelectMany(l => l.AppUsers.Select(appUser => new { l, appUser.Id }))
.GroupBy(x => x.Id)
.ToDictionary(g => g.Key, g => g.Select(x => x.l).ToList());
dto.Users ??= [];
var allUsers = await _unitOfWork.UserRepository.GetAllUsersAsync(AppUserIncludes.UserPreferences
| AppUserIncludes.ReadingLists | AppUserIncludes.Bookmarks
| AppUserIncludes.Collections | AppUserIncludes.Devices
| AppUserIncludes.Progress | AppUserIncludes.Ratings
| AppUserIncludes.SmartFilters | AppUserIncludes.WantToRead, false);
foreach (var user in allUsers)
{
var userDto = new UserStatV3();
userDto.HasMALToken = !string.IsNullOrEmpty(user.MalAccessToken);
userDto.HasAniListToken = !string.IsNullOrEmpty(user.AniListAccessToken);
userDto.AgeRestriction = new AgeRestriction()
{
AgeRating = user.AgeRestriction,
IncludeUnknowns = user.AgeRestrictionIncludeUnknowns
};
userDto.Locale = user.UserPreferences.Locale;
userDto.Roles = [.. _userManager.GetRolesAsync(user).Result];
userDto.LastLogin = user.LastActiveUtc;
userDto.HasValidEmail = user.Email != null && _emailService.IsValidEmail(user.Email);
userDto.IsEmailConfirmed = user.EmailConfirmed;
userDto.ActiveTheme = user.UserPreferences.Theme.Name;
userDto.CollectionsCreatedCount = user.Collections.Count;
userDto.ReadingListsCreatedCount = user.ReadingLists.Count;
userDto.LastReadTime = user.Progresses
.Select(p => p.LastModifiedUtc)
.DefaultIfEmpty()
.Max();
userDto.DevicePlatforms = user.Devices.Select(d => d.Platform).ToList();
userDto.SeriesBookmarksCreatedCount = user.Bookmarks.Count;
userDto.SmartFilterCreatedCount = user.SmartFilters.Count;
userDto.WantToReadSeriesCount = user.WantToRead.Count;
if (allLibraries.Count > 0 && userLibraryAccess.TryGetValue(user.Id, out var accessibleLibraries))
{
userDto.PercentageOfLibrariesHasAccess = (1f * accessibleLibraries.Count) / allLibraries.Count;
}
else
{
userDto.PercentageOfLibrariesHasAccess = 0;
}
dto.Users.Add(userDto);
}
#endregion
return dto;
}
private Task<bool> GetUsingRestrictedProfiles()
private async Task OpenRandomFile(ServerInfoV3Dto dto)
{
return _context.Users.AnyAsync(u => u.AgeRestriction > AgeRating.NotApplicable);
var random = new Random();
List<string> extensions = [".cbz", ".zip"];
// Count the total number of files that match the criteria
var count = await _context.MangaFile.AsNoTracking()
.Where(r => r.Extension != null && extensions.Contains(r.Extension))
.CountAsync();
if (count == 0)
{
dto.TimeToOpeCbzMs = 0;
dto.TimeToOpenCbzPages = 0;
return;
}
// Generate a random skip value
var skip = random.Next(count);
// Fetch the random file
var randomFile = await _context.MangaFile.AsNoTracking()
.Where(r => r.Extension != null && extensions.Contains(r.Extension))
.Skip(skip)
.Take(1)
.FirstAsync();
var sw = Stopwatch.StartNew();
await _cacheService.Ensure(randomFile.ChapterId);
var time = sw.ElapsedMilliseconds;
sw.Stop();
dto.TimeToOpeCbzMs = time;
dto.TimeToOpenCbzPages = randomFile.Pages;
}
}

517
API/Services/Tasks/VersionUpdaterService.cs
View file

@ -1,8 +1,13 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.IO;
using System.Linq;
using System.Text.Json;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using API.DTOs.Update;
using API.Extensions;
using API.SignalR;
using Flurl.Http;
using Kavita.Common.EnvironmentInfo;
@ -30,7 +35,7 @@ internal class GithubReleaseMetadata
/// </summary>
public required string Body { get; init; }
/// <summary>
/// Url of the release on Github
/// Url of the release on GitHub
/// </summary>
// ReSharper disable once InconsistentNaming
public required string Html_Url { get; init; }
@ -45,11 +50,12 @@ public interface IVersionUpdaterService
{
Task<UpdateNotificationDto?> CheckForUpdate();
Task PushUpdate(UpdateNotificationDto update);
Task<IList<UpdateNotificationDto>> GetAllReleases();
Task<int> GetNumberOfReleasesBehind();
Task<IList<UpdateNotificationDto>> GetAllReleases(int count = 0);
Task<int> GetNumberOfReleasesBehind(bool stableOnly = false);
}
public class VersionUpdaterService : IVersionUpdaterService
public partial class VersionUpdaterService : IVersionUpdaterService
{
private readonly ILogger<VersionUpdaterService> _logger;
private readonly IEventHub _eventHub;
@ -57,50 +63,387 @@ public class VersionUpdaterService : IVersionUpdaterService
#pragma warning disable S1075
private const string GithubLatestReleasesUrl = "https://api.github.com/repos/Kareadita/Kavita/releases/latest";
private const string GithubAllReleasesUrl = "https://api.github.com/repos/Kareadita/Kavita/releases";
private const string GithubPullsUrl = "https://api.github.com/repos/Kareadita/Kavita/pulls/";
private const string GithubBranchCommitsUrl = "https://api.github.com/repos/Kareadita/Kavita/commits?sha=develop";
#pragma warning restore S1075
public VersionUpdaterService(ILogger<VersionUpdaterService> logger, IEventHub eventHub)
[GeneratedRegex(@"^\n*(.*?)\n+#{1,2}\s", RegexOptions.Singleline)]
private static partial Regex BlogPartRegex();
private readonly string _cacheFilePath;
/// <summary>
/// The latest release cache
/// </summary>
private readonly string _cacheLatestReleaseFilePath;
private static readonly TimeSpan CacheDuration = TimeSpan.FromHours(1);
private static readonly JsonSerializerOptions JsonOptions = new() { WriteIndented = true };
public VersionUpdaterService(ILogger<VersionUpdaterService> logger, IEventHub eventHub, IDirectoryService directoryService)
{
_logger = logger;
_eventHub = eventHub;
_cacheFilePath = Path.Combine(directoryService.LongTermCacheDirectory, "github_releases_cache.json");
_cacheLatestReleaseFilePath = Path.Combine(directoryService.LongTermCacheDirectory, "github_latest_release_cache.json");
FlurlHttp.ConfigureClient(GithubLatestReleasesUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
FlurlHttp.ConfigureClient(GithubAllReleasesUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
FlurlConfiguration.ConfigureClientForUrl(GithubLatestReleasesUrl);
FlurlConfiguration.ConfigureClientForUrl(GithubAllReleasesUrl);
}
/// <summary>
/// Fetches the latest release from Github
/// Fetches the latest (stable) release from GitHub. Does not do any extra nightly release parsing.
/// </summary>
/// <returns>Latest update</returns>
public async Task<UpdateNotificationDto?> CheckForUpdate()
{
// Attempt to fetch from cache
var cachedRelease = await TryGetCachedLatestRelease();
if (cachedRelease != null)
{
return cachedRelease;
}
var update = await GetGithubRelease();
return CreateDto(update);
var dto = CreateDto(update);
if (dto != null)
{
await CacheLatestReleaseAsync(dto);
}
return dto;
}
public async Task<IList<UpdateNotificationDto>> GetAllReleases()
/// <summary>
/// Will add any extra (nightly) updates from the latest stable. Does not back-fill anything prior to the latest stable.
/// </summary>
/// <param name="dtos"></param>
private async Task EnrichWithNightlyInfo(List<UpdateNotificationDto> dtos)
{
var dto = dtos[0]; // Latest version
try
{
var currentVersion = new Version(dto.CurrentVersion);
var nightlyReleases = await GetNightlyReleases(currentVersion, Version.Parse(dto.UpdateVersion));
if (nightlyReleases.Count == 0) return;
// Create new DTOs for each nightly release and insert them at the beginning of the list
var nightlyDtos = new List<UpdateNotificationDto>();
foreach (var nightly in nightlyReleases)
{
var prInfo = await FetchPullRequestInfo(nightly.PrNumber);
if (prInfo == null) continue;
var sections = ParseReleaseBody(prInfo.Body);
var blogPart = ExtractBlogPart(prInfo.Body);
var nightlyDto = new UpdateNotificationDto
{
// TODO: I should pass Title to the FE so that Nightly Release can be localized
UpdateTitle = $"Nightly Release {nightly.Version} - {prInfo.Title}",
UpdateVersion = nightly.Version,
CurrentVersion = dto.CurrentVersion,
UpdateUrl = prInfo.Html_Url,
PublishDate = prInfo.Merged_At,
IsDocker = true, // Nightlies are always Docker Only
IsReleaseEqual = IsVersionEqualToBuildVersion(Version.Parse(nightly.Version)),
IsReleaseNewer = true, // Since we already filtered these in GetNightlyReleases
IsPrerelease = true, // All Nightlies are considered prerelease
Added = sections.TryGetValue("Added", out var added) ? added : [],
Changed = sections.TryGetValue("Changed", out var changed) ? changed : [],
Fixed = sections.TryGetValue("Fixed", out var bugfixes) ? bugfixes : [],
Removed = sections.TryGetValue("Removed", out var removed) ? removed : [],
Theme = sections.TryGetValue("Theme", out var theme) ? theme : [],
Developer = sections.TryGetValue("Developer", out var developer) ? developer : [],
KnownIssues = sections.TryGetValue("KnownIssues", out var knownIssues) ? knownIssues : [],
Api = sections.TryGetValue("Api", out var api) ? api : [],
FeatureRequests = sections.TryGetValue("Feature Requests", out var frs) ? frs : [],
BlogPart = _markdown.Transform(blogPart.Trim()),
UpdateBody = _markdown.Transform(prInfo.Body.Trim())
};
nightlyDtos.Add(nightlyDto);
}
// Insert nightly releases at the beginning of the list
var sortedNightlyDtos = nightlyDtos.OrderByDescending(x => x.PublishDate).ToList();
dtos.InsertRange(0, sortedNightlyDtos);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to enrich nightly release information");
}
}
private async Task<PullRequestInfo?> FetchPullRequestInfo(int prNumber)
{
try
{
return await $"{GithubPullsUrl}{prNumber}"
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.GetJsonAsync<PullRequestInfo>();
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to fetch PR information for #{PrNumber}", prNumber);
return null;
}
}
private async Task<List<NightlyInfo>> GetNightlyReleases(Version currentVersion, Version latestStableVersion)
{
try
{
var nightlyReleases = new List<NightlyInfo>();
var commits = await GithubBranchCommitsUrl
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.GetJsonAsync<IList<CommitInfo>>();
var commitList = commits.ToList();
bool foundLastStable = false;
for (var i = 0; i < commitList.Count - 1; i++)
{
var commit = commitList[i];
var message = commit.Commit.Message.Split('\n')[0]; // Take first line only
// Skip [skip ci] commits
if (message.Contains("[skip ci]")) continue;
// Check if this is a stable release
if (message.StartsWith('v'))
{
var stableMatch = Regex.Match(message, @"v(\d+\.\d+\.\d+\.\d+)");
if (stableMatch.Success)
{
var stableVersion = new Version(stableMatch.Groups[1].Value);
// If we find a stable version lower than current, we've gone too far back
if (stableVersion <= currentVersion)
{
foundLastStable = true;
break;
}
}
continue;
}
// Look for version bumps that follow PRs
if (!foundLastStable && message == "Bump versions by dotnet-bump-version.")
{
// Get the PR commit that triggered this version bump
if (i + 1 < commitList.Count)
{
var prCommit = commitList[i + 1];
var prMessage = prCommit.Commit.Message.Split('\n')[0];
// Extract PR number using improved regex
var prMatch = Regex.Match(prMessage, @"(?:^|\s)\(#(\d+)\)|\s#(\d+)");
if (!prMatch.Success) continue;
var prNumber = int.Parse(prMatch.Groups[1].Value != "" ?
prMatch.Groups[1].Value : prMatch.Groups[2].Value);
// Get the version from AssemblyInfo.cs in this commit
var version = await GetVersionFromCommit(commit.Sha);
if (version == null) continue;
// Parse version and compare with current version
if (Version.TryParse(version, out var parsedVersion) &&
parsedVersion > latestStableVersion)
{
nightlyReleases.Add(new NightlyInfo
{
Version = version,
PrNumber = prNumber,
Date = DateTime.Parse(commit.Commit.Author.Date, CultureInfo.InvariantCulture)
});
}
}
}
}
return nightlyReleases.OrderByDescending(x => x.Date).ToList();
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to get nightly releases");
return [];
}
}
public async Task<IList<UpdateNotificationDto>> GetAllReleases(int count = 0)
{
// Attempt to fetch from cache
var cachedReleases = await TryGetCachedReleases();
// If there is a cached release and the current version is within it, use it, otherwise regenerate
if (cachedReleases != null && cachedReleases.Any(r => IsVersionEqual(r.UpdateVersion, BuildInfo.Version.ToString())))
{
if (count > 0)
{
// NOTE: We may want to allow the admin to clear Github cache
return cachedReleases.Take(count).ToList();
}
return cachedReleases;
}
var updates = await GetGithubReleases();
var updateDtos = updates.Select(CreateDto)
var query = updates.Select(CreateDto)
.Where(d => d != null)
.OrderByDescending(d => d!.PublishDate)
.Select(d => d!)
.ToList();
.Select(d => d!);
var updateDtos = query.ToList();
// Sometimes a release can be 0.8.5.0 on disk, but 0.8.5 from Github
var versionParts = updateDtos[0].UpdateVersion.Split('.');
if (versionParts.Length < 4)
{
updateDtos[0].UpdateVersion += ".0"; // Append missing parts
}
// If we're on a nightly build, enrich the information
if (updateDtos.Count != 0) // && BuildInfo.Version > new Version(updateDtos[0].UpdateVersion)
{
await EnrichWithNightlyInfo(updateDtos);
}
// Find the latest dto
var latestRelease = updateDtos[0]!;
var updateVersion = new Version(latestRelease.UpdateVersion);
var isNightly = BuildInfo.Version > new Version(latestRelease.UpdateVersion);
// isNightly can be true when we compare something like v0.8.1 vs v0.8.1.0
if (IsVersionEqualToBuildVersion(updateVersion))
{
isNightly = false;
}
latestRelease.IsOnNightlyInRelease = isNightly;
// Cache the fetched data
if (updateDtos.Count > 0)
{
await CacheReleasesAsync(updateDtos);
}
if (count > 0)
{
return updateDtos.Take(count).ToList();
}
return updateDtos;
}
public async Task<int> GetNumberOfReleasesBehind()
/// <summary>
/// Compares 2 versions and ensures that the minor is always there
/// </summary>
/// <param name="v1"></param>
/// <param name="v2"></param>
/// <returns></returns>
private static bool IsVersionEqual(string v1, string v2)
{
var versionParts = v1.Split('.');
if (versionParts.Length < 4)
{
v1 += ".0"; // Append missing parts
}
versionParts = v2.Split('.');
if (versionParts.Length < 4)
{
v2 += ".0"; // Append missing parts
}
return string.Equals(v2, v2, StringComparison.OrdinalIgnoreCase);
}
private async Task<IList<UpdateNotificationDto>?> TryGetCachedReleases()
{
if (!File.Exists(_cacheFilePath)) return null;
var fileInfo = new FileInfo(_cacheFilePath);
if (DateTime.UtcNow - fileInfo.LastWriteTimeUtc <= CacheDuration)
{
var cachedData = await File.ReadAllTextAsync(_cacheFilePath);
return JsonSerializer.Deserialize<IList<UpdateNotificationDto>>(cachedData);
}
return null;
}
private async Task<UpdateNotificationDto?> TryGetCachedLatestRelease()
{
if (!File.Exists(_cacheLatestReleaseFilePath)) return null;
var fileInfo = new FileInfo(_cacheLatestReleaseFilePath);
if (DateTime.UtcNow - fileInfo.LastWriteTimeUtc <= CacheDuration)
{
var cachedData = await File.ReadAllTextAsync(_cacheLatestReleaseFilePath);
return System.Text.Json.JsonSerializer.Deserialize<UpdateNotificationDto>(cachedData);
}
return null;
}
private async Task CacheReleasesAsync(IList<UpdateNotificationDto> updates)
{
try
{
var json = JsonSerializer.Serialize(updates, JsonOptions);
await File.WriteAllTextAsync(_cacheFilePath, json);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to cache releases");
}
}
private async Task CacheLatestReleaseAsync(UpdateNotificationDto update)
{
try
{
var json = System.Text.Json.JsonSerializer.Serialize(update, JsonOptions);
await File.WriteAllTextAsync(_cacheLatestReleaseFilePath, json);
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to cache latest release");
}
}
private static bool IsVersionEqualToBuildVersion(Version updateVersion)
{
return updateVersion == BuildInfo.Version || (updateVersion.Revision < 0 && BuildInfo.Version.Revision == 0 &&
BuildInfo.Version.CompareWithoutRevision(updateVersion));
}
/// <summary>
/// Returns the number of releases ahead of this install version. If this install version is on a nightly,
/// then include nightly releases, otherwise only count Stable releases.
/// </summary>
/// <param name="stableOnly">Only count Stable releases </param>
/// <returns></returns>
public async Task<int> GetNumberOfReleasesBehind(bool stableOnly = false)
{
var updates = await GetAllReleases();
return updates.TakeWhile(update => update.UpdateVersion != update.CurrentVersion).Count();
// If the user is on nightly, then we need to handle releases behind differently
if (!stableOnly && (updates[0].IsPrerelease || updates[0].IsOnNightlyInRelease))
{
return updates.Count(u => u.IsReleaseNewer);
}
return updates
.Where(update => !update.IsPrerelease)
.Count(u => u.IsReleaseNewer);
}
private UpdateNotificationDto? CreateDto(GithubReleaseMetadata? update)
@ -109,17 +452,33 @@ public class VersionUpdaterService : IVersionUpdaterService
var updateVersion = new Version(update.Tag_Name.Replace("v", string.Empty));
var currentVersion = BuildInfo.Version.ToString(4);
var bodyHtml = _markdown.Transform(update.Body.Trim());
var parsedSections = ParseReleaseBody(update.Body);
var blogPart = _markdown.Transform(ExtractBlogPart(update.Body).Trim());
return new UpdateNotificationDto()
{
CurrentVersion = currentVersion,
UpdateVersion = updateVersion.ToString(),
UpdateBody = _markdown.Transform(update.Body.Trim()),
UpdateBody = bodyHtml,
UpdateTitle = update.Name,
UpdateUrl = update.Html_Url,
IsDocker = OsInfo.IsDocker,
PublishDate = update.Published_At,
IsReleaseEqual = BuildInfo.Version == updateVersion,
IsReleaseEqual = IsVersionEqualToBuildVersion(updateVersion),
IsReleaseNewer = BuildInfo.Version < updateVersion,
IsPrerelease = false,
Added = parsedSections.TryGetValue("Added", out var added) ? added : [],
Removed = parsedSections.TryGetValue("Removed", out var removed) ? removed : [],
Changed = parsedSections.TryGetValue("Changed", out var changed) ? changed : [],
Fixed = parsedSections.TryGetValue("Fixed", out var fixes) ? fixes : [],
Theme = parsedSections.TryGetValue("Theme", out var theme) ? theme : [],
Developer = parsedSections.TryGetValue("Developer", out var developer) ? developer : [],
KnownIssues = parsedSections.TryGetValue("Known Issues", out var knownIssues) ? knownIssues : [],
Api = parsedSections.TryGetValue("Api", out var api) ? api : [],
FeatureRequests = parsedSections.TryGetValue("Feature Requests", out var frs) ? frs : [],
BlogPart = blogPart
};
}
@ -138,6 +497,26 @@ public class VersionUpdaterService : IVersionUpdaterService
}
}
private async Task<string?> GetVersionFromCommit(string commitSha)
{
try
{
// Use the raw GitHub URL format for the csproj file
var content = await $"https://raw.githubusercontent.com/Kareadita/Kavita/{commitSha}/Kavita.Common/Kavita.Common.csproj"
.WithHeader("User-Agent", "Kavita")
.GetStringAsync();
var versionMatch = Regex.Match(content, @"<AssemblyVersion>([0-9\.]+)</AssemblyVersion>");
return versionMatch.Success ? versionMatch.Groups[1].Value : null;
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to get version from commit {Sha}: {Message}", commitSha, ex.Message);
return null;
}
}
private static async Task<GithubReleaseMetadata> GetGithubRelease()
{
@ -149,13 +528,109 @@ public class VersionUpdaterService : IVersionUpdaterService
return update;
}
private static async Task<IEnumerable<GithubReleaseMetadata>> GetGithubReleases()
private static async Task<IList<GithubReleaseMetadata>> GetGithubReleases()
{
var update = await GithubAllReleasesUrl
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.GetJsonAsync<IEnumerable<GithubReleaseMetadata>>();
.GetJsonAsync<IList<GithubReleaseMetadata>>();
return update;
}
private static string ExtractBlogPart(string body)
{
if (body.StartsWith('#')) return string.Empty;
var match = BlogPartRegex().Match(body);
return match.Success ? match.Groups[1].Value.Trim() : body.Trim();
}
private static Dictionary<string, List<string>> ParseReleaseBody(string body)
{
var sections = new Dictionary<string, List<string>>(StringComparer.OrdinalIgnoreCase);
var lines = body.Split('\n');
string? currentSection = null;
foreach (var line in lines)
{
var trimmedLine = line.Trim();
// Check for section headers (case-insensitive)
if (trimmedLine.StartsWith('#'))
{
currentSection = trimmedLine.TrimStart('#').Trim();
sections[currentSection] = [];
continue;
}
// Parse items under a section
if (currentSection != null &&
trimmedLine.StartsWith("- ") &&
!string.IsNullOrWhiteSpace(trimmedLine))
{
// Remove "Fixed:", "Added:" etc. if present
var cleanedItem = CleanSectionItem(trimmedLine);
// Some sections like API/Developer/Removed don't have the title repeated, so we need to check for an additional cleaning
if (cleanedItem.StartsWith("- "))
{
cleanedItem = trimmedLine.Substring(2);
}
// Only add non-empty items
if (!string.IsNullOrWhiteSpace(cleanedItem))
{
sections[currentSection].Add(cleanedItem);
}
}
}
return sections;
}
private static string CleanSectionItem(string item)
{
// Remove everything up to and including the first ":"
var colonIndex = item.IndexOf(':');
if (colonIndex != -1)
{
item = item.Substring(colonIndex + 1).Trim();
}
return item;
}
private sealed class PullRequestInfo
{
public required string Title { get; init; }
public required string Body { get; init; }
public required string Html_Url { get; init; }
public required string Merged_At { get; init; }
public required int Number { get; init; }
}
private sealed class CommitInfo
{
public required string Sha { get; init; }
public required CommitDetail Commit { get; init; }
public required string Html_Url { get; init; }
}
private sealed class CommitDetail
{
public required string Message { get; init; }
public required CommitAuthor Author { get; init; }
}
private sealed class CommitAuthor
{
public required string Date { get; init; }
}
private sealed class NightlyInfo
{
public required string Version { get; init; }
public required int PrNumber { get; init; }
public required DateTime Date { get; init; }
}
}