AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merged develop into main

Browse source
This commit is contained in:
Joseph Milazzo 2021-11-27 09:46:57 -06:00
commit 8a19c1da9e
103 changed files with 1242 additions and 900 deletions
Download patch file Download diff file Expand all files Collapse all files

13
API/API.csproj
View file

@ -107,6 +107,11 @@
<EmbeddedResource Remove="logs\**" />
<EmbeddedResource Remove="temp\**" />
<EmbeddedResource Remove="covers\**" />
<EmbeddedResource Remove="config\covers\**" />
<EmbeddedResource Remove="config\backups\**" />
<EmbeddedResource Remove="config\logs\**" />
<EmbeddedResource Remove="config\temp\**" />
<EmbeddedResource Remove="config\stats\**" />
</ItemGroup>
<ItemGroup>
@ -115,12 +120,18 @@
<Content Remove="backups\**" />
<Content Remove="logs\**" />
<Content Remove="temp\**" />
<Content Remove="stats\**" />
<Content Remove="config\stats\**" />
<Content Remove="config\cache\**" />
<Content Remove="config\backups\**" />
<Content Remove="config\logs\**" />
<Content Remove="config\temp\**" />
<Content Remove="config\stats\**" />
<Content Condition=" '$(Configuration)' == 'Release' " Remove="appsettings.Development.json" />
<Content Update="appsettings.json">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</Content>
<Content Remove="covers\**" />
<Content Remove="config\covers\**" />
</ItemGroup>
<ItemGroup>

2
API/Controllers/AccountController.cs
View file

@ -95,7 +95,7 @@ namespace API.Controllers
var settings = await _unitOfWork.SettingsRepository.GetSettingsDtoAsync();
if (!settings.EnableAuthentication && !registerDto.IsAdmin)
{
_logger.LogInformation("User {UserName} is being registered as non-admin with no server authentication. Using default password.", registerDto.Username);
_logger.LogInformation("User {UserName} is being registered as non-admin with no server authentication. Using default password", registerDto.Username);
registerDto.Password = AccountService.DefaultPassword;
}

79
API/Controllers/OPDSController.cs
View file

@ -17,6 +17,7 @@ using API.Interfaces.Services;
using API.Services;
using Kavita.Common;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace API.Controllers
{
@ -48,7 +49,6 @@ namespace API.Controllers
_cacheService = cacheService;
_readerService = readerService;
_xmlSerializer = new XmlSerializer(typeof(Feed));
_xmlOpenSearchSerializer = new XmlSerializer(typeof(OpenSearchDescription));
@ -62,18 +62,18 @@ namespace API.Controllers
if (!(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EnableOpds)
return BadRequest("OPDS is not enabled on this server");
var feed = CreateFeed("Kavita", string.Empty, apiKey);
feed.Id = "root";
SetFeedId(feed, "root");
feed.Entries.Add(new FeedEntry()
{
Id = "inProgress",
Title = "In Progress",
Id = "onDeck",
Title = "On Deck",
Content = new FeedEntryContent()
{
Text = "Browse by In Progress"
Text = "Browse by On Deck"
},
Links = new List<FeedLink>()
{
CreateLink(FeedLinkRelation.SubSection, FeedLinkType.AtomNavigation, Prefix + $"{apiKey}/in-progress"),
CreateLink(FeedLinkRelation.SubSection, FeedLinkType.AtomNavigation, Prefix + $"{apiKey}/on-deck"),
}
});
feed.Entries.Add(new FeedEntry()
@ -140,9 +140,8 @@ namespace API.Controllers
return BadRequest("OPDS is not enabled on this server");
var userId = await GetUser(apiKey);
var libraries = await _unitOfWork.LibraryRepository.GetLibrariesForUserIdAsync(userId);
var feed = CreateFeed("All Libraries", $"{apiKey}/libraries", apiKey);
SetFeedId(feed, "libraries");
foreach (var library in libraries)
{
feed.Entries.Add(new FeedEntry()
@ -181,7 +180,7 @@ namespace API.Controllers
var feed = CreateFeed("All Collections", $"{apiKey}/collections", apiKey);
SetFeedId(feed, "collections");
foreach (var tag in tags)
{
feed.Entries.Add(new FeedEntry()
@ -198,14 +197,6 @@ namespace API.Controllers
});
}
if (tags.Count == 0)
{
feed.Entries.Add(new FeedEntry()
{
Title = "Nothing here",
});
}
return CreateXmlResult(SerializeXml(feed));
}
@ -243,6 +234,7 @@ namespace API.Controllers
});
var feed = CreateFeed(tag.Title + " Collection", $"{apiKey}/collections/{collectionId}", apiKey);
SetFeedId(feed, $"collections-{collectionId}");
AddPagination(feed, series, $"{Prefix}{apiKey}/collections/{collectionId}");
foreach (var seriesDto in series)
@ -269,7 +261,7 @@ namespace API.Controllers
var feed = CreateFeed("All Reading Lists", $"{apiKey}/reading-list", apiKey);
SetFeedId(feed, "reading-list");
foreach (var readingListDto in readingLists)
{
feed.Entries.Add(new FeedEntry()
@ -304,6 +296,7 @@ namespace API.Controllers
}
var feed = CreateFeed(readingList.Title + " Reading List", $"{apiKey}/reading-list/{readingListId}", apiKey);
SetFeedId(feed, $"reading-list-{readingListId}");
var items = (await _unitOfWork.ReadingListRepository.GetReadingListItemDtosByIdAsync(readingListId, userId)).ToList();
foreach (var item in items)
@ -320,16 +313,6 @@ namespace API.Controllers
});
}
if (items.Count == 0)
{
feed.Entries.Add(new FeedEntry()
{
Title = "Nothing here",
});
}
return CreateXmlResult(SerializeXml(feed));
}
@ -355,6 +338,7 @@ namespace API.Controllers
}, _filterDto);
var feed = CreateFeed(library.Name, $"{apiKey}/libraries/{libraryId}", apiKey);
SetFeedId(feed, $"library-{library.Name}");
AddPagination(feed, series, $"{Prefix}{apiKey}/libraries/{libraryId}");
foreach (var seriesDto in series)
@ -379,6 +363,7 @@ namespace API.Controllers
}, _filterDto);
var feed = CreateFeed("Recently Added", $"{apiKey}/recently-added", apiKey);
SetFeedId(feed, "recently-added");
AddPagination(feed, recentlyAdded, $"{Prefix}{apiKey}/recently-added");
foreach (var seriesDto in recentlyAdded)
@ -386,21 +371,12 @@ namespace API.Controllers
feed.Entries.Add(CreateSeries(seriesDto, apiKey));
}
if (recentlyAdded.Count == 0)
{
feed.Entries.Add(new FeedEntry()
{
Title = "Nothing here",
});
}
return CreateXmlResult(SerializeXml(feed));
}
[HttpGet("{apiKey}/in-progress")]
[HttpGet("{apiKey}/on-deck")]
[Produces("application/xml")]
public async Task<IActionResult> GetInProgress(string apiKey, [FromQuery] int pageNumber = 1)
public async Task<IActionResult> GetOnDeck(string apiKey, [FromQuery] int pageNumber = 1)
{
if (!(await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).EnableOpds)
return BadRequest("OPDS is not enabled on this server");
@ -410,29 +386,22 @@ namespace API.Controllers
PageNumber = pageNumber,
PageSize = 20
};
var results = await _unitOfWork.SeriesRepository.GetInProgress(userId, 0, userParams, _filterDto);
var results = await _unitOfWork.SeriesRepository.GetOnDeck(userId, 0, userParams, _filterDto);
var listResults = results.DistinctBy(s => s.Name).Skip((userParams.PageNumber - 1) * userParams.PageSize)
.Take(userParams.PageSize).ToList();
var pagedList = new PagedList<SeriesDto>(listResults, listResults.Count, userParams.PageNumber, userParams.PageSize);
Response.AddPaginationHeader(pagedList.CurrentPage, pagedList.PageSize, pagedList.TotalCount, pagedList.TotalPages);
var feed = CreateFeed("In Progress", $"{apiKey}/in-progress", apiKey);
AddPagination(feed, pagedList, $"{Prefix}{apiKey}/in-progress");
var feed = CreateFeed("On Deck", $"{apiKey}/on-deck", apiKey);
SetFeedId(feed, "on-deck");
AddPagination(feed, pagedList, $"{Prefix}{apiKey}/on-deck");
foreach (var seriesDto in pagedList)
{
feed.Entries.Add(CreateSeries(seriesDto, apiKey));
}
if (pagedList.Count == 0)
{
feed.Entries.Add(new FeedEntry()
{
Title = "Nothing here",
});
}
return CreateXmlResult(SerializeXml(feed));
}
@ -456,7 +425,7 @@ namespace API.Controllers
var series = await _unitOfWork.SeriesRepository.SearchSeries(libraries.Select(l => l.Id).ToArray(), query);
var feed = CreateFeed(query, $"{apiKey}/series?query=" + query, apiKey);
SetFeedId(feed, "search-series");
foreach (var seriesDto in series)
{
feed.Entries.Add(CreateSeries(seriesDto, apiKey));
@ -465,6 +434,11 @@ namespace API.Controllers
return CreateXmlResult(SerializeXml(feed));
}
private static void SetFeedId(Feed feed, string id)
{
feed.Id = id;
}
[HttpGet("{apiKey}/search")]
[Produces("application/xml")]
public async Task<IActionResult> GetSearchDescriptor(string apiKey)
@ -498,6 +472,7 @@ namespace API.Controllers
var series = await _unitOfWork.SeriesRepository.GetSeriesDtoByIdAsync(seriesId, userId);
var volumes = await _unitOfWork.VolumeRepository.GetVolumesDtoAsync(seriesId, userId);
var feed = CreateFeed(series.Name + " - Volumes", $"{apiKey}/series/{series.Id}", apiKey);
SetFeedId(feed, $"series-{series.Id}");
feed.Links.Add(CreateLink(FeedLinkRelation.Image, FeedLinkType.Image, $"/api/image/series-cover?seriesId={seriesId}"));
foreach (var volumeDto in volumes)
{
@ -521,6 +496,7 @@ namespace API.Controllers
_chapterSortComparer);
var feed = CreateFeed(series.Name + " - Volume " + volume.Name + " - Chapters ", $"{apiKey}/series/{seriesId}/volume/{volumeId}", apiKey);
SetFeedId(feed, $"series-{series.Id}-volume-{volume.Id}-chapters");
foreach (var chapter in chapters)
{
feed.Entries.Add(new FeedEntry()
@ -551,6 +527,7 @@ namespace API.Controllers
var files = await _unitOfWork.ChapterRepository.GetFilesForChapterAsync(chapterId);
var feed = CreateFeed(series.Name + " - Volume " + volume.Name + " - Chapters ", $"{apiKey}/series/{seriesId}/volume/{volumeId}/chapter/{chapterId}", apiKey);
SetFeedId(feed, $"series-{series.Id}-volume-{volume.Id}-chapter-{chapter.Id}-files");
foreach (var mangaFile in files)
{
feed.Entries.Add(CreateChapter(seriesId, volumeId, chapterId, mangaFile, series, volume, chapter, apiKey));

15
API/Controllers/SeriesController.cs
View file

@ -180,7 +180,7 @@ namespace API.Controllers
if (series == null) return BadRequest("Series does not exist");
if (series.Name != updateSeries.Name && await _unitOfWork.SeriesRepository.DoesSeriesNameExistInLibrary(updateSeries.Name))
if (series.Name != updateSeries.Name && await _unitOfWork.SeriesRepository.DoesSeriesNameExistInLibrary(updateSeries.Name, series.Format))
{
return BadRequest("A series already exists in this library with this name. Series Names must be unique to a library.");
}
@ -230,12 +230,19 @@ namespace API.Controllers
return Ok(series);
}
[HttpPost("in-progress")]
public async Task<ActionResult<IEnumerable<SeriesDto>>> GetInProgress(FilterDto filterDto, [FromQuery] UserParams userParams, [FromQuery] int libraryId = 0)
/// <summary>
/// Fetches series that are on deck aka have progress on them.
/// </summary>
/// <param name="filterDto"></param>
/// <param name="userParams"></param>
/// <param name="libraryId">Default of 0 meaning all libraries</param>
/// <returns></returns>
[HttpPost("on-deck")]
public async Task<ActionResult<IEnumerable<SeriesDto>>> GetOnDeck(FilterDto filterDto, [FromQuery] UserParams userParams, [FromQuery] int libraryId = 0)
{
// NOTE: This has to be done manually like this due to the DistinctBy requirement
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
var results = await _unitOfWork.SeriesRepository.GetInProgress(userId, libraryId, userParams, filterDto);
var results = await _unitOfWork.SeriesRepository.GetOnDeck(userId, libraryId, userParams, filterDto);
var listResults = results.DistinctBy(s => s.Name).Skip((userParams.PageNumber - 1) * userParams.PageSize)
.Take(userParams.PageSize).ToList();

8
API/Controllers/ServerController.cs
View file

@ -26,10 +26,11 @@ namespace API.Controllers
private readonly IArchiveService _archiveService;
private readonly ICacheService _cacheService;
private readonly IVersionUpdaterService _versionUpdaterService;
private readonly IStatsService _statsService;
public ServerController(IHostApplicationLifetime applicationLifetime, ILogger<ServerController> logger, IConfiguration config,
IBackupService backupService, IArchiveService archiveService, ICacheService cacheService,
IVersionUpdaterService versionUpdaterService)
IVersionUpdaterService versionUpdaterService, IStatsService statsService)
{
_applicationLifetime = applicationLifetime;
_logger = logger;
@ -38,6 +39,7 @@ namespace API.Controllers
_archiveService = archiveService;
_cacheService = cacheService;
_versionUpdaterService = versionUpdaterService;
_statsService = statsService;
}
/// <summary>
@ -84,9 +86,9 @@ namespace API.Controllers
/// </summary>
/// <returns></returns>
[HttpGet("server-info")]
public ActionResult<ServerInfoDto> GetVersion()
public async Task<ActionResult<ServerInfoDto>> GetVersion()
{
return Ok(StatsService.GetServerInfo());
return Ok(await _statsService.GetServerInfo());
}
[HttpGet("logs")]

39
API/Controllers/StatsController.cs
View file

@ -1,39 +0,0 @@
using System;
using System.Threading.Tasks;
using API.DTOs.Stats;
using API.Interfaces.Services;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.Extensions.Logging;
namespace API.Controllers
{
public class StatsController : BaseApiController
{
private readonly ILogger<StatsController> _logger;
private readonly IStatsService _statsService;
public StatsController(ILogger<StatsController> logger, IStatsService statsService)
{
_logger = logger;
_statsService = statsService;
}
[AllowAnonymous]
[HttpPost("client-info")]
public async Task<IActionResult> AddClientInfo([FromBody] ClientInfoDto clientInfoDto)
{
try
{
await _statsService.RecordClientInfo(clientInfoDto);
return Ok();
}
catch (Exception ex)
{
_logger.LogError(ex, "Error updating the usage statistics");
throw;
}
}
}
}

7
API/DTOs/ChapterDto.cs
View file

@ -1,4 +1,5 @@
using System.Collections.Generic;
using System;
using System.Collections.Generic;
namespace API.DTOs
{
@ -45,5 +46,9 @@ namespace API.DTOs
/// Volume Id this Chapter belongs to
/// </summary>
public int VolumeId { get; init; }
/// <summary>
/// When chapter was created
/// </summary>
public DateTime Created { get; init; }
}
}

37
API/DTOs/Stats/ClientInfoDto.cs
View file

@ -1,37 +0,0 @@
using System;
namespace API.DTOs.Stats
{
public class ClientInfoDto
{
public ClientInfoDto()
{
CollectedAt = DateTime.UtcNow;
}
public string KavitaUiVersion { get; set; }
public string ScreenResolution { get; set; }
public string PlatformType { get; set; }
public DetailsVersion Browser { get; set; }
public DetailsVersion Os { get; set; }
public DateTime? CollectedAt { get; set; }
public bool UsingDarkTheme { get; set; }
public bool IsTheSameDevice(ClientInfoDto clientInfoDto)
{
return (clientInfoDto.ScreenResolution ?? string.Empty).Equals(ScreenResolution) &&
(clientInfoDto.PlatformType ?? string.Empty).Equals(PlatformType) &&
(clientInfoDto.Browser?.Name ?? string.Empty).Equals(Browser?.Name) &&
(clientInfoDto.Os?.Name ?? string.Empty).Equals(Os?.Name) &&
clientInfoDto.CollectedAt.GetValueOrDefault().ToString("yyyy-MM-dd")
.Equals(CollectedAt.GetValueOrDefault().ToString("yyyy-MM-dd"));
}
}
public class DetailsVersion
{
public string Name { get; set; }
public string Version { get; set; }
}
}

8
API/DTOs/Stats/ServerInfoDto.cs
View file

@ -2,13 +2,11 @@
{
public class ServerInfoDto
{
public string InstallId { get; set; }
public string Os { get; set; }
public string DotNetVersion { get; set; }
public string RunTimeVersion { get; set; }
public string KavitaVersion { get; set; }
public string BuildBranch { get; set; }
public string Culture { get; set; }
public bool IsDocker { get; set; }
public string DotnetVersion { get; set; }
public string KavitaVersion { get; set; }
public int NumOfCores { get; set; }
}
}

24
API/DTOs/Stats/UsageInfoDto.cs
View file

@ -1,24 +0,0 @@
using System.Collections.Generic;
using API.Entities.Enums;
namespace API.DTOs.Stats
{
public class UsageInfoDto
{
public UsageInfoDto()
{
FileTypes = new HashSet<string>();
LibraryTypesCreated = new HashSet<LibInfo>();
}
public int UsersCount { get; set; }
public IEnumerable<string> FileTypes { get; set; }
public IEnumerable<LibInfo> LibraryTypesCreated { get; set; }
}
public class LibInfo
{
public LibraryType Type { get; set; }
public int Count { get; set; }
}
}

33
API/DTOs/Stats/UsageStatisticsDto.cs
View file

@ -1,33 +0,0 @@
using System;
using System.Collections.Generic;
using System.Linq;
namespace API.DTOs.Stats
{
public class UsageStatisticsDto
{
public UsageStatisticsDto()
{
MarkAsUpdatedNow();
ClientsInfo = new List<ClientInfoDto>();
}
public string InstallId { get; set; }
public DateTime LastUpdate { get; set; }
public UsageInfoDto UsageInfo { get; set; }
public ServerInfoDto ServerInfo { get; set; }
public List<ClientInfoDto> ClientsInfo { get; set; }
public void MarkAsUpdatedNow()
{
LastUpdate = DateTime.UtcNow;
}
public void AddClientInfo(ClientInfoDto clientInfoDto)
{
if (ClientsInfo.Any(x => x.IsTheSameDevice(clientInfoDto))) return;
ClientsInfo.Add(clientInfoDto);
}
}
}

8
API/DTOs/Update/UpdateNotificationDto.cs
View file

@ -1,4 +1,6 @@
namespace API.DTOs.Update
using System;
namespace API.DTOs.Update
{
/// <summary>
/// Update Notification denoting a new release available for user to update to
@ -34,5 +36,9 @@
/// Is this a pre-release
/// </summary>
public bool IsPrerelease { get; init; }
/// <summary>
/// Date of the publish
/// </summary>
public string PublishDate { get; init; }
}
}

12
API/Data/MigrateConfigFiles.cs
View file

@ -26,8 +26,6 @@ namespace API.Data
"temp"
};
private static readonly string ConfigDirectory = Path.Join(Directory.GetCurrentDirectory(), "config");
/// <summary>
/// In v0.4.8 we moved all config files to config/ to match with how docker was setup. This will move all config files from current directory
@ -66,8 +64,8 @@ namespace API.Data
Console.WriteLine(
"Migrating files from pre-v0.4.8. All Kavita config files are now located in config/");
Console.WriteLine($"Creating {ConfigDirectory}");
DirectoryService.ExistOrCreate(ConfigDirectory);
Console.WriteLine($"Creating {DirectoryService.ConfigDirectory}");
DirectoryService.ExistOrCreate(DirectoryService.ConfigDirectory);
try
{
@ -116,13 +114,13 @@ namespace API.Data
foreach (var folderToMove in AppFolders)
{
if (new DirectoryInfo(Path.Join(ConfigDirectory, folderToMove)).Exists) continue;
if (new DirectoryInfo(Path.Join(DirectoryService.ConfigDirectory, folderToMove)).Exists) continue;
try
{
DirectoryService.CopyDirectoryToDirectory(
Path.Join(Directory.GetCurrentDirectory(), folderToMove),
Path.Join(ConfigDirectory, folderToMove));
Path.Join(DirectoryService.ConfigDirectory, folderToMove));
}
catch (Exception)
{
@ -144,7 +142,7 @@ namespace API.Data
{
try
{
fileInfo.CopyTo(Path.Join(ConfigDirectory, fileInfo.Name));
fileInfo.CopyTo(Path.Join(DirectoryService.ConfigDirectory, fileInfo.Name));
}
catch (Exception)
{

19
API/Data/Repositories/SeriesRepository.cs
View file

@ -7,6 +7,7 @@ using API.DTOs;
using API.DTOs.CollectionTags;
using API.DTOs.Filtering;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers;
using API.Interfaces.Repositories;
@ -47,16 +48,22 @@ namespace API.Data.Repositories
_context.Series.RemoveRange(series);
}
public async Task<bool> DoesSeriesNameExistInLibrary(string name)
/// <summary>
/// Returns if a series name and format exists already in a library
/// </summary>
/// <param name="name">Name of series</param>
/// <param name="format">Format of series</param>
/// <returns></returns>
public async Task<bool> DoesSeriesNameExistInLibrary(string name, MangaFormat format)
{
var libraries = _context.Series
.AsNoTracking()
.Where(x => x.Name == name)
.Where(x => x.Name.Equals(name) && x.Format == format)
.Select(s => s.LibraryId);
return await _context.Series
.AsNoTracking()
.Where(s => libraries.Contains(s.LibraryId) && s.Name == name)
.Where(s => libraries.Contains(s.LibraryId) && s.Name.Equals(name) && s.Format == format)
.CountAsync() > 1;
}
@ -312,14 +319,15 @@ namespace API.Data.Repositories
}
/// <summary>
/// Returns Series that the user has some partial progress on
/// Returns Series that the user has some partial progress on. Sorts based on activity. Sort first by User progress, but if a series
/// has been updated recently, bump it to the front.
/// </summary>
/// <param name="userId"></param>
/// <param name="libraryId">Library to restrict to, if 0, will apply to all libraries</param>
/// <param name="userParams">Pagination information</param>
/// <param name="filter">Optional (default null) filter on query</param>
/// <returns></returns>
public async Task<IEnumerable<SeriesDto>> GetInProgress(int userId, int libraryId, UserParams userParams, FilterDto filter)
public async Task<IEnumerable<SeriesDto>> GetOnDeck(int userId, int libraryId, UserParams userParams, FilterDto filter)
{
var formats = filter.GetSqlFilter();
IList<int> userLibraries;
@ -352,6 +360,7 @@ namespace API.Data.Repositories
&& s.PagesRead > 0
&& s.PagesRead < s.Series.Pages)
.OrderByDescending(s => s.LastModified)
.ThenByDescending(s => s.Series.LastModified)
.Select(s => s.Series)
.ProjectTo<SeriesDto>(_mapper.ConfigurationProvider)
.AsSplitQuery()

5
API/Data/Seed.cs
View file

@ -41,7 +41,7 @@ namespace API.Data
IList<ServerSetting> defaultSettings = new List<ServerSetting>()
{
new() {Key = ServerSettingKey.CacheDirectory, Value = DirectoryService.CacheDirectory},
new () {Key = ServerSettingKey.CacheDirectory, Value = DirectoryService.CacheDirectory},
new () {Key = ServerSettingKey.TaskScan, Value = "daily"},
new () {Key = ServerSettingKey.LoggingLevel, Value = "Information"}, // Not used from DB, but DB is sync with appSettings.json
new () {Key = ServerSettingKey.TaskBackup, Value = "weekly"},
@ -51,6 +51,7 @@ namespace API.Data
new () {Key = ServerSettingKey.EnableOpds, Value = "false"},
new () {Key = ServerSettingKey.EnableAuthentication, Value = "true"},
new () {Key = ServerSettingKey.BaseUrl, Value = "/"},
new () {Key = ServerSettingKey.InstallId, Value = HashUtil.AnonymousToken()},
};
foreach (var defaultSetting in defaultSettings)
@ -71,6 +72,8 @@ namespace API.Data
Configuration.LogLevel + string.Empty;
context.ServerSetting.First(s => s.Key == ServerSettingKey.CacheDirectory).Value =
DirectoryService.CacheDirectory + string.Empty;
context.ServerSetting.First(s => s.Key == ServerSettingKey.BackupDirectory).Value =
DirectoryService.BackupDirectory + string.Empty;
await context.SaveChangesAsync();

15
API/Entities/AppUserPreferences.cs
View file

@ -16,7 +16,7 @@ namespace API.Entities
/// <summary>
/// Manga Reader Option: Which side of a split image should we show first
/// </summary>
public PageSplitOption PageSplitOption { get; set; } = PageSplitOption.SplitRightToLeft;
public PageSplitOption PageSplitOption { get; set; } = PageSplitOption.FitSplit;
/// <summary>
/// Manga Reader Option: How the manga reader should perform paging or reading of the file
/// <example>
@ -25,14 +25,15 @@ namespace API.Entities
/// </example>
/// </summary>
public ReaderMode ReaderMode { get; set; }
/// <summary>
/// Manga Reader Option: Allow the menu to close after 6 seconds without interaction
/// </summary>
public bool AutoCloseMenu { get; set; }
public bool AutoCloseMenu { get; set; } = true;
/// <summary>
/// Book Reader Option: Should the background color be dark
/// </summary>
public bool BookReaderDarkMode { get; set; } = false;
public bool BookReaderDarkMode { get; set; } = true;
/// <summary>
/// Book Reader Option: Override extra Margin
/// </summary>
@ -62,10 +63,10 @@ namespace API.Entities
/// UI Site Global Setting: Whether the UI should render in Dark mode or not.
/// </summary>
public bool SiteDarkMode { get; set; } = true;
public AppUser AppUser { get; set; }
public int AppUserId { get; set; }
}
}
}

5
API/Entities/Enums/PageSplitOption.cs
View file

@ -4,6 +4,7 @@
{
SplitLeftToRight = 0,
SplitRightToLeft = 1,
NoSplit = 2
NoSplit = 2,
FitSplit = 3
}
}
}

37
API/Entities/Enums/ServerSettingKey.cs
View file

@ -4,26 +4,61 @@ namespace API.Entities.Enums
{
public enum ServerSettingKey
{
/// <summary>
/// Cron format for how often full library scans are performed.
/// </summary>
[Description("TaskScan")]
TaskScan = 0,
/// <summary>
/// Where files are cached. Not currently used.
/// </summary>
[Description("CacheDirectory")]
CacheDirectory = 1,
/// <summary>
/// Cron format for how often backups are taken.
/// </summary>
[Description("TaskBackup")]
TaskBackup = 2,
/// <summary>
/// Logging level for Server. Not managed in DB. Managed in appsettings.json and synced to DB.
/// </summary>
[Description("LoggingLevel")]
LoggingLevel = 3,
/// <summary>
/// Port server listens on. Not managed in DB. Managed in appsettings.json and synced to DB.
/// </summary>
[Description("Port")]
Port = 4,
/// <summary>
/// Where the backups are stored.
/// </summary>
[Description("BackupDirectory")]
BackupDirectory = 5,
/// <summary>
/// Allow anonymous data to be reported to KavitaStats
/// </summary>
[Description("AllowStatCollection")]
AllowStatCollection = 6,
/// <summary>
/// Is OPDS enabled for the server
/// </summary>
[Description("EnableOpds")]
EnableOpds = 7,
/// <summary>
/// Is Authentication needed for non-admin accounts
/// </summary>
[Description("EnableAuthentication")]
EnableAuthentication = 8,
/// <summary>
/// Base Url for the server. Not Implemented.
/// </summary>
[Description("BaseUrl")]
BaseUrl = 9
BaseUrl = 9,
/// <summary>
/// Represents this installation of Kavita. Is tied to Stat reporting but has no information about user or files.
/// </summary>
[Description("InstallId")]
InstallId = 10
}
}

2
API/Entities/Series.cs
View file

@ -23,7 +23,7 @@ namespace API.Entities
/// </summary>
public string SortName { get; set; }
/// <summary>
/// Name in Japanese. By default, will be same as Name.
/// Name in original language (Japanese for Manga). By default, will be same as Name.
/// </summary>
public string LocalizedName { get; set; }
/// <summary>

2
API/Interfaces/ITaskScheduler.cs
View file

@ -17,6 +17,6 @@ namespace API.Interfaces
void RefreshSeriesMetadata(int libraryId, int seriesId, bool forceUpdate = false);
void ScanSeries(int libraryId, int seriesId, bool forceUpdate = false);
void CancelStatsTasks();
void RunStatCollection();
Task RunStatCollection();
}
}

5
API/Interfaces/Repositories/ISeriesRepository.cs
View file

@ -4,6 +4,7 @@ using API.Data.Scanner;
using API.DTOs;
using API.DTOs.Filtering;
using API.Entities;
using API.Entities.Enums;
using API.Helpers;
namespace API.Interfaces.Repositories
@ -14,7 +15,7 @@ namespace API.Interfaces.Repositories
void Update(Series series);
void Remove(Series series);
void Remove(IEnumerable<Series> series);
Task<bool> DoesSeriesNameExistInLibrary(string name);
Task<bool> DoesSeriesNameExistInLibrary(string name, MangaFormat format);
/// <summary>
/// Adds user information like progress, ratings, etc
/// </summary>
@ -45,7 +46,7 @@ namespace API.Interfaces.Repositories
/// <returns></returns>
Task AddSeriesModifiers(int userId, List<SeriesDto> series);
Task<string> GetSeriesCoverImageAsync(int seriesId);
Task<IEnumerable<SeriesDto>> GetInProgress(int userId, int libraryId, UserParams userParams, FilterDto filter);
Task<IEnumerable<SeriesDto>> GetOnDeck(int userId, int libraryId, UserParams userParams, FilterDto filter);
Task<PagedList<SeriesDto>> GetRecentlyAdded(int libraryId, int userId, UserParams userParams, FilterDto filter); // NOTE: Probably put this in LibraryRepo
Task<SeriesMetadataDto> GetSeriesMetadata(int seriesId);
Task<PagedList<SeriesDto>> GetSeriesDtoForCollectionAsync(int collectionId, int userId, UserParams userParams);

2
API/Interfaces/Services/IStatsService.cs
View file

@ -5,7 +5,7 @@ namespace API.Interfaces.Services
{
public interface IStatsService
{
Task RecordClientInfo(ClientInfoDto clientInfoDto);
Task Send();
Task<ServerInfoDto> GetServerInfo();
}
}

22
API/Parser/Parser.cs
View file

@ -258,19 +258,19 @@ namespace API.Parser
MatchOptions, RegexTimeout),
// Teen Titans v1 001 (1966-02) (digital) (OkC.O.M.P.U.T.O.-Novus)
new Regex(
@"^(?<Series>.*)(?: |_)v\d+",
@"^(?<Series>.+?)(?: |_)v\d+",
MatchOptions, RegexTimeout),
// Amazing Man Comics chapter 25
new Regex(
@"^(?<Series>.*)(?: |_)c(hapter) \d+",
@"^(?<Series>.+?)(?: |_)c(hapter) \d+",
MatchOptions, RegexTimeout),
// Amazing Man Comics issue #25
new Regex(
@"^(?<Series>.*)(?: |_)i(ssue) #\d+",
@"^(?<Series>.+?)(?: |_)i(ssue) #\d+",
MatchOptions, RegexTimeout),
// Batman Wayne Family Adventures - Ep. 001 - Moving In
new Regex(
@"^(?<Series>.+?)(\s|_|-)?(?:Ep\.?)(\s|_|-)+\d+",
@"^(?<Series>.+?)(\s|_|-)(?:Ep\.?)(\s|_|-)+\d+",
MatchOptions, RegexTimeout),
// Batgirl Vol.2000 #57 (December, 2004)
new Regex(
@ -286,7 +286,7 @@ namespace API.Parser
MatchOptions, RegexTimeout),
// Scott Pilgrim 02 - Scott Pilgrim vs. The World (2005)
new Regex(
@"^(?<Series>.*)(?: |_)(?<Volume>\d+)",
@"^(?<Series>.+?)(?: |_)(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// The First Asterix Frieze (WebP by Doc MaKS)
new Regex(
@ -336,9 +336,13 @@ namespace API.Parser
new Regex(
@"^(?<Series>.+?)(?:\s|_)#(?<Chapter>\d+)",
MatchOptions, RegexTimeout),
// Batman 2016 - Chapter 01, Batman 2016 - Issue 01, Batman 2016 - Issue #01
new Regex(
@"^(?<Series>.+?)((c(hapter)?)|issue)(_|\s)#?(?<Chapter>(\d+(\.\d)?)-?(\d+(\.\d)?)?)",
MatchOptions, RegexTimeout),
// Invincible 070.5 - Invincible Returns 1 (2010) (digital) (Minutemen-InnerDemons).cbr
new Regex(
@"^(?<Series>.+?)(?: |_)(c? ?)(?<Chapter>(\d+(\.\d)?)-?(\d+(\.\d)?)?)(c? ?)-",
@"^(?<Series>.+?)(?:\s|_)(c? ?(chapter)?)(?<Chapter>(\d+(\.\d)?)-?(\d+(\.\d)?)?)(c? ?)-",
MatchOptions, RegexTimeout),
// Batgirl Vol.2000 #57 (December, 2004)
new Regex(
@ -883,7 +887,7 @@ namespace API.Parser
{
if (match.Success)
{
title = title.Replace(match.Value, "").Trim();
title = title.Replace(match.Value, string.Empty).Trim();
}
}
}
@ -900,7 +904,7 @@ namespace API.Parser
{
if (match.Success)
{
title = title.Replace(match.Value, "").Trim();
title = title.Replace(match.Value, string.Empty).Trim();
}
}
}
@ -946,7 +950,7 @@ namespace API.Parser
{
if (match.Success)
{
title = title.Replace(match.Value, "");
title = title.Replace(match.Value, string.Empty);
}
}
}

18
API/Services/ArchiveService.cs
View file

@ -138,6 +138,22 @@ namespace API.Services
if (!string.IsNullOrEmpty(nonNestedFile)) return nonNestedFile;
// Check the first folder and sort within that to see if we can find a file, else fallback to first file with basic sort.
// Get first folder, then sort within that
var firstDirectoryFile = fullNames.OrderBy(Path.GetDirectoryName, new NaturalSortComparer()).FirstOrDefault();
if (!string.IsNullOrEmpty(firstDirectoryFile))
{
var firstDirectory = Path.GetDirectoryName(firstDirectoryFile);
if (!string.IsNullOrEmpty(firstDirectory))
{
var firstDirectoryResult = fullNames.Where(f => firstDirectory.Equals(Path.GetDirectoryName(f)))
.OrderBy(Path.GetFileName, new NaturalSortComparer())
.FirstOrDefault();
if (!string.IsNullOrEmpty(firstDirectoryResult)) return firstDirectoryResult;
}
}
var result = fullNames
.OrderBy(Path.GetFileName, new NaturalSortComparer())
.FirstOrDefault();
@ -159,7 +175,7 @@ namespace API.Services
/// <returns></returns>
public string GetCoverImage(string archivePath, string fileName)
{
if (archivePath == null || !IsValidArchive(archivePath)) return String.Empty;
if (archivePath == null || !IsValidArchive(archivePath)) return string.Empty;
try
{
var libraryHandler = CanOpen(archivePath);

12
API/Services/BookService.cs
View file

@ -30,6 +30,7 @@ namespace API.Services
private readonly ILogger<BookService> _logger;
private readonly StylesheetParser _cssParser = new ();
private static readonly RecyclableMemoryStreamManager StreamManager = new ();
private const string CssScopeClass = ".book-content";
public BookService(ILogger<BookService> logger)
{
@ -152,22 +153,23 @@ namespace API.Services
EscapeCssImageReferences(ref stylesheetHtml, apiBase, book);
var styleContent = RemoveWhiteSpaceFromStylesheets(stylesheetHtml);
styleContent = styleContent.Replace("body", ".reading-section");
styleContent = styleContent.Replace("body", CssScopeClass);
if (string.IsNullOrEmpty(styleContent)) return string.Empty;
var stylesheet = await _cssParser.ParseAsync(styleContent);
foreach (var styleRule in stylesheet.StyleRules)
{
if (styleRule.Selector.Text == ".reading-section") continue;
if (styleRule.Selector.Text == CssScopeClass) continue;
if (styleRule.Selector.Text.Contains(","))
{
styleRule.Text = styleRule.Text.Replace(styleRule.SelectorText,
string.Join(", ",
styleRule.Selector.Text.Split(",").Select(s => ".reading-section " + s)));
styleRule.Selector.Text.Split(",").Select(s => $"{CssScopeClass} " + s)));
continue;
}
styleRule.Text = ".reading-section " + styleRule.Text;
styleRule.Text = $"{CssScopeClass} " + styleRule.Text;
}
return RemoveWhiteSpaceFromStylesheets(stylesheet.ToCss());
}
@ -371,7 +373,7 @@ namespace API.Services
FullFilePath = filePath,
IsSpecial = false,
Series = series.Trim(),
Volumes = seriesIndex.Split(".")[0]
Volumes = seriesIndex
};
}
}

23
API/Services/DirectoryService.cs
View file

@ -21,7 +21,7 @@ namespace API.Services
public static readonly string CacheDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "cache");
public static readonly string CoverImageDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "covers");
public static readonly string BackupDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "backups");
public static readonly string StatsDirectory = Path.Join(Directory.GetCurrentDirectory(), "config", "stats");
public static readonly string ConfigDirectory = Path.Join(Directory.GetCurrentDirectory(), "config");
public DirectoryService(ILogger<DirectoryService> logger)
{
@ -173,7 +173,15 @@ namespace API.Services
return true;
}
/// <summary>
/// Checks if the root path of a path exists or not.
/// </summary>
/// <param name="path"></param>
/// <returns></returns>
public static bool IsDriveMounted(string path)
{
return new DirectoryInfo(Path.GetPathRoot(path) ?? string.Empty).Exists;
}
public static string[] GetFilesWithExtension(string path, string searchPatternExpression = "")
{
@ -257,7 +265,7 @@ namespace API.Services
/// <param name="directoryPath"></param>
/// <param name="prepend">An optional string to prepend to the target file's name</param>
/// <returns></returns>
public bool CopyFilesToDirectory(IEnumerable<string> filePaths, string directoryPath, string prepend = "")
public static bool CopyFilesToDirectory(IEnumerable<string> filePaths, string directoryPath, string prepend = "", ILogger logger = null)
{
ExistOrCreate(directoryPath);
string currentFile = null;
@ -273,19 +281,24 @@ namespace API.Services
}
else
{
_logger.LogWarning("Tried to copy {File} but it doesn't exist", file);
logger?.LogWarning("Tried to copy {File} but it doesn't exist", file);
}
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Unable to copy {File} to {DirectoryPath}", currentFile, directoryPath);
logger?.LogError(ex, "Unable to copy {File} to {DirectoryPath}", currentFile, directoryPath);
return false;
}
return true;
}
public bool CopyFilesToDirectory(IEnumerable<string> filePaths, string directoryPath, string prepend = "")
{
return CopyFilesToDirectory(filePaths, directoryPath, prepend, _logger);
}
public IEnumerable<string> ListDirectory(string rootPath)
{
if (!Directory.Exists(rootPath)) return ImmutableList<string>.Empty;

2
API/Services/HostedServices/StartupTasksHostedService.cs
View file

@ -29,7 +29,7 @@ namespace API.Services.HostedServices
// These methods will automatically check if stat collection is disabled to prevent sending any data regardless
// of when setting was changed
await taskScheduler.ScheduleStatsTasks();
taskScheduler.RunStatCollection();
await taskScheduler.RunStatCollection();
}
catch (Exception)
{

13
API/Services/MetadataService.cs
View file

@ -218,14 +218,18 @@ namespace API.Services
var stopwatch = Stopwatch.StartNew();
var totalTime = 0L;
_logger.LogInformation("[MetadataService] Refreshing Library {LibraryName}. Total Items: {TotalSize}. Total Chunks: {TotalChunks} with {ChunkSize} size", library.Name, chunkInfo.TotalSize, chunkInfo.TotalChunks, chunkInfo.ChunkSize);
await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress,
MessageFactory.RefreshMetadataProgressEvent(library.Id, 0F));
for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++)
var i = 0;
for (var chunk = 1; chunk <= chunkInfo.TotalChunks; chunk++, i++)
{
if (chunkInfo.TotalChunks == 0) continue;
totalTime += stopwatch.ElapsedMilliseconds;
stopwatch.Restart();
_logger.LogInformation("[MetadataService] Processing chunk {ChunkNumber} / {TotalChunks} with size {ChunkSize}. Series ({SeriesStart} - {SeriesEnd}",
chunk, chunkInfo.TotalChunks, chunkInfo.ChunkSize, chunk * chunkInfo.ChunkSize, (chunk + 1) * chunkInfo.ChunkSize);
var nonLibrarySeries = await _unitOfWork.SeriesRepository.GetFullSeriesForLibraryIdAsync(library.Id,
new UserParams()
{
@ -233,6 +237,7 @@ namespace API.Services
PageSize = chunkInfo.ChunkSize
});
_logger.LogDebug("[MetadataService] Fetched {SeriesCount} series for refresh", nonLibrarySeries.Count);
Parallel.ForEach(nonLibrarySeries, series =>
{
try
@ -275,8 +280,14 @@ namespace API.Services
"[MetadataService] Processed {SeriesStart} - {SeriesEnd} out of {TotalSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}",
chunk * chunkInfo.ChunkSize, (chunk * chunkInfo.ChunkSize) + nonLibrarySeries.Count, chunkInfo.TotalSize, stopwatch.ElapsedMilliseconds, library.Name);
}
var progress = Math.Max(0F, Math.Min(1F, i * 1F / chunkInfo.TotalChunks));
await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress,
MessageFactory.RefreshMetadataProgressEvent(library.Id, progress));
}
await _messageHub.Clients.All.SendAsync(SignalREvents.RefreshMetadataProgress,
MessageFactory.RefreshMetadataProgressEvent(library.Id, 1F));
_logger.LogInformation("[MetadataService] Updated metadata for {SeriesNumber} series in library {LibraryName} in {ElapsedMilliseconds} milliseconds total", chunkInfo.TotalSize, library.Name, totalTime);
}

23
API/Services/TaskScheduler.cs
View file

@ -23,9 +23,9 @@ namespace API.Services
private readonly IStatsService _statsService;
private readonly IVersionUpdaterService _versionUpdaterService;
private const string SendDataTask = "finalize-stats";
public static BackgroundJobServer Client => new BackgroundJobServer();
private static readonly Random Rnd = new Random();
public TaskScheduler(ICacheService cacheService, ILogger<TaskScheduler> logger, IScannerService scannerService,
@ -73,7 +73,6 @@ namespace API.Services
RecurringJob.AddOrUpdate("cleanup", () => _cleanupService.Cleanup(), Cron.Daily, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate("check-for-updates", () => _scannerService.ScanLibraries(), Cron.Daily, TimeZoneInfo.Local);
}
#region StatsTasks
@ -89,19 +88,27 @@ namespace API.Services
}
_logger.LogDebug("Scheduling stat collection daily");
RecurringJob.AddOrUpdate(SendDataTask, () => _statsService.Send(), Cron.Daily, TimeZoneInfo.Local);
RecurringJob.AddOrUpdate("report-stats", () => _statsService.Send(), Cron.Daily(Rnd.Next(0, 22)), TimeZoneInfo.Local);
}
public void CancelStatsTasks()
{
_logger.LogDebug("Cancelling/Removing StatsTasks");
RecurringJob.RemoveIfExists(SendDataTask);
RecurringJob.RemoveIfExists("report-stats");
}
public void RunStatCollection()
/// <summary>
/// First time run stat collection. Executes immediately on a background thread. Does not block.
/// </summary>
public async Task RunStatCollection()
{
_logger.LogInformation("Enqueuing stat collection");
var allowStatCollection = (await _unitOfWork.SettingsRepository.GetSettingsDtoAsync()).AllowStatCollection;
if (!allowStatCollection)
{
_logger.LogDebug("User has opted out of stat collection, not sending stats");
return;
}
BackgroundJob.Enqueue(() => _statsService.Send());
}
@ -112,8 +119,8 @@ namespace API.Services
public void ScheduleUpdaterTasks()
{
_logger.LogInformation("Scheduling Auto-Update tasks");
RecurringJob.AddOrUpdate("check-updates", () => CheckForUpdate(), Cron.Weekly, TimeZoneInfo.Local);
// Schedule update check between noon and 6pm local time
RecurringJob.AddOrUpdate("check-updates", () => _versionUpdaterService.CheckForUpdate(), Cron.Daily(Rnd.Next(12, 18)), TimeZoneInfo.Local);
}
#endregion

65
API/Services/Tasks/BackupService.cs
View file

@ -8,8 +8,9 @@ using API.Entities.Enums;
using API.Extensions;
using API.Interfaces;
using API.Interfaces.Services;
using API.SignalR;
using Hangfire;
using Kavita.Common.EnvironmentInfo;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.Logging;
@ -20,45 +21,32 @@ namespace API.Services.Tasks
private readonly IUnitOfWork _unitOfWork;
private readonly ILogger<BackupService> _logger;
private readonly IDirectoryService _directoryService;
private readonly string _tempDirectory = DirectoryService.TempDirectory;
private readonly string _logDirectory = DirectoryService.LogDirectory;
private readonly IHubContext<MessageHub> _messageHub;
private readonly IList<string> _backupFiles;
public BackupService(IUnitOfWork unitOfWork, ILogger<BackupService> logger, IDirectoryService directoryService, IConfiguration config)
public BackupService(IUnitOfWork unitOfWork, ILogger<BackupService> logger,
IDirectoryService directoryService, IConfiguration config, IHubContext<MessageHub> messageHub)
{
_unitOfWork = unitOfWork;
_logger = logger;
_directoryService = directoryService;
_messageHub = messageHub;
var maxRollingFiles = config.GetMaxRollingFiles();
var loggingSection = config.GetLoggingFileName();
var files = LogFiles(maxRollingFiles, loggingSection);
if (new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker)
_backupFiles = new List<string>()
{
_backupFiles = new List<string>()
{
"data/appsettings.json",
"data/Hangfire.db",
"data/Hangfire-log.db",
"data/kavita.db",
"data/kavita.db-shm", // This wont always be there
"data/kavita.db-wal" // This wont always be there
};
}
else
{
_backupFiles = new List<string>()
{
"appsettings.json",
"Hangfire.db",
"Hangfire-log.db",
"kavita.db",
"kavita.db-shm", // This wont always be there
"kavita.db-wal" // This wont always be there
};
}
"appsettings.json",
"Hangfire.db", // This is not used atm
"Hangfire-log.db", // This is not used atm
"kavita.db",
"kavita.db-shm", // This wont always be there
"kavita.db-wal" // This wont always be there
};
foreach (var file in files.Select(f => (new FileInfo(f)).Name).ToList())
{
@ -72,7 +60,7 @@ namespace API.Services.Tasks
var fi = new FileInfo(logFileName);
var files = maxRollingFiles > 0
? DirectoryService.GetFiles(_logDirectory, $@"{Path.GetFileNameWithoutExtension(fi.Name)}{multipleFileRegex}\.log")
? DirectoryService.GetFiles(DirectoryService.LogDirectory, $@"{Path.GetFileNameWithoutExtension(fi.Name)}{multipleFileRegex}\.log")
: new[] {"kavita.log"};
return files;
}
@ -89,11 +77,13 @@ namespace API.Services.Tasks
_logger.LogDebug("Backing up to {BackupDirectory}", backupDirectory);
if (!DirectoryService.ExistOrCreate(backupDirectory))
{
_logger.LogError("Could not write to {BackupDirectory}; aborting backup", backupDirectory);
_logger.LogCritical("Could not write to {BackupDirectory}; aborting backup", backupDirectory);
return;
}
var dateString = DateTime.Now.ToShortDateString().Replace("/", "_");
await SendProgress(0F);
var dateString = $"{DateTime.Now.ToShortDateString()}_{DateTime.Now.ToLongTimeString()}".Replace("/", "_").Replace(":", "_");
var zipPath = Path.Join(backupDirectory, $"kavita_backup_{dateString}.zip");
if (File.Exists(zipPath))
@ -102,15 +92,19 @@ namespace API.Services.Tasks
return;
}
var tempDirectory = Path.Join(_tempDirectory, dateString);
var tempDirectory = Path.Join(DirectoryService.TempDirectory, dateString);
DirectoryService.ExistOrCreate(tempDirectory);
DirectoryService.ClearDirectory(tempDirectory);
_directoryService.CopyFilesToDirectory(
_backupFiles.Select(file => Path.Join(Directory.GetCurrentDirectory(), file)).ToList(), tempDirectory);
_backupFiles.Select(file => Path.Join(DirectoryService.ConfigDirectory, file)).ToList(), tempDirectory);
await SendProgress(0.25F);
await CopyCoverImagesToBackupDirectory(tempDirectory);
await SendProgress(0.75F);
try
{
ZipFile.CreateFromDirectory(tempDirectory, zipPath);
@ -122,6 +116,7 @@ namespace API.Services.Tasks
DirectoryService.ClearAndDeleteDirectory(tempDirectory);
_logger.LogInformation("Database backup completed");
await SendProgress(1F);
}
private async Task CopyCoverImagesToBackupDirectory(string tempDirectory)
@ -154,6 +149,12 @@ namespace API.Services.Tasks
}
}
private async Task SendProgress(float progress)
{
await _messageHub.Clients.All.SendAsync(SignalREvents.BackupDatabaseProgress,
MessageFactory.BackupDatabaseProgressEvent(progress));
}
/// <summary>
/// Removes Database backups older than 30 days. If all backups are older than 30 days, the latest is kept.
/// </summary>

22
API/Services/Tasks/CleanupService.cs
View file

@ -2,7 +2,9 @@
using System.Threading.Tasks;
using API.Interfaces;
using API.Interfaces.Services;
using API.SignalR;
using Hangfire;
using Microsoft.AspNetCore.SignalR;
using Microsoft.Extensions.Logging;
namespace API.Services.Tasks
@ -16,14 +18,16 @@ namespace API.Services.Tasks
private readonly ILogger<CleanupService> _logger;
private readonly IBackupService _backupService;
private readonly IUnitOfWork _unitOfWork;
private readonly IHubContext<MessageHub> _messageHub;
public CleanupService(ICacheService cacheService, ILogger<CleanupService> logger,
IBackupService backupService, IUnitOfWork unitOfWork)
IBackupService backupService, IUnitOfWork unitOfWork, IHubContext<MessageHub> messageHub)
{
_cacheService = cacheService;
_logger = logger;
_backupService = backupService;
_unitOfWork = unitOfWork;
_messageHub = messageHub;
}
public void CleanupCacheDirectory()
@ -39,19 +43,31 @@ namespace API.Services.Tasks
public async Task Cleanup()
{
_logger.LogInformation("Starting Cleanup");
await SendProgress(0F);
_logger.LogInformation("Cleaning temp directory");
var tempDirectory = DirectoryService.TempDirectory;
DirectoryService.ClearDirectory(tempDirectory);
DirectoryService.ClearDirectory(DirectoryService.TempDirectory);
await SendProgress(0.1F);
CleanupCacheDirectory();
await SendProgress(0.25F);
_logger.LogInformation("Cleaning old database backups");
_backupService.CleanupBackups();
await SendProgress(0.50F);
_logger.LogInformation("Cleaning deleted cover images");
await DeleteSeriesCoverImages();
await SendProgress(0.6F);
await DeleteChapterCoverImages();
await SendProgress(0.7F);
await DeleteTagCoverImages();
await SendProgress(1F);
_logger.LogInformation("Cleanup finished");
}
private async Task SendProgress(float progress)
{
await _messageHub.Clients.All.SendAsync(SignalREvents.CleanupProgress,
MessageFactory.CleanupProgressEvent(progress));
}
private async Task DeleteSeriesCoverImages()
{
var images = await _unitOfWork.SeriesRepository.GetAllCoverImagesAsync();

94
API/Services/Tasks/ScannerService.cs
View file

@ -56,6 +56,14 @@ namespace API.Services.Tasks
var chapterIds = await _unitOfWork.SeriesRepository.GetChapterIdsForSeriesAsync(new[] {seriesId});
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.Folders);
var folderPaths = library.Folders.Select(f => f.Path).ToList();
// Check if any of the folder roots are not available (ie disconnected from network, etc) and fail if any of them are
if (folderPaths.Any(f => !DirectoryService.IsDriveMounted(f)))
{
_logger.LogError("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted");
return;
}
var dirs = DirectoryService.FindHighestDirectoriesFromFiles(folderPaths, files.Select(f => f.FilePath).ToList());
_logger.LogInformation("Beginning file scan on {SeriesName}", series.Name);
@ -129,8 +137,7 @@ namespace API.Services.Tasks
await _unitOfWork.RollbackAsync();
}
// Tell UI that this series is done
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanSeries, MessageFactory.ScanSeriesEvent(seriesId, series.Name),
cancellationToken: token);
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanSeries, MessageFactory.ScanSeriesEvent(seriesId, series.Name), token);
await CleanupDbEntities();
BackgroundJob.Enqueue(() => _cacheService.CleanupChapters(chapterIds));
BackgroundJob.Enqueue(() => _metadataService.RefreshMetadataForSeries(libraryId, series.Id, false));
@ -195,6 +202,14 @@ namespace API.Services.Tasks
return;
}
// Check if any of the folder roots are not available (ie disconnected from network, etc) and fail if any of them are
if (library.Folders.Any(f => !DirectoryService.IsDriveMounted(f.Path)))
{
_logger.LogError("Some of the root folders for library are not accessible. Please check that drives are connected and rescan. Scan will be aborted");
return;
}
_logger.LogInformation("[ScannerService] Beginning file scan on {LibraryName}", library.Name);
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(libraryId, 0));
@ -228,7 +243,7 @@ namespace API.Services.Tasks
BackgroundJob.Enqueue(() => _metadataService.RefreshMetadata(libraryId, false));
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(libraryId, 100));
MessageFactory.ScanLibraryProgressEvent(libraryId, 1F));
}
/// <summary>
@ -326,7 +341,7 @@ namespace API.Services.Tasks
await _messageHub.Clients.All.SendAsync(SignalREvents.SeriesRemoved, MessageFactory.SeriesRemovedEvent(missing.Id, missing.Name, library.Id));
}
var progress = Math.Max(0, Math.Min(100, ((chunk + 1F) * chunkInfo.ChunkSize) / chunkInfo.TotalSize));
var progress = Math.Max(0, Math.Min(1, ((chunk + 1F) * chunkInfo.ChunkSize) / chunkInfo.TotalSize));
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(library.Id, progress));
}
@ -343,15 +358,14 @@ namespace API.Services.Tasks
// Key is normalized already
Series existingSeries;
try
{// NOTE: Maybe use .Equals() here
existingSeries = allSeries.SingleOrDefault(s =>
(s.NormalizedName == key.NormalizedName || Parser.Parser.Normalize(s.OriginalName) == key.NormalizedName)
&& (s.Format == key.Format || s.Format == MangaFormat.Unknown));
{
existingSeries = allSeries.SingleOrDefault(s => FindSeries(s, key));
}
catch (Exception e)
{
// NOTE: If I ever want to put Duplicates table, this is where it can go
_logger.LogCritical(e, "[ScannerService] There are multiple series that map to normalized key {Key}. You can manually delete the entity via UI and rescan to fix it. This will be skipped", key.NormalizedName);
var duplicateSeries = allSeries.Where(s => s.NormalizedName == key.NormalizedName || Parser.Parser.Normalize(s.OriginalName) == key.NormalizedName).ToList();
var duplicateSeries = allSeries.Where(s => FindSeries(s, key));
foreach (var series in duplicateSeries)
{
_logger.LogCritical("[ScannerService] Duplicate Series Found: {Key} maps with {Series}", key.Name, series.OriginalName);
@ -362,46 +376,38 @@ namespace API.Services.Tasks
if (existingSeries != null) continue;
existingSeries = DbFactory.Series(infos[0].Series);
existingSeries.Format = key.Format;
newSeries.Add(existingSeries);
var s = DbFactory.Series(infos[0].Series);
s.Format = key.Format;
s.LibraryId = library.Id; // We have to manually set this since we aren't adding the series to the Library's series.
newSeries.Add(s);
}
var i = 0;
foreach(var series in newSeries)
{
_logger.LogDebug("[ScannerService] Processing series {SeriesName}", series.OriginalName);
UpdateSeries(series, parsedSeries);
_unitOfWork.SeriesRepository.Attach(series);
try
{
_logger.LogDebug("[ScannerService] Processing series {SeriesName}", series.OriginalName);
UpdateVolumes(series, ParseScannedFiles.GetInfosByName(parsedSeries, series).ToArray());
series.Pages = series.Volumes.Sum(v => v.Pages);
series.LibraryId = library.Id; // We have to manually set this since we aren't adding the series to the Library's series.
_unitOfWork.SeriesRepository.Attach(series);
if (await _unitOfWork.CommitAsync())
{
_logger.LogInformation(
"[ScannerService] Added {NewSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}",
newSeries.Count, stopwatch.ElapsedMilliseconds, library.Name);
await _unitOfWork.CommitAsync();
_logger.LogInformation(
"[ScannerService] Added {NewSeries} series in {ElapsedScanTime} milliseconds for {LibraryName}",
newSeries.Count, stopwatch.ElapsedMilliseconds, library.Name);
// Inform UI of new series added
await _messageHub.Clients.All.SendAsync(SignalREvents.SeriesAdded, MessageFactory.SeriesAddedEvent(series.Id, series.Name, library.Id));
var progress = Math.Max(0F, Math.Min(100F, i * 1F / newSeries.Count));
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(library.Id, progress));
}
else
{
// This is probably not needed. Better to catch the exception.
_logger.LogCritical(
"[ScannerService] There was a critical error that resulted in a failed scan. Please check logs and rescan");
}
i++;
// Inform UI of new series added
await _messageHub.Clients.All.SendAsync(SignalREvents.SeriesAdded, MessageFactory.SeriesAddedEvent(series.Id, series.Name, library.Id));
}
catch (Exception ex)
{
_logger.LogError(ex, "[ScannerService] There was an exception updating volumes for {SeriesName}", series.Name);
_logger.LogCritical(ex, "[ScannerService] There was a critical exception adding new series entry for {SeriesName} with a duplicate index key: {IndexKey} ",
series.Name, $"{series.Name}_{series.NormalizedName}_{series.LocalizedName}_{series.LibraryId}_{series.Format}");
}
var progress = Math.Max(0F, Math.Min(1F, i * 1F / newSeries.Count));
await _messageHub.Clients.All.SendAsync(SignalREvents.ScanLibraryProgress,
MessageFactory.ScanLibraryProgressEvent(library.Id, progress));
i++;
}
_logger.LogInformation(
@ -409,13 +415,19 @@ namespace API.Services.Tasks
newSeries.Count, stopwatch.ElapsedMilliseconds, library.Name);
}
private static bool FindSeries(Series series, ParsedSeries parsedInfoKey)
{
return (series.NormalizedName.Equals(parsedInfoKey.NormalizedName) || Parser.Parser.Normalize(series.OriginalName).Equals(parsedInfoKey.NormalizedName))
&& (series.Format == parsedInfoKey.Format || series.Format == MangaFormat.Unknown);
}
private void UpdateSeries(Series series, Dictionary<ParsedSeries, List<ParserInfo>> parsedSeries)
{
try
{
_logger.LogInformation("[ScannerService] Processing series {SeriesName}", series.OriginalName);
var parsedInfos = ParseScannedFiles.GetInfosByName(parsedSeries, series).ToArray();
var parsedInfos = ParseScannedFiles.GetInfosByName(parsedSeries, series);
UpdateVolumes(series, parsedInfos);
series.Pages = series.Volumes.Sum(v => v.Pages);
@ -482,7 +494,7 @@ namespace API.Services.Tasks
/// <param name="missingSeries">Series not found on disk or can't be parsed</param>
/// <param name="removeCount"></param>
/// <returns>the updated existingSeries</returns>
public static IList<Series> RemoveMissingSeries(IList<Series> existingSeries, IEnumerable<Series> missingSeries, out int removeCount)
public static IEnumerable<Series> RemoveMissingSeries(IList<Series> existingSeries, IEnumerable<Series> missingSeries, out int removeCount)
{
var existingCount = existingSeries.Count;
var missingList = missingSeries.ToList();
@ -496,7 +508,7 @@ namespace API.Services.Tasks
return existingSeries;
}
private void UpdateVolumes(Series series, ParserInfo[] parsedInfos)
private void UpdateVolumes(Series series, IList<ParserInfo> parsedInfos)
{
var startingVolumeCount = series.Volumes.Count;
// Add new volumes and update chapters per volume
@ -550,7 +562,7 @@ namespace API.Services.Tasks
/// </summary>
/// <param name="volume"></param>
/// <param name="parsedInfos"></param>
private void UpdateChapters(Volume volume, ParserInfo[] parsedInfos)
private void UpdateChapters(Volume volume, IList<ParserInfo> parsedInfos)
{
// Add new chapters
foreach (var info in parsedInfos)

157
API/Services/Tasks/StatsService.cs
View file

@ -1,46 +1,31 @@
using System;
using System.IO;
using System.Linq;
using System.Net.Http;
using System.Runtime.InteropServices;
using System.Text.Json;
using System.Threading;
using System.Threading.Tasks;
using API.Data;
using API.DTOs.Stats;
using API.Entities.Enums;
using API.Interfaces;
using API.Interfaces.Services;
using Flurl.Http;
using Hangfire;
using Kavita.Common;
using Kavita.Common.EnvironmentInfo;
using Microsoft.AspNetCore.Http;
using Microsoft.EntityFrameworkCore;
using Microsoft.Extensions.Logging;
namespace API.Services.Tasks
{
public class StatsService : IStatsService
{
private const string StatFileName = "app_stats.json";
private readonly DataContext _dbContext;
private readonly ILogger<StatsService> _logger;
private readonly IUnitOfWork _unitOfWork;
private const string ApiUrl = "https://stats.kavitareader.com";
#pragma warning disable S1075
private const string ApiUrl = "http://stats.kavitareader.com";
#pragma warning restore S1075
private static readonly string StatsFilePath = Path.Combine(DirectoryService.StatsDirectory, StatFileName);
private static bool FileExists => File.Exists(StatsFilePath);
public StatsService(DataContext dbContext, ILogger<StatsService> logger,
IUnitOfWork unitOfWork)
public StatsService(ILogger<StatsService> logger, IUnitOfWork unitOfWork)
{
_dbContext = dbContext;
_logger = logger;
_unitOfWork = unitOfWork;
FlurlHttp.ConfigureClient(ApiUrl, cli =>
cli.Settings.HttpClientFactory = new UntrustedCertClientFactory());
}
/// <summary>
@ -55,17 +40,7 @@ namespace API.Services.Tasks
return;
}
var rnd = new Random();
var offset = rnd.Next(0, 6);
if (offset == 0)
{
await SendData();
}
else
{
_logger.LogInformation("KavitaStats upload has been schedule to run in {Offset} hours", offset);
BackgroundJob.Schedule(() => SendData(), DateTimeOffset.Now.AddHours(offset));
}
await SendData();
}
/// <summary>
@ -74,66 +49,30 @@ namespace API.Services.Tasks
// ReSharper disable once MemberCanBePrivate.Global
public async Task SendData()
{
await CollectRelevantData();
await FinalizeStats();
var data = await GetServerInfo();
await SendDataToStatsServer(data);
}
public async Task RecordClientInfo(ClientInfoDto clientInfoDto)
{
var statisticsDto = await GetData();
statisticsDto.AddClientInfo(clientInfoDto);
await SaveFile(statisticsDto);
}
private async Task CollectRelevantData()
{
var usageInfo = await GetUsageInfo();
var serverInfo = GetServerInfo();
await PathData(serverInfo, usageInfo);
}
private async Task FinalizeStats()
{
try
{
var data = await GetExistingData<UsageStatisticsDto>();
var successful = await SendDataToStatsServer(data);
if (successful)
{
ResetStats();
}
}
catch (Exception ex)
{
_logger.LogError(ex, "There was an exception while sending data to KavitaStats");
}
}
private async Task<bool> SendDataToStatsServer(UsageStatisticsDto data)
private async Task SendDataToStatsServer(ServerInfoDto data)
{
var responseContent = string.Empty;
try
{
var response = await (ApiUrl + "/api/InstallationStats")
var response = await (ApiUrl + "/api/v2/stats")
.WithHeader("Accept", "application/json")
.WithHeader("User-Agent", "Kavita")
.WithHeader("x-api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("api-key", "MsnvA2DfQqxSK5jh")
.WithHeader("x-kavita-version", BuildInfo.Version)
.WithHeader("Content-Type", "application/json")
.WithTimeout(TimeSpan.FromSeconds(30))
.PostJsonAsync(data);
if (response.StatusCode != StatusCodes.Status200OK)
{
_logger.LogError("KavitaStats did not respond successfully. {Content}", response);
return false;
}
return true;
}
catch (HttpRequestException e)
{
@ -149,84 +88,22 @@ namespace API.Services.Tasks
{
_logger.LogError(e, "An error happened during the request to KavitaStats");
}
return false;
}
private static void ResetStats()
{
if (FileExists) File.Delete(StatsFilePath);
}
private async Task PathData(ServerInfoDto serverInfoDto, UsageInfoDto usageInfoDto)
{
var data = await GetData();
data.ServerInfo = serverInfoDto;
data.UsageInfo = usageInfoDto;
data.MarkAsUpdatedNow();
await SaveFile(data);
}
private static async ValueTask<UsageStatisticsDto> GetData()
{
if (!FileExists) return new UsageStatisticsDto {InstallId = HashUtil.AnonymousToken()};
return await GetExistingData<UsageStatisticsDto>();
}
private async Task<UsageInfoDto> GetUsageInfo()
{
var usersCount = await _dbContext.Users.CountAsync();
var libsCountByType = await _dbContext.Library
.AsNoTracking()
.GroupBy(x => x.Type)
.Select(x => new LibInfo {Type = x.Key, Count = x.Count()})
.ToArrayAsync();
var uniqueFileTypes = await _unitOfWork.FileRepository.GetFileExtensions();
var usageInfo = new UsageInfoDto
{
UsersCount = usersCount,
LibraryTypesCreated = libsCountByType,
FileTypes = uniqueFileTypes
};
return usageInfo;
}
public static ServerInfoDto GetServerInfo()
public async Task<ServerInfoDto> GetServerInfo()
{
var installId = await _unitOfWork.SettingsRepository.GetSettingAsync(ServerSettingKey.InstallId);
var serverInfo = new ServerInfoDto
{
InstallId = installId.Value,
Os = RuntimeInformation.OSDescription,
DotNetVersion = Environment.Version.ToString(),
RunTimeVersion = RuntimeInformation.FrameworkDescription,
KavitaVersion = BuildInfo.Version.ToString(),
Culture = Thread.CurrentThread.CurrentCulture.Name,
BuildBranch = BuildInfo.Branch,
DotnetVersion = Environment.Version.ToString(),
IsDocker = new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker,
NumOfCores = Environment.ProcessorCount
NumOfCores = Math.Max(Environment.ProcessorCount, 1)
};
return serverInfo;
}
private static async Task<T> GetExistingData<T>()
{
var json = await File.ReadAllTextAsync(StatsFilePath);
return JsonSerializer.Deserialize<T>(json);
}
private static async Task SaveFile(UsageStatisticsDto statisticsDto)
{
DirectoryService.ExistOrCreate(DirectoryService.StatsDirectory);
await File.WriteAllTextAsync(StatsFilePath, JsonSerializer.Serialize(statisticsDto));
}
}
}

8
API/Services/Tasks/VersionUpdaterService.cs
View file

@ -38,6 +38,11 @@ namespace API.Services.Tasks
/// </summary>
// ReSharper disable once InconsistentNaming
public string Html_Url { get; init; }
/// <summary>
/// Date Release was Published
/// </summary>
// ReSharper disable once InconsistentNaming
public string Published_At { get; init; }
}
public class UntrustedCertClientFactory : DefaultHttpClientFactory
@ -109,7 +114,8 @@ namespace API.Services.Tasks
UpdateBody = _markdown.Transform(update.Body.Trim()),
UpdateTitle = update.Name,
UpdateUrl = update.Html_Url,
IsDocker = new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker
IsDocker = new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker,
PublishDate = update.Published_At
};
}

39
API/SignalR/MessageFactory.cs
View file

@ -60,6 +60,20 @@ namespace API.SignalR
};
}
public static SignalRMessage RefreshMetadataProgressEvent(int libraryId, float progress)
{
return new SignalRMessage()
{
Name = SignalREvents.RefreshMetadataProgress,
Body = new
{
LibraryId = libraryId,
Progress = progress,
EventTime = DateTime.Now
}
};
}
public static SignalRMessage RefreshMetadataEvent(int libraryId, int seriesId)
@ -75,6 +89,31 @@ namespace API.SignalR
};
}
public static SignalRMessage BackupDatabaseProgressEvent(float progress)
{
return new SignalRMessage()
{
Name = SignalREvents.BackupDatabaseProgress,
Body = new
{
Progress = progress
}
};
}
public static SignalRMessage CleanupProgressEvent(float progress)
{
return new SignalRMessage()
{
Name = SignalREvents.CleanupProgress,
Body = new
{
Progress = progress
}
};
}
public static SignalRMessage UpdateVersionEvent(UpdateNotificationDto update)
{
return new SignalRMessage

15
API/SignalR/SignalREvents.cs
View file

@ -4,7 +4,14 @@
{
public const string UpdateVersion = "UpdateVersion";
public const string ScanSeries = "ScanSeries";
/// <summary>
/// Event during Refresh Metadata for cover image change
/// </summary>
public const string RefreshMetadata = "RefreshMetadata";
/// <summary>
/// Event sent out during Refresh Metadata for progress tracking
/// </summary>
public const string RefreshMetadataProgress = "RefreshMetadataProgress";
public const string ScanLibrary = "ScanLibrary";
public const string SeriesAdded = "SeriesAdded";
public const string SeriesRemoved = "SeriesRemoved";
@ -12,5 +19,13 @@
public const string OnlineUsers = "OnlineUsers";
public const string SeriesAddedToCollection = "SeriesAddedToCollection";
public const string ScanLibraryError = "ScanLibraryError";
/// <summary>
/// Event sent out during backing up the database
/// </summary>
public const string BackupDatabaseProgress = "BackupDatabaseProgress";
/// <summary>
/// Event sent out during cleaning up temp and cache folders
/// </summary>
public const string CleanupProgress = "CleanupProgress";
}
}