AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Feature/local metadata more tags (#832)

Browse source 
* Stashing code

* removed some debug code on series detail page. Now detail is collapsed by default.

* Added AgeRating

* Fixed a crash when NetVips tries to write a cover file and cover directory is not existing.

* When a card is selected for bulk actions, show an outline in addition to select box

* Added AgeRating into the metadata parsing. Added a hack where ComicInfo uses Number in ComicInfo rather than Volume. This is to test out the effects on users libraries.

* Added AgeRating and ReleaseDate to the metadata implelentation.
This commit is contained in:
Joseph Milazzo 2021-12-06 13:59:47 -06:00 • committed by GitHub
parent 46f37069db
commit af24c928d7
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
31 changed files with 2825 additions and 101 deletions
Download patch file Download diff file Expand all files Collapse all files

11
API/Controllers/SeriesController.cs
View file

@ -3,15 +3,18 @@ using System.Collections.Generic;
using System.Linq;
using System.Threading.Tasks;
using API.Data;
using API.Data.Metadata;
using API.Data.Repositories;
using API.DTOs;
using API.DTOs.Filtering;
using API.Entities;
using API.Entities.Enums;
using API.Extensions;
using API.Helpers;
using API.Services;
using API.SignalR;
using Kavita.Common;
using Kavita.Common.Extensions;
using Microsoft.AspNetCore.Authorization;
using Microsoft.AspNetCore.Mvc;
using Microsoft.AspNetCore.SignalR;
@ -392,5 +395,13 @@ namespace API.Controllers
var userId = await _unitOfWork.UserRepository.GetUserIdByUsernameAsync(User.GetUsername());
return Ok(await _unitOfWork.SeriesRepository.GetSeriesDtoForIdsAsync(dto.SeriesIds, userId));
}
[HttpGet("age-rating")]
public ActionResult<string> GetAgeRating(int ageRating)
{
var val = (AgeRating) ageRating;
return Ok(val.ToDescription());
}
}
}

6
API/DTOs/ChapterDto.cs
View file

@ -51,8 +51,14 @@ namespace API.DTOs
/// </summary>
public DateTime Created { get; init; }
/// <summary>
/// When the chapter was released.
/// </summary>
/// <remarks>Metadata field</remarks>
public DateTime ReleaseDate { get; init; }
/// <summary>
/// Title of the Chapter/Issue
/// </summary>
/// <remarks>Metadata field</remarks>
public string TitleName { get; set; }
public ICollection<PersonDto> Writers { get; set; } = new List<PersonDto>();
public ICollection<PersonDto> Penciller { get; set; } = new List<PersonDto>();

9
API/DTOs/SeriesMetadataDto.cs
View file

@ -1,6 +1,7 @@
using System.Collections.Generic;
using API.DTOs.CollectionTags;
using API.DTOs.Metadata;
using API.Entities.Enums;
namespace API.DTOs
{
@ -19,6 +20,14 @@ namespace API.DTOs
public ICollection<PersonDto> Colorists { get; set; } = new List<PersonDto>();
public ICollection<PersonDto> Letterers { get; set; } = new List<PersonDto>();
public ICollection<PersonDto> Editors { get; set; } = new List<PersonDto>();
/// <summary>
/// Highest Age Rating from all Chapters
/// </summary>
public AgeRating AgeRating { get; set; } = AgeRating.Unknown;
/// <summary>
/// Earliest Year from all chapters
/// </summary>
public int ReleaseYear { get; set; }
public int SeriesId { get; set; }
}

27
API/Data/Metadata/ComicInfo.cs
View file

@ -1,4 +1,9 @@
namespace API.Data.Metadata
using System;
using System.Linq;
using API.Entities.Enums;
using Kavita.Common.Extensions;
namespace API.Data.Metadata
{
/// <summary>
/// A representation of a ComicInfo.xml file
@ -16,13 +21,25 @@
public int PageCount { get; set; }
// ReSharper disable once InconsistentNaming
public string LanguageISO { get; set; }
/// <summary>
/// This is the link to where the data was scraped from
/// </summary>
public string Web { get; set; }
public int Day { get; set; }
public int Month { get; set; }
public int Year { get; set; }
/// <summary>
/// Rating based on the content. Think PG-13, R for movies
/// Rating based on the content. Think PG-13, R for movies. See <see cref="AgeRating"/> for valid types
/// </summary>
public string AgeRating { get; set; }
// public AgeRating AgeRating
// {
// get => ConvertAgeRatingToEnum(_AgeRating);
// set => ConvertAgeRatingToEnum(value);
// }
/// <summary>
/// User's rating of the content
/// </summary>
@ -55,5 +72,11 @@
public string Editor { get; set; }
public string Publisher { get; set; }
public static AgeRating ConvertAgeRatingToEnum(string value)
{
return Enum.GetValues<AgeRating>()
.SingleOrDefault(t => t.ToDescription().ToUpperInvariant().Equals(value.ToUpperInvariant()), Entities.Enums.AgeRating.Unknown);
}
}
}

1199
API/Data/Migrations/20211205185207_MetadataAgeRating.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

26
API/Data/Migrations/20211205185207_MetadataAgeRating.cs Normal file
View file

@ -0,0 +1,26 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
public partial class MetadataAgeRating : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<int>(
name: "AgeRating",
table: "SeriesMetadata",
type: "INTEGER",
nullable: false,
defaultValue: 0);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "AgeRating",
table: "SeriesMetadata");
}
}
}

1208
API/Data/Migrations/20211206193225_AgeRatingAndReleaseDate.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

49
API/Data/Migrations/20211206193225_AgeRatingAndReleaseDate.cs Normal file
View file

@ -0,0 +1,49 @@
using System;
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
public partial class AgeRatingAndReleaseDate : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.AddColumn<int>(
name: "ReleaseYear",
table: "SeriesMetadata",
type: "INTEGER",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<int>(
name: "AgeRating",
table: "Chapter",
type: "INTEGER",
nullable: false,
defaultValue: 0);
migrationBuilder.AddColumn<DateTime>(
name: "ReleaseDate",
table: "Chapter",
type: "TEXT",
nullable: false,
defaultValue: new DateTime(1, 1, 1, 0, 0, 0, 0, DateTimeKind.Unspecified));
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropColumn(
name: "ReleaseYear",
table: "SeriesMetadata");
migrationBuilder.DropColumn(
name: "AgeRating",
table: "Chapter");
migrationBuilder.DropColumn(
name: "ReleaseDate",
table: "Chapter");
}
}
}

12
API/Data/Migrations/DataContextModelSnapshot.cs
View file

@ -289,6 +289,9 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<int>("AgeRating")
.HasColumnType("INTEGER");
b.Property<string>("CoverImage")
.HasColumnType("TEXT");
@ -316,6 +319,9 @@ namespace API.Data.Migrations
b.Property<string>("Range")
.HasColumnType("TEXT");
b.Property<DateTime>("ReleaseDate")
.HasColumnType("TEXT");
b.Property<string>("Title")
.HasColumnType("TEXT");
@ -477,6 +483,12 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<int>("AgeRating")
.HasColumnType("INTEGER");
b.Property<int>("ReleaseYear")
.HasColumnType("INTEGER");
b.Property<uint>("RowVersion")
.IsConcurrencyToken()
.HasColumnType("INTEGER");

9
API/Entities/Chapter.cs
View file

@ -41,6 +41,10 @@ namespace API.Entities
/// Used for books/specials to display custom title. For non-specials/books, will be set to <see cref="Range"/>
/// </summary>
public string Title { get; set; }
/// <summary>
/// Age Rating for the issue/chapter
/// </summary>
public AgeRating AgeRating { get; set; }
/// <summary>
@ -48,7 +52,10 @@ namespace API.Entities
/// </summary>
/// <remarks>This should not be confused with Title which is used for special filenames.</remarks>
public string TitleName { get; set; } = string.Empty;
// public string Year { get; set; } // Only time I can think this will be more than 1 year is for a volume which will be a spread
/// <summary>
/// Date which chapter was released
/// </summary>
public DateTime ReleaseDate { get; set; }
/// <summary>

35
API/Entities/Enums/AgeRating.cs Normal file
View file

@ -0,0 +1,35 @@
using System.ComponentModel;
namespace API.Entities.Enums;
public enum AgeRating
{
[Description("Unknown")]
Unknown = 0,
[Description("Rating Pending")]
RatingPending = 1,
[Description("Early Childhood")]
EarlyChildhood = 2,
[Description("Everyone")]
Everyone = 3,
[Description("G")]
G = 4,
[Description("Everyone 10+")]
Everyone10Plus = 5,
[Description("Kids to Adults")]
KidsToAdults = 6,
[Description("Teen")]
Teen = 7,
[Description("Mature 15+")]
Mature15Plus = 8,
[Description("Mature 17+")]
Mature17Plus = 9,
[Description("Mature")]
Mature = 10,
[Description("Adults Only 18+")]
AdultsOnly = 11,
[Description("X 18+")]
X18Plus = 12
}

9
API/Entities/Metadata/SeriesMetadata.cs
View file

@ -1,5 +1,6 @@
using System.Collections.Generic;
using System.ComponentModel.DataAnnotations;
using API.Entities.Enums;
using API.Entities.Interfaces;
using Microsoft.EntityFrameworkCore;
@ -21,6 +22,14 @@ namespace API.Entities.Metadata
/// </summary>
public ICollection<Person> People { get; set; } = new List<Person>();
/// <summary>
/// Highest Age Rating from all Chapters
/// </summary>
public AgeRating AgeRating { get; set; }
/// <summary>
/// Earliest Year from all chapters
/// </summary>
public int ReleaseYear { get; set; }
// Relationship
public Series Series { get; set; }

4
API/Parser/Parser.cs
View file

@ -1071,13 +1071,13 @@ namespace API.Parser
/// <summary>
/// Cleans an author's name
/// </summary>
/// <remarks>If the author is Last, First, this will reverse</remarks>
/// <remarks>If the author is Last, First, this will not reverse</remarks>
/// <param name="author"></param>
/// <returns></returns>
public static string CleanAuthor(string author)
{
if (string.IsNullOrEmpty(author)) return string.Empty;
return string.Join(" ", author.Split(",").Reverse().Select(s => s.Trim()));
return author.Trim();
}
}
}

18
API/Program.cs
View file

@ -16,7 +16,6 @@ using Microsoft.Extensions.Configuration;
using Microsoft.Extensions.DependencyInjection;
using Microsoft.Extensions.Hosting;
using Microsoft.Extensions.Logging;
using NetVips;
namespace API
{
@ -33,19 +32,10 @@ namespace API
Console.OutputEncoding = System.Text.Encoding.UTF8;
var isDocker = new OsInfo(Array.Empty<IOsVersionAdapter>()).IsDocker;
// var migrateLogger = LoggerFactory.Create(builder =>
// {
// builder
// //.AddConfiguration(Configuration.GetSection("Logging"))
// .AddFilter("Microsoft", LogLevel.Warning)
// .AddFilter("System", LogLevel.Warning)
// .AddFilter("SampleApp.Program", LogLevel.Debug)
// .AddConsole()
// .AddEventLog();
// });
// var mLogger = migrateLogger.CreateLogger<DirectoryService>();
// TODO: Figure out a solution for this migration and logger.
MigrateConfigFiles.Migrate(isDocker, new DirectoryService(null, new FileSystem()));
var directoryService = new DirectoryService(null, new FileSystem());
MigrateConfigFiles.Migrate(isDocker, directoryService);
// Before anything, check if JWT has been generated properly or if user still has default
if (!Configuration.CheckIfJwtTokenSet() &&
@ -76,7 +66,7 @@ namespace API
// This doesn't work either
//var directoryService = services.GetRequiredService<DirectoryService>();
var directoryService = new DirectoryService(null, new FileSystem());

49
API/Services/ArchiveService.cs
View file

@ -335,6 +335,33 @@ namespace API.Services
return null;
}
public static void CleanComicInfo(ComicInfo info)
{
if (info != null)
{
info.Writer = Parser.Parser.CleanAuthor(info.Writer);
info.Colorist = Parser.Parser.CleanAuthor(info.Colorist);
info.Editor = Parser.Parser.CleanAuthor(info.Editor);
info.Inker = Parser.Parser.CleanAuthor(info.Inker);
info.Letterer = Parser.Parser.CleanAuthor(info.Letterer);
info.Penciller = Parser.Parser.CleanAuthor(info.Penciller);
info.Publisher = Parser.Parser.CleanAuthor(info.Publisher);
if (!string.IsNullOrEmpty(info.Web))
{
// TODO: Validate this works through testing
// ComicVine stores the Issue number in Number field and does not use Volume.
if (info.Web.Contains("https://comicvine.gamespot.com/"))
{
if (info.Volume.Equals("1"))
{
info.Volume = Parser.Parser.DefaultVolume;
}
}
}
}
}
/// <summary>
/// This can be null if nothing is found or any errors occur during access
/// </summary>
@ -365,16 +392,7 @@ namespace API.Services
using var stream = entry.Open();
var serializer = new XmlSerializer(typeof(ComicInfo));
var info = (ComicInfo) serializer.Deserialize(stream);
if (info != null)
{
info.Writer = Parser.Parser.CleanAuthor(info.Writer);
info.Colorist = Parser.Parser.CleanAuthor(info.Colorist);
info.Editor = Parser.Parser.CleanAuthor(info.Editor);
info.Inker = Parser.Parser.CleanAuthor(info.Inker);
info.Letterer = Parser.Parser.CleanAuthor(info.Letterer);
info.Penciller = Parser.Parser.CleanAuthor(info.Penciller);
info.Publisher = Parser.Parser.CleanAuthor(info.Publisher);
}
CleanComicInfo(info);
return info;
}
@ -394,16 +412,7 @@ namespace API.Services
.Parser
.MacOsMetadataFileStartsWith)
&& Parser.Parser.IsXml(entry.Key)));
if (info != null)
{
info.Writer = Parser.Parser.CleanAuthor(info.Writer);
info.Colorist = Parser.Parser.CleanAuthor(info.Colorist);
info.Editor = Parser.Parser.CleanAuthor(info.Editor);
info.Inker = Parser.Parser.CleanAuthor(info.Inker);
info.Letterer = Parser.Parser.CleanAuthor(info.Letterer);
info.Penciller = Parser.Parser.CleanAuthor(info.Penciller);
info.Publisher = Parser.Parser.CleanAuthor(info.Publisher);
}
CleanComicInfo(info);
return info;
}

3
API/Services/ImageService.cs
View file

@ -133,7 +133,8 @@ public class ImageService : IImageService
{
using var thumbnail = Image.ThumbnailStream(stream, ThumbnailWidth);
var filename = fileName + ".png";
thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, fileName + ".png"));
_directoryService.ExistOrCreate(_directoryService.CoverImageDirectory);
thumbnail.WriteToFile(_directoryService.FileSystem.Path.Join(_directoryService.CoverImageDirectory, filename));
return filename;
}

22
API/Services/MetadataService.cs
View file

@ -6,6 +6,7 @@ using System.Linq;
using System.Threading.Tasks;
using API.Comparators;
using API.Data;
using API.Data.Metadata;
using API.Data.Repositories;
using API.Data.Scanner;
using API.Entities;
@ -90,11 +91,20 @@ public class MetadataService : IMetadataService
var comicInfo = _readingItemService.GetComicInfo(firstFile.FilePath, firstFile.Format);
if (comicInfo == null) return;
chapter.AgeRating = ComicInfo.ConvertAgeRatingToEnum(comicInfo.AgeRating);
if (!string.IsNullOrEmpty(comicInfo.Title))
{
chapter.TitleName = comicInfo.Title.Trim();
}
if (comicInfo.Year > 0 && comicInfo.Month > 0)
{
var day = Math.Max(comicInfo.Day, 1);
var month = Math.Max(comicInfo.Month, 1);
chapter.ReleaseDate = DateTime.Parse($"{month}/{day}/{comicInfo.Year}");
}
if (!string.IsNullOrEmpty(comicInfo.Colorist))
{
var people = comicInfo.Colorist.Split(",");
@ -230,7 +240,8 @@ public class MetadataService : IMetadataService
// Summary Info
if (!string.IsNullOrEmpty(comicInfo.Summary))
{
series.Metadata.Summary = comicInfo.Summary; // NOTE: I can move this to the bottom as I have a comicInfo selection, save me an extra read
// PERF: I can move this to the bottom as I have a comicInfo selection, save me an extra read
series.Metadata.Summary = comicInfo.Summary;
}
foreach (var chapter in series.Volumes.SelectMany(volume => volume.Chapters))
@ -270,6 +281,13 @@ public class MetadataService : IMetadataService
.Where(ci => ci != null)
.ToList();
//var firstComicInfo = comicInfos.First(i => i.)
// Set the AgeRating as highest in all the comicInfos
series.Metadata.AgeRating = comicInfos.Max(i => ComicInfo.ConvertAgeRatingToEnum(comicInfo.AgeRating));
series.Metadata.ReleaseYear = series.Volumes
.SelectMany(volume => volume.Chapters).Min(c => c.ReleaseDate.Year);
var genres = comicInfos.SelectMany(i => i.Genre.Split(",")).Distinct().ToList();
var people = series.Volumes.SelectMany(volume => volume.Chapters).SelectMany(c => c.People).ToList();
@ -280,7 +298,6 @@ public class MetadataService : IMetadataService
GenreHelper.UpdateGenre(allGenres, genres, false, genre => GenreHelper.AddGenreIfNotExists(series.Metadata.Genres, genre));
GenreHelper.KeepOnlySameGenreBetweenLists(series.Metadata.Genres, genres.Select(g => DbFactory.Genre(g, false)).ToList(),
genre => series.Metadata.Genres.Remove(genre));
}
/// <summary>
@ -324,6 +341,7 @@ public class MetadataService : IMetadataService
/// <param name="forceUpdate">Force updating cover image even if underlying file has not been modified or chapter already has a cover image</param>
public async Task RefreshMetadata(int libraryId, bool forceUpdate = false)
{
// TODO: Think about splitting the comicinfo stuff into a separate task
var library = await _unitOfWork.LibraryRepository.GetLibraryForIdAsync(libraryId, LibraryIncludes.None);
_logger.LogInformation("[MetadataService] Beginning metadata refresh of {LibraryName}", library.Name);