AZEN-SGG/Kavita
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Scanner Performance Improvements (#1774)

Browse source 
* Refactored the Genre code to be faster and used a dictonary to avoid some lookups. May fix the rare foreign constraint issue.

* Refactored tag to the same implementation as Genre. Ensure when grabbing tags from ComicInfo, we normalize and throw out duplicates.

* Removed an internal "external" field that was planned for Genres and Tags, but now with new plugin architecture, not needed.
This commit is contained in:
Joe Milazzo 2023-02-03 04:52:51 -08:00 • committed by GitHub
parent 48aebfc3c2
commit 8a0a2f0961
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 1925 additions and 152 deletions
Download patch file Download diff file Expand all files Collapse all files

6
API/Data/DbFactory.cs
View file

@ -121,23 +121,21 @@ public static class DbFactory
};
}
public static Genre Genre(string name, bool external)
public static Genre Genre(string name)
{
return new Genre()
{
Title = name.Trim().SentenceCase(),
NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(name),
ExternalTag = external
};
}
public static Tag Tag(string name, bool external)
public static Tag Tag(string name)
{
return new Tag()
{
Title = name.Trim().SentenceCase(),
NormalizedTitle = Services.Tasks.Scanner.Parser.Parser.Normalize(name),
ExternalTag = external
};
}

1748
API/Data/Migrations/20230203112022_RemoveExternalFromTagAndGenre.Designer.cs generated Normal file
View file

File diff suppressed because it is too large Load diff

77
API/Data/Migrations/20230203112022_RemoveExternalFromTagAndGenre.cs Normal file
View file

@ -0,0 +1,77 @@
using Microsoft.EntityFrameworkCore.Migrations;
#nullable disable
namespace API.Data.Migrations
{
public partial class RemoveExternalFromTagAndGenre : Migration
{
protected override void Up(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_Tag_NormalizedTitle_ExternalTag",
table: "Tag");
migrationBuilder.DropIndex(
name: "IX_Genre_NormalizedTitle_ExternalTag",
table: "Genre");
migrationBuilder.DropColumn(
name: "ExternalTag",
table: "Tag");
migrationBuilder.DropColumn(
name: "ExternalTag",
table: "Genre");
migrationBuilder.CreateIndex(
name: "IX_Tag_NormalizedTitle",
table: "Tag",
column: "NormalizedTitle",
unique: true);
migrationBuilder.CreateIndex(
name: "IX_Genre_NormalizedTitle",
table: "Genre",
column: "NormalizedTitle",
unique: true);
}
protected override void Down(MigrationBuilder migrationBuilder)
{
migrationBuilder.DropIndex(
name: "IX_Tag_NormalizedTitle",
table: "Tag");
migrationBuilder.DropIndex(
name: "IX_Genre_NormalizedTitle",
table: "Genre");
migrationBuilder.AddColumn<bool>(
name: "ExternalTag",
table: "Tag",
type: "INTEGER",
nullable: false,
defaultValue: false);
migrationBuilder.AddColumn<bool>(
name: "ExternalTag",
table: "Genre",
type: "INTEGER",
nullable: false,
defaultValue: false);
migrationBuilder.CreateIndex(
name: "IX_Tag_NormalizedTitle_ExternalTag",
table: "Tag",
columns: new[] { "NormalizedTitle", "ExternalTag" },
unique: true);
migrationBuilder.CreateIndex(
name: "IX_Genre_NormalizedTitle_ExternalTag",
table: "Genre",
columns: new[] { "NormalizedTitle", "ExternalTag" },
unique: true);
}
}
}

10
API/Data/Migrations/DataContextModelSnapshot.cs
View file

@ -528,9 +528,6 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<bool>("ExternalTag")
.HasColumnType("INTEGER");
b.Property<string>("NormalizedTitle")
.HasColumnType("TEXT");
@ -539,7 +536,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.HasIndex("NormalizedTitle", "ExternalTag")
b.HasIndex("NormalizedTitle")
.IsUnique();
b.ToTable("Genre");
@ -1036,9 +1033,6 @@ namespace API.Data.Migrations
.ValueGeneratedOnAdd()
.HasColumnType("INTEGER");
b.Property<bool>("ExternalTag")
.HasColumnType("INTEGER");
b.Property<string>("NormalizedTitle")
.HasColumnType("TEXT");
@ -1047,7 +1041,7 @@ namespace API.Data.Migrations
b.HasKey("Id");
b.HasIndex("NormalizedTitle", "ExternalTag")
b.HasIndex("NormalizedTitle")
.IsUnique();
b.ToTable("Tag");

2
API/Data/Repositories/GenreRepository.cs
View file

@ -56,7 +56,7 @@ public class GenreRepository : IGenreRepository
var genresWithNoConnections = await _context.Genre
.Include(p => p.SeriesMetadatas)
.Include(p => p.Chapters)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0 && p.ExternalTag == removeExternal)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0)
.AsSplitQuery()
.ToListAsync();

2
API/Data/Repositories/TagRepository.cs
View file

@ -46,7 +46,7 @@ public class TagRepository : ITagRepository
var tagsWithNoConnections = await _context.Tag
.Include(p => p.SeriesMetadatas)
.Include(p => p.Chapters)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0 && p.ExternalTag == removeExternal)
.Where(p => p.SeriesMetadatas.Count == 0 && p.Chapters.Count == 0)
.AsSplitQuery()
.ToListAsync();