Commit critical import artifacts
This commit is contained in:
13
src/RolemasterDb.ImportTool/CriticalImageArtifactMetadata.cs
Normal file
13
src/RolemasterDb.ImportTool/CriticalImageArtifactMetadata.cs
Normal file
@@ -0,0 +1,13 @@
|
||||
using RolemasterDb.ImportTool.Parsing;
|
||||
|
||||
namespace RolemasterDb.ImportTool;
|
||||
|
||||
public sealed class CriticalImageArtifactMetadata(
|
||||
int resultId,
|
||||
string relativePath,
|
||||
CriticalSourceImageCrop crop)
|
||||
{
|
||||
public int ResultId { get; } = resultId;
|
||||
public string RelativePath { get; } = relativePath;
|
||||
public CriticalSourceImageCrop Crop { get; } = crop;
|
||||
}
|
||||
@@ -9,6 +9,7 @@ public sealed class CriticalImportCommandRunner
|
||||
private readonly PdfXmlExtractor pdfXmlExtractor = new();
|
||||
private readonly StandardOcrBootstrapper standardOcrBootstrapper = new();
|
||||
private readonly CriticalSourceImageArtifactGenerator sourceImageArtifactGenerator;
|
||||
private readonly CriticalSourceImageArtifactRegenerator sourceImageArtifactRegenerator;
|
||||
private readonly StandardCriticalTableParser standardParser = new();
|
||||
private readonly VariantColumnCriticalTableParser variantColumnParser = new();
|
||||
private readonly GroupedVariantCriticalTableParser groupedVariantParser = new();
|
||||
@@ -16,6 +17,7 @@ public sealed class CriticalImportCommandRunner
|
||||
public CriticalImportCommandRunner()
|
||||
{
|
||||
sourceImageArtifactGenerator = new CriticalSourceImageArtifactGenerator(pdfXmlExtractor);
|
||||
sourceImageArtifactRegenerator = new CriticalSourceImageArtifactRegenerator(pdfXmlExtractor);
|
||||
}
|
||||
|
||||
public async Task<int> RunAsync(ResetOptions options)
|
||||
@@ -97,35 +99,70 @@ public sealed class CriticalImportCommandRunner
|
||||
|
||||
public async Task<int> RunAsync(ReimportImagesOptions options)
|
||||
{
|
||||
var entry = GetManifestEntry(options.Table);
|
||||
var artifactPaths = CreateArtifactPaths(entry.Slug);
|
||||
var extractor = CreateSourceExtractor(entry);
|
||||
if (!File.Exists(artifactPaths.GetSourceArtifactPath(entry.ExtractionMethod)))
|
||||
var entries = GetManifestEntries(options).ToList();
|
||||
CriticalImportLoader? loader = null;
|
||||
if (options.UpdateMetadata)
|
||||
{
|
||||
await extractor.ExtractAsync(ResolveRepositoryPath(entry.PdfPath), artifactPaths, CancellationToken.None);
|
||||
loader = new CriticalImportLoader(ResolveDatabasePath(options.DatabasePath));
|
||||
}
|
||||
|
||||
var extractedSource = await extractor.LoadAsync(ResolveRepositoryPath(entry.PdfPath), artifactPaths, CancellationToken.None);
|
||||
var parseResult = Parse(entry, extractedSource);
|
||||
await sourceImageArtifactGenerator.GenerateAsync(
|
||||
ResolveRepositoryPath(entry.PdfPath),
|
||||
artifactPaths,
|
||||
parseResult,
|
||||
CancellationToken.None);
|
||||
await artifactWriter.WriteAsync(artifactPaths, parseResult, CancellationToken.None);
|
||||
|
||||
if (!parseResult.ValidationReport.IsValid)
|
||||
foreach (var entry in entries)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Validation failed for '{entry.Slug}'. See {artifactPaths.ValidationReportPath} for details.");
|
||||
var artifactPaths = CreateArtifactPaths(entry.Slug);
|
||||
try
|
||||
{
|
||||
var extractor = CreateSourceExtractor(entry);
|
||||
if (!File.Exists(artifactPaths.GetSourceArtifactPath(entry.ExtractionMethod)))
|
||||
{
|
||||
await extractor.ExtractAsync(ResolveRepositoryPath(entry.PdfPath), artifactPaths,
|
||||
CancellationToken.None);
|
||||
}
|
||||
|
||||
var extractedSource = await extractor.LoadAsync(ResolveRepositoryPath(entry.PdfPath), artifactPaths,
|
||||
CancellationToken.None);
|
||||
var parseResult = Parse(entry, extractedSource);
|
||||
await sourceImageArtifactGenerator.GenerateAsync(
|
||||
ResolveRepositoryPath(entry.PdfPath),
|
||||
artifactPaths,
|
||||
parseResult,
|
||||
CancellationToken.None);
|
||||
await artifactWriter.WriteAsync(artifactPaths, parseResult, CancellationToken.None);
|
||||
|
||||
if (!parseResult.ValidationReport.IsValid)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Validation failed for '{entry.Slug}'. See {artifactPaths.ValidationReportPath} for details.");
|
||||
}
|
||||
|
||||
if (loader is null)
|
||||
{
|
||||
Console.WriteLine(
|
||||
$"Refreshed image artifacts for {entry.Slug}: metadata unchanged.");
|
||||
continue;
|
||||
}
|
||||
|
||||
var refreshedCount = await loader.RefreshImageArtifactsAsync(parseResult.Table);
|
||||
Console.WriteLine(
|
||||
$"Refreshed image artifacts for {entry.Slug}: {refreshedCount} results updated.");
|
||||
}
|
||||
catch (Exception exception) when (CanUseDatabaseMetadataFallback(entry, options))
|
||||
{
|
||||
Console.Error.WriteLine(
|
||||
$"Falling back to database image metadata for {entry.Slug}: {exception.Message}");
|
||||
|
||||
var metadataLoader = loader ?? new CriticalImportLoader(ResolveDatabasePath(options.DatabasePath));
|
||||
var imageArtifacts = await metadataLoader.LoadImageArtifactMetadataAsync(entry.Slug);
|
||||
await sourceImageArtifactRegenerator.GenerateAsync(
|
||||
ResolveRepositoryPath(entry.PdfPath),
|
||||
artifactPaths,
|
||||
imageArtifacts,
|
||||
CancellationToken.None);
|
||||
|
||||
Console.WriteLine(
|
||||
$"Refreshed image artifacts for {entry.Slug}: regenerated {imageArtifacts.Count} images from existing database metadata.");
|
||||
}
|
||||
}
|
||||
|
||||
var loader = new CriticalImportLoader(ResolveDatabasePath(options.DatabasePath));
|
||||
var refreshedCount = await loader.RefreshImageArtifactsAsync(parseResult.Table);
|
||||
|
||||
Console.WriteLine(
|
||||
$"Refreshed image artifacts for {entry.Slug}: {refreshedCount} results updated.");
|
||||
|
||||
return 0;
|
||||
}
|
||||
|
||||
@@ -133,12 +170,47 @@ public sealed class CriticalImportCommandRunner
|
||||
{
|
||||
var manifest = manifestLoader.Load(RepositoryPaths.Discover().ManifestPath);
|
||||
return manifest.Tables
|
||||
.Where(item => item.Enabled)
|
||||
.SingleOrDefault(item => string.Equals(item.Slug, tableSlug.Trim(), StringComparison.OrdinalIgnoreCase))
|
||||
?? throw new InvalidOperationException($"No enabled manifest entry was found for '{tableSlug}'.");
|
||||
.Where(item => item.Enabled)
|
||||
.SingleOrDefault(item =>
|
||||
string.Equals(item.Slug, tableSlug.Trim(), StringComparison.OrdinalIgnoreCase))
|
||||
?? throw new InvalidOperationException($"No enabled manifest entry was found for '{tableSlug}'.");
|
||||
}
|
||||
|
||||
private async Task<ExtractedCriticalSource> LoadExtractedSourceAsync(CriticalImportManifestEntry entry, ImportArtifactPaths artifactPaths)
|
||||
private IReadOnlyList<CriticalImportManifestEntry> GetManifestEntries(ReimportImagesOptions options)
|
||||
{
|
||||
if (options.All)
|
||||
{
|
||||
var manifest = manifestLoader.Load(RepositoryPaths.Discover().ManifestPath);
|
||||
return manifest.Tables.Where(item => item.Enabled).ToList();
|
||||
}
|
||||
|
||||
var normalizedTable = NormalizeTableArgument(options.Table);
|
||||
if (string.IsNullOrWhiteSpace(normalizedTable))
|
||||
{
|
||||
throw new InvalidOperationException("Specify a table slug or pass --all.");
|
||||
}
|
||||
|
||||
return new[] { GetManifestEntry(normalizedTable) };
|
||||
}
|
||||
|
||||
private static string? NormalizeTableArgument(string? table)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(table))
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
var trimmed = table.Trim();
|
||||
return trimmed[0] == '-' ? null : trimmed;
|
||||
}
|
||||
|
||||
private static bool
|
||||
CanUseDatabaseMetadataFallback(CriticalImportManifestEntry entry, ReimportImagesOptions options) =>
|
||||
!options.UpdateMetadata
|
||||
&& string.Equals(entry.ExtractionMethod, "ocr", StringComparison.OrdinalIgnoreCase);
|
||||
|
||||
private async Task<ExtractedCriticalSource> LoadExtractedSourceAsync(CriticalImportManifestEntry entry,
|
||||
ImportArtifactPaths artifactPaths)
|
||||
{
|
||||
var extractor = CreateSourceExtractor(entry);
|
||||
var sourceArtifactPath = artifactPaths.GetSourceArtifactPath(entry.ExtractionMethod);
|
||||
@@ -190,7 +262,8 @@ public sealed class CriticalImportCommandRunner
|
||||
return new OcrCriticalSourceExtractor(pdfXmlExtractor);
|
||||
}
|
||||
|
||||
throw new InvalidOperationException($"Extraction method '{entry.ExtractionMethod}' is not supported by the importer.");
|
||||
throw new InvalidOperationException(
|
||||
$"Extraction method '{entry.ExtractionMethod}' is not supported by the importer.");
|
||||
}
|
||||
|
||||
private static ImportArtifactPaths CreateArtifactPaths(string slug) =>
|
||||
@@ -201,4 +274,4 @@ public sealed class CriticalImportCommandRunner
|
||||
|
||||
private static string ResolveRepositoryPath(string path) =>
|
||||
Path.GetFullPath(Path.Combine(RepositoryPaths.Discover().RootPath, path));
|
||||
}
|
||||
}
|
||||
@@ -1,6 +1,5 @@
|
||||
using Microsoft.EntityFrameworkCore;
|
||||
using System.Text.Json;
|
||||
|
||||
using RolemasterDb.App.Data;
|
||||
using RolemasterDb.App.Domain;
|
||||
using RolemasterDb.ImportTool.Parsing;
|
||||
@@ -35,7 +34,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
return removedTableCount;
|
||||
}
|
||||
|
||||
public async Task<ImportCommandResult> LoadAsync(ParsedCriticalTable table, CancellationToken cancellationToken = default)
|
||||
public async Task<ImportCommandResult> LoadAsync(ParsedCriticalTable table,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var dbContext = CreateDbContext();
|
||||
await dbContext.Database.EnsureCreatedAsync(cancellationToken);
|
||||
@@ -48,16 +48,16 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
.Include(item => item.Columns)
|
||||
.Include(item => item.RollBands)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalGroup)
|
||||
.ThenInclude(result => result.CriticalGroup)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalColumn)
|
||||
.ThenInclude(result => result.CriticalColumn)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalRollBand)
|
||||
.ThenInclude(result => result.CriticalRollBand)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.Effects)
|
||||
.ThenInclude(result => result.Effects)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.Branches)
|
||||
.ThenInclude(branch => branch.Effects)
|
||||
.ThenInclude(result => result.Branches)
|
||||
.ThenInclude(branch => branch.Effects)
|
||||
.SingleOrDefaultAsync(item => item.Slug == table.Slug, cancellationToken);
|
||||
|
||||
if (entity is null)
|
||||
@@ -78,7 +78,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
var columnsByKey = SynchronizeColumns(entity, table);
|
||||
var rollBandsByLabel = SynchronizeRollBands(entity, table);
|
||||
var existingResultsByKey = entity.Results.ToDictionary(
|
||||
item => CreateResultKey(item.CriticalGroup?.GroupKey, item.CriticalColumn.ColumnKey, item.CriticalRollBand.Label),
|
||||
item => CreateResultKey(item.CriticalGroup?.GroupKey, item.CriticalColumn.ColumnKey,
|
||||
item.CriticalRollBand.Label),
|
||||
StringComparer.Ordinal);
|
||||
var importedKeys = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
@@ -109,7 +110,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
}
|
||||
|
||||
foreach (var unmatchedResult in entity.Results
|
||||
.Where(item => !importedKeys.Contains(CreateResultKey(item.CriticalGroup?.GroupKey, item.CriticalColumn.ColumnKey, item.CriticalRollBand.Label)))
|
||||
.Where(item => !importedKeys.Contains(CreateResultKey(item.CriticalGroup?.GroupKey,
|
||||
item.CriticalColumn.ColumnKey, item.CriticalRollBand.Label)))
|
||||
.ToList())
|
||||
{
|
||||
if (unmatchedResult.IsCurated)
|
||||
@@ -128,7 +130,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
return new ImportCommandResult(entity.Slug, entity.Columns.Count, entity.RollBands.Count, entity.Results.Count);
|
||||
}
|
||||
|
||||
public async Task<int> RefreshImageArtifactsAsync(ParsedCriticalTable table, CancellationToken cancellationToken = default)
|
||||
public async Task<int> RefreshImageArtifactsAsync(ParsedCriticalTable table,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var dbContext = CreateDbContext();
|
||||
await dbContext.Database.EnsureCreatedAsync(cancellationToken);
|
||||
@@ -138,20 +141,22 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
var entity = await dbContext.CriticalTables
|
||||
.AsSplitQuery()
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalGroup)
|
||||
.ThenInclude(result => result.CriticalGroup)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalColumn)
|
||||
.ThenInclude(result => result.CriticalColumn)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalRollBand)
|
||||
.ThenInclude(result => result.CriticalRollBand)
|
||||
.SingleOrDefaultAsync(item => item.Slug == table.Slug, cancellationToken);
|
||||
|
||||
if (entity is null)
|
||||
{
|
||||
throw new InvalidOperationException($"Critical table '{table.Slug}' does not exist in the target database.");
|
||||
throw new InvalidOperationException(
|
||||
$"Critical table '{table.Slug}' does not exist in the target database.");
|
||||
}
|
||||
|
||||
var existingResultsByKey = entity.Results.ToDictionary(
|
||||
item => CreateResultKey(item.CriticalGroup?.GroupKey, item.CriticalColumn.ColumnKey, item.CriticalRollBand.Label),
|
||||
item => CreateResultKey(item.CriticalGroup?.GroupKey, item.CriticalColumn.ColumnKey,
|
||||
item.CriticalRollBand.Label),
|
||||
StringComparer.Ordinal);
|
||||
|
||||
var refreshedCount = 0;
|
||||
@@ -172,6 +177,66 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
return refreshedCount;
|
||||
}
|
||||
|
||||
public async Task<IReadOnlyList<CriticalImageArtifactMetadata>> LoadImageArtifactMetadataAsync(
|
||||
string tableSlug,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
await using var dbContext = CreateDbContext();
|
||||
await dbContext.Database.EnsureCreatedAsync(cancellationToken);
|
||||
await RolemasterDbSchemaUpgrader.EnsureLatestAsync(dbContext, cancellationToken);
|
||||
|
||||
var rows = await dbContext.CriticalResults
|
||||
.AsNoTracking()
|
||||
.Where(item => item.CriticalTable.Slug == tableSlug)
|
||||
.OrderBy(item => item.Id)
|
||||
.Select(item => new
|
||||
{
|
||||
item.Id,
|
||||
item.SourcePageNumber,
|
||||
item.SourceImagePath,
|
||||
item.SourceImageCropJson
|
||||
})
|
||||
.ToListAsync(cancellationToken);
|
||||
|
||||
if (rows.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException($"Critical table '{tableSlug}' does not exist in the target database.");
|
||||
}
|
||||
|
||||
var metadata = new List<CriticalImageArtifactMetadata>(rows.Count);
|
||||
foreach (var row in rows)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(row.SourceImagePath))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Critical result {row.Id} in table '{tableSlug}' is missing SourceImagePath.");
|
||||
}
|
||||
|
||||
if (string.IsNullOrWhiteSpace(row.SourceImageCropJson))
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Critical result {row.Id} in table '{tableSlug}' is missing SourceImageCropJson.");
|
||||
}
|
||||
|
||||
var crop = JsonSerializer.Deserialize<CriticalSourceImageCrop>(row.SourceImageCropJson);
|
||||
if (crop is null)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Critical result {row.Id} in table '{tableSlug}' has invalid SourceImageCropJson.");
|
||||
}
|
||||
|
||||
if (row.SourcePageNumber is not null && row.SourcePageNumber != crop.PageNumber)
|
||||
{
|
||||
throw new InvalidOperationException(
|
||||
$"Critical result {row.Id} in table '{tableSlug}' has mismatched source page metadata.");
|
||||
}
|
||||
|
||||
metadata.Add(new CriticalImageArtifactMetadata(row.Id, row.SourceImagePath, crop));
|
||||
}
|
||||
|
||||
return metadata;
|
||||
}
|
||||
|
||||
private RolemasterDbContext CreateDbContext()
|
||||
{
|
||||
var options = new DbContextOptionsBuilder<RolemasterDbContext>()
|
||||
@@ -205,7 +270,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
return groupsByKey;
|
||||
}
|
||||
|
||||
private static Dictionary<string, CriticalColumn> SynchronizeColumns(CriticalTable entity, ParsedCriticalTable table)
|
||||
private static Dictionary<string, CriticalColumn> SynchronizeColumns(CriticalTable entity,
|
||||
ParsedCriticalTable table)
|
||||
{
|
||||
var columnsByKey = entity.Columns.ToDictionary(item => item.ColumnKey, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
@@ -230,7 +296,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
return columnsByKey;
|
||||
}
|
||||
|
||||
private static Dictionary<string, CriticalRollBand> SynchronizeRollBands(CriticalTable entity, ParsedCriticalTable table)
|
||||
private static Dictionary<string, CriticalRollBand> SynchronizeRollBands(CriticalTable entity,
|
||||
ParsedCriticalTable table)
|
||||
{
|
||||
var rollBandsByLabel = entity.RollBands.ToDictionary(item => item.Label, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
@@ -300,7 +367,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
: JsonSerializer.Serialize(item.SourceImageCrop, JsonOptions);
|
||||
}
|
||||
|
||||
private static void ReplaceResultChildren(RolemasterDbContext dbContext, CriticalResult result, ParsedCriticalResult item)
|
||||
private static void ReplaceResultChildren(RolemasterDbContext dbContext, CriticalResult result,
|
||||
ParsedCriticalResult item)
|
||||
{
|
||||
foreach (var branch in result.Branches)
|
||||
{
|
||||
@@ -346,7 +414,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var group in entity.Groups
|
||||
.Where(item => !activeGroupKeys.Contains(item.GroupKey) && !importedGroupKeys.Contains(item.GroupKey))
|
||||
.Where(item =>
|
||||
!activeGroupKeys.Contains(item.GroupKey) && !importedGroupKeys.Contains(item.GroupKey))
|
||||
.ToList())
|
||||
{
|
||||
dbContext.CriticalGroups.Remove(group);
|
||||
@@ -361,7 +430,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var column in entity.Columns
|
||||
.Where(item => !activeColumnKeys.Contains(item.ColumnKey) && !importedColumnKeys.Contains(item.ColumnKey))
|
||||
.Where(item =>
|
||||
!activeColumnKeys.Contains(item.ColumnKey) && !importedColumnKeys.Contains(item.ColumnKey))
|
||||
.ToList())
|
||||
{
|
||||
dbContext.CriticalColumns.Remove(column);
|
||||
@@ -376,7 +446,8 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var rollBand in entity.RollBands
|
||||
.Where(item => !activeRollBandLabels.Contains(item.Label) && !importedRollBandLabels.Contains(item.Label))
|
||||
.Where(item =>
|
||||
!activeRollBandLabels.Contains(item.Label) && !importedRollBandLabels.Contains(item.Label))
|
||||
.ToList())
|
||||
{
|
||||
dbContext.CriticalRollBands.Remove(rollBand);
|
||||
@@ -437,4 +508,4 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
|
||||
private static string NormalizeKey(string? value) =>
|
||||
string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim().ToUpperInvariant();
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,60 @@
|
||||
namespace RolemasterDb.ImportTool;
|
||||
|
||||
public sealed class CriticalSourceImageArtifactRegenerator(PdfXmlExtractor pdfXmlExtractor)
|
||||
{
|
||||
public async Task GenerateAsync(
|
||||
string pdfPath,
|
||||
ImportArtifactPaths artifactPaths,
|
||||
IReadOnlyList<CriticalImageArtifactMetadata> imageArtifacts,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
if (imageArtifacts.Count == 0)
|
||||
{
|
||||
throw new InvalidOperationException("No image artifact metadata was supplied.");
|
||||
}
|
||||
|
||||
Directory.CreateDirectory(artifactPaths.PagesDirectoryPath);
|
||||
Directory.CreateDirectory(artifactPaths.CellsDirectoryPath);
|
||||
|
||||
var renderProfiles = imageArtifacts
|
||||
.Select(item => (item.Crop.RenderDpi, item.Crop.ScaleFactor))
|
||||
.Distinct()
|
||||
.ToList();
|
||||
if (renderProfiles.Count != 1)
|
||||
{
|
||||
throw new InvalidOperationException("Image artifact metadata contains inconsistent render settings.");
|
||||
}
|
||||
|
||||
var renderDpi = renderProfiles[0].RenderDpi;
|
||||
foreach (var pageNumber in imageArtifacts
|
||||
.Select(item => item.Crop.PageNumber)
|
||||
.Distinct()
|
||||
.OrderBy(item => item))
|
||||
{
|
||||
await pdfXmlExtractor.RenderPagePngAsync(
|
||||
pdfPath,
|
||||
pageNumber,
|
||||
artifactPaths.GetPageImagePath(pageNumber),
|
||||
renderDpi,
|
||||
cancellationToken);
|
||||
}
|
||||
|
||||
foreach (var imageArtifact in imageArtifacts)
|
||||
{
|
||||
var crop = imageArtifact.Crop;
|
||||
var fullPath = artifactPaths.ResolveRelativePath(imageArtifact.RelativePath);
|
||||
Directory.CreateDirectory(Path.GetDirectoryName(fullPath)!);
|
||||
|
||||
await pdfXmlExtractor.RenderCropPngAsync(
|
||||
pdfPath,
|
||||
crop.PageNumber,
|
||||
crop.CropLeft,
|
||||
crop.CropTop,
|
||||
crop.CropWidth,
|
||||
crop.CropHeight,
|
||||
fullPath,
|
||||
crop.RenderDpi,
|
||||
cancellationToken);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -32,18 +32,11 @@ public sealed class PdfXmlExtractor
|
||||
startInfo.ArgumentList.Add(pdfPath);
|
||||
startInfo.ArgumentList.Add(outputPath);
|
||||
|
||||
using var process = new Process { StartInfo = startInfo };
|
||||
process.Start();
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
var error = await process.StandardError.ReadToEndAsync(cancellationToken);
|
||||
throw new InvalidOperationException($"pdftohtml failed for '{pdfPath}': {error}");
|
||||
}
|
||||
await RunProcessAsync(startInfo, cancellationToken);
|
||||
}
|
||||
|
||||
public async Task<PdfDocumentInfo> ReadDocumentInfoAsync(string pdfPath, CancellationToken cancellationToken = default)
|
||||
public async Task<PdfDocumentInfo> ReadDocumentInfoAsync(string pdfPath,
|
||||
CancellationToken cancellationToken = default)
|
||||
{
|
||||
var startInfo = new ProcessStartInfo
|
||||
{
|
||||
@@ -56,16 +49,7 @@ public sealed class PdfXmlExtractor
|
||||
|
||||
startInfo.ArgumentList.Add(pdfPath);
|
||||
|
||||
using var process = new Process { StartInfo = startInfo };
|
||||
process.Start();
|
||||
var output = await process.StandardOutput.ReadToEndAsync(cancellationToken);
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
var error = await process.StandardError.ReadToEndAsync(cancellationToken);
|
||||
throw new InvalidOperationException($"pdfinfo failed for '{pdfPath}': {error}");
|
||||
}
|
||||
var (output, error) = await RunProcessAsync(startInfo, cancellationToken);
|
||||
|
||||
var pageCountMatch = Regex.Match(output, @"Pages:\s*(\d+)", RegexOptions.Multiline);
|
||||
var sizeMatch = Regex.Match(output, @"Page size:\s*([0-9.]+)\s*x\s*([0-9.]+)\s*pts", RegexOptions.Multiline);
|
||||
@@ -104,7 +88,8 @@ public sealed class PdfXmlExtractor
|
||||
int height,
|
||||
string outputPath,
|
||||
CancellationToken cancellationToken = default) =>
|
||||
RenderCropPngAsync(pdfPath, pageNumber, left, top, width, height, outputPath, ScaledRenderDpi, cancellationToken);
|
||||
RenderCropPngAsync(pdfPath, pageNumber, left, top, width, height, outputPath, ScaledRenderDpi,
|
||||
cancellationToken);
|
||||
|
||||
public Task RenderCropPngAsync(
|
||||
string pdfPath,
|
||||
@@ -162,17 +147,10 @@ public sealed class PdfXmlExtractor
|
||||
}
|
||||
|
||||
startInfo.ArgumentList.Add(pdfPath);
|
||||
startInfo.ArgumentList.Add(Path.Combine(Path.GetDirectoryName(outputPath)!, Path.GetFileNameWithoutExtension(outputPath)));
|
||||
startInfo.ArgumentList.Add(Path.Combine(Path.GetDirectoryName(outputPath)!,
|
||||
Path.GetFileNameWithoutExtension(outputPath)));
|
||||
|
||||
using var process = new Process { StartInfo = startInfo };
|
||||
process.Start();
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
var error = await process.StandardError.ReadToEndAsync(cancellationToken);
|
||||
throw new InvalidOperationException($"pdftoppm failed for '{pdfPath}': {error}");
|
||||
}
|
||||
await RunProcessAsync(startInfo, cancellationToken);
|
||||
|
||||
if (!File.Exists(outputPath))
|
||||
{
|
||||
@@ -180,6 +158,34 @@ public sealed class PdfXmlExtractor
|
||||
}
|
||||
}
|
||||
|
||||
private static async Task<(string Output, string Error)> RunProcessAsync(
|
||||
ProcessStartInfo startInfo,
|
||||
CancellationToken cancellationToken)
|
||||
{
|
||||
using var process = new Process { StartInfo = startInfo };
|
||||
process.Start();
|
||||
|
||||
var outputTask = startInfo.RedirectStandardOutput
|
||||
? process.StandardOutput.ReadToEndAsync(cancellationToken)
|
||||
: Task.FromResult(string.Empty);
|
||||
var errorTask = startInfo.RedirectStandardError
|
||||
? process.StandardError.ReadToEndAsync(cancellationToken)
|
||||
: Task.FromResult(string.Empty);
|
||||
|
||||
await process.WaitForExitAsync(cancellationToken);
|
||||
var output = await outputTask;
|
||||
var error = await errorTask;
|
||||
|
||||
if (process.ExitCode != 0)
|
||||
{
|
||||
throw new InvalidOperationException(string.IsNullOrWhiteSpace(error)
|
||||
? $"{Path.GetFileNameWithoutExtension(startInfo.FileName)} failed with exit code {process.ExitCode}."
|
||||
: error);
|
||||
}
|
||||
|
||||
return (output, error);
|
||||
}
|
||||
|
||||
private static string ResolveExecutable(string environmentVariableName, string executableName)
|
||||
{
|
||||
var configuredPath = Environment.GetEnvironmentVariable(environmentVariableName);
|
||||
@@ -196,4 +202,4 @@ public sealed class PdfXmlExtractor
|
||||
|
||||
return Path.GetFileNameWithoutExtension(executableName);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -2,12 +2,26 @@ using CommandLine;
|
||||
|
||||
namespace RolemasterDb.ImportTool;
|
||||
|
||||
[Verb("reimport-images", HelpText = "Regenerate critical table page and cell images and refresh only image metadata in SQLite.")]
|
||||
[Verb("reimport-images",
|
||||
HelpText = "Regenerate critical table page and cell images, with optional SQLite metadata refresh.")]
|
||||
public sealed class ReimportImagesOptions
|
||||
{
|
||||
[Value(0, MetaName = "table", Required = true, HelpText = "The manifest slug of the critical table to refresh.")]
|
||||
public string Table { get; set; } = string.Empty;
|
||||
[Value(0, MetaName = "table", Required = false, HelpText = "The manifest slug of the critical table to refresh.")]
|
||||
public string? Table { get; set; }
|
||||
|
||||
[Option('d', "db", HelpText = "Optional SQLite database path.")]
|
||||
public string? DatabasePath { get; set; }
|
||||
}
|
||||
|
||||
[Option('a', "all", Default = false, HelpText = "Refresh every enabled manifest entry instead of one table.")]
|
||||
public bool All { get; set; }
|
||||
|
||||
[Option("update-metadata", Default = "true",
|
||||
HelpText =
|
||||
"Pass true or false to control whether SQLite source-image metadata is refreshed after regenerating artifacts.")]
|
||||
public string UpdateMetadataText { get; set; } = "true";
|
||||
|
||||
public bool UpdateMetadata =>
|
||||
bool.TryParse(UpdateMetadataText, out var updateMetadata)
|
||||
? updateMetadata
|
||||
: throw new InvalidOperationException("The --update-metadata option must be either 'true' or 'false'.");
|
||||
}
|
||||
@@ -7,7 +7,7 @@ public sealed class RepositoryPaths
|
||||
RootPath = rootPath;
|
||||
ManifestPath = Path.Combine(rootPath, "sources", "critical-import-manifest.json");
|
||||
DefaultDatabasePath = Path.Combine(rootPath, "src", "RolemasterDb.App", "rolemaster.db");
|
||||
ArtifactsRootPath = Path.Combine(rootPath, "artifacts", "import", "critical");
|
||||
ArtifactsRootPath = Path.Combine(rootPath, "src", "RolemasterDb.App", "import-artifacts", "critical");
|
||||
}
|
||||
|
||||
public string RootPath { get; }
|
||||
@@ -31,4 +31,4 @@ public sealed class RepositoryPaths
|
||||
|
||||
throw new InvalidOperationException("Could not discover the repository root from the current directory.");
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user