Merge critical imports without overwriting curation
This commit is contained in:
@@ -42,85 +42,86 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
await RolemasterDbSchemaUpgrader.EnsureLatestAsync(dbContext, cancellationToken);
|
||||
await using var transaction = await dbContext.Database.BeginTransactionAsync(cancellationToken);
|
||||
|
||||
await DeleteTableAsync(dbContext, table.Slug, cancellationToken);
|
||||
var entity = await dbContext.CriticalTables
|
||||
.AsSplitQuery()
|
||||
.Include(item => item.Groups)
|
||||
.Include(item => item.Columns)
|
||||
.Include(item => item.RollBands)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalGroup)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalColumn)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.CriticalRollBand)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.Effects)
|
||||
.Include(item => item.Results)
|
||||
.ThenInclude(result => result.Branches)
|
||||
.ThenInclude(branch => branch.Effects)
|
||||
.SingleOrDefaultAsync(item => item.Slug == table.Slug, cancellationToken);
|
||||
|
||||
var entity = new CriticalTable
|
||||
if (entity is null)
|
||||
{
|
||||
Slug = table.Slug,
|
||||
DisplayName = table.DisplayName,
|
||||
Family = table.Family,
|
||||
SourceDocument = table.SourceDocument,
|
||||
Notes = table.Notes
|
||||
};
|
||||
|
||||
entity.Groups = table.Groups
|
||||
.Select(item => new CriticalGroup
|
||||
entity = new CriticalTable
|
||||
{
|
||||
GroupKey = item.GroupKey,
|
||||
Label = item.Label,
|
||||
SortOrder = item.SortOrder
|
||||
})
|
||||
.ToList();
|
||||
Slug = table.Slug
|
||||
};
|
||||
dbContext.CriticalTables.Add(entity);
|
||||
}
|
||||
|
||||
entity.Columns = table.Columns
|
||||
.Select(item => new CriticalColumn
|
||||
entity.DisplayName = table.DisplayName;
|
||||
entity.Family = table.Family;
|
||||
entity.SourceDocument = table.SourceDocument;
|
||||
entity.Notes = table.Notes;
|
||||
|
||||
var groupsByKey = SynchronizeGroups(entity, table);
|
||||
var columnsByKey = SynchronizeColumns(entity, table);
|
||||
var rollBandsByLabel = SynchronizeRollBands(entity, table);
|
||||
var existingResultsByKey = entity.Results.ToDictionary(
|
||||
item => CreateResultKey(item.CriticalGroup?.GroupKey, item.CriticalColumn.ColumnKey, item.CriticalRollBand.Label),
|
||||
StringComparer.Ordinal);
|
||||
var importedKeys = new HashSet<string>(StringComparer.Ordinal);
|
||||
|
||||
foreach (var item in table.Results)
|
||||
{
|
||||
var resultKey = CreateResultKey(item.GroupKey, item.ColumnKey, item.RollBandLabel);
|
||||
importedKeys.Add(resultKey);
|
||||
|
||||
if (!existingResultsByKey.TryGetValue(resultKey, out var existingResult))
|
||||
{
|
||||
ColumnKey = item.ColumnKey,
|
||||
Label = item.Label,
|
||||
Role = item.Role,
|
||||
SortOrder = item.SortOrder
|
||||
})
|
||||
.ToList();
|
||||
var newResult = CreateResultEntity(item, entity, groupsByKey, columnsByKey, rollBandsByLabel);
|
||||
entity.Results.Add(newResult);
|
||||
continue;
|
||||
}
|
||||
|
||||
entity.RollBands = table.RollBands
|
||||
.Select(item => new CriticalRollBand
|
||||
existingResult.CriticalGroup = item.GroupKey is null ? null : groupsByKey[item.GroupKey];
|
||||
existingResult.CriticalColumn = columnsByKey[item.ColumnKey];
|
||||
existingResult.CriticalRollBand = rollBandsByLabel[item.RollBandLabel];
|
||||
ApplyImporterProvenance(existingResult, item);
|
||||
|
||||
if (existingResult.IsCurated)
|
||||
{
|
||||
Label = item.Label,
|
||||
MinRoll = item.MinRoll,
|
||||
MaxRoll = item.MaxRoll,
|
||||
SortOrder = item.SortOrder
|
||||
})
|
||||
.ToList();
|
||||
continue;
|
||||
}
|
||||
|
||||
var groupsByKey = entity.Groups.ToDictionary(item => item.GroupKey, StringComparer.OrdinalIgnoreCase);
|
||||
var columnsByKey = entity.Columns.ToDictionary(item => item.ColumnKey, StringComparer.OrdinalIgnoreCase);
|
||||
var rollBandsByLabel = entity.RollBands.ToDictionary(item => item.Label, StringComparer.OrdinalIgnoreCase);
|
||||
ApplyImportedContent(existingResult, item);
|
||||
ReplaceResultChildren(dbContext, existingResult, item);
|
||||
}
|
||||
|
||||
entity.Results = table.Results
|
||||
.Select(item => new CriticalResult
|
||||
foreach (var unmatchedResult in entity.Results
|
||||
.Where(item => !importedKeys.Contains(CreateResultKey(item.CriticalGroup?.GroupKey, item.CriticalColumn.ColumnKey, item.CriticalRollBand.Label)))
|
||||
.ToList())
|
||||
{
|
||||
if (unmatchedResult.IsCurated)
|
||||
{
|
||||
CriticalGroup = item.GroupKey is null ? null : groupsByKey[item.GroupKey],
|
||||
CriticalColumn = columnsByKey[item.ColumnKey],
|
||||
CriticalRollBand = rollBandsByLabel[item.RollBandLabel],
|
||||
RawCellText = item.RawCellText,
|
||||
DescriptionText = item.DescriptionText,
|
||||
RawAffixText = item.RawAffixText,
|
||||
ParsedJson = SerializeParsedEffects(item.Effects),
|
||||
ParseStatus = ResolveParseStatus(item.Effects, item.Branches),
|
||||
Effects = item.Effects
|
||||
.Select(CreateEffectEntity)
|
||||
.ToList(),
|
||||
Branches = item.Branches
|
||||
.Select(branch => new CriticalBranch
|
||||
{
|
||||
BranchKind = branch.BranchKind,
|
||||
ConditionKey = branch.ConditionKey,
|
||||
ConditionText = branch.ConditionText,
|
||||
ConditionJson = "{}",
|
||||
RawText = branch.RawText,
|
||||
DescriptionText = branch.DescriptionText,
|
||||
RawAffixText = branch.RawAffixText,
|
||||
ParsedJson = SerializeParsedEffects(branch.Effects),
|
||||
SortOrder = branch.SortOrder,
|
||||
Effects = branch.Effects
|
||||
.Select(CreateEffectEntity)
|
||||
.ToList()
|
||||
})
|
||||
.ToList()
|
||||
})
|
||||
.ToList();
|
||||
continue;
|
||||
}
|
||||
|
||||
dbContext.CriticalTables.Add(entity);
|
||||
RemoveResultGraph(dbContext, unmatchedResult);
|
||||
entity.Results.Remove(unmatchedResult);
|
||||
}
|
||||
|
||||
RemoveUnusedAxes(dbContext, entity, table);
|
||||
await dbContext.SaveChangesAsync(cancellationToken);
|
||||
await transaction.CommitAsync(cancellationToken);
|
||||
|
||||
@@ -136,52 +137,207 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
return new RolemasterDbContext(options);
|
||||
}
|
||||
|
||||
private static async Task DeleteTableAsync(
|
||||
RolemasterDbContext dbContext,
|
||||
string slug,
|
||||
CancellationToken cancellationToken)
|
||||
private static Dictionary<string, CriticalGroup> SynchronizeGroups(CriticalTable entity, ParsedCriticalTable table)
|
||||
{
|
||||
var tableId = await dbContext.CriticalTables
|
||||
.Where(item => item.Slug == slug)
|
||||
.Select(item => (int?)item.Id)
|
||||
.SingleOrDefaultAsync(cancellationToken);
|
||||
var groupsByKey = entity.Groups.ToDictionary(item => item.GroupKey, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
if (tableId is null)
|
||||
foreach (var parsedGroup in table.Groups)
|
||||
{
|
||||
return;
|
||||
if (!groupsByKey.TryGetValue(parsedGroup.GroupKey, out var group))
|
||||
{
|
||||
group = new CriticalGroup
|
||||
{
|
||||
CriticalTable = entity,
|
||||
GroupKey = parsedGroup.GroupKey
|
||||
};
|
||||
entity.Groups.Add(group);
|
||||
groupsByKey[parsedGroup.GroupKey] = group;
|
||||
}
|
||||
|
||||
group.Label = parsedGroup.Label;
|
||||
group.SortOrder = parsedGroup.SortOrder;
|
||||
}
|
||||
|
||||
await dbContext.CriticalEffects
|
||||
.Where(item => item.CriticalBranch != null && item.CriticalBranch.CriticalResult.CriticalTableId == tableId.Value)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
return groupsByKey;
|
||||
}
|
||||
|
||||
await dbContext.CriticalEffects
|
||||
.Where(item => item.CriticalResult != null && item.CriticalResult.CriticalTableId == tableId.Value)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
private static Dictionary<string, CriticalColumn> SynchronizeColumns(CriticalTable entity, ParsedCriticalTable table)
|
||||
{
|
||||
var columnsByKey = entity.Columns.ToDictionary(item => item.ColumnKey, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
await dbContext.CriticalBranches
|
||||
.Where(item => item.CriticalResult.CriticalTableId == tableId.Value)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
foreach (var parsedColumn in table.Columns)
|
||||
{
|
||||
if (!columnsByKey.TryGetValue(parsedColumn.ColumnKey, out var column))
|
||||
{
|
||||
column = new CriticalColumn
|
||||
{
|
||||
CriticalTable = entity,
|
||||
ColumnKey = parsedColumn.ColumnKey
|
||||
};
|
||||
entity.Columns.Add(column);
|
||||
columnsByKey[parsedColumn.ColumnKey] = column;
|
||||
}
|
||||
|
||||
await dbContext.CriticalResults
|
||||
.Where(item => item.CriticalTableId == tableId.Value)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
column.Label = parsedColumn.Label;
|
||||
column.Role = parsedColumn.Role;
|
||||
column.SortOrder = parsedColumn.SortOrder;
|
||||
}
|
||||
|
||||
await dbContext.CriticalGroups
|
||||
.Where(item => item.CriticalTableId == tableId.Value)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
return columnsByKey;
|
||||
}
|
||||
|
||||
await dbContext.CriticalColumns
|
||||
.Where(item => item.CriticalTableId == tableId.Value)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
private static Dictionary<string, CriticalRollBand> SynchronizeRollBands(CriticalTable entity, ParsedCriticalTable table)
|
||||
{
|
||||
var rollBandsByLabel = entity.RollBands.ToDictionary(item => item.Label, StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
await dbContext.CriticalRollBands
|
||||
.Where(item => item.CriticalTableId == tableId.Value)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
foreach (var parsedRollBand in table.RollBands)
|
||||
{
|
||||
if (!rollBandsByLabel.TryGetValue(parsedRollBand.Label, out var rollBand))
|
||||
{
|
||||
rollBand = new CriticalRollBand
|
||||
{
|
||||
CriticalTable = entity,
|
||||
Label = parsedRollBand.Label
|
||||
};
|
||||
entity.RollBands.Add(rollBand);
|
||||
rollBandsByLabel[parsedRollBand.Label] = rollBand;
|
||||
}
|
||||
|
||||
await dbContext.CriticalTables
|
||||
.Where(item => item.Id == tableId.Value)
|
||||
.ExecuteDeleteAsync(cancellationToken);
|
||||
rollBand.MinRoll = parsedRollBand.MinRoll;
|
||||
rollBand.MaxRoll = parsedRollBand.MaxRoll;
|
||||
rollBand.SortOrder = parsedRollBand.SortOrder;
|
||||
}
|
||||
|
||||
return rollBandsByLabel;
|
||||
}
|
||||
|
||||
private static CriticalResult CreateResultEntity(
|
||||
ParsedCriticalResult item,
|
||||
CriticalTable entity,
|
||||
IReadOnlyDictionary<string, CriticalGroup> groupsByKey,
|
||||
IReadOnlyDictionary<string, CriticalColumn> columnsByKey,
|
||||
IReadOnlyDictionary<string, CriticalRollBand> rollBandsByLabel)
|
||||
{
|
||||
var result = new CriticalResult
|
||||
{
|
||||
CriticalTable = entity,
|
||||
CriticalGroup = item.GroupKey is null ? null : groupsByKey[item.GroupKey],
|
||||
CriticalColumn = columnsByKey[item.ColumnKey],
|
||||
CriticalRollBand = rollBandsByLabel[item.RollBandLabel],
|
||||
IsCurated = false
|
||||
};
|
||||
|
||||
ApplyImportedContent(result, item);
|
||||
ApplyImporterProvenance(result, item);
|
||||
result.Effects = item.Effects
|
||||
.Select(CreateEffectEntity)
|
||||
.ToList();
|
||||
result.Branches = item.Branches
|
||||
.Select(CreateBranchEntity)
|
||||
.ToList();
|
||||
return result;
|
||||
}
|
||||
|
||||
private static void ApplyImportedContent(CriticalResult result, ParsedCriticalResult item)
|
||||
{
|
||||
result.RawCellText = item.RawCellText;
|
||||
result.DescriptionText = item.DescriptionText;
|
||||
result.RawAffixText = item.RawAffixText;
|
||||
result.ParsedJson = SerializeParsedEffects(item.Effects);
|
||||
result.ParseStatus = ResolveParseStatus(item.Effects, item.Branches);
|
||||
}
|
||||
|
||||
private static void ApplyImporterProvenance(CriticalResult result, ParsedCriticalResult item)
|
||||
{
|
||||
result.SourcePageNumber = item.SourceBounds.PageNumber;
|
||||
result.SourceImagePath = NormalizeOptionalText(item.SourceImagePath);
|
||||
result.SourceImageCropJson = item.SourceImageCrop is null
|
||||
? null
|
||||
: JsonSerializer.Serialize(item.SourceImageCrop, JsonOptions);
|
||||
}
|
||||
|
||||
private static void ReplaceResultChildren(RolemasterDbContext dbContext, CriticalResult result, ParsedCriticalResult item)
|
||||
{
|
||||
foreach (var branch in result.Branches)
|
||||
{
|
||||
dbContext.CriticalEffects.RemoveRange(branch.Effects);
|
||||
}
|
||||
|
||||
dbContext.CriticalEffects.RemoveRange(result.Effects);
|
||||
dbContext.CriticalBranches.RemoveRange(result.Branches);
|
||||
result.Effects.Clear();
|
||||
result.Branches.Clear();
|
||||
|
||||
foreach (var effect in item.Effects)
|
||||
{
|
||||
result.Effects.Add(CreateEffectEntity(effect));
|
||||
}
|
||||
|
||||
foreach (var branch in item.Branches)
|
||||
{
|
||||
result.Branches.Add(CreateBranchEntity(branch));
|
||||
}
|
||||
}
|
||||
|
||||
private static void RemoveResultGraph(RolemasterDbContext dbContext, CriticalResult result)
|
||||
{
|
||||
foreach (var branch in result.Branches)
|
||||
{
|
||||
dbContext.CriticalEffects.RemoveRange(branch.Effects);
|
||||
}
|
||||
|
||||
dbContext.CriticalEffects.RemoveRange(result.Effects);
|
||||
dbContext.CriticalBranches.RemoveRange(result.Branches);
|
||||
dbContext.CriticalResults.Remove(result);
|
||||
}
|
||||
|
||||
private static void RemoveUnusedAxes(RolemasterDbContext dbContext, CriticalTable entity, ParsedCriticalTable table)
|
||||
{
|
||||
var activeGroupKeys = entity.Results
|
||||
.Where(item => item.CriticalGroup is not null)
|
||||
.Select(item => item.CriticalGroup!.GroupKey)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var importedGroupKeys = table.Groups
|
||||
.Select(item => item.GroupKey)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var group in entity.Groups
|
||||
.Where(item => !activeGroupKeys.Contains(item.GroupKey) && !importedGroupKeys.Contains(item.GroupKey))
|
||||
.ToList())
|
||||
{
|
||||
dbContext.CriticalGroups.Remove(group);
|
||||
entity.Groups.Remove(group);
|
||||
}
|
||||
|
||||
var activeColumnKeys = entity.Results
|
||||
.Select(item => item.CriticalColumn.ColumnKey)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var importedColumnKeys = table.Columns
|
||||
.Select(item => item.ColumnKey)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var column in entity.Columns
|
||||
.Where(item => !activeColumnKeys.Contains(item.ColumnKey) && !importedColumnKeys.Contains(item.ColumnKey))
|
||||
.ToList())
|
||||
{
|
||||
dbContext.CriticalColumns.Remove(column);
|
||||
entity.Columns.Remove(column);
|
||||
}
|
||||
|
||||
var activeRollBandLabels = entity.Results
|
||||
.Select(item => item.CriticalRollBand.Label)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
var importedRollBandLabels = table.RollBands
|
||||
.Select(item => item.Label)
|
||||
.ToHashSet(StringComparer.OrdinalIgnoreCase);
|
||||
|
||||
foreach (var rollBand in entity.RollBands
|
||||
.Where(item => !activeRollBandLabels.Contains(item.Label) && !importedRollBandLabels.Contains(item.Label))
|
||||
.ToList())
|
||||
{
|
||||
dbContext.CriticalRollBands.Remove(rollBand);
|
||||
entity.RollBands.Remove(rollBand);
|
||||
}
|
||||
}
|
||||
|
||||
private static CriticalEffect CreateEffectEntity(ParsedCriticalEffect effect) =>
|
||||
@@ -200,6 +356,23 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
SourceText = effect.SourceText
|
||||
};
|
||||
|
||||
private static CriticalBranch CreateBranchEntity(ParsedCriticalBranch branch) =>
|
||||
new()
|
||||
{
|
||||
BranchKind = branch.BranchKind,
|
||||
ConditionKey = branch.ConditionKey,
|
||||
ConditionText = branch.ConditionText,
|
||||
ConditionJson = "{}",
|
||||
RawText = branch.RawText,
|
||||
DescriptionText = branch.DescriptionText,
|
||||
RawAffixText = branch.RawAffixText,
|
||||
ParsedJson = SerializeParsedEffects(branch.Effects),
|
||||
SortOrder = branch.SortOrder,
|
||||
Effects = branch.Effects
|
||||
.Select(CreateEffectEntity)
|
||||
.ToList()
|
||||
};
|
||||
|
||||
private static string SerializeParsedEffects(IReadOnlyList<ParsedCriticalEffect> effects) =>
|
||||
effects.Count == 0
|
||||
? "{}"
|
||||
@@ -211,4 +384,13 @@ public sealed class CriticalImportLoader(string databasePath)
|
||||
effects.Count > 0 || branches.Any(branch => branch.Effects.Count > 0)
|
||||
? "partial"
|
||||
: "raw";
|
||||
|
||||
private static string? NormalizeOptionalText(string? value) =>
|
||||
string.IsNullOrWhiteSpace(value) ? null : value.Trim();
|
||||
|
||||
private static string CreateResultKey(string? groupKey, string columnKey, string rollBandLabel) =>
|
||||
$"{NormalizeKey(groupKey)}|{NormalizeKey(columnKey)}|{NormalizeKey(rollBandLabel)}";
|
||||
|
||||
private static string NormalizeKey(string? value) =>
|
||||
string.IsNullOrWhiteSpace(value) ? string.Empty : value.Trim().ToUpperInvariant();
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user