xTr1m/RolemasterDB
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Surface parser token review explicitly

Browse Source
This commit is contained in:
Frank Tovar
2026-03-15 15:33:19 +01:00
parent 47b72419ad
commit 74a9436c92
14 changed files with 355 additions and 41 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
docs/player_gm_ux_redesign_plan.md
View File

@@ -719,6 +719,17 @@ Acceptance criteria:
### Phase 7: Parser trustworthiness and token review
Status:
- implemented in the web app on March 15, 2026
Implemented model:
- quick-parse generation now records explicit token-review items for unknown or partially parsed affix tokens instead of relying only on generic note strings
- advanced review calls out unresolved tokens directly alongside generated comparison so curators can see what needs manual attention
- shared affix parsing no longer consumes unsupported legacy matches silently, preventing recognized content from disappearing without review
- regression coverage now includes unknown-token surfacing and keeps Arcane Aether B16 stable through load and re-parse flows
Scope:
- remove silent-loss behavior from generation

4
src/RolemasterDb.App/Components/Pages/Api.razor
View File

@@ -74,7 +74,7 @@
"effects": [],
"branches": []
}</pre>
<p class="panel-copy">Use this to retrieve the full editable result graph for one critical-table cell, including nested branches and normalized effects.</p>
<p class="panel-copy">Use this to retrieve the full editable result graph for one critical-table cell, including nested branches, normalized effects, and review notes for unresolved quick-parse tokens.</p>
</section>
<section class="panel">
@@ -95,7 +95,7 @@
"branches": []
}
}</pre>
<p class="panel-copy">Re-runs the shared single-cell parser, merges the generated result with the current override state, and returns the refreshed editor payload without saving changes.</p>
<p class="panel-copy">Re-runs the shared single-cell parser, merges the generated result with the current override state, and returns the refreshed editor payload without saving changes. Unknown or partially parsed tokens are surfaced explicitly in the returned review data.</p>
</section>
<section class="panel">

24
src/RolemasterDb.App/Components/Shared/CriticalCellEditorDialog.razor
View File

@@ -255,6 +255,25 @@
}
</div>
</div>
@if (Model.GeneratedState.TokenReviewItems.Count > 0)
{
<div class="critical-editor-card nested">
<div class="critical-editor-card-header">
<div>
<strong>Token Review</strong>
<p class="muted critical-editor-inline-copy">These tokens were unknown or only partially understood during generation and need manual review.</p>
</div>
</div>
<div class="critical-editor-validation-list">
@foreach (var issue in Model.GeneratedState.TokenReviewItems)
{
<p class="critical-editor-validation-item">@issue.ReviewText</p>
}
</div>
</div>
}
}
</div>
</details>
@@ -621,6 +640,11 @@
items.Add("Current card matches the fresh generation");
}
if (model.GeneratedState.TokenReviewItems.Count > 0)
{
items.Add($"{model.GeneratedState.TokenReviewItems.Count} token review item{(model.GeneratedState.TokenReviewItems.Count == 1 ? string.Empty : "s")}");
}
if (model.GeneratedState.ValidationMessages.Count > 0)
{
items.Add($"{model.GeneratedState.ValidationMessages.Count} parser note{(model.GeneratedState.ValidationMessages.Count == 1 ? string.Empty : "s")}");

3
src/RolemasterDb.App/Features/CriticalCellComparisonState.cs
View File

@@ -6,4 +6,5 @@ public sealed record CriticalCellComparisonState(
string DescriptionText,
IReadOnlyList<CriticalEffectLookupResponse> Effects,
IReadOnlyList<CriticalBranchLookupResponse> Branches,
IReadOnlyList<string> ValidationMessages);
IReadOnlyList<string> ValidationMessages,
IReadOnlyList<CriticalTokenReviewItem> TokenReviewItems);

9
src/RolemasterDb.App/Features/CriticalTokenReviewItem.cs Normal file
View File

@@ -0,0 +1,9 @@
using System.Collections.Generic;
namespace RolemasterDb.App.Features;
public sealed record CriticalTokenReviewItem(
string Scope,
string? ConditionText,
string Token,
string ReviewText);

12
src/RolemasterDb.App/Features/LookupService.cs
View File

@@ -535,7 +535,17 @@ public sealed class LookupService(IDbContextFactory<RolemasterDbContext> dbConte
.OrderBy(branch => branch.SortOrder)
.Select(CreateBranchLookupResponse)
.ToList(),
content.ValidationErrors.ToList());
content.ValidationErrors.ToList(),
content.TokenReviewIssues
.Select(CreateTokenReviewItem)
.ToList());
private static CriticalTokenReviewItem CreateTokenReviewItem(SharedParsing.CriticalTokenReviewIssue issue) =>
new(
issue.Scope,
issue.ConditionText,
issue.Token,
issue.ReviewText);
private static CriticalEffectEditorItem CreateEffectEditorItem(CriticalEffect effect, string originKey) =>
new(

115
src/RolemasterDb.CriticalParsing/AffixEffectParser.cs
View File

@@ -35,39 +35,40 @@ public static class AffixEffectParser
return effects;
}
public static bool TryParseAffixToken(
internal static AffixTokenParseResult ParseAffixToken(
string token,
AffixLegend affixLegend,
out IReadOnlyList<ParsedCriticalEffect> effects,
out string normalizedToken)
AffixLegend affixLegend)
{
normalizedToken = CriticalCellParserSupport.CollapseWhitespace(token);
var normalizedToken = CriticalCellParserSupport.CollapseWhitespace(token);
if (string.IsNullOrWhiteSpace(normalizedToken) || normalizedToken is "-" or "–" or "—")
{
return new AffixTokenParseResult([], normalizedToken, []);
}
if (TryParseQuickToken(normalizedToken, out var quickEffects, out var canonicalToken))
{
normalizedToken = canonicalToken;
effects = quickEffects;
return true;
return new AffixTokenParseResult(quickEffects, canonicalToken, []);
}
var legacyEffects = Parse(normalizedToken, affixLegend);
if (legacyEffects.Count > 0)
{
effects = legacyEffects;
return true;
}
effects = [];
return false;
return ParseLegacyToken(normalizedToken, affixLegend);
}
private static void ParseLine(string line, AffixLegend affixLegend, List<ParsedCriticalEffect> effects)
{
ParseLine(line, affixLegend, effects, []);
}
private static void ParseLine(
string line,
AffixLegend affixLegend,
List<ParsedCriticalEffect> effects,
List<(int Start, int End)> consumedRanges)
{
if (string.IsNullOrWhiteSpace(line) || line is "-" or "–" or "—")
{
return;
}
var consumedRanges = new List<(int Start, int End)>();
var matchedEffects = new List<(int Index, ParsedCriticalEffect Effect)>();
AddMatches(
@@ -180,6 +181,7 @@ public static class AffixEffectParser
continue;
}
var effectCountBeforeMatch = matchedEffects.Count;
var magnitude = match.Groups["count"].Success
? int.Parse(match.Groups["count"].Value)
: 1;
@@ -195,6 +197,11 @@ public static class AffixEffectParser
matchedEffects.Add((match.Index, CreateSymbolEffect(effectCode, magnitude, matchedText)));
}
if (matchedEffects.Count > effectCountBeforeMatch)
{
consumedRanges.Add((match.Index, match.Index + match.Length));
}
}
if (matchedEffects.Count > 0)
@@ -459,16 +466,84 @@ public static class AffixEffectParser
continue;
}
consumedRanges.Add((match.Index, match.Index + match.Length));
var effect = createEffect(match);
if (effect is not null)
{
consumedRanges.Add((match.Index, match.Index + match.Length));
matchedEffects.Add((match.Index, effect));
}
}
}
private static AffixTokenParseResult ParseLegacyToken(string normalizedToken, AffixLegend affixLegend)
{
var effects = new List<ParsedCriticalEffect>();
var consumedRanges = new List<(int Start, int End)>();
ParseLine(normalizedToken, affixLegend, effects, consumedRanges);
var unresolvedFragments = ExtractUnresolvedFragments(normalizedToken, consumedRanges);
var canonicalToken = effects
.Select(effect => effect.SourceText)
.Where(sourceText => !string.IsNullOrWhiteSpace(sourceText))
.Distinct(StringComparer.Ordinal)
.ToList();
return new AffixTokenParseResult(
effects,
canonicalToken.Count == 0 ? normalizedToken : string.Join(", ", canonicalToken),
unresolvedFragments);
}
private static IReadOnlyList<string> ExtractUnresolvedFragments(
string normalizedToken,
IReadOnlyList<(int Start, int End)> consumedRanges)
{
if (string.IsNullOrWhiteSpace(normalizedToken))
{
return [];
}
if (consumedRanges.Count == 0)
{
return [normalizedToken];
}
var unresolved = new List<string>();
var orderedRanges = consumedRanges
.OrderBy(range => range.Start)
.ToList();
var currentIndex = 0;
foreach (var range in orderedRanges)
{
if (currentIndex < range.Start)
{
AddUnresolvedFragment(normalizedToken[currentIndex..range.Start], unresolved);
}
currentIndex = Math.Max(currentIndex, range.End);
}
if (currentIndex < normalizedToken.Length)
{
AddUnresolvedFragment(normalizedToken[currentIndex..], unresolved);
}
return unresolved;
}
private static void AddUnresolvedFragment(string candidate, List<string> unresolved)
{
var normalizedFragment = candidate
.Trim()
.Trim(',', ';', '/', '\\');
if (!string.IsNullOrWhiteSpace(normalizedFragment))
{
unresolved.Add(normalizedFragment);
}
}
private static bool OverlapsConsumedRange(Match match, IReadOnlyList<(int Start, int End)> consumedRanges) =>
consumedRanges.Any(range => match.Index < range.End && range.Start < match.Index + match.Length);

6
src/RolemasterDb.CriticalParsing/AffixTokenParseResult.cs Normal file
View File

@@ -0,0 +1,6 @@
namespace RolemasterDb.CriticalParsing;
internal sealed record AffixTokenParseResult(
IReadOnlyList<ParsedCriticalEffect> Effects,
string NormalizedToken,
IReadOnlyList<string> UnresolvedFragments);

4
src/RolemasterDb.CriticalParsing/CriticalCellParseContent.cs
View File

@@ -7,7 +7,8 @@ public sealed class CriticalCellParseContent(
string? rawAffixText,
IReadOnlyList<ParsedCriticalEffect> effects,
IReadOnlyList<ParsedCriticalBranch> branches,
IReadOnlyList<string> validationErrors)
IReadOnlyList<string> validationErrors,
IReadOnlyList<CriticalTokenReviewIssue> tokenReviewIssues)
{
public IReadOnlyList<string> BaseLines { get; } = baseLines;
public string RawCellText { get; } = rawCellText;
@@ -16,4 +17,5 @@ public sealed class CriticalCellParseContent(
public IReadOnlyList<ParsedCriticalEffect> Effects { get; } = effects;
public IReadOnlyList<ParsedCriticalBranch> Branches { get; } = branches;
public IReadOnlyList<string> ValidationErrors { get; } = validationErrors;
public IReadOnlyList<CriticalTokenReviewIssue> TokenReviewIssues { get; } = tokenReviewIssues;
}

2
src/RolemasterDb.CriticalParsing/CriticalCellTextParser.cs
View File

@@ -37,7 +37,7 @@ public static class CriticalCellTextParser
var (rawText, descriptionText, rawAffixText) = BuildTextSections(baseLines, affixLegendSymbols);
var effects = AffixEffectParser.Parse(rawAffixText, affixLegend);
return new CriticalCellParseContent(baseLines, rawText, descriptionText, rawAffixText, effects, branches, validationErrors);
return new CriticalCellParseContent(baseLines, rawText, descriptionText, rawAffixText, effects, branches, validationErrors, []);
}
private static ParsedCriticalBranch ParseBranch(

122
src/RolemasterDb.CriticalParsing/CriticalQuickNotationParser.cs
View File

@@ -12,10 +12,11 @@ public static class CriticalQuickNotationParser
if (lines.Count == 0)
{
return new CriticalCellParseContent([], string.Empty, string.Empty, null, [], [], []);
return new CriticalCellParseContent([], string.Empty, string.Empty, null, [], [], [], []);
}
var validationErrors = new List<string>();
var tokenReviewIssues = new List<CriticalTokenReviewIssue>();
var descriptionText = lines[0];
var rawAffixLines = new List<string>();
var effects = new List<ParsedCriticalEffect>();
@@ -25,11 +26,11 @@ public static class CriticalQuickNotationParser
{
if (line.Contains(':', StringComparison.Ordinal))
{
branches.Add(ParseBranch(line, branches.Count + 1, affixLegend, validationErrors));
branches.Add(ParseBranch(line, branches.Count + 1, affixLegend, validationErrors, tokenReviewIssues));
continue;
}
var normalizedAffixLine = ParseBaseAffixLine(line, affixLegend, validationErrors, effects);
var normalizedAffixLine = ParseBaseAffixLine(line, affixLegend, validationErrors, tokenReviewIssues, effects);
if (!string.IsNullOrWhiteSpace(normalizedAffixLine))
{
rawAffixLines.Add(normalizedAffixLine);
@@ -43,13 +44,15 @@ public static class CriticalQuickNotationParser
rawAffixLines.Count == 0 ? null : string.Join(Environment.NewLine, rawAffixLines),
effects,
branches,
validationErrors);
validationErrors,
tokenReviewIssues);
}
private static string? ParseBaseAffixLine(
string line,
AffixLegend affixLegend,
List<string> validationErrors,
List<CriticalTokenReviewIssue> tokenReviewIssues,
List<ParsedCriticalEffect> effects)
{
var tokens = SplitPayload(line);
@@ -57,14 +60,20 @@ public static class CriticalQuickNotationParser
foreach (var token in tokens)
{
if (!AffixEffectParser.TryParseAffixToken(token, affixLegend, out var parsedEffects, out var normalizedToken))
var parseResult = AffixEffectParser.ParseAffixToken(token, affixLegend);
if (parseResult.Effects.Count > 0)
{
validationErrors.Add($"Base content has unrecognized affix token '{token}'.");
continue;
normalizedTokens.Add(parseResult.NormalizedToken);
effects.AddRange(parseResult.Effects);
}
normalizedTokens.Add(normalizedToken);
effects.AddRange(parsedEffects);
AddTokenReviewIssues(
tokenReviewIssues,
validationErrors,
scope: "base",
conditionText: null,
originalToken: token,
parseResult);
}
return normalizedTokens.Count == 0 ? null : string.Join(", ", normalizedTokens);
@@ -74,7 +83,8 @@ public static class CriticalQuickNotationParser
string line,
int sortOrder,
AffixLegend affixLegend,
List<string> validationErrors)
List<string> validationErrors,
List<CriticalTokenReviewIssue> tokenReviewIssues)
{
var separatorIndex = line.IndexOf(':', StringComparison.Ordinal);
var conditionText = CriticalCellParserSupport.CollapseWhitespace(line[..separatorIndex]);
@@ -87,17 +97,31 @@ public static class CriticalQuickNotationParser
foreach (var token in tokens)
{
if (AffixEffectParser.TryParseAffixToken(token, affixLegend, out var parsedEffects, out var normalizedToken))
var parseResult = AffixEffectParser.ParseAffixToken(token, affixLegend);
if (parseResult.Effects.Count > 0)
{
seenAffix = true;
normalizedAffixTokens.Add(normalizedToken);
effects.AddRange(parsedEffects);
normalizedAffixTokens.Add(parseResult.NormalizedToken);
effects.AddRange(parseResult.Effects);
AddTokenReviewIssues(
tokenReviewIssues,
validationErrors,
scope: "branch",
conditionText,
originalToken: token,
parseResult);
continue;
}
if (seenAffix)
if (seenAffix || LooksLikeAffixToken(token, affixLegend))
{
validationErrors.Add($"Branch '{conditionText}' has unrecognized affix token '{token}'.");
AddTokenReviewIssues(
tokenReviewIssues,
validationErrors,
scope: "branch",
conditionText,
originalToken: token,
parseResult);
continue;
}
@@ -128,4 +152,72 @@ public static class CriticalQuickNotationParser
.Select(CriticalCellParserSupport.CollapseWhitespace)
.Where(token => !string.IsNullOrWhiteSpace(token))
.ToList();
private static void AddTokenReviewIssues(
List<CriticalTokenReviewIssue> tokenReviewIssues,
List<string> validationErrors,
string scope,
string? conditionText,
string originalToken,
AffixTokenParseResult parseResult)
{
if (parseResult.UnresolvedFragments.Count == 0)
{
return;
}
var reviewText = BuildReviewText(scope, conditionText, originalToken, parseResult);
var issue = new CriticalTokenReviewIssue(
scope,
conditionText,
CriticalCellParserSupport.CollapseWhitespace(originalToken),
reviewText);
tokenReviewIssues.Add(issue);
validationErrors.Add(reviewText);
}
private static string BuildReviewText(
string scope,
string? conditionText,
string originalToken,
AffixTokenParseResult parseResult)
{
var normalizedToken = CriticalCellParserSupport.CollapseWhitespace(originalToken);
var scopeLabel = scope == "branch"
? $"Branch '{conditionText}'"
: "Base content";
if (parseResult.Effects.Count == 0)
{
return $"{scopeLabel} has unrecognized affix token '{normalizedToken}'.";
}
var unresolvedText = string.Join(", ", parseResult.UnresolvedFragments.Select(fragment => $"'{fragment}'"));
var fragmentLabel = parseResult.UnresolvedFragments.Count == 1 ? "fragment" : "fragments";
return $"{scopeLabel} token '{normalizedToken}' still has unresolved {fragmentLabel} {unresolvedText}.";
}
private static bool LooksLikeAffixToken(string token, AffixLegend affixLegend)
{
var normalized = CriticalCellParserSupport.CollapseWhitespace(token);
if (string.IsNullOrWhiteSpace(normalized))
{
return false;
}
if (normalized.Any(char.IsDigit) ||
normalized.StartsWith('+') ||
normalized.StartsWith('-'))
{
return true;
}
if (normalized.Contains("pp", StringComparison.OrdinalIgnoreCase))
{
return true;
}
return affixLegend.EffectSymbols.Any(symbol => normalized.Contains(symbol, StringComparison.Ordinal));
}
}

7
src/RolemasterDb.CriticalParsing/CriticalTokenReviewIssue.cs Normal file
View File

@@ -0,0 +1,7 @@
namespace RolemasterDb.CriticalParsing;
public sealed record CriticalTokenReviewIssue(
string Scope,
string? ConditionText,
string Token,
string ReviewText);

1
src/RolemasterDb.ImportTool.Tests/CriticalCellComparisonEvaluatorTests.cs
View File

@@ -77,6 +77,7 @@ public sealed class CriticalCellComparisonEvaluatorTests
"+10")
],
[],
[],
[]);
var differenceCount = CriticalCellComparisonEvaluator.GetDifferenceCount(

76
src/RolemasterDb.ImportTool.Tests/CriticalCellReparseIntegrationTests.cs
View File

@@ -65,6 +65,53 @@ public sealed class CriticalCellReparseIntegrationTests
Assert.Empty(content.ValidationErrors);
}
[Fact]
public void Shared_quick_notation_parser_surfaces_unknown_base_tokens_explicitly()
{
var legend = new AffixLegend(
new Dictionary<string, string>(StringComparer.Ordinal)
{
["π"] = AppCriticalEffectCodes.MustParryRounds
},
[],
supportsFoePenalty: false,
supportsAttackerBonus: false,
supportsPowerPointModifier: false);
var content = CriticalQuickNotationParser.Parse(
"Foe reels away from the blast.\r\n+5, mystery",
legend);
Assert.Contains(content.Effects, effect => effect.EffectCode == AppCriticalEffectCodes.DirectHits && effect.ValueInteger == 5);
Assert.Single(content.TokenReviewIssues);
Assert.Equal("mystery", content.TokenReviewIssues[0].Token);
Assert.Contains("unrecognized affix token 'mystery'", content.TokenReviewIssues[0].ReviewText, StringComparison.Ordinal);
Assert.Contains(content.ValidationErrors, message => message.Contains("'mystery'", StringComparison.Ordinal));
}
[Fact]
public void Shared_quick_notation_parser_keeps_recognized_effects_when_a_token_is_only_partially_understood()
{
var legend = new AffixLegend(
new Dictionary<string, string>(StringComparer.Ordinal)
{
["π"] = AppCriticalEffectCodes.MustParryRounds
},
[],
supportsFoePenalty: false,
supportsAttackerBonus: false,
supportsPowerPointModifier: false);
var content = CriticalQuickNotationParser.Parse(
"Foe hesitates.\r\nπ?",
legend);
Assert.Contains(content.Effects, effect => effect.EffectCode == AppCriticalEffectCodes.MustParryRounds && effect.DurationRounds == 1);
Assert.Single(content.TokenReviewIssues);
Assert.Equal("π?", content.TokenReviewIssues[0].Token);
Assert.Contains("unresolved fragment '?'", content.TokenReviewIssues[0].ReviewText, StringComparison.Ordinal);
}
[Fact]
public void Shared_symbol_affix_parser_supports_negative_power_point_notation()
{
@@ -176,6 +223,8 @@ public sealed class CriticalCellReparseIntegrationTests
branch.DescriptionText == "glancing blow." &&
branch.Effects.Any(effect => effect.EffectCode == AppCriticalEffectCodes.StunnedRounds && effect.DurationRounds == 2));
Assert.Empty(response.ValidationMessages);
Assert.NotNull(response.GeneratedState);
Assert.Empty(response.GeneratedState!.TokenReviewItems);
}
[Fact]
@@ -322,6 +371,7 @@ public sealed class CriticalCellReparseIntegrationTests
response!.QuickParseInput.Replace("\r\n", "\n", StringComparison.Ordinal));
Assert.NotNull(response.GeneratedState);
Assert.Contains(response.GeneratedState!.Effects, effect => effect.EffectCode == AppCriticalEffectCodes.MustParryRounds && effect.DurationRounds == 1);
Assert.Empty(response.GeneratedState.TokenReviewItems);
var reparsed = await lookupService.ReparseCriticalCellAsync(
"arcane-aether",
@@ -331,6 +381,32 @@ public sealed class CriticalCellReparseIntegrationTests
Assert.NotNull(reparsed);
Assert.NotNull(reparsed!.GeneratedState);
Assert.Contains(reparsed.GeneratedState!.Effects, effect => effect.EffectCode == AppCriticalEffectCodes.MustParryRounds && effect.DurationRounds == 1);
Assert.Empty(reparsed.GeneratedState.TokenReviewItems);
}
[Fact]
public async Task Lookup_service_reparse_surfaces_unknown_tokens_in_generated_review()
{
var databasePath = Path.Combine(Path.GetTempPath(), $"rolemaster-reparse-unknown-{Guid.NewGuid():N}.db");
await SeedSlashResultAsync(databasePath);
var lookupService = new LookupService(CreateDbContextFactory(databasePath));
var resultId = await GetSlashResultIdAsync(databasePath);
var response = await lookupService.ReparseCriticalCellAsync(
"slash",
resultId,
CreateEditorRequest(
"Imported OCR source",
"Strike to thigh.",
quickParseInput: "Strike to thigh.\n+10, mystery"));
Assert.NotNull(response);
Assert.NotNull(response!.GeneratedState);
Assert.Single(response.GeneratedState!.TokenReviewItems);
Assert.Equal("mystery", response.GeneratedState.TokenReviewItems[0].Token);
Assert.Contains("unrecognized affix token 'mystery'", response.GeneratedState.TokenReviewItems[0].ReviewText, StringComparison.Ordinal);
Assert.Contains(response.ValidationMessages, message => message.Contains("'mystery'", StringComparison.Ordinal));
}
[Fact]