From 8afdb1e727c12df0677717dd60f3cd5eefd5687f Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 10:58:56 +0000 Subject: [PATCH 01/21] WIP layout model: TokenStream source, named line-break derivation, RefuseOnMatch / AfterLineBreak grammar arms, controls Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/compiler/01_tokenize.dag | 4 +- src/v2/compiler/02_parse.dag | 426 +++++++++++++++--- src/v2/compiler/body_lowering_fold.dag | 4 +- src/v2/compiler/namespace_graft.dag | 4 +- src/v2/compiler/occurrence_role.dag | 4 +- src/v2/std/compilers/lexing.dag | 69 ++- src/v2/std/grammar.dag | 38 ++ src/v2/std/parse_refusal_reason.dag | 6 + ...parse_table_memo_governed_witness_test.dag | 14 +- .../test/claim/parse/grammar_expect_test.dag | 6 +- .../claim/parse/layout_line_break_test.dag | 262 +++++++++++ .../parse/parse_table_content_key_test.dag | 4 +- .../parse_memo_frame_replay_test.dag | 8 +- 13 files changed, 772 insertions(+), 77 deletions(-) create mode 100644 src/v2/test/claim/parse/layout_line_break_test.dag diff --git a/src/v2/compiler/01_tokenize.dag b/src/v2/compiler/01_tokenize.dag index 7c84a2c83bd..7cc0cc0d1b4 100644 --- a/src/v2/compiler/01_tokenize.dag +++ b/src/v2/compiler/01_tokenize.dag @@ -33,7 +33,7 @@ import v2.std.compilers.lexing { lex_artifact_new, lex_artifact_semantic_tokens, token_stream_empty, - token_stream_new, + token_stream_lexed, TriviaRule, UnboundSourceAnnotation, VoidLexRules, @@ -843,7 +843,7 @@ fn lex_walk_artifact(source: v2.std.text.String, file: Symbol, rules: LexRuleSet } else { v2.std.diagnostic.Accepted { value: lex_artifact_new( - tokens: token_stream_new(all: result.tokens.reverse()), + tokens: token_stream_lexed(all: result.tokens.reverse(), source: source), annotations: result.annotations.reverse() ), diagnostics: v2.std.diagnostic.None diff --git a/src/v2/compiler/02_parse.dag b/src/v2/compiler/02_parse.dag index 6be20f3f105..47e76e2b27b 100644 --- a/src/v2/compiler/02_parse.dag +++ b/src/v2/compiler/02_parse.dag @@ -99,6 +99,7 @@ import v2.std.witness { Holds, Violates, Witness, witness_from_optional } import std.cache_identity { ArtifactKindId, CacheInterfaceId } import std.realization { Materialization } import v2.std.integer { integer_int_content_digest } +import std.unicode.types { unicode_char_code_point } import v2.compiler.materialization_carriers { parse_table_memo_artifact_kind, parse_table_memo_provider_id, @@ -112,16 +113,24 @@ import v2.extdeps.languages.dag { dag_string_literal_node_from_lexeme } import v2.std.compilers.lexing { + LineBreakAnswer, + LineBreakInGap, + LineBreakUnknownWithoutSource, Lexeme, + NoLineBreakInGap, Token, TokenStream, + token_stream_line_break_between, token_stream_remaining, token_stream_remaining_count, + token_stream_source, } -import v2.std.parse_refusal_reason { ParseRefusalReason, parse_refusal_reason_symbol } +import v2.std.parse_refusal_reason { LayoutUnknownWithoutSource, ParseRefusalReason, parse_refusal_reason_symbol } import v2.std.grammar { + AfterLineBreak, Choice, Expect, + RefuseOnMatch, LiteralTerminal, ParseGrammar, GrammarExpr, @@ -287,9 +296,10 @@ type ParseMemoEntry captured: Node, diagnostics: List, prov: ParseProvenanceState, - furthest: ParseFurthestFailure + furthest: ParseFurthestFailure, + refusal: Optional } - | ParseMemoRejected { diagnostics: List, furthest: ParseFurthestFailure } + | ParseMemoRejected { diagnostics: List, furthest: ParseFurthestFailure, refusal: Optional } type ParseExprResult = ParseExprAccepted { @@ -340,6 +350,8 @@ type ParseTableRealization { first_rows: Map> productions_by_name: Map prepared_exprs: Map + source: Optional + refusal: Optional } type ParseTable = ParseTableRealization @@ -639,10 +651,11 @@ fn parse_memo_entry_from_expr(result: ParseExprResult) -> ParseMemoEntry { captured: captured, diagnostics: diagnostics, prov: prov, - furthest: parse_expr_result_table(result: result).furthest + furthest: parse_expr_result_table(result: result).furthest, + refusal: parse_expr_result_table(result: result).refusal } ParseExprRejected { diagnostics: diagnostics, table: table } => - ParseMemoRejected { diagnostics: diagnostics, furthest: table.furthest } + ParseMemoRejected { diagnostics: diagnostics, furthest: table.furthest, refusal: table.refusal } } } @@ -703,7 +716,9 @@ fn parse_table_carry_analysis(table: ParseTable, entries: Map) -> ParseTable { + match table.refusal { + Present { value: _ } => table + Absent => + match refusal { + Absent => table + Present { value: d } => parse_table_with_refusal(table: table, refusal: d) + } + } +} + +fn parse_table_with_refusal(table: ParseTable, refusal: Diagnostic) -> ParseTable { + ParseTableRealization { + artifact_kind: table.artifact_kind, + provider_id: table.provider_id, + materialization: table.materialization, + grammar_digest: table.grammar_digest, + token_stream_digest: table.token_stream_digest, + entries: table.entries, + memo_hits: table.memo_hits, + memo_misses: table.memo_misses, + memo_lookup_calls: table.memo_lookup_calls, + furthest: table.furthest, + nullable_set: table.nullable_set, + first_rows: table.first_rows, + productions_by_name: table.productions_by_name, + prepared_exprs: table.prepared_exprs, + source: table.source, + refusal: optional_present(value: refusal) + } +} + fn parse_table_record_lookup_call(table: ParseTable) -> ParseTable { parse_table_carry_analysis( table: table, @@ -841,6 +892,21 @@ fn parse_grammar_digest(validated: ValidatedGrammarRoot) -> Hash { // byte 255 and every non-Latin-1 codepoint collide in the seed interpreter's cross-parse memo key // (v1_interpreter parse_table_memo_scope_and_key reads this field). Offsets and positions now go // through the Int content authority and the lexeme through its interned symbol. +fn parse_token_stream_source_digest(all_tokens: List, source: Optional) -> Hash { + match source { + Absent => parse_token_stream_digest(all_tokens: all_tokens) + Present { value: text } => + combine_hash( + a: parse_token_stream_digest(all_tokens: all_tokens), + b: fold_list( + xs: text, + empty: symbol_identity_digest(sym: ^parse_token_stream_source_digest_nil), + cons: fn(acc, c) { combine_hash(a: acc, b: integer_int_content_digest(value: unicode_char_code_point(c: c))) } + ) + ) + } +} + fn parse_token_stream_digest(all_tokens: List) -> Hash { fold_list( xs: all_tokens, @@ -876,13 +942,33 @@ fn parse_table_for_production_analyzed( analysis: GrammarFirstAnalysis, all_tokens: List, materialization: Materialization +) -> ParseTable { + parse_table_for_stream_analyzed( + validated: validated, + analysis: analysis, + all_tokens: all_tokens, + source: optional_absent(), + materialization: materialization + ) +} + +// The stream's source rides on the table so a layout-conditioned production can ask its question at +// the point of use (v2.std.compilers.lexing token_stream_line_break_between), and it is part of the +// stream's digest: two sources can lex to byte-identical tokens and differ only in layout, and a memo +// entry must never serve one's answer to the other. +fn parse_table_for_stream_analyzed( + validated: ValidatedGrammarRoot, + analysis: GrammarFirstAnalysis, + all_tokens: List, + source: Optional, + materialization: Materialization ) -> ParseTable { ParseTableRealization { artifact_kind: v2.compiler.materialization_carriers.parse_table_memo_artifact_kind(), provider_id: v2.compiler.materialization_carriers.parse_table_memo_provider_id(), materialization: materialization, grammar_digest: analysis.grammar_digest, - token_stream_digest: parse_token_stream_digest(all_tokens: all_tokens), + token_stream_digest: parse_token_stream_source_digest(all_tokens: all_tokens, source: source), entries: empty_map(), memo_hits: 0, memo_misses: 0, @@ -891,7 +977,9 @@ fn parse_table_for_production_analyzed( nullable_set: analysis.nullable_set, first_rows: analysis.first_rows, productions_by_name: analysis.productions_by_name, - prepared_exprs: analysis.prepared_exprs + prepared_exprs: analysis.prepared_exprs, + source: source, + refusal: optional_absent() } } @@ -927,7 +1015,9 @@ fn parse_table_empty(materialization: Materialization) -> ParseTable { nullable_set: std.algebra.Empty, first_rows: empty_map(), productions_by_name: empty_map(), - prepared_exprs: empty_map() + prepared_exprs: empty_map(), + source: optional_absent(), + refusal: optional_absent() } } @@ -1485,9 +1575,12 @@ fn parse_nonterminal_memoized_core( Holds { value: inner } => let table_hit = parse_table_record_hit(table: table_at_lookup) match inner { - ParseMemoRejected { diagnostics: diags, furthest: served } => - ParseExprRejected { diagnostics: diags, table: parse_table_join_furthest(table: table_hit, furthest: served) } - ParseMemoAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: inner_prov, furthest: served } => + ParseMemoRejected { diagnostics: diags, furthest: served, refusal: served_refusal } => + ParseExprRejected { + diagnostics: diags, + table: parse_table_join_refusal(table: parse_table_join_furthest(table: table_hit, furthest: served), refusal: served_refusal) + } + ParseMemoAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: inner_prov, furthest: served, refusal: served_refusal } => let merged_prov = parse_prov_merge(base: prov, incoming: inner_prov) let stamped = parse_wrap_production_captured( production: production, @@ -1499,7 +1592,7 @@ fn parse_nonterminal_memoized_core( captured: stamped.node, diagnostics: diags, prov: stamped.prov, - table: parse_table_join_furthest(table: table_hit, furthest: served) + table: parse_table_join_refusal(table: parse_table_join_furthest(table: table_hit, furthest: served), refusal: served_refusal) } } Violates { diagnostic: _ } => @@ -1561,7 +1654,9 @@ fn expr_is_nullable_given_set(expr: GrammarExpr, nullable_set: List) -> choice: fn(left_nullable, right_nullable) { left_nullable || right_nullable }, optional: fn(_) { true }, repeat: fn(_) { true }, - expect: fn(element_nullable, _reason) { element_nullable } + expect: fn(element_nullable, _reason) { element_nullable }, + refuse_on_match: fn(element_nullable, _reason) { element_nullable }, + after_line_break: fn(element_nullable) { element_nullable } } ) } @@ -1846,7 +1941,9 @@ fn expr_first_fold( choice: expr_first_fold_choice, optional: expr_first_fold_optional, repeat: expr_first_fold_repeat, - expect: fn(element, _reason) { element } + expect: fn(element, _reason) { element }, + refuse_on_match: fn(element, _reason) { element }, + after_line_break: fn(element) { element } } ) } @@ -1981,6 +2078,8 @@ type PreparedGrammarExpr | PreparedOptional { element: PreparedGrammarExpr } | PreparedRepeat { element: PreparedGrammarExpr } | PreparedExpect { element: PreparedGrammarExpr, reason: ParseRefusalReason } + | PreparedRefuseOnMatch { element: PreparedGrammarExpr, reason: ParseRefusalReason } + | PreparedAfterLineBreak { element: PreparedGrammarExpr } type PreparedExprFold { tree: PreparedGrammarExpr @@ -2024,6 +2123,8 @@ fn flatten_choice_exprs(expr: GrammarExpr) -> List { Optional { element: _ } => Cons { head: expr, tail: std.algebra.Empty } Repeat { element: _ } => Cons { head: expr, tail: std.algebra.Empty } Expect { element: _, reason: _ } => Cons { head: expr, tail: std.algebra.Empty } + RefuseOnMatch { element: _, reason: _ } => Cons { head: expr, tail: std.algebra.Empty } + AfterLineBreak { element: _ } => Cons { head: expr, tail: std.algebra.Empty } } } @@ -2063,6 +2164,8 @@ fn prepared_prefix_atom_eq(left: PreparedGrammarExpr, right: PreparedGrammarExpr PreparedOptional { element: _ } => false PreparedRepeat { element: _ } => false PreparedExpect { element: _, reason: _ } => false + PreparedRefuseOnMatch { element: _, reason: _ } => false + PreparedAfterLineBreak { element: _ } => false } PreparedLiteralTerminal { token_class: lc, lexeme: lw } => match right { @@ -2077,6 +2180,8 @@ fn prepared_prefix_atom_eq(left: PreparedGrammarExpr, right: PreparedGrammarExpr PreparedOptional { element: _ } => false PreparedRepeat { element: _ } => false PreparedExpect { element: _, reason: _ } => false + PreparedRefuseOnMatch { element: _, reason: _ } => false + PreparedAfterLineBreak { element: _ } => false } PreparedNonterminal { production: lp } => match right { @@ -2088,12 +2193,16 @@ fn prepared_prefix_atom_eq(left: PreparedGrammarExpr, right: PreparedGrammarExpr PreparedOptional { element: _ } => false PreparedRepeat { element: _ } => false PreparedExpect { element: _, reason: _ } => false + PreparedRefuseOnMatch { element: _, reason: _ } => false + PreparedAfterLineBreak { element: _ } => false } PreparedSequence { left: _, right: _ } => false PreparedChoice { plan: _, first: _ } => false PreparedOptional { element: _ } => false PreparedRepeat { element: _ } => false PreparedExpect { element: _, reason: _ } => false + PreparedRefuseOnMatch { element: _, reason: _ } => false + PreparedAfterLineBreak { element: _ } => false } } @@ -2109,6 +2218,8 @@ fn prepared_sequence_left_atom(expr: PreparedGrammarExpr) -> Optional optional_absent() PreparedRepeat { element: _ } => optional_absent() PreparedExpect { element: _, reason: _ } => optional_absent() + PreparedRefuseOnMatch { element: _, reason: _ } => optional_absent() + PreparedAfterLineBreak { element: _ } => optional_absent() } PreparedTerminal { token_class: _, stamp: _ } => optional_absent() PreparedLiteralTerminal { token_class: _, lexeme: _ } => optional_absent() @@ -2117,6 +2228,8 @@ fn prepared_sequence_left_atom(expr: PreparedGrammarExpr) -> Optional optional_absent() PreparedRepeat { element: _ } => optional_absent() PreparedExpect { element: _, reason: _ } => optional_absent() + PreparedRefuseOnMatch { element: _, reason: _ } => optional_absent() + PreparedAfterLineBreak { element: _ } => optional_absent() } } @@ -2246,12 +2359,16 @@ fn prepared_terminal_prefix(expr: PreparedGrammarExpr, need: Int) -> Optional
  • optional_absent() PreparedRepeat { element: _ } => optional_absent() PreparedExpect { element: _, reason: _ } => optional_absent() + PreparedRefuseOnMatch { element: _, reason: _ } => optional_absent() + PreparedAfterLineBreak { element: _ } => optional_absent() } PreparedNonterminal { production: _ } => optional_absent() PreparedChoice { plan: _, first: _ } => optional_absent() PreparedOptional { element: _ } => optional_absent() PreparedRepeat { element: _ } => optional_absent() PreparedExpect { element: _, reason: _ } => optional_absent() + PreparedRefuseOnMatch { element: _, reason: _ } => optional_absent() + PreparedAfterLineBreak { element: _ } => optional_absent() } } } @@ -2371,6 +2488,8 @@ fn choice_shared_prefix_suffixes_from( PreparedOptional { element: _ } => optional_absent() PreparedRepeat { element: _ } => optional_absent() PreparedExpect { element: _, reason: _ } => optional_absent() + PreparedRefuseOnMatch { element: _, reason: _ } => optional_absent() + PreparedAfterLineBreak { element: _ } => optional_absent() } } } @@ -2403,6 +2522,10 @@ fn expr_first_fold_from_prepared( ) PreparedExpect { element: element, reason: _ } => expr_first_fold_from_prepared(expr: element, nullable_set: nullable_set, first_rows: first_rows) + PreparedRefuseOnMatch { element: element, reason: _ } => + expr_first_fold_from_prepared(expr: element, nullable_set: nullable_set, first_rows: first_rows) + PreparedAfterLineBreak { element: element } => + expr_first_fold_from_prepared(expr: element, nullable_set: nullable_set, first_rows: first_rows) } } @@ -2701,6 +2824,20 @@ fn prepare_grammar_expr( first: element_prepared.first, residue: element_prepared.residue } + RefuseOnMatch { element: element, reason: reason } => + let element_prepared = prepare_grammar_expr(expr: element, nullable_set: nullable_set, first_rows: first_rows, production: production, token_dispatch: token_dispatch) + PreparedExprFold { + tree: PreparedRefuseOnMatch { element: element_prepared.tree, reason: reason }, + first: element_prepared.first, + residue: element_prepared.residue + } + AfterLineBreak { element: element } => + let element_prepared = prepare_grammar_expr(expr: element, nullable_set: nullable_set, first_rows: first_rows, production: production, token_dispatch: token_dispatch) + PreparedExprFold { + tree: PreparedAfterLineBreak { element: element_prepared.tree }, + first: element_prepared.first, + residue: element_prepared.residue + } } } @@ -2828,7 +2965,9 @@ fn expr_parse_demand_fold(expr: GrammarExpr) -> ParseDemandFold { discarding_sites: e.discarding_sites + parse_demand_fold_site(attempted: e) } }, - expect: fn(e, _reason) { e } + expect: fn(e, _reason) { e }, + refuse_on_match: fn(e, _reason) { e }, + after_line_break: fn(e) { e } } ) } @@ -2980,7 +3119,9 @@ fn expr_corner_nullable_fold(expr: GrammarExpr, nullable_set: List) -> G choice: expr_corner_fold_choice, optional: expr_corner_fold_optional, repeat: expr_corner_fold_repeat, - expect: fn(element, _reason) { element } + expect: fn(element, _reason) { element }, + refuse_on_match: fn(element, _reason) { element }, + after_line_break: fn(element) { element } } ) } @@ -3144,7 +3285,9 @@ fn expr_has_undefined_nonterminal(expr: GrammarExpr, root: GrammarRoot) -> Bool choice: fn(left_undef, right_undef) { left_undef || right_undef }, optional: fn(element_undef) { element_undef }, repeat: fn(element_undef) { element_undef }, - expect: fn(element_undef, _reason) { element_undef } + expect: fn(element_undef, _reason) { element_undef }, + refuse_on_match: fn(element_undef, _reason) { element_undef }, + after_line_break: fn(element_undef) { element_undef } } ) } @@ -3835,6 +3978,120 @@ fn parse_expr_expect( } } +// A MATCH THAT IS A REFUSAL. Where the element matches, the parse refuses with `reason` located at +// the first token of the match, and the table carries that refusal from here on: every later +// attempt rejects at once (parse_expr_with_first), so no alternative can succeed around it, and the +// production door reports it as the outcome. Where the element does not match, this is an ordinary +// failed alternative. +fn parse_expr_refuse_on_match( + element: PreparedGrammarExpr, + reason: ParseRefusalReason, + tokens: List, + all_tokens: List, + root: GrammarRoot, + table: ParseTable, + prov: ParseProvenanceState +) -> ParseExprResult { + match parse_expr_with_first(expr: element, tokens: tokens, all_tokens: all_tokens, root: root, table: table, prov: prov) { + ParseExprRejected { diagnostics: diags, table: table1 } => ParseExprRejected { diagnostics: diags, table: table1 } + ParseExprAccepted { remaining: _, captured: _, diagnostics: _, prov: _, table: table1 } => + ParseExprRejected { + diagnostics: [], + table: parse_table_join_refusal( + table: table1, + refusal: optional_present(value: parse_refusal_at(reason: reason, tokens: tokens, all_tokens: all_tokens)) + ) + } + } +} + +// A MATCH CONDITIONED ON LAYOUT. The element is parsed first, so a layout question is asked only of +// a match; then the source decides (v2.std.compilers.lexing token_stream_line_break_between) whether a +// line break separates the token before the match from its first token. No line break: an ordinary +// failed alternative. A stream that carries no source cannot answer, and the parse refuses with +// LayoutUnknownWithoutSource rather than assume one line. +fn parse_expr_after_line_break( + element: PreparedGrammarExpr, + tokens: List, + all_tokens: List, + root: GrammarRoot, + table: ParseTable, + prov: ParseProvenanceState +) -> ParseExprResult { + match parse_expr_with_first(expr: element, tokens: tokens, all_tokens: all_tokens, root: root, table: table, prov: prov) { + ParseExprRejected { diagnostics: diags, table: table1 } => ParseExprRejected { diagnostics: diags, table: table1 } + ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: prov1, table: table1 } => + match parse_line_break_before_first(tokens: tokens, all_tokens: all_tokens, source: table1.source) { + LineBreakInGap => + ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: prov1, table: table1 } + NoLineBreakInGap => ParseExprRejected { diagnostics: [], table: table1 } + LineBreakUnknownWithoutSource => + ParseExprRejected { + diagnostics: [], + table: parse_table_join_refusal( + table: table1, + refusal: optional_present(value: parse_refusal_at(reason: LayoutUnknownWithoutSource, tokens: tokens, all_tokens: all_tokens)) + ) + } + } + } +} + +// The gap before the first token of `tokens`: from the end of the token before it in the stream (the +// start of the source when it is the first token) to its own start. An empty remainder has no first +// token and so no gap; it answers NoLineBreakInGap, a failed match rather than a layout question. +fn parse_line_break_before_first( + tokens: List, + all_tokens: List, + source: Optional +) -> LineBreakAnswer { + match parse_token_first(tokens: tokens) { + Absent => NoLineBreakInGap + Present { value: first } => + token_stream_line_break_between( + source: source, + from: parse_token_end_before(all_tokens: all_tokens, index: length(xs: all_tokens) - length(xs: tokens)), + to: first.start + ) + } +} + +fn parse_token_end_before(all_tokens: List, index: Int) -> Int { + if index <= 0 { + 0 + } else { + match list_at_optional(xs: all_tokens, index: index - 1) { + Present { value: prev } => prev.end + Absent => 0 + } + } +} + +// A refusal is located at the first token of the match it refuses; with no token left it is located +// zero-width at the end of the last token. +fn parse_refusal_at(reason: ParseRefusalReason, tokens: List, all_tokens: List) -> Diagnostic { + match parse_token_first(tokens: tokens) { + Present { value: tok } => parse_refusal_reason_diagnostic(reason: reason, file: tok.file, start: tok.start, end: tok.end) + Absent => + match list_at_optional(xs: all_tokens, index: length(xs: all_tokens) - 1) { + Present { value: last } => parse_refusal_reason_diagnostic(reason: reason, file: last.file, start: last.end, end: last.end) + Absent => Diagnostic { + reason: parse_refusal_reason_symbol(reason: reason), + at: invariant_port_locus(port: ^parse_stage_locus_port, invariant: ^parse_refusal_without_a_token), + correction: Unavailable { reason: ExternalContractUnknown } + } + } + } +} + +fn parse_refusal_reason_diagnostic(reason: ParseRefusalReason, file: Symbol, start: Int, end: Int) -> Diagnostic { + Diagnostic { + reason: parse_refusal_reason_symbol(reason: reason), + at: source_port_locus(port: ^parse_stage_locus_port, file: file, extent: ByteRange { start: start, end: end }), + correction: Unavailable { reason: UserInputBoundary } + } +} + // A PreparedLiteralTerminal is a MATCH constraint in front of an ordinary class terminal: it // captures and publishes exactly what a StampClass terminal of that class does, never its own text. fn parse_expr_with_first( @@ -3845,11 +4102,15 @@ fn parse_expr_with_first( table: ParseTable, prov: ParseProvenanceState ) -> ParseExprResult { - match parse_expr_dispatch(expr: expr, tokens: tokens, all_tokens: all_tokens, root: root, table: table, prov: prov) { - ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: prov1, table: table1 } => - ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: prov1, table: table1 } - ParseExprRejected { diagnostics: diags, table: table1 } => - ParseExprRejected { diagnostics: diags, table: parse_table_join_furthest(table: table1, furthest: parse_furthest_at(tokens: tokens)) } + match table.refusal { + Present { value: _ } => ParseExprRejected { diagnostics: [], table: table } + Absent => + match parse_expr_dispatch(expr: expr, tokens: tokens, all_tokens: all_tokens, root: root, table: table, prov: prov) { + ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: prov1, table: table1 } => + ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: prov1, table: table1 } + ParseExprRejected { diagnostics: diags, table: table1 } => + ParseExprRejected { diagnostics: diags, table: parse_table_join_furthest(table: table1, furthest: parse_furthest_at(tokens: tokens)) } + } } } @@ -3946,6 +4207,25 @@ fn parse_expr_dispatch( table: table, prov: prov ) + PreparedRefuseOnMatch { element: element, reason: reason } => + parse_expr_refuse_on_match( + element: element, + reason: reason, + tokens: tokens, + all_tokens: all_tokens, + root: root, + table: table, + prov: prov + ) + PreparedAfterLineBreak { element: element } => + parse_expr_after_line_break( + element: element, + tokens: tokens, + all_tokens: all_tokens, + root: root, + table: table, + prov: prov + ) } } @@ -4038,6 +4318,7 @@ fn parse_production_prepared_measured( analysis: analysis, production_name: production_name, seed: occurrence_id_allocator_initial(), + source: optional_absent(), materialization: materialization ) } @@ -4048,62 +4329,35 @@ fn parse_production_prepared_measured_seeded( analysis: GrammarFirstAnalysis, production_name: Symbol, seed: OccurrenceIdAllocator, + source: Optional, materialization: Materialization ) -> ParseProductionMeasured { let root = validated.root match grammar_lookup_production(root: root, name: production_name) { Holds { value: production } => let prov = parse_prov_seeded(seed: seed) - let table = parse_table_for_production_analyzed( + let table = parse_table_for_stream_analyzed( validated: validated, analysis: analysis, + source: source, all_tokens: tokens, materialization: materialization ) - match parse_expr( + let parsed = parse_expr( expr: production.expression, tokens: tokens, all_tokens: tokens, root: root, table: table, prov: prov - ) { - ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: inner_prov, table: final_table } => - if is_empty(xs: remaining) { - let stamped = parse_wrap_production_captured( - production: production, - captured: captured, - prov: inner_prov - ) - ParseProductionMeasured { - outcome: Accepted { - value: ParseArtifact { tree: stamped.node, span_index: stamped.prov.index, alloc: stamped.prov.alloc }, - diagnostics: parse_diags_to_diagnostics(diags: diags) - }, - memo: parse_table_memo_stats(table: final_table) - } - } else { - ParseProductionMeasured { - outcome: Rejected { - diagnostics: diagnostics_singleton( - d: parse_g0_tokens_remain_diagnostic_at(tokens: remaining, furthest: final_table.furthest) - ) - }, - memo: parse_table_memo_stats(table: final_table) - } - } - ParseExprRejected { diagnostics: diags, table: final_table } => + ) + match parse_expr_result_table(result: parsed).refusal { + Present { value: refusal } => ParseProductionMeasured { - outcome: Rejected { - diagnostics: parse_diags_to_non_empty( - diags: diags, - default: parse_e1_syntax_error_diagnostic( - at: node_locus(node: grammar_to_node(grammar: ModeledGrammar { root: root })) - ) - ) - }, - memo: parse_table_memo_stats(table: final_table) + outcome: Rejected { diagnostics: diagnostics_singleton(d: refusal) }, + memo: parse_table_memo_stats(table: parse_expr_result_table(result: parsed)) } + Absent => parse_production_measured_unrefused(parsed: parsed, production: production, root: root) } Violates { diagnostic: d } => ParseProductionMeasured { @@ -4113,6 +4367,55 @@ fn parse_production_prepared_measured_seeded( } } +// A REFUSAL RECORDED ANYWHERE IN THE PARSE IS THE OUTCOME, whether or not the rest of the input +// happened to parse around it: RefuseOnMatch names source the seed refuses although another reading +// succeeds, so an Accepted here would be exactly the success it exists to deny. Only an unrefused +// parse reaches the ordinary accept / leftover-token / no-match reading below. +fn parse_production_measured_unrefused( + parsed: ParseExprResult, + production: GrammarProduction, + root: GrammarRoot +) -> ParseProductionMeasured { + match parsed { + ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: inner_prov, table: final_table } => + if is_empty(xs: remaining) { + let stamped = parse_wrap_production_captured( + production: production, + captured: captured, + prov: inner_prov + ) + ParseProductionMeasured { + outcome: Accepted { + value: ParseArtifact { tree: stamped.node, span_index: stamped.prov.index, alloc: stamped.prov.alloc }, + diagnostics: parse_diags_to_diagnostics(diags: diags) + }, + memo: parse_table_memo_stats(table: final_table) + } + } else { + ParseProductionMeasured { + outcome: Rejected { + diagnostics: diagnostics_singleton( + d: parse_g0_tokens_remain_diagnostic_at(tokens: remaining, furthest: final_table.furthest) + ) + }, + memo: parse_table_memo_stats(table: final_table) + } + } + ParseExprRejected { diagnostics: diags, table: final_table } => + ParseProductionMeasured { + outcome: Rejected { + diagnostics: parse_diags_to_non_empty( + diags: diags, + default: parse_e1_syntax_error_diagnostic( + at: node_locus(node: grammar_to_node(grammar: ModeledGrammar { root: root })) + ) + ) + }, + memo: parse_table_memo_stats(table: final_table) + } + } +} + fn parse_production( tokens: List, validated: ValidatedGrammarRoot, @@ -4299,6 +4602,7 @@ fn parse_module_prepared_measured_seeded( analysis: modeled.analysis, production_name: modeled.validated.root.start, seed: seed, + source: token_stream_source(stream: tokens), materialization: materialization ) match measured.outcome { diff --git a/src/v2/compiler/body_lowering_fold.dag b/src/v2/compiler/body_lowering_fold.dag index 3a629c3f1ce..c0172183122 100644 --- a/src/v2/compiler/body_lowering_fold.dag +++ b/src/v2/compiler/body_lowering_fold.dag @@ -1430,7 +1430,9 @@ fn body_lower_grammar_terminals(expr: GrammarExpr) -> List { choice: fn(l, r) { list_append(left: l, right: r) }, optional: fn(e) { e }, repeat: fn(e) { e }, - expect: fn(e, _reason) { e } + expect: fn(e, _reason) { e }, + refuse_on_match: fn(e, _reason) { e }, + after_line_break: fn(e) { e } } ) } diff --git a/src/v2/compiler/namespace_graft.dag b/src/v2/compiler/namespace_graft.dag index 3023953b27e..402789dbba4 100644 --- a/src/v2/compiler/namespace_graft.dag +++ b/src/v2/compiler/namespace_graft.dag @@ -178,7 +178,9 @@ fn namespace_graft_grammar_expr_nonterminal_productions(expr: GrammarExpr) -> Li choice: fn(left, right) { list_append(left: left, right: right) }, optional: fn(xs) { xs }, repeat: fn(xs) { xs }, - expect: fn(xs, _reason) { xs } + expect: fn(xs, _reason) { xs }, + refuse_on_match: fn(xs, _reason) { xs }, + after_line_break: fn(xs) { xs } } ) } diff --git a/src/v2/compiler/occurrence_role.dag b/src/v2/compiler/occurrence_role.dag index 872093e2646..d55099f09ae 100644 --- a/src/v2/compiler/occurrence_role.dag +++ b/src/v2/compiler/occurrence_role.dag @@ -265,7 +265,9 @@ fn production_holds_name_terminal(expr: GrammarExpr) -> Bool { choice: fn(a, b) { a || b }, optional: fn(a) { a }, repeat: fn(a) { a }, - expect: fn(a, _reason) { a } + expect: fn(a, _reason) { a }, + refuse_on_match: fn(a, _reason) { a }, + after_line_break: fn(a) { a } }) } diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 73578a0a054..0201010ffce 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -22,6 +22,7 @@ import v2.std.logic { Bool } import v2.std.node { Atom, Conj, Node, Symbol, TypeNode } import v2.std.node_shape { stamp_node_shape_synthetic, type_shape } import v2.std.text { Char, String } +import std.unicode.types { unicode_char_code_point, unicode_scalar_utf8_octet_count } type Lexeme = String @@ -149,17 +150,79 @@ type Token { end: Int } +// THE STREAM CARRIES THE SOURCE ITS TOKENS INDEX, AND LAYOUT IS DERIVED FROM IT. A token's +// `start`/`end` say THAT two tokens have a gap between them, not what the gap holds, and the grammar +// sometimes needs to know (a `-` that begins a new line after a complete operand is ambiguous +// between infix and prefix, and the seed refuses it: v1.compiler.parse +// is_ambiguous_prefix_infix_newline_boundary). The source text stays the ONE authority for that +// fact: nothing about layout is stored on a Token or as a token of its own, and +// `token_stream_line_break_between` reads the gap at the point a question is asked. A stream built +// from tokens alone (fixtures) carries `Absent`, and a layout question on it answers +// `LineBreakUnknownWithoutSource` -- never an assumed single line. type TokenStream { all: List pos: Int + source: Optional } fn token_stream_new(all: List) -> TokenStream { - TokenStream { all: all, pos: 0 } + TokenStream { all: all, pos: 0, source: optional_absent() } +} + +// The tokenizer's constructor: the tokens together with the exact scalar sequence they were lexed +// from, whose UTF-8 octet offsets their `start`/`end` name. +fn token_stream_lexed(all: List, source: String) -> TokenStream { + TokenStream { all: all, pos: 0, source: optional_present(value: source) } } fn token_stream_empty() -> TokenStream { - TokenStream { all: Empty, pos: 0 } + TokenStream { all: Empty, pos: 0, source: optional_absent() } +} + +fn token_stream_source(stream: TokenStream) -> Optional { + stream.source +} + +type LineBreakAnswer + = LineBreakInGap + | NoLineBreakInGap + | LineBreakUnknownWithoutSource + +type LineBreakScan { + octet: Int + found: Bool +} + +// Whether the source holds a line feed (U+000A) at an octet offset in [from, to). Offsets are UTF-8 +// octets, as token extents are, so each scalar advances the position by its own encoded width. +fn source_has_line_break_between(source: String, from: Int, to: Int) -> Bool { + fold_list( + xs: source, + empty: LineBreakScan { octet: 0, found: false }, + cons: fn(acc, c) { line_break_scan_step(acc: acc, c: c, from: from, to: to) } + ).found +} + +fn line_break_scan_step(acc: LineBreakScan, c: Char, from: Int, to: Int) -> LineBreakScan { + let code = unicode_char_code_point(c: c) + LineBreakScan { + octet: acc.octet + unicode_scalar_utf8_octet_count(code_point: code), + found: acc.found || ((code == 10) && (acc.octet >= from) && (acc.octet < to)) + } +} + +// THE NAMED DERIVATION. Is there a line break in the source between octet `from` (the end of the +// token before) and octet `to` (the start of the token asked about)? +fn token_stream_line_break_between(source: Optional, from: Int, to: Int) -> LineBreakAnswer { + match source { + Absent => LineBreakUnknownWithoutSource + Present { value: text } => + if source_has_line_break_between(source: text, from: from, to: to) { + LineBreakInGap + } else { + NoLineBreakInGap + } + } } fn token_stream_position(stream: TokenStream) -> Int { @@ -187,7 +250,7 @@ fn token_stream_get(stream: TokenStream, offset: Int) -> Optional { } fn token_stream_advance(stream: TokenStream, n: Int) -> TokenStream { - TokenStream { all: stream.all, pos: stream.pos + n } + TokenStream { all: stream.all, pos: stream.pos + n, source: stream.source } } // AnnotationPlacement is NOT declared here. It is the language-agnostic concept and its authority diff --git a/src/v2/std/grammar.dag b/src/v2/std/grammar.dag index d33415c607d..faede0487f2 100644 --- a/src/v2/std/grammar.dag +++ b/src/v2/std/grammar.dag @@ -231,6 +231,27 @@ type GrammarExpr | Optional { element: GrammarExpr } | Repeat { element: GrammarExpr } | Expect { element: GrammarExpr, reason: ParseRefusalReason } + | RefuseOnMatch { element: GrammarExpr, reason: ParseRefusalReason } + | AfterLineBreak { element: GrammarExpr } + +// A PRODUCTION WHOSE MATCH IS A REFUSAL. `RefuseOnMatch { element, reason }` parses `element` and, +// where it MATCHES, refuses the whole parse with `reason`, located at the first token of the match: +// the refusal is not a failed alternative that ordered choice may backtrack past, it stops the +// parse. Where `element` does not match, it fails like any alternative. It exists for source the +// seed refuses although some other reading would succeed -- the success is the defect -- so a +// relabel (`Expect`) cannot express it. Emit never selects it: it derives no tree. +fn grammar_expr_refuse_on_match(element: GrammarExpr, reason: ParseRefusalReason) -> GrammarExpr { + RefuseOnMatch { element: element, reason: reason } +} + +// A LAYOUT-CONDITIONED MATCH. `AfterLineBreak { element }` matches what `element` matches only when +// the source holds a line break between the token before the match and its first token +// (v2.std.compilers.lexing token_stream_line_break_between); otherwise it fails. On a stream that +// carries no source the question has no answer, and the parse refuses with +// LayoutUnknownWithoutSource rather than assuming one line. +fn grammar_expr_after_line_break(element: GrammarExpr) -> GrammarExpr { + AfterLineBreak { element: element } +} // AN EXPECTATION IS A LABEL, NEVER A CUT. `Expect { element, reason }` matches exactly what // `element` matches and captures exactly what it captures, in every direction: ingest, emit and @@ -293,6 +314,8 @@ type GrammarExprFold { optional: fn(R) -> R repeat: fn(R) -> R expect: fn(R, ParseRefusalReason) -> R + refuse_on_match: fn(R, ParseRefusalReason) -> R + after_line_break: fn(R) -> R } fn fold_grammar_expr(expr: GrammarExpr, algebra: GrammarExprFold) -> R { @@ -318,6 +341,10 @@ fn fold_grammar_expr(expr: GrammarExpr, algebra: GrammarExprFold) -> R { algebra.repeat(fold_grammar_expr(expr: element, algebra: algebra)) Expect { element: element, reason: reason } => algebra.expect(fold_grammar_expr(expr: element, algebra: algebra), reason) + RefuseOnMatch { element: element, reason: reason } => + algebra.refuse_on_match(fold_grammar_expr(expr: element, algebra: algebra), reason) + AfterLineBreak { element: element } => + algebra.after_line_break(fold_grammar_expr(expr: element, algebra: algebra)) } } @@ -2276,6 +2303,17 @@ fn grammar_expr_to_node(expr: GrammarExpr) -> Node { grammar_named_edge(name: ^grammar_expect_element_node_projection, target: element_node), grammar_named_edge(name: ^grammar_expect_reason_node_projection, target: grammar_atom(identity: parse_refusal_reason_symbol(reason: reason))) ]) + }, + refuse_on_match: fn(element_node, reason) { + grammar_expr_node(children: [ + grammar_named_edge(name: ^grammar_refuse_on_match_element_node_projection, target: element_node), + grammar_named_edge(name: ^grammar_refuse_on_match_reason_node_projection, target: grammar_atom(identity: parse_refusal_reason_symbol(reason: reason))) + ]) + }, + after_line_break: fn(element_node) { + grammar_expr_node(children: [ + grammar_named_edge(name: ^grammar_after_line_break_element_node_projection, target: element_node) + ]) } } ) diff --git a/src/v2/std/parse_refusal_reason.dag b/src/v2/std/parse_refusal_reason.dag index 7e9237996b6..4e7a5230335 100644 --- a/src/v2/std/parse_refusal_reason.dag +++ b/src/v2/std/parse_refusal_reason.dag @@ -10,14 +10,20 @@ import v2.std.node { Symbol } // // FnExpressionBodyMissingAfterEq -- an expression-bodied `fn ... =` with no expression after `=`. // DataValueMissingAfterEq -- a `data name: Type =` with no value expression after `=`. +// LayoutUnknownWithoutSource -- a layout-conditioned production (v2.std.grammar +// AfterLineBreak) was reached on a token stream that carries no +// source, so whether a line break precedes the match has no +// answer; the parse refuses rather than assume one line. type ParseRefusalReason = FnExpressionBodyMissingAfterEq | DataValueMissingAfterEq + | LayoutUnknownWithoutSource // The diagnostic identity each reason refuses under (`v2.std.diagnostic` `Diagnostic.reason`). fn parse_refusal_reason_symbol(reason: ParseRefusalReason) -> Symbol { match reason { FnExpressionBodyMissingAfterEq => ^parse_fn_expression_body_missing_after_eq DataValueMissingAfterEq => ^parse_data_value_missing_after_eq + LayoutUnknownWithoutSource => ^parse_layout_unknown_without_source } } diff --git a/src/v2/test/claim/long/parse_table_memo_governed_witness_test.dag b/src/v2/test/claim/long/parse_table_memo_governed_witness_test.dag index 0ebb7ff8581..6e50c8aeb5e 100644 --- a/src/v2/test/claim/long/parse_table_memo_governed_witness_test.dag +++ b/src/v2/test/claim/long/parse_table_memo_governed_witness_test.dag @@ -8,7 +8,7 @@ fn governed_witness_probe_key() -> ParsePositionKey { } fn governed_witness_dummy_entry() -> ParseMemoEntry { - ParseMemoRejected { diagnostics: [], furthest: FurthestNone } + ParseMemoRejected { diagnostics: [], furthest: FurthestNone, refusal: optional_absent() } } fn parse_table_recompute_fixture() -> ParseTable { @@ -27,7 +27,9 @@ fn parse_table_recompute_fixture() -> ParseTable { nullable_set: base.nullable_set, first_rows: base.first_rows, productions_by_name: base.productions_by_name, - prepared_exprs: base.prepared_exprs + prepared_exprs: base.prepared_exprs, + source: base.source, + refusal: base.refusal } } @@ -47,7 +49,9 @@ fn parse_table_memoize_fixture() -> ParseTable { nullable_set: base.nullable_set, first_rows: base.first_rows, productions_by_name: base.productions_by_name, - prepared_exprs: base.prepared_exprs + prepared_exprs: base.prepared_exprs, + source: base.source, + refusal: base.refusal } } @@ -110,7 +114,9 @@ fn witness_bypass_store_reds_non_memoize_carrier() -> Bool { nullable_set: base.nullable_set, first_rows: base.first_rows, productions_by_name: base.productions_by_name, - prepared_exprs: base.prepared_exprs + prepared_exprs: base.prepared_exprs, + source: base.source, + refusal: base.refusal } !parse_table_memo_store_well_formed_at(table: bypass, key: key) } diff --git a/src/v2/test/claim/parse/grammar_expect_test.dag b/src/v2/test/claim/parse/grammar_expect_test.dag index f0b64328017..d2aae2a0335 100644 --- a/src/v2/test/claim/parse/grammar_expect_test.dag +++ b/src/v2/test/claim/parse/grammar_expect_test.dag @@ -17,6 +17,7 @@ import v2.std.diagnostic { diagnostics_fatal } import v2.std.grammar { + AfterLineBreak, Choice, Expect, GrammarExpr, @@ -27,6 +28,7 @@ import v2.std.grammar { ModeledGrammar, Nonterminal, Optional, + RefuseOnMatch, Repeat, Sequence, StampClass, @@ -154,7 +156,9 @@ fn ge_wrap_all(expr: GrammarExpr) -> GrammarExpr { choice: fn(l, r) { Expect { element: Choice { left: l, right: r }, reason: FnExpressionBodyMissingAfterEq } }, optional: fn(e) { Expect { element: Optional { element: e }, reason: FnExpressionBodyMissingAfterEq } }, repeat: fn(e) { Expect { element: Repeat { element: e }, reason: FnExpressionBodyMissingAfterEq } }, - expect: fn(e, reason) { Expect { element: Expect { element: e, reason: reason }, reason: FnExpressionBodyMissingAfterEq } } + expect: fn(e, reason) { Expect { element: Expect { element: e, reason: reason }, reason: FnExpressionBodyMissingAfterEq } }, + refuse_on_match: fn(e, reason) { Expect { element: RefuseOnMatch { element: e, reason: reason }, reason: FnExpressionBodyMissingAfterEq } }, + after_line_break: fn(e) { Expect { element: AfterLineBreak { element: e }, reason: FnExpressionBodyMissingAfterEq } } } ) } diff --git a/src/v2/test/claim/parse/layout_line_break_test.dag b/src/v2/test/claim/parse/layout_line_break_test.dag new file mode 100644 index 00000000000..421f3ea7f4e --- /dev/null +++ b/src/v2/test/claim/parse/layout_line_break_test.dag @@ -0,0 +1,262 @@ +module v2.test.parse.layout_line_break + +import v2.compiler.parse { ParseArtifact, parse_module } +import v2.compiler.tokenize { tokenize } +import v2.extdeps.languages.dag { dag_lex, dag_grammar } +import v2.std.algebra { length } +import v2.std.collection { List, list_at_optional } +import v2.std.compilers.lexing { + LineBreakAnswer, + LineBreakInGap, + LineBreakUnknownWithoutSource, + NoLineBreakInGap, + Token, + TokenStream, + token_stream_all, + token_stream_line_break_between, + token_stream_new, + token_stream_source +} +import v2.std.diagnostic { + Accepted, + ByteRange, + DeclarationLocus, + InvariantPortLocus, + NodeLocus, + Outcome, + Rejected, + SourcePortLocus, + Textual, + WholeFile, + diagnostics_fatal +} +import v2.std.grammar { + AfterLineBreak, + GrammarExpr, + GrammarProduction, + GrammarRoot, + ModeledGrammar, + RefuseOnMatch, + Repeat, + Sequence, + Choice, + StampClass, + Terminal, + grammar_atom, + grammar_empty_sync_tokens +} +import v2.std.integer { Int } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import v2.std.logic { Bool } +import v2.std.node { Symbol, content_hash } +import v2.std.optional { Absent, Present } +import v2.std.parse_refusal_reason { + FnExpressionBodyMissingAfterEq, + LayoutUnknownWithoutSource, + ParseRefusalReason, + parse_refusal_reason_symbol +} +import v2.std.text { String } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// LAYOUT IS DERIVED FROM THE SOURCE THE STREAM CARRIES, AND IT IS READ ONLY WHERE A PRODUCTION ASKS. +// `x - 1` and `x` newline `- 1` lex to the same tokens at the same octets (a space and a line feed are +// each one octet), so the token stream alone cannot tell them apart; the source can. These claims hold +// the carrier (v2.std.compilers.lexing TokenStream.source, token_stream_line_break_between) and the two +// grammar arms that read it (v2.std.grammar AfterLineBreak, RefuseOnMatch) to that, over hand grammars. + +data same_line_text: String = "x - 1" + +data next_line_text: String = "x\n- 1" + +fn lexed(text: String, file: Symbol) -> Outcome { + tokenize(text: text, file: file, rules: dag_lex()) +} + +// The gap before the second token (`-`): from the end of `x` to the start of `-`. +fn gap_before_second(stream: TokenStream) -> LineBreakAnswer { + match list_at_optional(xs: token_stream_all(stream: stream), index: 0) { + Absent => NoLineBreakInGap + Present { value: first } => + match list_at_optional(xs: token_stream_all(stream: stream), index: 1) { + Absent => NoLineBreakInGap + Present { value: second } => + token_stream_line_break_between(source: token_stream_source(stream: stream), from: first.end, to: second.start) + } + } +} + +fn token_at_same(a: List, b: List, index: Int) -> Bool { + match list_at_optional(xs: a, index: index) { + Absent => false + Present { value: ta } => + match list_at_optional(xs: b, index: index) { + Absent => false + Present { value: tb } => token_same(a: ta, b: tb) + } + } +} + +fn tokens_agree(a: List, b: List) -> Bool { + (length(xs: a) == 3) && (length(xs: b) == 3) + && token_at_same(a: a, b: b, index: 0) + && token_at_same(a: a, b: b, index: 1) + && token_at_same(a: a, b: b, index: 2) +} + +fn token_same(a: Token, b: Token) -> Bool { + (a.class == b.class) && (a.start == b.start) && (a.end == b.end) +} + +// THE LEXING CONTROL: same tokens, different layout, and the derivation tells them apart. +test fn same_tokens_different_layout_is_told_apart_by_the_source_holds() -> Bool { + match lexed(text: same_line_text, file: ^layout_same) { + Rejected { diagnostics: _ } => false + Accepted { value: same, diagnostics: _ } => + match lexed(text: next_line_text, file: ^layout_same) { + Rejected { diagnostics: _ } => false + Accepted { value: next, diagnostics: _ } => + tokens_agree(a: token_stream_all(stream: same), b: token_stream_all(stream: next)) + && match gap_before_second(stream: same) { + NoLineBreakInGap => true + LineBreakInGap => false + LineBreakUnknownWithoutSource => false + } + && match gap_before_second(stream: next) { + LineBreakInGap => true + NoLineBreakInGap => false + LineBreakUnknownWithoutSource => false + } + } + } +} + +// THE FAIL-CLOSED ARM OF THE DERIVATION: tokens alone (no source) answer Unknown, never one line. +test fn a_stream_without_source_answers_unknown_holds() -> Bool { + match lexed(text: next_line_text, file: ^layout_unknown) { + Rejected { diagnostics: _ } => false + Accepted { value: next, diagnostics: _ } => + match gap_before_second(stream: token_stream_new(all: token_stream_all(stream: next))) { + LineBreakUnknownWithoutSource => true + LineBreakInGap => false + NoLineBreakInGap => false + } + } +} + +// THE IDENTITY CONTROL: carrying the source changes nothing the grammar did not ask about. The dag +// grammar asks no layout question, so the same tokens parse to the same tree (content hash) with the +// source attached and with it absent. +data identity_module_text: String = "module probe.layout_identity\n\nfn f(x: Int) -> Int {\n x - 1\n}\n" + +test fn carrying_the_source_changes_no_parse_tree_holds() -> Bool { + match lexed(text: identity_module_text, file: ^layout_identity) { + Rejected { diagnostics: _ } => false + Accepted { value: with_source, diagnostics: _ } => + match parse_module(tokens: with_source, grammar: dag_grammar()) { + Rejected { diagnostics: _ } => false + Accepted { value: a, diagnostics: _ } => + match parse_module(tokens: token_stream_new(all: token_stream_all(stream: with_source)), grammar: dag_grammar()) { + Rejected { diagnostics: _ } => false + Accepted { value: b, diagnostics: _ } => content_hash(n: a.tree) == content_hash(n: b.tree) + } + } + } +} + +// A hand grammar in the dag token classes: start = operand (continuation)*, operand = ident, and a +// continuation either REFUSES (a `-` that follows a line break) or is `-` operand. The refusal arm is +// tried first, so a same-line `-` falls through to subtraction and a next-line `-` refuses at itself. +// The reason is borrowed from the closed vocabulary only to exercise the mechanism. +fn lb_terminal(class: Symbol) -> GrammarExpr { + Terminal { token_class: class, stamp: StampClass } +} + +fn lb_root() -> GrammarRoot { + GrammarRoot { + start: ^lb_start, + productions: [ + GrammarProduction { + name: ^lb_start, + expression: Sequence { + left: lb_terminal(class: ^dag_token_ident), + right: Repeat { + element: Choice { + left: RefuseOnMatch { + element: AfterLineBreak { element: lb_terminal(class: ^dag_token_minus) }, + reason: FnExpressionBodyMissingAfterEq + }, + right: Sequence { left: lb_terminal(class: ^dag_token_minus), right: lb_terminal(class: ^dag_token_int_literal) } + } + } + }, + emitted: grammar_atom(identity: ^lb_start) + } + ], + sync_tokens: grammar_empty_sync_tokens() + } +} + +fn lb_parse(stream: TokenStream) -> Outcome { + parse_module(tokens: stream, grammar: ModeledGrammar { root: lb_root() }) +} + +fn fatal_is(parsed: Outcome, reason: ParseRefusalReason, file: Symbol, start: Int, end: Int) -> Bool { + match parsed { + Accepted { value: _, diagnostics: _ } => false + Rejected { diagnostics: d } => + let fatal = diagnostics_fatal(d: d) + (fatal.reason == parse_refusal_reason_symbol(reason: reason)) + && match fatal.at { + SourcePortLocus { anchor: _, file: f, extent: extent } => + (f == file) + && match extent { + ByteRange { start: s, end: e } => (s == start) && (e == end) + WholeFile => false + } + Textual { file: _, extent: _ } => false + NodeLocus { anchor: _ } => false + InvariantPortLocus { anchor: _, invariant: _ } => false + DeclarationLocus { declaration: _ } => false + } + } +} + +// Same line: the refusal arm's layout question answers no, and `- 1` is the continuation. +test fn a_same_line_minus_continues_holds() -> Bool { + match lexed(text: same_line_text, file: ^lb_same) { + Rejected { diagnostics: _ } => false + Accepted { value: stream, diagnostics: _ } => + match lb_parse(stream: stream) { + Accepted { value: _, diagnostics: _ } => true + Rejected { diagnostics: _ } => false + } + } +} + +// Next line: the `-` (bytes 2..3) matches after a line break, and the parse REFUSES there. No +// alternative succeeds around it, although `- 1` as subtraction would have parsed. +test fn a_next_line_minus_refuses_at_the_minus_holds() -> Bool { + match lexed(text: next_line_text, file: ^lb_next) { + Rejected { diagnostics: _ } => false + Accepted { value: stream, diagnostics: _ } => + fatal_is(parsed: lb_parse(stream: stream), reason: FnExpressionBodyMissingAfterEq, file: ^lb_next, start: 2, end: 3) + } +} + +// No source: the layout question has no answer, and the parse refuses with LayoutUnknownWithoutSource +// at the `-` instead of treating the tokens as one line (which would have accepted). +test fn a_layout_question_without_source_refuses_holds() -> Bool { + match lexed(text: same_line_text, file: ^lb_unknown) { + Rejected { diagnostics: _ } => false + Accepted { value: stream, diagnostics: _ } => + fatal_is( + parsed: lb_parse(stream: token_stream_new(all: token_stream_all(stream: stream))), + reason: LayoutUnknownWithoutSource, + file: ^lb_unknown, + start: 2, + end: 3 + ) + } +} diff --git a/src/v2/test/claim/parse/parse_table_content_key_test.dag b/src/v2/test/claim/parse/parse_table_content_key_test.dag index c494858bff3..0e9416ecaed 100644 --- a/src/v2/test/claim/parse/parse_table_content_key_test.dag +++ b/src/v2/test/claim/parse/parse_table_content_key_test.dag @@ -18,7 +18,9 @@ fn mk_table(grammar: Hash, tokens: Hash) -> ParseTable { nullable_set: base.nullable_set, first_rows: base.first_rows, productions_by_name: base.productions_by_name, - prepared_exprs: base.prepared_exprs + prepared_exprs: base.prepared_exprs, + source: base.source, + refusal: base.refusal } } diff --git a/src/v2/test/claim/provenance/parse_memo_frame_replay_test.dag b/src/v2/test/claim/provenance/parse_memo_frame_replay_test.dag index 0b04b511a9c..1278c3d96c6 100644 --- a/src/v2/test/claim/provenance/parse_memo_frame_replay_test.dag +++ b/src/v2/test/claim/provenance/parse_memo_frame_replay_test.dag @@ -50,7 +50,9 @@ fn parse_memo_frame_replay_run_modeled( nullable_set: base.nullable_set, first_rows: base.first_rows, productions_by_name: base.productions_by_name, - prepared_exprs: base.prepared_exprs + prepared_exprs: base.prepared_exprs, + source: base.source, + refusal: base.refusal } match grammar_lookup_production(root: root, name: root.start) { Violates { diagnostic: _ } => optional_absent() @@ -479,7 +481,9 @@ fn linear_run(forced: Optional) -> Optional { nullable_set: base.nullable_set, first_rows: base.first_rows, productions_by_name: base.productions_by_name, - prepared_exprs: base.prepared_exprs + prepared_exprs: base.prepared_exprs, + source: base.source, + refusal: base.refusal } match grammar_lookup_production(root: root, name: root.start) { Violates { diagnostic: _ } => optional_absent() From 8371ea006b509855fb4bca9d5e6250769728f2f0 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 11:12:48 +0000 Subject: [PATCH 02/21] Layout control: same tokens, different layout, different stream digests Co-Authored-By: Claude Opus 5.5 (1M context) --- .../claim/parse/layout_line_break_test.dag | 21 ++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/src/v2/test/claim/parse/layout_line_break_test.dag b/src/v2/test/claim/parse/layout_line_break_test.dag index 421f3ea7f4e..392c694d715 100644 --- a/src/v2/test/claim/parse/layout_line_break_test.dag +++ b/src/v2/test/claim/parse/layout_line_break_test.dag @@ -1,6 +1,6 @@ module v2.test.parse.layout_line_break -import v2.compiler.parse { ParseArtifact, parse_module } +import v2.compiler.parse { ParseArtifact, parse_module, parse_token_stream_digest, parse_token_stream_source_digest } import v2.compiler.tokenize { tokenize } import v2.extdeps.languages.dag { dag_lex, dag_grammar } import v2.std.algebra { length } @@ -260,3 +260,22 @@ test fn a_layout_question_without_source_refuses_holds() -> Bool { ) } } + +// THE MEMO CANNOT SERVE ONE LAYOUT'S ANSWER TO THE OTHER. `x - 1` and `x` newline `- 1` lex to the +// same tokens at the same octets, so a stream digest over tokens alone would key them together and a +// warm memo entry could replay a same-line acceptance for a next-line refusal. The source is part of +// the stream's digest (v2.compiler.parse parse_token_stream_source_digest): same tokens, different +// digest; and the token-only digest, which is what a sourceless stream keys by, is unchanged. +test fn same_tokens_different_layout_have_different_stream_digests_holds() -> Bool { + match lexed(text: same_line_text, file: ^layout_digest) { + Rejected { diagnostics: _ } => false + Accepted { value: same, diagnostics: _ } => + match lexed(text: next_line_text, file: ^layout_digest) { + Rejected { diagnostics: _ } => false + Accepted { value: next, diagnostics: _ } => + (parse_token_stream_digest(all_tokens: token_stream_all(stream: same)) == parse_token_stream_digest(all_tokens: token_stream_all(stream: next))) + && (parse_token_stream_source_digest(all_tokens: token_stream_all(stream: same), source: token_stream_source(stream: same)) + != parse_token_stream_source_digest(all_tokens: token_stream_all(stream: next), source: token_stream_source(stream: next))) + } + } +} From ef567abcb48541db37a90352fd9ec10dcffddf47 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 11:58:09 +0000 Subject: [PATCH 03/21] Layout: derive a per-token line-break index once per stream; a layout question is a lookup (review 73189) The gap question folded over the whole source per question: O(tokens x source). StreamLayout pairs the source with a line-break-before answer for every token, derived in one walk over source and tokens by its only constructor; a line feed inside a token is not layout. The digest reads the source through the layout. Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/compiler/02_parse.dag | 77 +++++------- src/v2/std/compilers/lexing.dag | 116 +++++++++++++----- ...parse_table_memo_governed_witness_test.dag | 6 +- .../claim/parse/layout_line_break_test.dag | 37 ++++-- .../parse/parse_table_content_key_test.dag | 2 +- .../parse_memo_frame_replay_test.dag | 4 +- 6 files changed, 147 insertions(+), 95 deletions(-) diff --git a/src/v2/compiler/02_parse.dag b/src/v2/compiler/02_parse.dag index 47e76e2b27b..cacc65c0815 100644 --- a/src/v2/compiler/02_parse.dag +++ b/src/v2/compiler/02_parse.dag @@ -118,12 +118,13 @@ import v2.std.compilers.lexing { LineBreakUnknownWithoutSource, Lexeme, NoLineBreakInGap, + StreamLayout, Token, TokenStream, - token_stream_line_break_between, + token_stream_line_break_before, token_stream_remaining, token_stream_remaining_count, - token_stream_source, + token_stream_layout, } import v2.std.parse_refusal_reason { LayoutUnknownWithoutSource, ParseRefusalReason, parse_refusal_reason_symbol } import v2.std.grammar { @@ -350,7 +351,7 @@ type ParseTableRealization { first_rows: Map> productions_by_name: Map prepared_exprs: Map - source: Optional + layout: Optional refusal: Optional } @@ -717,7 +718,7 @@ fn parse_table_carry_analysis(table: ParseTable, entries: Map ParseTabl first_rows: table.first_rows, productions_by_name: table.productions_by_name, prepared_exprs: table.prepared_exprs, - source: table.source, + layout: table.layout, refusal: optional_present(value: refusal) } } @@ -892,14 +893,14 @@ fn parse_grammar_digest(validated: ValidatedGrammarRoot) -> Hash { // byte 255 and every non-Latin-1 codepoint collide in the seed interpreter's cross-parse memo key // (v1_interpreter parse_table_memo_scope_and_key reads this field). Offsets and positions now go // through the Int content authority and the lexeme through its interned symbol. -fn parse_token_stream_source_digest(all_tokens: List, source: Optional) -> Hash { - match source { +fn parse_token_stream_source_digest(all_tokens: List, layout: Optional) -> Hash { + match layout { Absent => parse_token_stream_digest(all_tokens: all_tokens) - Present { value: text } => + Present { value: l } => combine_hash( a: parse_token_stream_digest(all_tokens: all_tokens), b: fold_list( - xs: text, + xs: l.source, empty: symbol_identity_digest(sym: ^parse_token_stream_source_digest_nil), cons: fn(acc, c) { combine_hash(a: acc, b: integer_int_content_digest(value: unicode_char_code_point(c: c))) } ) @@ -947,20 +948,20 @@ fn parse_table_for_production_analyzed( validated: validated, analysis: analysis, all_tokens: all_tokens, - source: optional_absent(), + layout: optional_absent(), materialization: materialization ) } -// The stream's source rides on the table so a layout-conditioned production can ask its question at -// the point of use (v2.std.compilers.lexing token_stream_line_break_between), and it is part of the +// The stream's layout rides on the table so a layout-conditioned production can ask its question at +// the point of use (v2.std.compilers.lexing token_stream_line_break_before), and its source is part of the // stream's digest: two sources can lex to byte-identical tokens and differ only in layout, and a memo // entry must never serve one's answer to the other. fn parse_table_for_stream_analyzed( validated: ValidatedGrammarRoot, analysis: GrammarFirstAnalysis, all_tokens: List, - source: Optional, + layout: Optional, materialization: Materialization ) -> ParseTable { ParseTableRealization { @@ -968,7 +969,7 @@ fn parse_table_for_stream_analyzed( provider_id: v2.compiler.materialization_carriers.parse_table_memo_provider_id(), materialization: materialization, grammar_digest: analysis.grammar_digest, - token_stream_digest: parse_token_stream_source_digest(all_tokens: all_tokens, source: source), + token_stream_digest: parse_token_stream_source_digest(all_tokens: all_tokens, layout: layout), entries: empty_map(), memo_hits: 0, memo_misses: 0, @@ -978,7 +979,7 @@ fn parse_table_for_stream_analyzed( first_rows: analysis.first_rows, productions_by_name: analysis.productions_by_name, prepared_exprs: analysis.prepared_exprs, - source: source, + layout: layout, refusal: optional_absent() } } @@ -1016,7 +1017,7 @@ fn parse_table_empty(materialization: Materialization) -> ParseTable { first_rows: empty_map(), productions_by_name: empty_map(), prepared_exprs: empty_map(), - source: optional_absent(), + layout: optional_absent(), refusal: optional_absent() } } @@ -4006,7 +4007,7 @@ fn parse_expr_refuse_on_match( } // A MATCH CONDITIONED ON LAYOUT. The element is parsed first, so a layout question is asked only of -// a match; then the source decides (v2.std.compilers.lexing token_stream_line_break_between) whether a +// a match; then the stream's layout decides (v2.std.compilers.lexing token_stream_line_break_before) whether a // line break separates the token before the match from its first token. No line break: an ordinary // failed alternative. A stream that carries no source cannot answer, and the parse refuses with // LayoutUnknownWithoutSource rather than assume one line. @@ -4021,7 +4022,7 @@ fn parse_expr_after_line_break( match parse_expr_with_first(expr: element, tokens: tokens, all_tokens: all_tokens, root: root, table: table, prov: prov) { ParseExprRejected { diagnostics: diags, table: table1 } => ParseExprRejected { diagnostics: diags, table: table1 } ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: prov1, table: table1 } => - match parse_line_break_before_first(tokens: tokens, all_tokens: all_tokens, source: table1.source) { + match parse_line_break_before_first(tokens: tokens, layout: table1.layout) { LineBreakInGap => ParseExprAccepted { remaining: remaining, captured: captured, diagnostics: diags, prov: prov1, table: table1 } NoLineBreakInGap => ParseExprRejected { diagnostics: [], table: table1 } @@ -4037,33 +4038,13 @@ fn parse_expr_after_line_break( } } -// The gap before the first token of `tokens`: from the end of the token before it in the stream (the -// start of the source when it is the first token) to its own start. An empty remainder has no first -// token and so no gap; it answers NoLineBreakInGap, a failed match rather than a layout question. -fn parse_line_break_before_first( - tokens: List, - all_tokens: List, - source: Optional -) -> LineBreakAnswer { +// The gap before the first token of `tokens`, answered by the stream's layout (a lookup by that +// token's start). An empty remainder has no first token and so no gap; it answers NoLineBreakInGap, a +// failed match rather than a layout question. +fn parse_line_break_before_first(tokens: List, layout: Optional) -> LineBreakAnswer { match parse_token_first(tokens: tokens) { Absent => NoLineBreakInGap - Present { value: first } => - token_stream_line_break_between( - source: source, - from: parse_token_end_before(all_tokens: all_tokens, index: length(xs: all_tokens) - length(xs: tokens)), - to: first.start - ) - } -} - -fn parse_token_end_before(all_tokens: List, index: Int) -> Int { - if index <= 0 { - 0 - } else { - match list_at_optional(xs: all_tokens, index: index - 1) { - Present { value: prev } => prev.end - Absent => 0 - } + Present { value: first } => token_stream_line_break_before(layout: layout, token_start: first.start) } } @@ -4318,7 +4299,7 @@ fn parse_production_prepared_measured( analysis: analysis, production_name: production_name, seed: occurrence_id_allocator_initial(), - source: optional_absent(), + layout: optional_absent(), materialization: materialization ) } @@ -4329,7 +4310,7 @@ fn parse_production_prepared_measured_seeded( analysis: GrammarFirstAnalysis, production_name: Symbol, seed: OccurrenceIdAllocator, - source: Optional, + layout: Optional, materialization: Materialization ) -> ParseProductionMeasured { let root = validated.root @@ -4339,7 +4320,7 @@ fn parse_production_prepared_measured_seeded( let table = parse_table_for_stream_analyzed( validated: validated, analysis: analysis, - source: source, + layout: layout, all_tokens: tokens, materialization: materialization ) @@ -4602,7 +4583,7 @@ fn parse_module_prepared_measured_seeded( analysis: modeled.analysis, production_name: modeled.validated.root.start, seed: seed, - source: token_stream_source(stream: tokens), + layout: token_stream_layout(stream: tokens), materialization: materialization ) match measured.outcome { diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 0201010ffce..3d5694db072 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -9,7 +9,8 @@ import v2.std.collection { Map, empty_map, list_at_optional, - map_insert + map_insert, + map_lookup } import v2.std.optional { Absent, @@ -155,32 +156,46 @@ type Token { // sometimes needs to know (a `-` that begins a new line after a complete operand is ambiguous // between infix and prefix, and the seed refuses it: v1.compiler.parse // is_ambiguous_prefix_infix_newline_boundary). The source text stays the ONE authority for that -// fact: nothing about layout is stored on a Token or as a token of its own, and -// `token_stream_line_break_between` reads the gap at the point a question is asked. A stream built -// from tokens alone (fixtures) carries `Absent`, and a layout question on it answers -// `LineBreakUnknownWithoutSource` -- never an assumed single line. +// fact: nothing about layout is stored on a Token or as a token of its own. `StreamLayout` pairs the +// source with the answer to "is there a line break in the gap before this token?" for every token, +// DERIVED from that source and those tokens in one walk by its only constructor +// (`stream_layout_of`), so the two cannot disagree and a layout question is a lookup rather than a +// rescan of the file. A stream built from tokens alone (fixtures) carries no layout, and a layout +// question on it answers `LineBreakUnknownWithoutSource` -- never an assumed single line. +type StreamLayout { + source: String + line_break_before: Map +} + type TokenStream { all: List pos: Int - source: Optional + layout: Optional } fn token_stream_new(all: List) -> TokenStream { - TokenStream { all: all, pos: 0, source: optional_absent() } + TokenStream { all: all, pos: 0, layout: optional_absent() } } // The tokenizer's constructor: the tokens together with the exact scalar sequence they were lexed // from, whose UTF-8 octet offsets their `start`/`end` name. fn token_stream_lexed(all: List, source: String) -> TokenStream { - TokenStream { all: all, pos: 0, source: optional_present(value: source) } + TokenStream { all: all, pos: 0, layout: optional_present(value: stream_layout_of(tokens: all, source: source)) } } fn token_stream_empty() -> TokenStream { - TokenStream { all: Empty, pos: 0, source: optional_absent() } + TokenStream { all: Empty, pos: 0, layout: optional_absent() } +} + +fn token_stream_layout(stream: TokenStream) -> Optional { + stream.layout } fn token_stream_source(stream: TokenStream) -> Optional { - stream.source + match stream.layout { + Absent => optional_absent() + Present { value: layout } => optional_present(value: layout.source) + } } type LineBreakAnswer @@ -188,39 +203,74 @@ type LineBreakAnswer | NoLineBreakInGap | LineBreakUnknownWithoutSource -type LineBreakScan { +// ONE WALK: the source's scalars in order, with the tokens still ahead and the end of the last token +// passed. A line feed (U+000A) counts only in a GAP -- at or after the previous token's end -- so a +// line feed inside a token (a multi-line string) is not layout. When the walk reaches a token's +// first octet, whether a line feed was seen since the previous token is that token's answer. +// Offsets are UTF-8 octets, as token extents are, so each scalar advances by its encoded width. +type StreamLayoutWalk { octet: Int - found: Bool + ahead: List + last_end: Int + seen: Bool + answers: Map } -// Whether the source holds a line feed (U+000A) at an octet offset in [from, to). Offsets are UTF-8 -// octets, as token extents are, so each scalar advances the position by its own encoded width. -fn source_has_line_break_between(source: String, from: Int, to: Int) -> Bool { - fold_list( +fn stream_layout_of(tokens: List, source: String) -> StreamLayout { + let walked = fold_list( xs: source, - empty: LineBreakScan { octet: 0, found: false }, - cons: fn(acc, c) { line_break_scan_step(acc: acc, c: c, from: from, to: to) } - ).found + empty: StreamLayoutWalk { octet: 0, ahead: tokens, last_end: 0, seen: false, answers: empty_map() }, + cons: fn(acc, c) { stream_layout_step(acc: acc, c: c) } + ) + StreamLayout { source: source, line_break_before: walked.answers } } -fn line_break_scan_step(acc: LineBreakScan, c: Char, from: Int, to: Int) -> LineBreakScan { +fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { + let at_token = stream_layout_record_token(acc: acc) let code = unicode_char_code_point(c: c) - LineBreakScan { - octet: acc.octet + unicode_scalar_utf8_octet_count(code_point: code), - found: acc.found || ((code == 10) && (acc.octet >= from) && (acc.octet < to)) + StreamLayoutWalk { + octet: at_token.octet + unicode_scalar_utf8_octet_count(code_point: code), + ahead: at_token.ahead, + last_end: at_token.last_end, + seen: at_token.seen || ((code == 10) && (at_token.octet >= at_token.last_end)), + answers: at_token.answers } } -// THE NAMED DERIVATION. Is there a line break in the source between octet `from` (the end of the -// token before) and octet `to` (the start of the token asked about)? -fn token_stream_line_break_between(source: Optional, from: Int, to: Int) -> LineBreakAnswer { - match source { - Absent => LineBreakUnknownWithoutSource - Present { value: text } => - if source_has_line_break_between(source: text, from: from, to: to) { - LineBreakInGap +// At a token's first octet: record the gap's answer for it, then start a new gap at its end. +fn stream_layout_record_token(acc: StreamLayoutWalk) -> StreamLayoutWalk { + match list_at_optional(xs: acc.ahead, index: 0) { + Absent => acc + Present { value: tok } => + if tok.start == acc.octet { + StreamLayoutWalk { + octet: acc.octet, + ahead: skip(xs: acc.ahead, n: 1), + last_end: tok.end, + seen: false, + answers: map_insert(m: acc.answers, key: tok.start, value: acc.seen) + } } else { - NoLineBreakInGap + acc + } + } +} + +// THE NAMED DERIVATION, READ AT THE POINT OF USE: is there a line break in the source between the +// token before the one starting at octet `token_start` and that token? A stream without layout +// cannot answer; neither can a start no token of the stream has. +fn token_stream_line_break_before(layout: Optional, token_start: Int) -> LineBreakAnswer { + match layout { + Absent => LineBreakUnknownWithoutSource + Present { value: l } => + match map_lookup(m: l.line_break_before, key: token_start) { + Absent => LineBreakUnknownWithoutSource + Present { value: seen } => + if seen { + LineBreakInGap + } else { + NoLineBreakInGap + } } } } @@ -250,7 +300,7 @@ fn token_stream_get(stream: TokenStream, offset: Int) -> Optional { } fn token_stream_advance(stream: TokenStream, n: Int) -> TokenStream { - TokenStream { all: stream.all, pos: stream.pos + n, source: stream.source } + TokenStream { all: stream.all, pos: stream.pos + n, layout: stream.layout } } // AnnotationPlacement is NOT declared here. It is the language-agnostic concept and its authority diff --git a/src/v2/test/claim/long/parse_table_memo_governed_witness_test.dag b/src/v2/test/claim/long/parse_table_memo_governed_witness_test.dag index 6e50c8aeb5e..49a31f42ed7 100644 --- a/src/v2/test/claim/long/parse_table_memo_governed_witness_test.dag +++ b/src/v2/test/claim/long/parse_table_memo_governed_witness_test.dag @@ -28,7 +28,7 @@ fn parse_table_recompute_fixture() -> ParseTable { first_rows: base.first_rows, productions_by_name: base.productions_by_name, prepared_exprs: base.prepared_exprs, - source: base.source, + layout: base.layout, refusal: base.refusal } } @@ -50,7 +50,7 @@ fn parse_table_memoize_fixture() -> ParseTable { first_rows: base.first_rows, productions_by_name: base.productions_by_name, prepared_exprs: base.prepared_exprs, - source: base.source, + layout: base.layout, refusal: base.refusal } } @@ -115,7 +115,7 @@ fn witness_bypass_store_reds_non_memoize_carrier() -> Bool { first_rows: base.first_rows, productions_by_name: base.productions_by_name, prepared_exprs: base.prepared_exprs, - source: base.source, + layout: base.layout, refusal: base.refusal } !parse_table_memo_store_well_formed_at(table: bypass, key: key) diff --git a/src/v2/test/claim/parse/layout_line_break_test.dag b/src/v2/test/claim/parse/layout_line_break_test.dag index 392c694d715..8c2b0a91e0d 100644 --- a/src/v2/test/claim/parse/layout_line_break_test.dag +++ b/src/v2/test/claim/parse/layout_line_break_test.dag @@ -13,9 +13,9 @@ import v2.std.compilers.lexing { Token, TokenStream, token_stream_all, - token_stream_line_break_between, - token_stream_new, - token_stream_source + token_stream_layout, + token_stream_line_break_before, + token_stream_new } import v2.std.diagnostic { Accepted, @@ -63,7 +63,7 @@ data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly // LAYOUT IS DERIVED FROM THE SOURCE THE STREAM CARRIES, AND IT IS READ ONLY WHERE A PRODUCTION ASKS. // `x - 1` and `x` newline `- 1` lex to the same tokens at the same octets (a space and a line feed are // each one octet), so the token stream alone cannot tell them apart; the source can. These claims hold -// the carrier (v2.std.compilers.lexing TokenStream.source, token_stream_line_break_between) and the two +// the carrier (v2.std.compilers.lexing TokenStream.layout, token_stream_line_break_before) and the two // grammar arms that read it (v2.std.grammar AfterLineBreak, RefuseOnMatch) to that, over hand grammars. data same_line_text: String = "x - 1" @@ -78,11 +78,11 @@ fn lexed(text: String, file: Symbol) -> Outcome { fn gap_before_second(stream: TokenStream) -> LineBreakAnswer { match list_at_optional(xs: token_stream_all(stream: stream), index: 0) { Absent => NoLineBreakInGap - Present { value: first } => + Present { value: _ } => match list_at_optional(xs: token_stream_all(stream: stream), index: 1) { Absent => NoLineBreakInGap Present { value: second } => - token_stream_line_break_between(source: token_stream_source(stream: stream), from: first.end, to: second.start) + token_stream_line_break_before(layout: token_stream_layout(stream: stream), token_start: second.start) } } } @@ -274,8 +274,29 @@ test fn same_tokens_different_layout_have_different_stream_digests_holds() -> Bo Rejected { diagnostics: _ } => false Accepted { value: next, diagnostics: _ } => (parse_token_stream_digest(all_tokens: token_stream_all(stream: same)) == parse_token_stream_digest(all_tokens: token_stream_all(stream: next))) - && (parse_token_stream_source_digest(all_tokens: token_stream_all(stream: same), source: token_stream_source(stream: same)) - != parse_token_stream_source_digest(all_tokens: token_stream_all(stream: next), source: token_stream_source(stream: next))) + && (parse_token_stream_source_digest(all_tokens: token_stream_all(stream: same), layout: token_stream_layout(stream: same)) + != parse_token_stream_source_digest(all_tokens: token_stream_all(stream: next), layout: token_stream_layout(stream: next))) + } + } +} + +// A LINE FEED INSIDE A TOKEN IS NOT LAYOUT. The layout walk counts a line feed only at or after the +// previous token's end, so a multi-line string literal between two tokens on one line leaves the next +// token's gap line-break-free; only the gap after the string is asked about. +data string_line_feed_text: String = "x \"a\nb\" - 1" + +test fn a_line_feed_inside_a_token_is_not_layout_holds() -> Bool { + match lexed(text: string_line_feed_text, file: ^layout_string) { + Rejected { diagnostics: _ } => false + Accepted { value: stream, diagnostics: _ } => + match list_at_optional(xs: token_stream_all(stream: stream), index: 2) { + Absent => false + Present { value: minus } => + match token_stream_line_break_before(layout: token_stream_layout(stream: stream), token_start: minus.start) { + NoLineBreakInGap => true + LineBreakInGap => false + LineBreakUnknownWithoutSource => false + } } } } diff --git a/src/v2/test/claim/parse/parse_table_content_key_test.dag b/src/v2/test/claim/parse/parse_table_content_key_test.dag index 0e9416ecaed..c2d02aa297c 100644 --- a/src/v2/test/claim/parse/parse_table_content_key_test.dag +++ b/src/v2/test/claim/parse/parse_table_content_key_test.dag @@ -19,7 +19,7 @@ fn mk_table(grammar: Hash, tokens: Hash) -> ParseTable { first_rows: base.first_rows, productions_by_name: base.productions_by_name, prepared_exprs: base.prepared_exprs, - source: base.source, + layout: base.layout, refusal: base.refusal } } diff --git a/src/v2/test/claim/provenance/parse_memo_frame_replay_test.dag b/src/v2/test/claim/provenance/parse_memo_frame_replay_test.dag index 1278c3d96c6..387137bba6a 100644 --- a/src/v2/test/claim/provenance/parse_memo_frame_replay_test.dag +++ b/src/v2/test/claim/provenance/parse_memo_frame_replay_test.dag @@ -51,7 +51,7 @@ fn parse_memo_frame_replay_run_modeled( first_rows: base.first_rows, productions_by_name: base.productions_by_name, prepared_exprs: base.prepared_exprs, - source: base.source, + layout: base.layout, refusal: base.refusal } match grammar_lookup_production(root: root, name: root.start) { @@ -482,7 +482,7 @@ fn linear_run(forced: Optional) -> Optional { first_rows: base.first_rows, productions_by_name: base.productions_by_name, prepared_exprs: base.prepared_exprs, - source: base.source, + layout: base.layout, refusal: base.refusal } match grammar_lookup_production(root: root, name: root.start) { From 22131ceb51fa46dea4523607189c68dc2e72cd8d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 12:10:00 +0000 Subject: [PATCH 04/21] Layout identity control parses against the warmed dag_prepared_grammar (floor budget: 97k steps vs 72.3k) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../claim/parse/layout_line_break_test.dag | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/src/v2/test/claim/parse/layout_line_break_test.dag b/src/v2/test/claim/parse/layout_line_break_test.dag index 8c2b0a91e0d..13d4f629996 100644 --- a/src/v2/test/claim/parse/layout_line_break_test.dag +++ b/src/v2/test/claim/parse/layout_line_break_test.dag @@ -1,8 +1,9 @@ module v2.test.parse.layout_line_break -import v2.compiler.parse { ParseArtifact, parse_module, parse_token_stream_digest, parse_token_stream_source_digest } +import v2.compiler.parse { ParseArtifact, parse_module, parse_module_prepared, parse_token_stream_digest, parse_token_stream_source_digest } +import v2.compiler.program_assembly { dag_prepared_grammar } import v2.compiler.tokenize { tokenize } -import v2.extdeps.languages.dag { dag_lex, dag_grammar } +import v2.extdeps.languages.dag { dag_lex } import v2.std.algebra { length } import v2.std.collection { List, list_at_optional } import v2.std.compilers.lexing { @@ -150,14 +151,24 @@ test fn a_stream_without_source_answers_unknown_holds() -> Bool { // source attached and with it absent. data identity_module_text: String = "module probe.layout_identity\n\nfn f(x: Int) -> Int {\n x - 1\n}\n" +// Against the prepared dag grammar the floor warms once (v2.compiler.program_assembly +// dag_prepared_grammar), so the claim pays two parses and no grammar preparation. +fn identity_parse(stream: TokenStream) -> Outcome { + match dag_prepared_grammar() { + Rejected { diagnostics: d } => Rejected { diagnostics: d } + Accepted { value: prepared, diagnostics: residue } => + parse_module_prepared(tokens: stream, prepared: prepared, validation_residue: residue) + } +} + test fn carrying_the_source_changes_no_parse_tree_holds() -> Bool { match lexed(text: identity_module_text, file: ^layout_identity) { Rejected { diagnostics: _ } => false Accepted { value: with_source, diagnostics: _ } => - match parse_module(tokens: with_source, grammar: dag_grammar()) { + match identity_parse(stream: with_source) { Rejected { diagnostics: _ } => false Accepted { value: a, diagnostics: _ } => - match parse_module(tokens: token_stream_new(all: token_stream_all(stream: with_source)), grammar: dag_grammar()) { + match identity_parse(stream: token_stream_new(all: token_stream_all(stream: with_source))) { Rejected { diagnostics: _ } => false Accepted { value: b, diagnostics: _ } => content_hash(n: a.tree) == content_hash(n: b.tree) } From 24b7716e8d95fc699ee9ac9e0f5e6180069e1677 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 12:11:09 +0000 Subject: [PATCH 05/21] Layout identity control: a module that reaches no layout question (+, not -) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/test/claim/parse/layout_line_break_test.dag | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/v2/test/claim/parse/layout_line_break_test.dag b/src/v2/test/claim/parse/layout_line_break_test.dag index 13d4f629996..bc893c05a49 100644 --- a/src/v2/test/claim/parse/layout_line_break_test.dag +++ b/src/v2/test/claim/parse/layout_line_break_test.dag @@ -146,10 +146,10 @@ test fn a_stream_without_source_answers_unknown_holds() -> Bool { } } -// THE IDENTITY CONTROL: carrying the source changes nothing the grammar did not ask about. The dag -// grammar asks no layout question, so the same tokens parse to the same tree (content hash) with the -// source attached and with it absent. -data identity_module_text: String = "module probe.layout_identity\n\nfn f(x: Int) -> Int {\n x - 1\n}\n" +// THE IDENTITY CONTROL: carrying the source changes nothing the grammar did not ask about. This module +// reaches no layout-conditioned production (it has no `-` after an operand, where the dag grammar asks +// one), so the same tokens parse to the same tree (content hash) with the source attached and absent. +data identity_module_text: String = "module probe.layout_identity\n\nfn f(x: Int) -> Int {\n x + 1\n}\n" // Against the prepared dag grammar the floor warms once (v2.compiler.program_assembly // dag_prepared_grammar), so the claim pays two parses and no grammar preparation. From 0f04f7fd9e21db31a99cc0ffff8e8d6259f535ca Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 12:54:56 +0000 Subject: [PATCH 06/21] Layout: keep only line-break-preceded starts; digest covers layout by those, not by every character (floor budget) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/compiler/02_parse.dag | 8 ++--- src/v2/std/compilers/lexing.dag | 31 ++++++++++--------- .../claim/parse/layout_line_break_test.dag | 2 +- 3 files changed, 21 insertions(+), 20 deletions(-) diff --git a/src/v2/compiler/02_parse.dag b/src/v2/compiler/02_parse.dag index cacc65c0815..d31a5107ac7 100644 --- a/src/v2/compiler/02_parse.dag +++ b/src/v2/compiler/02_parse.dag @@ -99,7 +99,6 @@ import v2.std.witness { Holds, Violates, Witness, witness_from_optional } import std.cache_identity { ArtifactKindId, CacheInterfaceId } import std.realization { Materialization } import v2.std.integer { integer_int_content_digest } -import std.unicode.types { unicode_char_code_point } import v2.compiler.materialization_carriers { parse_table_memo_artifact_kind, parse_table_memo_provider_id, @@ -900,9 +899,9 @@ fn parse_token_stream_source_digest(all_tokens: List, layout: Optional line_break_before: Map } @@ -213,16 +215,17 @@ type StreamLayoutWalk { ahead: List last_end: Int seen: Bool + starts: List answers: Map } fn stream_layout_of(tokens: List, source: String) -> StreamLayout { let walked = fold_list( xs: source, - empty: StreamLayoutWalk { octet: 0, ahead: tokens, last_end: 0, seen: false, answers: empty_map() }, + empty: StreamLayoutWalk { octet: 0, ahead: tokens, last_end: 0, seen: false, starts: Empty, answers: empty_map() }, cons: fn(acc, c) { stream_layout_step(acc: acc, c: c) } ) - StreamLayout { source: source, line_break_before: walked.answers } + StreamLayout { source: source, line_break_starts: walked.starts, line_break_before: walked.answers } } fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { @@ -233,6 +236,7 @@ fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { ahead: at_token.ahead, last_end: at_token.last_end, seen: at_token.seen || ((code == 10) && (at_token.octet >= at_token.last_end)), + starts: at_token.starts, answers: at_token.answers } } @@ -248,7 +252,8 @@ fn stream_layout_record_token(acc: StreamLayoutWalk) -> StreamLayoutWalk { ahead: skip(xs: acc.ahead, n: 1), last_end: tok.end, seen: false, - answers: map_insert(m: acc.answers, key: tok.start, value: acc.seen) + starts: if acc.seen { list_snoc_item(xs: acc.starts, item: tok.start) } else { acc.starts }, + answers: if acc.seen { map_insert(m: acc.answers, key: tok.start, value: true) } else { acc.answers } } } else { acc @@ -258,19 +263,15 @@ fn stream_layout_record_token(acc: StreamLayoutWalk) -> StreamLayoutWalk { // THE NAMED DERIVATION, READ AT THE POINT OF USE: is there a line break in the source between the // token before the one starting at octet `token_start` and that token? A stream without layout -// cannot answer; neither can a start no token of the stream has. +// cannot answer. `token_start` is always a start of one of the stream's own tokens (the parser asks +// only about the token it is at), so a start the walk did not record is a token with no break before it. fn token_stream_line_break_before(layout: Optional, token_start: Int) -> LineBreakAnswer { match layout { Absent => LineBreakUnknownWithoutSource Present { value: l } => match map_lookup(m: l.line_break_before, key: token_start) { - Absent => LineBreakUnknownWithoutSource - Present { value: seen } => - if seen { - LineBreakInGap - } else { - NoLineBreakInGap - } + Absent => NoLineBreakInGap + Present { value: _ } => LineBreakInGap } } } diff --git a/src/v2/test/claim/parse/layout_line_break_test.dag b/src/v2/test/claim/parse/layout_line_break_test.dag index bc893c05a49..6d3e1f0e6db 100644 --- a/src/v2/test/claim/parse/layout_line_break_test.dag +++ b/src/v2/test/claim/parse/layout_line_break_test.dag @@ -149,7 +149,7 @@ test fn a_stream_without_source_answers_unknown_holds() -> Bool { // THE IDENTITY CONTROL: carrying the source changes nothing the grammar did not ask about. This module // reaches no layout-conditioned production (it has no `-` after an operand, where the dag grammar asks // one), so the same tokens parse to the same tree (content hash) with the source attached and absent. -data identity_module_text: String = "module probe.layout_identity\n\nfn f(x: Int) -> Int {\n x + 1\n}\n" +data identity_module_text: String = "module probe.layout_identity\n\ndata d: Int = x + 1\n" // Against the prepared dag grammar the floor warms once (v2.compiler.program_assembly // dag_prepared_grammar), so the claim pays two parses and no grammar preparation. From dcc31bb9421767c866d1860cc8fb727a369a90af Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 13:41:55 +0000 Subject: [PATCH 07/21] grammar: cite token_stream_line_break_before (stale symbol, review 73221) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/std/grammar.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v2/std/grammar.dag b/src/v2/std/grammar.dag index faede0487f2..0830ed10082 100644 --- a/src/v2/std/grammar.dag +++ b/src/v2/std/grammar.dag @@ -246,7 +246,7 @@ fn grammar_expr_refuse_on_match(element: GrammarExpr, reason: ParseRefusalReason // A LAYOUT-CONDITIONED MATCH. `AfterLineBreak { element }` matches what `element` matches only when // the source holds a line break between the token before the match and its first token -// (v2.std.compilers.lexing token_stream_line_break_between); otherwise it fails. On a stream that +// (v2.std.compilers.lexing token_stream_line_break_before); otherwise it fails. On a stream that // carries no source the question has no answer, and the parse refuses with // LayoutUnknownWithoutSource rather than assuming one line. fn grammar_expr_after_line_break(element: GrammarExpr) -> GrammarExpr { From a30b62f0fc1842f902daac5ef60661d5184acab9 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 14:27:02 +0000 Subject: [PATCH 08/21] lexing: delete token_stream_source, no consumer after the layout rework (review 73234) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/std/compilers/lexing.dag | 7 ------- 1 file changed, 7 deletions(-) diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index a1670faa94e..71f4b3d8eed 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -193,13 +193,6 @@ fn token_stream_layout(stream: TokenStream) -> Optional { stream.layout } -fn token_stream_source(stream: TokenStream) -> Optional { - match stream.layout { - Absent => optional_absent() - Present { value: layout } => optional_present(value: layout.source) - } -} - type LineBreakAnswer = LineBreakInGap | NoLineBreakInGap From e4e899baeddecab04d92106599894162e8347f20 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 14:28:05 +0000 Subject: [PATCH 09/21] StreamLayout: drop the unread source field (the source is read once, at lexing); digest renamed to what it covers Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/compiler/02_parse.dag | 6 +++--- src/v2/std/compilers/lexing.dag | 20 +++++++++---------- .../claim/parse/layout_line_break_test.dag | 8 ++++---- 3 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/v2/compiler/02_parse.dag b/src/v2/compiler/02_parse.dag index d31a5107ac7..01e4b723e7a 100644 --- a/src/v2/compiler/02_parse.dag +++ b/src/v2/compiler/02_parse.dag @@ -892,7 +892,7 @@ fn parse_grammar_digest(validated: ValidatedGrammarRoot) -> Hash { // byte 255 and every non-Latin-1 codepoint collide in the seed interpreter's cross-parse memo key // (v1_interpreter parse_table_memo_scope_and_key reads this field). Offsets and positions now go // through the Int content authority and the lexeme through its interned symbol. -fn parse_token_stream_source_digest(all_tokens: List, layout: Optional) -> Hash { +fn parse_token_stream_layout_digest(all_tokens: List, layout: Optional) -> Hash { match layout { Absent => parse_token_stream_digest(all_tokens: all_tokens) Present { value: l } => @@ -900,7 +900,7 @@ fn parse_token_stream_source_digest(all_tokens: List, layout: Optional line_break_before: Map } @@ -218,7 +218,7 @@ fn stream_layout_of(tokens: List, source: String) -> StreamLayout { empty: StreamLayoutWalk { octet: 0, ahead: tokens, last_end: 0, seen: false, starts: Empty, answers: empty_map() }, cons: fn(acc, c) { stream_layout_step(acc: acc, c: c) } ) - StreamLayout { source: source, line_break_starts: walked.starts, line_break_before: walked.answers } + StreamLayout { line_break_starts: walked.starts, line_break_before: walked.answers } } fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { diff --git a/src/v2/test/claim/parse/layout_line_break_test.dag b/src/v2/test/claim/parse/layout_line_break_test.dag index 6d3e1f0e6db..7c51f27d8dc 100644 --- a/src/v2/test/claim/parse/layout_line_break_test.dag +++ b/src/v2/test/claim/parse/layout_line_break_test.dag @@ -1,6 +1,6 @@ module v2.test.parse.layout_line_break -import v2.compiler.parse { ParseArtifact, parse_module, parse_module_prepared, parse_token_stream_digest, parse_token_stream_source_digest } +import v2.compiler.parse { ParseArtifact, parse_module, parse_module_prepared, parse_token_stream_digest, parse_token_stream_layout_digest } import v2.compiler.program_assembly { dag_prepared_grammar } import v2.compiler.tokenize { tokenize } import v2.extdeps.languages.dag { dag_lex } @@ -275,7 +275,7 @@ test fn a_layout_question_without_source_refuses_holds() -> Bool { // THE MEMO CANNOT SERVE ONE LAYOUT'S ANSWER TO THE OTHER. `x - 1` and `x` newline `- 1` lex to the // same tokens at the same octets, so a stream digest over tokens alone would key them together and a // warm memo entry could replay a same-line acceptance for a next-line refusal. The source is part of -// the stream's digest (v2.compiler.parse parse_token_stream_source_digest): same tokens, different +// the stream's digest (v2.compiler.parse parse_token_stream_layout_digest): same tokens, different // digest; and the token-only digest, which is what a sourceless stream keys by, is unchanged. test fn same_tokens_different_layout_have_different_stream_digests_holds() -> Bool { match lexed(text: same_line_text, file: ^layout_digest) { @@ -285,8 +285,8 @@ test fn same_tokens_different_layout_have_different_stream_digests_holds() -> Bo Rejected { diagnostics: _ } => false Accepted { value: next, diagnostics: _ } => (parse_token_stream_digest(all_tokens: token_stream_all(stream: same)) == parse_token_stream_digest(all_tokens: token_stream_all(stream: next))) - && (parse_token_stream_source_digest(all_tokens: token_stream_all(stream: same), layout: token_stream_layout(stream: same)) - != parse_token_stream_source_digest(all_tokens: token_stream_all(stream: next), layout: token_stream_layout(stream: next))) + && (parse_token_stream_layout_digest(all_tokens: token_stream_all(stream: same), layout: token_stream_layout(stream: same)) + != parse_token_stream_layout_digest(all_tokens: token_stream_all(stream: next), layout: token_stream_layout(stream: next))) } } } From 39bbbd420491ab7f577b12638bb93e73b2b9a34c Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 14:28:50 +0000 Subject: [PATCH 10/21] layout test: the digest covers layout, not source (comment) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/test/claim/parse/layout_line_break_test.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v2/test/claim/parse/layout_line_break_test.dag b/src/v2/test/claim/parse/layout_line_break_test.dag index 7c51f27d8dc..ed96a158787 100644 --- a/src/v2/test/claim/parse/layout_line_break_test.dag +++ b/src/v2/test/claim/parse/layout_line_break_test.dag @@ -274,7 +274,7 @@ test fn a_layout_question_without_source_refuses_holds() -> Bool { // THE MEMO CANNOT SERVE ONE LAYOUT'S ANSWER TO THE OTHER. `x - 1` and `x` newline `- 1` lex to the // same tokens at the same octets, so a stream digest over tokens alone would key them together and a -// warm memo entry could replay a same-line acceptance for a next-line refusal. The source is part of +// warm memo entry could replay a same-line acceptance for a next-line refusal. The layout (its line-break-preceded starts) is part of // the stream's digest (v2.compiler.parse parse_token_stream_layout_digest): same tokens, different // digest; and the token-only digest, which is what a sourceless stream keys by, is unchanged. test fn same_tokens_different_layout_have_different_stream_digests_holds() -> Bool { From 66b9ff1dd1567b98dcdef866c63636ccba7a83cc Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 15:39:31 +0000 Subject: [PATCH 11/21] lexing: spell the tokenizer's source as v2.std.text.String (emit-build: emitted as host String) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/std/compilers/lexing.dag | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 2cbe2f979e3..934c703285a 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -181,7 +181,7 @@ fn token_stream_new(all: List) -> TokenStream { // The tokenizer's constructor: the tokens together with the exact scalar sequence they were lexed // from, whose UTF-8 octet offsets their `start`/`end` name. -fn token_stream_lexed(all: List, source: String) -> TokenStream { +fn token_stream_lexed(all: List, source: v2.std.text.String) -> TokenStream { TokenStream { all: all, pos: 0, layout: optional_present(value: stream_layout_of(tokens: all, source: source)) } } @@ -212,7 +212,7 @@ type StreamLayoutWalk { answers: Map } -fn stream_layout_of(tokens: List, source: String) -> StreamLayout { +fn stream_layout_of(tokens: List, source: v2.std.text.String) -> StreamLayout { let walked = fold_list( xs: source, empty: StreamLayoutWalk { octet: 0, ahead: tokens, last_end: 0, seen: false, starts: Empty, answers: empty_map() }, From f7888e65eb4a5edbd4a48c8eaefa1e653dab6b02 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Wed, 30 Sep 2026 16:22:32 +0000 Subject: [PATCH 12/21] StreamLayout: one representation, start -> line of each line-break-preceded token (review 73282) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/compiler/02_parse.dag | 13 +++++++++++-- src/v2/std/compilers/lexing.dag | 29 ++++++++++++++--------------- 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/src/v2/compiler/02_parse.dag b/src/v2/compiler/02_parse.dag index 01e4b723e7a..ea74bcac346 100644 --- a/src/v2/compiler/02_parse.dag +++ b/src/v2/compiler/02_parse.dag @@ -892,6 +892,9 @@ fn parse_grammar_digest(validated: ValidatedGrammarRoot) -> Hash { // byte 255 and every non-Latin-1 codepoint collide in the seed interpreter's cross-parse memo key // (v1_interpreter parse_table_memo_scope_and_key reads this field). Offsets and positions now go // through the Int content authority and the lexeme through its interned symbol. +// Over the same tokens the token digest folds: each line-break-preceded token contributes its start +// and the line it begins (the layout's one representation, read by lookup). Only a sourceless stream +// keys by the token digest alone. fn parse_token_stream_layout_digest(all_tokens: List, layout: Optional) -> Hash { match layout { Absent => parse_token_stream_digest(all_tokens: all_tokens) @@ -899,9 +902,15 @@ fn parse_token_stream_layout_digest(all_tokens: List, layout: Optional acc + Present { value: line } => + combine_hash(a: acc, b: combine_hash(a: integer_int_content_digest(value: tok.start), b: integer_int_content_digest(value: line))) + } + } ) ) } diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 934c703285a..41c38276d75 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -160,13 +160,12 @@ type Token { // starts of the tokens that have a line break in the gap before them, DERIVED from the source and the // tokens in one walk by its only constructor (`stream_layout_of`, called by the tokenizer, the one // place the source is read), so a layout question is a lookup rather than a rescan of the file. Only -// the line-break-preceded starts are kept (a line is many tokens), in source order as well as keyed, -// so the stream digest can cover layout by folding over them rather than over every character. A -// stream built from tokens alone (fixtures) carries no layout, and a layout question on it answers -// `LineBreakUnknownWithoutSource` -- never an assumed single line. +// the line-break-preceded tokens are kept (a line is many tokens), each keyed by its start to the +// 1-based line it begins -- ONE representation of the set, with a value that is a fact about the token +// rather than a placeholder. A stream built from tokens alone (fixtures) carries no layout, and a +// layout question on it answers `LineBreakUnknownWithoutSource` -- never an assumed single line. type StreamLayout { - line_break_starts: List - line_break_before: Map + line_break_before: Map } type TokenStream { @@ -198,27 +197,27 @@ type LineBreakAnswer | NoLineBreakInGap | LineBreakUnknownWithoutSource -// ONE WALK: the source's scalars in order, with the tokens still ahead and the end of the last token -// passed. A line feed (U+000A) counts only in a GAP -- at or after the previous token's end -- so a +// ONE WALK: the source's scalars in order, with the line being walked, the tokens still ahead and the +// end of the last token passed. A line feed (U+000A) counts only in a GAP -- at or after the previous token's end -- so a // line feed inside a token (a multi-line string) is not layout. When the walk reaches a token's // first octet, whether a line feed was seen since the previous token is that token's answer. // Offsets are UTF-8 octets, as token extents are, so each scalar advances by its encoded width. type StreamLayoutWalk { octet: Int + line: Int ahead: List last_end: Int seen: Bool - starts: List - answers: Map + answers: Map } fn stream_layout_of(tokens: List, source: v2.std.text.String) -> StreamLayout { let walked = fold_list( xs: source, - empty: StreamLayoutWalk { octet: 0, ahead: tokens, last_end: 0, seen: false, starts: Empty, answers: empty_map() }, + empty: StreamLayoutWalk { octet: 0, line: 1, ahead: tokens, last_end: 0, seen: false, answers: empty_map() }, cons: fn(acc, c) { stream_layout_step(acc: acc, c: c) } ) - StreamLayout { line_break_starts: walked.starts, line_break_before: walked.answers } + StreamLayout { line_break_before: walked.answers } } fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { @@ -226,10 +225,10 @@ fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { let code = unicode_char_code_point(c: c) StreamLayoutWalk { octet: at_token.octet + unicode_scalar_utf8_octet_count(code_point: code), + line: if code == 10 { at_token.line + 1 } else { at_token.line }, ahead: at_token.ahead, last_end: at_token.last_end, seen: at_token.seen || ((code == 10) && (at_token.octet >= at_token.last_end)), - starts: at_token.starts, answers: at_token.answers } } @@ -242,11 +241,11 @@ fn stream_layout_record_token(acc: StreamLayoutWalk) -> StreamLayoutWalk { if tok.start == acc.octet { StreamLayoutWalk { octet: acc.octet, + line: acc.line, ahead: skip(xs: acc.ahead, n: 1), last_end: tok.end, seen: false, - starts: if acc.seen { list_snoc_item(xs: acc.starts, item: tok.start) } else { acc.starts }, - answers: if acc.seen { map_insert(m: acc.answers, key: tok.start, value: true) } else { acc.answers } + answers: if acc.seen { map_insert(m: acc.answers, key: tok.start, value: acc.line) } else { acc.answers } } } else { acc From 0cb9c37fa3a4c285508380d47f4a9b1aa74c4e01 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 05:06:56 +0000 Subject: [PATCH 13/21] v2 parse: refuse a line-break-preceded '-' at the additive continuation (NewlineBeforeDualRoleOperator); controls, emit control, mutation witness Co-Authored-By: Claude Opus 5.5 (1M context) --- ...al_role_operator_mutation_witness_test.dag | 136 ++++++++++++++++ src/v2/extdeps/languages/dag.dag | 26 ++- src/v2/std/parse_refusal_reason.dag | 7 + .../newline_dual_role_operator_parse_test.dag | 152 ++++++++++++++++++ 4 files changed, 317 insertions(+), 4 deletions(-) create mode 100644 dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag create mode 100644 src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag diff --git a/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag b/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag new file mode 100644 index 00000000000..df7f85ba887 --- /dev/null +++ b/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag @@ -0,0 +1,136 @@ +module test.claim.long.newline_dual_role_operator_mutation_witness_test + +import v2.compiler.parse { parse_module } +import v2.compiler.tokenize { tokenize } +import v2.extdeps.languages.dag { dag_grammar, dag_lex } +import v2.std.algebra { fold_list } +import v2.std.grammar { + AfterLineBreak, + Choice, + Expect, + GrammarExpr, + GrammarExprFold, + GrammarProduction, + GrammarRoot, + LiteralTerminal, + ModeledGrammar, + Nonterminal, + Optional, + ParseGrammar, + RefuseOnMatch, + Repeat, + Sequence, + StampClass, + Terminal, + VoidGrammar, + fold_grammar_expr +} +import v2.std.diagnostic { Accepted, Rejected } +import v2.std.integer { Int } +import v2.std.logic { Bool } +import v2.std.node { Symbol } +import v2.std.text { String } +import v2.test.parse.newline_dual_role_operator_parse { + newline_minus_after_block_source, + newline_minus_after_operand_source, + same_line_minus_source +} + +// THE MUTATION THAT REDS THE NEWLINE-`-` REFUSAL (v2.test.parse.newline_dual_role_operator_parse). +// It rebuilds the live dag grammar with every RefuseOnMatch arm replaced by a terminal no token +// carries -- an alternative that can never match -- and asserts there is exactly ONE such arm, so the +// mutant differs from the live grammar in that refusal and nothing else. Under the mutant the +// next-line sources parse again (the silent merge into one subtraction is back); under the live +// grammar they refuse. The same-line control parses under both. Long lane: preparing the mutant +// grammar is past the floor's per-claim budget. + +fn strip_refusals(expr: GrammarExpr) -> GrammarExpr { + fold_grammar_expr( + expr: expr, + algebra: GrammarExprFold { + terminal: fn(token_class, stamp) { Terminal { token_class: token_class, stamp: stamp } }, + literal_terminal: fn(token_class, lexeme) { LiteralTerminal { token_class: token_class, lexeme: lexeme } }, + nonterminal: fn(production) { Nonterminal { production: production } }, + sequence: fn(l, r) { Sequence { left: l, right: r } }, + choice: fn(l, r) { Choice { left: l, right: r } }, + optional: fn(e) { Optional { element: e } }, + repeat: fn(e) { Repeat { element: e } }, + expect: fn(e, reason) { Expect { element: e, reason: reason } }, + refuse_on_match: fn(_e, _reason) { Terminal { token_class: ^newline_mutation_no_token_carries_this_class, stamp: StampClass } }, + after_line_break: fn(e) { AfterLineBreak { element: e } } + } + ) +} + +fn refusal_arm_count(expr: GrammarExpr) -> Int { + fold_grammar_expr( + expr: expr, + algebra: GrammarExprFold { + terminal: fn(_, _) { 0 }, + literal_terminal: fn(_, _) { 0 }, + nonterminal: fn(_) { 0 }, + sequence: fn(l, r) { l + r }, + choice: fn(l, r) { l + r }, + optional: fn(e) { e }, + repeat: fn(e) { e }, + expect: fn(e, _reason) { e }, + refuse_on_match: fn(e, _reason) { e + 1 }, + after_line_break: fn(e) { e } + } + ) +} + +fn no_productions() -> List { + [] +} + +fn mutant_grammar() -> ParseGrammar { + match dag_grammar() { + ModeledGrammar { root: r } => + ModeledGrammar { + root: GrammarRoot { + start: r.start, + productions: fold_list(xs: r.productions, empty: no_productions(), cons: fn(acc, prod) { + concat(acc, [GrammarProduction { name: prod.name, expression: strip_refusals(expr: prod.expression), emitted: prod.emitted }]) + }), + sync_tokens: r.sync_tokens + } + } + VoidGrammar => VoidGrammar + } +} + +fn live_refusal_arm_count() -> Int { + match dag_grammar() { + ModeledGrammar { root: r } => fold_list(xs: r.productions, empty: 0, cons: fn(acc, prod) { acc + refusal_arm_count(expr: prod.expression) }) + VoidGrammar => 0 + } +} + +fn accepts_with(text: String, file: Symbol, grammar: ParseGrammar) -> Bool { + match tokenize(text: text, file: file, rules: dag_lex()) { + Rejected { diagnostics: _ } => false + Accepted { value: stream, diagnostics: _ } => + match parse_module(tokens: stream, grammar: grammar) { + Accepted { value: _, diagnostics: _ } => true + Rejected { diagnostics: _ } => false + } + } +} + +test fn witness_the_live_grammar_carries_exactly_one_refusal_arm() -> Bool { + live_refusal_arm_count() == 1 +} + +test fn witness_the_live_grammar_refuses_both_next_line_forms() -> Bool { + (accepts_with(text: newline_minus_after_operand_source, file: ^nlm_plain, grammar: dag_grammar()) == false) + && (accepts_with(text: newline_minus_after_block_source, file: ^nlm_block, grammar: dag_grammar()) == false) + && accepts_with(text: same_line_minus_source, file: ^nlm_same, grammar: dag_grammar()) +} + +test fn witness_without_the_refusal_both_next_line_forms_merge_RED() -> Bool { + let mutant = mutant_grammar() + accepts_with(text: newline_minus_after_operand_source, file: ^nlm_plain, grammar: mutant) + && accepts_with(text: newline_minus_after_block_source, file: ^nlm_block, grammar: mutant) + && accepts_with(text: same_line_minus_source, file: ^nlm_same, grammar: mutant) +} diff --git a/src/v2/extdeps/languages/dag.dag b/src/v2/extdeps/languages/dag.dag index 3a1e6447e0b..6898f299f44 100644 --- a/src/v2/extdeps/languages/dag.dag +++ b/src/v2/extdeps/languages/dag.dag @@ -20,7 +20,7 @@ import v2.std.optional { optional_absent, optional_present } -import v2.std.parse_refusal_reason { DataValueMissingAfterEq, FnExpressionBodyMissingAfterEq, ParseRefusalReason } +import v2.std.parse_refusal_reason { DataValueMissingAfterEq, FnExpressionBodyMissingAfterEq, NewlineBeforeDualRoleOperator, ParseRefusalReason } import v2.std.grammar { grammar_formal_terminal, grammar_formal_terminal_bound, @@ -56,7 +56,9 @@ import v2.std.grammar { grammar_production_to_node, grammar_relation_tokens_node, grammar_empty_sync_tokens, + grammar_expr_after_line_break, grammar_expr_expect, + grammar_expr_refuse_on_match, derive_grammar_relation_row, derive_grammar_relation_row_node, formal_production_for_lhs_exact, @@ -1283,13 +1285,29 @@ fn dag_grammar_multiplicative_expr_helper() -> GrammarExpr { ) } +// A `-` THAT BEGINS A LINE AFTER A COMPLETE OPERAND REFUSES. `-` is both infix (subtraction) and +// prefix (negation), and this grammar carries no statement terminator, so `x` then a line `-1` could +// be `x - 1` or two statements; neither reading is safe to pick silently. The seed refuses it +// (v1.compiler.parse is_ambiguous_prefix_infix_newline_boundary, whose dual-role set is exactly +// `-`), and so does this row: the refusal alternative is tried FIRST at every additive +// continuation, it matches only a `-` preceded by a line break (v2.std.grammar AfterLineBreak, read +// from the stream's layout), and where it matches the parse refuses at the `-` with +// NewlineBeforeDualRoleOperator. A same-line `-` falls through to subtraction. Every expression +// reaches this continuation, so the rule holds for every statement form -- including a block-headed +// statement, which is an operand since block forms are primaries of binary_expr (#12817). fn dag_grammar_additive_expr_helper() -> GrammarExpr { dag_grammar_sequence( left: dag_grammar_multiplicative_expr_helper(), right: dag_grammar_repeat( - element: dag_grammar_sequence( - left: dag_grammar_additive_op_choice(), - right: dag_grammar_multiplicative_expr_helper() + element: dag_grammar_choice( + left: grammar_expr_refuse_on_match( + element: grammar_expr_after_line_break(element: dag_grammar_terminal(token_class: ^dag_token_minus)), + reason: NewlineBeforeDualRoleOperator + ), + right: dag_grammar_sequence( + left: dag_grammar_additive_op_choice(), + right: dag_grammar_multiplicative_expr_helper() + ) ) ) ) diff --git a/src/v2/std/parse_refusal_reason.dag b/src/v2/std/parse_refusal_reason.dag index 4e7a5230335..2902f1cb1f2 100644 --- a/src/v2/std/parse_refusal_reason.dag +++ b/src/v2/std/parse_refusal_reason.dag @@ -14,10 +14,16 @@ import v2.std.node { Symbol } // AfterLineBreak) was reached on a token stream that carries no // source, so whether a line break precedes the match has no // answer; the parse refuses rather than assume one line. +// NewlineBeforeDualRoleOperator -- a `-` (both prefix and infix) begins a line after a complete +// operand. It could continue the expression or start a new one, +// so it refuses, as the seed does (v1.compiler.parse +// is_ambiguous_prefix_infix_newline_boundary): move the operator +// before the newline, or parenthesize the unary expression. type ParseRefusalReason = FnExpressionBodyMissingAfterEq | DataValueMissingAfterEq | LayoutUnknownWithoutSource + | NewlineBeforeDualRoleOperator // The diagnostic identity each reason refuses under (`v2.std.diagnostic` `Diagnostic.reason`). fn parse_refusal_reason_symbol(reason: ParseRefusalReason) -> Symbol { @@ -25,5 +31,6 @@ fn parse_refusal_reason_symbol(reason: ParseRefusalReason) -> Symbol { FnExpressionBodyMissingAfterEq => ^parse_fn_expression_body_missing_after_eq DataValueMissingAfterEq => ^parse_data_value_missing_after_eq LayoutUnknownWithoutSource => ^parse_layout_unknown_without_source + NewlineBeforeDualRoleOperator => ^parse_newline_before_dual_role_operator } } diff --git a/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag b/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag new file mode 100644 index 00000000000..d7aaa974633 --- /dev/null +++ b/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag @@ -0,0 +1,152 @@ +module v2.test.parse.newline_dual_role_operator_parse + +import v2.compiler.normalize { normalize } +import v2.compiler.parse { parse_module_prepared } +import v2.compiler.program_assembly { dag_prepared_grammar } +import v2.compiler.tokenize { tokenize } +import v2.extdeps.languages.dag { dag_lex, dag_lex_rules, dag_target_model } +import v2.std.algebra { fold_list } +import v2.std.collection { map_lookup } +import v2.std.compilers.lexing { AnnotationRule, TokenRule, TriviaRule } +import v2.std.compilers.target_model { + EmitLineBreaks, + EmitSpellingConstant, + EmitSpellingIdentity, + EmitSpellingQuote, + EmitSpellingWrap +} +import v2.std.diagnostic { + Accepted, + ByteRange, + DeclarationLocus, + InvariantPortLocus, + NodeLocus, + Outcome, + Rejected, + SourcePortLocus, + Textual, + WholeFile, + bind_outcome, + diagnostics_fatal +} +import v2.std.integer { Int } +import v2.std.live_tree { LiveTreeDisposition, SubstrateInputsOnly } +import v2.std.logic { Bool } +import v2.std.node { Node, Symbol } +import v2.std.optional { Absent, Present } +import v2.std.parse_refusal_reason { NewlineBeforeDualRoleOperator, parse_refusal_reason_symbol } +import v2.std.text { String } + +data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly + +// A `-` THAT BEGINS A LINE AFTER A COMPLETE OPERAND REFUSES, AS THE SEED REFUSES IT +// (v1.compiler.parse is_ambiguous_prefix_infix_newline_boundary). `-` is infix and prefix, and the +// dag grammar has no statement terminator, so `x` then a line `- 1` is either one subtraction or two +// statements; v2 read it as one, silently, and since block forms became operands (#12817) a block +// statement followed by a `-` line merged the same way. The dag grammar's additive continuation now +// refuses a line-break-preceded `-` with NewlineBeforeDualRoleOperator at the `-` +// (v2.extdeps.languages.dag dag_grammar_additive_expr_helper). Each side of the rule, through the +// live prepared grammar on a lexed stream (which carries its layout): the next-line forms refuse at +// the `-`, for a plain operand and for a block, and the same-line form stays one subtraction. The +// mutation that reds the refusals is test.claim.long.newline_dual_role_operator_mutation_witness. + +fn n_tokenize_parse(text: String, file: Symbol) -> Outcome { + match dag_prepared_grammar() { + Rejected { diagnostics: d } => Rejected { diagnostics: d } + Accepted { value: prepared, diagnostics: residue } => + bind_outcome( + o: tokenize(text: text, file: file, rules: dag_lex()), + f: fn(token_stream) { + match parse_module_prepared(tokens: token_stream, prepared: prepared, validation_residue: residue) { + Rejected { diagnostics: r } => Rejected { diagnostics: r } + Accepted { value: artifact, diagnostics: d } => Accepted { value: artifact.tree, diagnostics: d } + } + } + ) + } +} + +fn n_refuses_at_minus(text: String, file: Symbol, start: Int, end: Int) -> Bool { + match n_tokenize_parse(text: text, file: file) { + Accepted { value: _, diagnostics: _ } => false + Rejected { diagnostics: d } => + let fatal = diagnostics_fatal(d: d) + (fatal.reason == parse_refusal_reason_symbol(reason: NewlineBeforeDualRoleOperator)) + && match fatal.at { + SourcePortLocus { anchor: _, file: f, extent: extent } => + (f == file) + && match extent { + ByteRange { start: s, end: e } => (s == start) && (e == end) + WholeFile => false + } + Textual { file: _, extent: _ } => false + NodeLocus { anchor: _ } => false + InvariantPortLocus { anchor: _, invariant: _ } => false + DeclarationLocus { declaration: _ } => false + } + } +} + +fn n_parses_and_normalizes(text: String, file: Symbol) -> Bool { + match n_tokenize_parse(text: text, file: file) { + Rejected { diagnostics: _ } => false + Accepted { value: tree, diagnostics: _ } => + match normalize(parse_tree: tree) { + Rejected { diagnostics: _ } => false + Accepted { value: _, diagnostics: _ } => true + } + } +} + +// `-` at bytes 51..52. +data newline_minus_after_operand_source: String = "module probe.nl_plain\n\nfn f(x: Int) -> Int {\n x\n - 1\n}\n" + +// `-` at bytes 101..102. +data newline_minus_after_block_source: String = "module probe.nl_block\n\nfn f(w: Bool, n: Int) -> Int {\n match w {\n true => n\n false => 0\n }\n - n\n}\n" + +data same_line_minus_source: String = "module probe.nl_same\n\nfn f(x: Int) -> Int {\n x - 1\n}\n" + +test fn a_newline_minus_after_an_operand_refuses_holds() -> Bool { + n_refuses_at_minus(text: newline_minus_after_operand_source, file: ^probe_nl_plain, start: 51, end: 52) +} + +test fn a_newline_minus_after_a_block_refuses_holds() -> Bool { + n_refuses_at_minus(text: newline_minus_after_block_source, file: ^probe_nl_block, start: 101, end: 102) +} + +test fn a_same_line_minus_is_subtraction_holds() -> Bool { + n_parses_and_normalizes(text: same_line_minus_source, file: ^probe_nl_same) +} + +// THE EMIT DIRECTION (DESIGN section 4: one grammar, read both ways). The refusal is ingest-only: +// RefuseOnMatch derives no tree, so emit never selects it, and emit reads the formal productions, not +// these rows. What must hold on the emit side is that the dag emitter never WRITES what ingest now +// refuses -- a `-` that begins a line. Emitted text carries a line break only where a token class's +// emit transform is EmitLineBreaks (v2.std.compilers.target_model target_text_line_breaks, the one +// source of DocLine in a token fold), and no dag token class has one: the dag emitter never breaks a +// line, so a line-break-preceded `-` is unwritable by construction. This control reds the moment a +// dag class gains a line-breaking transform, which is when the emitter would have to be taught to keep +// a `-` on its operand's line. +fn dag_class_breaks_no_line(class: Symbol) -> Bool { + match map_lookup(m: dag_target_model().token_class_emit_transforms, key: class) { + Absent => true + Present { value: transform } => + match transform { + EmitLineBreaks { before: _, after: _ } => false + EmitSpellingIdentity => true + EmitSpellingConstant { value: _ } => true + EmitSpellingWrap { prefix: _, suffix: _ } => true + EmitSpellingQuote { prefix: _, open: _, close: _, escapes: _ } => true + } + } +} + +test fn no_dag_token_class_emits_a_line_break_holds() -> Bool { + fold_list(xs: dag_lex_rules().rules, empty: true, cons: fn(acc, rule) { + acc && match rule { + TokenRule { token_class: c, pattern: _ } => dag_class_breaks_no_line(class: c) + TriviaRule { pattern: _ } => true + AnnotationRule { annotation_class: _, pattern: _ } => true + } + }) +} From 8e02e4524f38c556c3cc7784bf5aaf8f918956df Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 05:20:21 +0000 Subject: [PATCH 14/21] Mutation witness: every live refusal arm is the newline refusal (the additive row is inlined, so arms > 1) Co-Authored-By: Claude Opus 5.5 (1M context) --- ...al_role_operator_mutation_witness_test.dag | 49 +++++++++++++++++-- 1 file changed, 45 insertions(+), 4 deletions(-) diff --git a/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag b/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag index df7f85ba887..14dc5cd70e6 100644 --- a/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag +++ b/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag @@ -26,6 +26,12 @@ import v2.std.grammar { fold_grammar_expr } import v2.std.diagnostic { Accepted, Rejected } +import v2.std.parse_refusal_reason { + DataValueMissingAfterEq, + FnExpressionBodyMissingAfterEq, + LayoutUnknownWithoutSource, + NewlineBeforeDualRoleOperator +} import v2.std.integer { Int } import v2.std.logic { Bool } import v2.std.node { Symbol } @@ -38,8 +44,11 @@ import v2.test.parse.newline_dual_role_operator_parse { // THE MUTATION THAT REDS THE NEWLINE-`-` REFUSAL (v2.test.parse.newline_dual_role_operator_parse). // It rebuilds the live dag grammar with every RefuseOnMatch arm replaced by a terminal no token -// carries -- an alternative that can never match -- and asserts there is exactly ONE such arm, so the -// mutant differs from the live grammar in that refusal and nothing else. Under the mutant the +// carries -- an alternative that can never match -- and asserts that such arms exist and that EVERY +// one of them is the newline refusal (NewlineBeforeDualRoleOperator), so the mutant differs from the +// live grammar in that refusal and nothing else. There is more than one arm because the additive row +// is a helper inlined into each operand position of the comparison and equality rows; it is one +// authored row. Under the mutant the // next-line sources parse again (the silent merge into one subtraction is back); under the live // grammar they refuse. The same-line control parses under both. Long lane: preparing the mutant // grammar is past the floor's per-claim budget. @@ -80,6 +89,31 @@ fn refusal_arm_count(expr: GrammarExpr) -> Int { ) } +fn other_refusal_arm_count(expr: GrammarExpr) -> Int { + fold_grammar_expr( + expr: expr, + algebra: GrammarExprFold { + terminal: fn(_, _) { 0 }, + literal_terminal: fn(_, _) { 0 }, + nonterminal: fn(_) { 0 }, + sequence: fn(l, r) { l + r }, + choice: fn(l, r) { l + r }, + optional: fn(e) { e }, + repeat: fn(e) { e }, + expect: fn(e, _reason) { e }, + refuse_on_match: fn(e, reason) { + match reason { + NewlineBeforeDualRoleOperator => e + FnExpressionBodyMissingAfterEq => e + 1 + DataValueMissingAfterEq => e + 1 + LayoutUnknownWithoutSource => e + 1 + } + }, + after_line_break: fn(e) { e } + } + ) +} + fn no_productions() -> List { [] } @@ -107,6 +141,13 @@ fn live_refusal_arm_count() -> Int { } } +fn live_other_refusal_arm_count() -> Int { + match dag_grammar() { + ModeledGrammar { root: r } => fold_list(xs: r.productions, empty: 0, cons: fn(acc, prod) { acc + other_refusal_arm_count(expr: prod.expression) }) + VoidGrammar => 0 + } +} + fn accepts_with(text: String, file: Symbol, grammar: ParseGrammar) -> Bool { match tokenize(text: text, file: file, rules: dag_lex()) { Rejected { diagnostics: _ } => false @@ -118,8 +159,8 @@ fn accepts_with(text: String, file: Symbol, grammar: ParseGrammar) -> Bool { } } -test fn witness_the_live_grammar_carries_exactly_one_refusal_arm() -> Bool { - live_refusal_arm_count() == 1 +test fn witness_every_live_refusal_arm_is_the_newline_refusal() -> Bool { + (live_refusal_arm_count() > 0) && (live_other_refusal_arm_count() == 0) } test fn witness_the_live_grammar_refuses_both_next_line_forms() -> Bool { From 26ad3f442bdfd6646abfac7d4df9212c031e9ba1 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 05:56:39 +0000 Subject: [PATCH 15/21] Layout digest folded once in the layout walk; the parse table reads it (no lookup per token; floor budget) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/compiler/02_parse.dag | 22 ++++------------------ src/v2/std/compilers/lexing.dag | 21 ++++++++++++++++----- 2 files changed, 20 insertions(+), 23 deletions(-) diff --git a/src/v2/compiler/02_parse.dag b/src/v2/compiler/02_parse.dag index bcc7c50f5e2..5dd0aaa08e9 100644 --- a/src/v2/compiler/02_parse.dag +++ b/src/v2/compiler/02_parse.dag @@ -892,27 +892,13 @@ fn parse_grammar_digest(validated: ValidatedGrammarRoot) -> Hash { // byte 255 and every non-Latin-1 codepoint collide in the seed interpreter's cross-parse memo key // (v1_interpreter parse_table_memo_scope_and_key reads this field). Offsets and positions now go // through the Int content authority and the lexeme through its interned symbol. -// Over the same tokens the token digest folds: each line-break-preceded token contributes its start -// and the line it begins (the layout's one representation, read by lookup). Only a sourceless stream -// keys by the token digest alone. +// The token digest, combined with the stream layout's own digest (folded once, in the layout walk, +// over the line-break-preceded tokens' starts and lines). Only a sourceless stream keys by the token +// digest alone. fn parse_token_stream_layout_digest(all_tokens: List, layout: Optional) -> Hash { match layout { Absent => parse_token_stream_digest(all_tokens: all_tokens) - Present { value: l } => - combine_hash( - a: parse_token_stream_digest(all_tokens: all_tokens), - b: fold_list( - xs: all_tokens, - empty: symbol_identity_digest(sym: ^parse_token_stream_layout_digest_nil), - cons: fn(acc, tok) { - match map_lookup(m: l.line_break_before, key: tok.start) { - Absent => acc - Present { value: line } => - combine_hash(a: acc, b: combine_hash(a: integer_int_content_digest(value: tok.start), b: integer_int_content_digest(value: line))) - } - } - ) - ) + Present { value: l } => combine_hash(a: parse_token_stream_digest(all_tokens: all_tokens), b: l.digest) } } diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 41c38276d75..c9ff971c270 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -20,7 +20,8 @@ import v2.std.optional { optional_present } import v2.std.logic { Bool } -import v2.std.node { Atom, Conj, Node, Symbol, TypeNode } +import v2.std.node { Atom, Conj, Hash, Node, Symbol, TypeNode, combine_hash, symbol_identity_digest } +import v2.std.integer { integer_int_content_digest } import v2.std.node_shape { stamp_node_shape_synthetic, type_shape } import v2.std.text { Char, String } import std.unicode.types { unicode_char_code_point, unicode_scalar_utf8_octet_count } @@ -164,8 +165,11 @@ type Token { // 1-based line it begins -- ONE representation of the set, with a value that is a fact about the token // rather than a placeholder. A stream built from tokens alone (fixtures) carries no layout, and a // layout question on it answers `LineBreakUnknownWithoutSource` -- never an assumed single line. +// `digest` summarizes exactly the entries of `line_break_before`, folded in the walk as each one is +// recorded (start, then line), so the parse memo key can cover layout without a lookup per token. type StreamLayout { line_break_before: Map + digest: Hash } type TokenStream { @@ -209,15 +213,16 @@ type StreamLayoutWalk { last_end: Int seen: Bool answers: Map + digest: Hash } fn stream_layout_of(tokens: List, source: v2.std.text.String) -> StreamLayout { let walked = fold_list( xs: source, - empty: StreamLayoutWalk { octet: 0, line: 1, ahead: tokens, last_end: 0, seen: false, answers: empty_map() }, + empty: StreamLayoutWalk { octet: 0, line: 1, ahead: tokens, last_end: 0, seen: false, answers: empty_map(), digest: symbol_identity_digest(sym: ^stream_layout_digest_nil) }, cons: fn(acc, c) { stream_layout_step(acc: acc, c: c) } ) - StreamLayout { line_break_before: walked.answers } + StreamLayout { line_break_before: walked.answers, digest: walked.digest } } fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { @@ -229,7 +234,8 @@ fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { ahead: at_token.ahead, last_end: at_token.last_end, seen: at_token.seen || ((code == 10) && (at_token.octet >= at_token.last_end)), - answers: at_token.answers + answers: at_token.answers, + digest: at_token.digest } } @@ -245,7 +251,12 @@ fn stream_layout_record_token(acc: StreamLayoutWalk) -> StreamLayoutWalk { ahead: skip(xs: acc.ahead, n: 1), last_end: tok.end, seen: false, - answers: if acc.seen { map_insert(m: acc.answers, key: tok.start, value: acc.line) } else { acc.answers } + answers: if acc.seen { map_insert(m: acc.answers, key: tok.start, value: acc.line) } else { acc.answers }, + digest: if acc.seen { + combine_hash(a: acc.digest, b: combine_hash(a: integer_int_content_digest(value: tok.start), b: integer_int_content_digest(value: acc.line))) + } else { + acc.digest + } } } else { acc From 6094f437cbcaa5803fee776dea7d479da5fa8938 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 06:06:17 +0000 Subject: [PATCH 16/21] lexing: type the layout digest as Fnv1a64Structural, what combine_hash returns (emit-build) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/std/compilers/lexing.dag | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index c9ff971c270..74596a0ebed 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -20,7 +20,8 @@ import v2.std.optional { optional_present } import v2.std.logic { Bool } -import v2.std.node { Atom, Conj, Hash, Node, Symbol, TypeNode, combine_hash, symbol_identity_digest } +import v2.std.node { Atom, Conj, Node, Symbol, TypeNode, combine_hash, symbol_identity_digest } +import std.content_hash { Fnv1a64Structural } import v2.std.integer { integer_int_content_digest } import v2.std.node_shape { stamp_node_shape_synthetic, type_shape } import v2.std.text { Char, String } @@ -169,7 +170,7 @@ type Token { // recorded (start, then line), so the parse memo key can cover layout without a lookup per token. type StreamLayout { line_break_before: Map - digest: Hash + digest: Fnv1a64Structural } type TokenStream { @@ -213,7 +214,7 @@ type StreamLayoutWalk { last_end: Int seen: Bool answers: Map - digest: Hash + digest: Fnv1a64Structural } fn stream_layout_of(tokens: List, source: v2.std.text.String) -> StreamLayout { From def354aaa4f1b11f238f873cf7b8f173fa8d1a0b Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 07:12:11 +0000 Subject: [PATCH 17/21] Layout recorded in the tokenizer's own walk (no second pass over the source); value = the octet the gap begins Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/compiler/01_tokenize.dag | 54 ++++++++++++++---- src/v2/std/compilers/lexing.dag | 98 +++++++++------------------------ 2 files changed, 70 insertions(+), 82 deletions(-) diff --git a/src/v2/compiler/01_tokenize.dag b/src/v2/compiler/01_tokenize.dag index 7cc0cc0d1b4..8adb959a31d 100644 --- a/src/v2/compiler/01_tokenize.dag +++ b/src/v2/compiler/01_tokenize.dag @@ -33,6 +33,10 @@ import v2.std.compilers.lexing { lex_artifact_new, lex_artifact_semantic_tokens, token_stream_empty, + lexeme_has_line_feed, + stream_layout_empty, + stream_layout_with_break, + StreamLayout, token_stream_lexed, TriviaRule, UnboundSourceAnnotation, @@ -659,6 +663,9 @@ type LexWalkAcc { error: Bool file: Symbol dispatch: LexRuleDispatch + gap_start: Int + gap_has_break: Bool + layout: StreamLayout } fn lex_walk_init(src: v2.std.text.String, file: Symbol, rules: LexRuleSet) -> LexWalkAcc { @@ -671,7 +678,10 @@ fn lex_walk_init(src: v2.std.text.String, file: Symbol, rules: LexRuleSet) -> Le done: false, error: false, file: file, - dispatch: lex_rule_dispatch(compiled_rules: lex_compile_rules(rules: rules.rules)) + dispatch: lex_rule_dispatch(compiled_rules: lex_compile_rules(rules: rules.rules)), + gap_start: 0, + gap_has_break: false, + layout: stream_layout_empty() } } @@ -691,7 +701,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc { done: true, error: false, file: acc.file, - dispatch: acc.dispatch + dispatch: acc.dispatch, + gap_start: acc.gap_start, + gap_has_break: acc.gap_has_break, + layout: acc.layout } false => match lex_try_rules(rules: lex_dispatch_candidates(dispatch: acc.dispatch, source: acc.remaining), source: acc.remaining) { @@ -705,7 +718,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc { done: true, error: true, file: acc.file, - dispatch: acc.dispatch + dispatch: acc.dispatch, + gap_start: acc.gap_start, + gap_has_break: acc.gap_has_break, + layout: acc.layout } LexRuleTrivia { lexeme: lexeme, remaining: remaining } => match string_is_empty(s: lexeme) { @@ -719,7 +735,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc { done: true, error: true, file: acc.file, - dispatch: acc.dispatch + dispatch: acc.dispatch, + gap_start: acc.gap_start, + gap_has_break: acc.gap_has_break, + layout: acc.layout } false => LexWalkAcc { @@ -731,7 +750,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc { done: false, error: false, file: acc.file, - dispatch: acc.dispatch + dispatch: acc.dispatch, + gap_start: acc.gap_start, + gap_has_break: acc.gap_has_break || lexeme_has_line_feed(lexeme: lexeme), + layout: acc.layout } } LexRuleToken { class: class, lexeme: lexeme, remaining: remaining } => @@ -746,7 +768,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc { done: true, error: true, file: acc.file, - dispatch: acc.dispatch + dispatch: acc.dispatch, + gap_start: acc.gap_start, + gap_has_break: acc.gap_has_break, + layout: acc.layout } false => LexWalkAcc { @@ -767,7 +792,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc { done: false, error: false, file: acc.file, - dispatch: acc.dispatch + dispatch: acc.dispatch, + gap_start: acc.pos + lexeme_utf8_octet_count(lexeme: lexeme), + gap_has_break: false, + layout: if acc.gap_has_break { stream_layout_with_break(layout: acc.layout, token_start: acc.pos, gap_start: acc.gap_start) } else { acc.layout } } } LexRuleAnnotation { class: class, lexeme: lexeme, remaining: remaining } => @@ -782,7 +810,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc { done: true, error: true, file: acc.file, - dispatch: acc.dispatch + dispatch: acc.dispatch, + gap_start: acc.gap_start, + gap_has_break: acc.gap_has_break, + layout: acc.layout } false => LexWalkAcc { @@ -804,7 +835,10 @@ fn lex_walk_step(acc: LexWalkAcc) -> LexWalkAcc { done: false, error: false, file: acc.file, - dispatch: acc.dispatch + dispatch: acc.dispatch, + gap_start: acc.gap_start, + gap_has_break: acc.gap_has_break || lexeme_has_line_feed(lexeme: lexeme), + layout: acc.layout } } } @@ -843,7 +877,7 @@ fn lex_walk_artifact(source: v2.std.text.String, file: Symbol, rules: LexRuleSet } else { v2.std.diagnostic.Accepted { value: lex_artifact_new( - tokens: token_stream_lexed(all: result.tokens.reverse(), source: source), + tokens: token_stream_lexed(all: result.tokens.reverse(), layout: result.layout), annotations: result.annotations.reverse() ), diagnostics: v2.std.diagnostic.None diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 74596a0ebed..9d9933ebd33 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -25,7 +25,7 @@ import std.content_hash { Fnv1a64Structural } import v2.std.integer { integer_int_content_digest } import v2.std.node_shape { stamp_node_shape_synthetic, type_shape } import v2.std.text { Char, String } -import std.unicode.types { unicode_char_code_point, unicode_scalar_utf8_octet_count } +import std.unicode.types { unicode_char_code_point } type Lexeme = String @@ -153,21 +153,22 @@ type Token { end: Int } -// THE STREAM CARRIES ITS LAYOUT, DERIVED FROM THE SOURCE ITS TOKENS WERE LEXED FROM. A token's +// THE STREAM CARRIES ITS LAYOUT, RECORDED BY THE LEXER AS IT READS THE SOURCE. A token's // `start`/`end` say THAT two tokens have a gap between them, not what the gap holds, and the grammar // sometimes needs to know (a `-` that begins a new line after a complete operand is ambiguous // between infix and prefix, and the seed refuses it: v1.compiler.parse // is_ambiguous_prefix_infix_newline_boundary). The source text stays the ONE authority for that // fact: nothing about layout is stored on a Token or as a token of its own. `StreamLayout` is the -// starts of the tokens that have a line break in the gap before them, DERIVED from the source and the -// tokens in one walk by its only constructor (`stream_layout_of`, called by the tokenizer, the one -// place the source is read), so a layout question is a lookup rather than a rescan of the file. Only -// the line-break-preceded tokens are kept (a line is many tokens), each keyed by its start to the -// 1-based line it begins -- ONE representation of the set, with a value that is a fact about the token -// rather than a placeholder. A stream built from tokens alone (fixtures) carries no layout, and a -// layout question on it answers `LineBreakUnknownWithoutSource` -- never an assumed single line. -// `digest` summarizes exactly the entries of `line_break_before`, folded in the walk as each one is -// recorded (start, then line), so the parse memo key can cover layout without a lookup per token. +// set of tokens that have a line break in the gap before them, keyed by each token's start to the +// octet where that gap begins (the end of the token before it) -- ONE representation of the set, whose +// value is a fact about the gap rather than a placeholder. It is recorded in the tokenizer's own walk +// (v2.compiler.tokenize lex_walk_step), the one place the source is read, through the two builders +// below, so there is no second pass over the source and a layout question is a lookup. A line feed +// counts when it is in a gap -- in trivia or an annotation between tokens -- never inside a token (a +// multi-line string). `digest` summarizes exactly the recorded entries, folded as each is recorded, so +// the parse memo key covers layout without a lookup per token. A stream built from tokens alone +// (fixtures) carries no layout, and a layout question on it answers `LineBreakUnknownWithoutSource` +// -- never an assumed single line. type StreamLayout { line_break_before: Map digest: Fnv1a64Structural @@ -183,10 +184,9 @@ fn token_stream_new(all: List) -> TokenStream { TokenStream { all: all, pos: 0, layout: optional_absent() } } -// The tokenizer's constructor: the tokens together with the exact scalar sequence they were lexed -// from, whose UTF-8 octet offsets their `start`/`end` name. -fn token_stream_lexed(all: List, source: v2.std.text.String) -> TokenStream { - TokenStream { all: all, pos: 0, layout: optional_present(value: stream_layout_of(tokens: all, source: source)) } +// The tokenizer's constructor: the tokens together with the layout recorded while they were lexed. +fn token_stream_lexed(all: List, layout: StreamLayout) -> TokenStream { + TokenStream { all: all, pos: 0, layout: optional_present(value: layout) } } fn token_stream_empty() -> TokenStream { @@ -202,73 +202,27 @@ type LineBreakAnswer | NoLineBreakInGap | LineBreakUnknownWithoutSource -// ONE WALK: the source's scalars in order, with the line being walked, the tokens still ahead and the -// end of the last token passed. A line feed (U+000A) counts only in a GAP -- at or after the previous token's end -- so a -// line feed inside a token (a multi-line string) is not layout. When the walk reaches a token's -// first octet, whether a line feed was seen since the previous token is that token's answer. -// Offsets are UTF-8 octets, as token extents are, so each scalar advances by its encoded width. -type StreamLayoutWalk { - octet: Int - line: Int - ahead: List - last_end: Int - seen: Bool - answers: Map - digest: Fnv1a64Structural +fn stream_layout_empty() -> StreamLayout { + StreamLayout { line_break_before: empty_map(), digest: symbol_identity_digest(sym: ^stream_layout_digest_nil) } } -fn stream_layout_of(tokens: List, source: v2.std.text.String) -> StreamLayout { - let walked = fold_list( - xs: source, - empty: StreamLayoutWalk { octet: 0, line: 1, ahead: tokens, last_end: 0, seen: false, answers: empty_map(), digest: symbol_identity_digest(sym: ^stream_layout_digest_nil) }, - cons: fn(acc, c) { stream_layout_step(acc: acc, c: c) } - ) - StreamLayout { line_break_before: walked.answers, digest: walked.digest } -} - -fn stream_layout_step(acc: StreamLayoutWalk, c: Char) -> StreamLayoutWalk { - let at_token = stream_layout_record_token(acc: acc) - let code = unicode_char_code_point(c: c) - StreamLayoutWalk { - octet: at_token.octet + unicode_scalar_utf8_octet_count(code_point: code), - line: if code == 10 { at_token.line + 1 } else { at_token.line }, - ahead: at_token.ahead, - last_end: at_token.last_end, - seen: at_token.seen || ((code == 10) && (at_token.octet >= at_token.last_end)), - answers: at_token.answers, - digest: at_token.digest +// The token starting at octet `token_start` has a line break in its gap, which began at `gap_start`. +fn stream_layout_with_break(layout: StreamLayout, token_start: Int, gap_start: Int) -> StreamLayout { + StreamLayout { + line_break_before: map_insert(m: layout.line_break_before, key: token_start, value: gap_start), + digest: combine_hash(a: layout.digest, b: combine_hash(a: integer_int_content_digest(value: token_start), b: integer_int_content_digest(value: gap_start))) } } -// At a token's first octet: record the gap's answer for it, then start a new gap at its end. -fn stream_layout_record_token(acc: StreamLayoutWalk) -> StreamLayoutWalk { - match list_at_optional(xs: acc.ahead, index: 0) { - Absent => acc - Present { value: tok } => - if tok.start == acc.octet { - StreamLayoutWalk { - octet: acc.octet, - line: acc.line, - ahead: skip(xs: acc.ahead, n: 1), - last_end: tok.end, - seen: false, - answers: if acc.seen { map_insert(m: acc.answers, key: tok.start, value: acc.line) } else { acc.answers }, - digest: if acc.seen { - combine_hash(a: acc.digest, b: combine_hash(a: integer_int_content_digest(value: tok.start), b: integer_int_content_digest(value: acc.line))) - } else { - acc.digest - } - } - } else { - acc - } - } +// Whether a lexeme (trivia or annotation text between tokens) holds a line feed, U+000A. +fn lexeme_has_line_feed(lexeme: String) -> Bool { + fold_list(xs: lexeme, empty: false, cons: fn(acc, c) { acc || (unicode_char_code_point(c: c) == 10) }) } // THE NAMED DERIVATION, READ AT THE POINT OF USE: is there a line break in the source between the // token before the one starting at octet `token_start` and that token? A stream without layout // cannot answer. `token_start` is always a start of one of the stream's own tokens (the parser asks -// only about the token it is at), so a start the walk did not record is a token with no break before it. +// only about the token it is at), so a start the lexer did not record is a token with no break before it. fn token_stream_line_break_before(layout: Optional, token_start: Int) -> LineBreakAnswer { match layout { Absent => LineBreakUnknownWithoutSource From 7a73163ca40ae42188cb0bc519e2c0cc7549a2a3 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 07:30:56 +0000 Subject: [PATCH 18/21] lexing: lexeme_has_line_feed takes v2.std.text.String (emit) Co-Authored-By: Claude Opus 5.5 (1M context) --- src/v2/std/compilers/lexing.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 9d9933ebd33..64e2f226563 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -215,7 +215,7 @@ fn stream_layout_with_break(layout: StreamLayout, token_start: Int, gap_start: I } // Whether a lexeme (trivia or annotation text between tokens) holds a line feed, U+000A. -fn lexeme_has_line_feed(lexeme: String) -> Bool { +fn lexeme_has_line_feed(lexeme: v2.std.text.String) -> Bool { fold_list(xs: lexeme, empty: false, cons: fn(acc, c) { acc || (unicode_char_code_point(c: c) == 10) }) } From 06f42f2ac8ac816e80d78220bb2d06682008ff0a Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 11:37:17 +0000 Subject: [PATCH 19/21] Newline controls: one data declaration each, same-line control parse-only (floor new-witness budget) Co-Authored-By: Claude Opus 5.5 (1M context) --- .../newline_dual_role_operator_parse_test.dag | 35 ++++++++----------- 1 file changed, 14 insertions(+), 21 deletions(-) diff --git a/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag b/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag index d7aaa974633..d142e581e46 100644 --- a/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag +++ b/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag @@ -1,6 +1,5 @@ module v2.test.parse.newline_dual_role_operator_parse -import v2.compiler.normalize { normalize } import v2.compiler.parse { parse_module_prepared } import v2.compiler.program_assembly { dag_prepared_grammar } import v2.compiler.tokenize { tokenize } @@ -47,7 +46,7 @@ data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly // refuses a line-break-preceded `-` with NewlineBeforeDualRoleOperator at the `-` // (v2.extdeps.languages.dag dag_grammar_additive_expr_helper). Each side of the rule, through the // live prepared grammar on a lexed stream (which carries its layout): the next-line forms refuse at -// the `-`, for a plain operand and for a block, and the same-line form stays one subtraction. The +// the `-`, for a plain operand and for a block, and the same-line form parses. The // mutation that reds the refusals is test.claim.long.newline_dual_role_operator_mutation_witness. fn n_tokenize_parse(text: String, file: Symbol) -> Outcome { @@ -87,35 +86,29 @@ fn n_refuses_at_minus(text: String, file: Symbol, start: Int, end: Int) -> Bool } } -fn n_parses_and_normalizes(text: String, file: Symbol) -> Bool { - match n_tokenize_parse(text: text, file: file) { - Rejected { diagnostics: _ } => false - Accepted { value: tree, diagnostics: _ } => - match normalize(parse_tree: tree) { - Rejected { diagnostics: _ } => false - Accepted { value: _, diagnostics: _ } => true - } - } -} - -// `-` at bytes 51..52. -data newline_minus_after_operand_source: String = "module probe.nl_plain\n\nfn f(x: Int) -> Int {\n x\n - 1\n}\n" +// One `data` declaration each: the rule is about an expression, and a declaration is its smallest +// carrier, so each claim pays one tokenize and one parse of a few lines. +// `-` at bytes 39..40. +data newline_minus_after_operand_source: String = "module probe.nl_plain\n\ndata d: Int = x\n- 1\n" -// `-` at bytes 101..102. -data newline_minus_after_block_source: String = "module probe.nl_block\n\nfn f(w: Bool, n: Int) -> Int {\n match w {\n true => n\n false => 0\n }\n - n\n}\n" +// `-` at bytes 74..75. +data newline_minus_after_block_source: String = "module probe.nl_block\n\ndata d: Int = match w {\n true => 1\n false => 0\n}\n- 1\n" -data same_line_minus_source: String = "module probe.nl_same\n\nfn f(x: Int) -> Int {\n x - 1\n}\n" +data same_line_minus_source: String = "module probe.nl_same\n\ndata d: Int = x - 1\n" test fn a_newline_minus_after_an_operand_refuses_holds() -> Bool { - n_refuses_at_minus(text: newline_minus_after_operand_source, file: ^probe_nl_plain, start: 51, end: 52) + n_refuses_at_minus(text: newline_minus_after_operand_source, file: ^probe_nl_plain, start: 39, end: 40) } test fn a_newline_minus_after_a_block_refuses_holds() -> Bool { - n_refuses_at_minus(text: newline_minus_after_block_source, file: ^probe_nl_block, start: 101, end: 102) + n_refuses_at_minus(text: newline_minus_after_block_source, file: ^probe_nl_block, start: 74, end: 75) } test fn a_same_line_minus_is_subtraction_holds() -> Bool { - n_parses_and_normalizes(text: same_line_minus_source, file: ^probe_nl_same) + match n_tokenize_parse(text: same_line_minus_source, file: ^probe_nl_same) { + Accepted { value: _, diagnostics: _ } => true + Rejected { diagnostics: _ } => false + } } // THE EMIT DIRECTION (DESIGN section 4: one grammar, read both ways). The refusal is ingest-only: From f2fd2998a701fbb19c27733aa4303f17b1c8984d Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 11:38:22 +0000 Subject: [PATCH 20/21] Newline mutation: supply the mutant grammar and the arm census as warm producers (no per-claim preparation); move to the floor Co-Authored-By: Claude Opus 5.5 (1M context) --- ...wline_dual_role_operator_mutation_test.dag | 71 ++++++++++++------- .../newline_dual_role_operator_parse_test.dag | 2 +- src/v2/workflow/floor_pure_producer_share.dag | 10 +++ 3 files changed, 57 insertions(+), 26 deletions(-) rename dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag => src/v2/test/claim/parse/newline_dual_role_operator_mutation_test.dag (64%) diff --git a/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag b/src/v2/test/claim/parse/newline_dual_role_operator_mutation_test.dag similarity index 64% rename from dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag rename to src/v2/test/claim/parse/newline_dual_role_operator_mutation_test.dag index 14dc5cd70e6..702f4b0d3fa 100644 --- a/dag/test/claim/long/newline_dual_role_operator_mutation_witness_test.dag +++ b/src/v2/test/claim/parse/newline_dual_role_operator_mutation_test.dag @@ -1,6 +1,8 @@ -module test.claim.long.newline_dual_role_operator_mutation_witness_test +module v2.test.parse.newline_dual_role_operator_mutation -import v2.compiler.parse { parse_module } +import v2.compiler.parse { PreparedGrammar, parse_module_prepared, prepare_grammar } +import v2.compiler.program_assembly { dag_prepared_grammar } +import v2.std.diagnostic { Outcome } import v2.compiler.tokenize { tokenize } import v2.extdeps.languages.dag { dag_grammar, dag_lex } import v2.std.algebra { fold_list } @@ -50,8 +52,13 @@ import v2.test.parse.newline_dual_role_operator_parse { // is a helper inlined into each operand position of the comparison and equality rows; it is one // authored row. Under the mutant the // next-line sources parse again (the silent merge into one subtraction is back); under the live -// grammar they refuse. The same-line control parses under both. Long lane: preparing the mutant -// grammar is past the floor's per-claim budget. +// grammar they refuse. The same-line control parses under both. +// +// THE GRAMMARS ARE SUPPLIED, NOT RE-PREPARED PER CLAIM (DESIGN section 3, a witness discriminates at +// one interface). The live grammar is the floor-shared dag_prepared_grammar; the mutant is prepared +// once by the nullary producer nlm_mutant_prepared and the refusal-arm counts are taken once by +// nlm_refusal_arm_census, both enrolled WARM in v2.workflow.floor_pure_producer_share. Each claim pays +// one tokenize and one parse of a one-line declaration. fn strip_refusals(expr: GrammarExpr) -> GrammarExpr { fold_grammar_expr( @@ -134,44 +141,58 @@ fn mutant_grammar() -> ParseGrammar { } } -fn live_refusal_arm_count() -> Int { - match dag_grammar() { - ModeledGrammar { root: r } => fold_list(xs: r.productions, empty: 0, cons: fn(acc, prod) { acc + refusal_arm_count(expr: prod.expression) }) - VoidGrammar => 0 - } +type RefusalArmCensus { + total: Int + other: Int } -fn live_other_refusal_arm_count() -> Int { +// WARM PRODUCER: the refusal arms of the live dag grammar, counted once. +fn nlm_refusal_arm_census() -> RefusalArmCensus { match dag_grammar() { - ModeledGrammar { root: r } => fold_list(xs: r.productions, empty: 0, cons: fn(acc, prod) { acc + other_refusal_arm_count(expr: prod.expression) }) - VoidGrammar => 0 + ModeledGrammar { root: r } => + RefusalArmCensus { + total: fold_list(xs: r.productions, empty: 0, cons: fn(acc, prod) { acc + refusal_arm_count(expr: prod.expression) }), + other: fold_list(xs: r.productions, empty: 0, cons: fn(acc, prod) { acc + other_refusal_arm_count(expr: prod.expression) }) + } + VoidGrammar => RefusalArmCensus { total: 0, other: 0 } } } -fn accepts_with(text: String, file: Symbol, grammar: ParseGrammar) -> Bool { - match tokenize(text: text, file: file, rules: dag_lex()) { +// WARM PRODUCER: the mutant grammar, prepared once. +fn nlm_mutant_prepared() -> Outcome { + prepare_grammar(grammar: mutant_grammar()) +} + +fn accepts_prepared(text: String, file: Symbol, prepared: Outcome) -> Bool { + match prepared { Rejected { diagnostics: _ } => false - Accepted { value: stream, diagnostics: _ } => - match parse_module(tokens: stream, grammar: grammar) { - Accepted { value: _, diagnostics: _ } => true + Accepted { value: p, diagnostics: residue } => + match tokenize(text: text, file: file, rules: dag_lex()) { Rejected { diagnostics: _ } => false + Accepted { value: stream, diagnostics: _ } => + match parse_module_prepared(tokens: stream, prepared: p, validation_residue: residue) { + Accepted { value: _, diagnostics: _ } => true + Rejected { diagnostics: _ } => false + } } } } test fn witness_every_live_refusal_arm_is_the_newline_refusal() -> Bool { - (live_refusal_arm_count() > 0) && (live_other_refusal_arm_count() == 0) + let census = nlm_refusal_arm_census() + (census.total > 0) && (census.other == 0) } test fn witness_the_live_grammar_refuses_both_next_line_forms() -> Bool { - (accepts_with(text: newline_minus_after_operand_source, file: ^nlm_plain, grammar: dag_grammar()) == false) - && (accepts_with(text: newline_minus_after_block_source, file: ^nlm_block, grammar: dag_grammar()) == false) - && accepts_with(text: same_line_minus_source, file: ^nlm_same, grammar: dag_grammar()) + let live = dag_prepared_grammar() + (accepts_prepared(text: newline_minus_after_operand_source, file: ^nlm_plain, prepared: live) == false) + && (accepts_prepared(text: newline_minus_after_block_source, file: ^nlm_block, prepared: live) == false) + && accepts_prepared(text: same_line_minus_source, file: ^nlm_same, prepared: live) } test fn witness_without_the_refusal_both_next_line_forms_merge_RED() -> Bool { - let mutant = mutant_grammar() - accepts_with(text: newline_minus_after_operand_source, file: ^nlm_plain, grammar: mutant) - && accepts_with(text: newline_minus_after_block_source, file: ^nlm_block, grammar: mutant) - && accepts_with(text: same_line_minus_source, file: ^nlm_same, grammar: mutant) + let mutant = nlm_mutant_prepared() + accepts_prepared(text: newline_minus_after_operand_source, file: ^nlm_plain, prepared: mutant) + && accepts_prepared(text: newline_minus_after_block_source, file: ^nlm_block, prepared: mutant) + && accepts_prepared(text: same_line_minus_source, file: ^nlm_same, prepared: mutant) } diff --git a/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag b/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag index d142e581e46..9aa73ef67ec 100644 --- a/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag +++ b/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag @@ -47,7 +47,7 @@ data live_tree_disposition: LiveTreeDisposition = SubstrateInputsOnly // (v2.extdeps.languages.dag dag_grammar_additive_expr_helper). Each side of the rule, through the // live prepared grammar on a lexed stream (which carries its layout): the next-line forms refuse at // the `-`, for a plain operand and for a block, and the same-line form parses. The -// mutation that reds the refusals is test.claim.long.newline_dual_role_operator_mutation_witness. +// mutation that reds the refusals is v2.test.parse.newline_dual_role_operator_mutation. fn n_tokenize_parse(text: String, file: Symbol) -> Outcome { match dag_prepared_grammar() { diff --git a/src/v2/workflow/floor_pure_producer_share.dag b/src/v2/workflow/floor_pure_producer_share.dag index 26b41a60398..a009adf3f75 100644 --- a/src/v2/workflow/floor_pure_producer_share.dag +++ b/src/v2/workflow/floor_pure_producer_share.dag @@ -760,6 +760,14 @@ import v2.std.collection { List } // the two nullary producers over it. The values are an Optional list of QualifiedNames and a record of // four Optional -- portable, no closure, no resolution context -- and both are WARM for the // ceiling arithmetic above: one front end is more than one claim's budget. +// THE NEWLINE-REFUSAL MUTATION'S TWO GRAMMAR FACTS ARE SUPPLIED, NOT RE-DERIVED PER CLAIM. +// v2.test.parse.newline_dual_role_operator_mutation discriminates the newline-`-` refusal by parsing +// three one-line sources under the live grammar and under a mutant with every RefuseOnMatch arm +// stripped. Preparing that mutant is a whole grammar preparation, past one claim's new-witness budget +// (gunbc#12881's floor: 183k and 229k steps when each claim prepared its own), so the mutant's +// PreparedGrammar is one nullary producer, and the refusal-arm counts over the live grammar are +// another. A PreparedGrammar is the value dag_prepared_grammar already serves; the census is a record +// of two Ints. Both WARM: one grammar preparation is more than one claim's budget. data floor_cross_claim_pure_producers_warm: List = [ "v2.test.claim.parameter_reference.pr_program_parameter_paths", "v2.test.claim.parameter_reference.pr_program_grounding", @@ -1084,6 +1092,8 @@ data floor_cross_claim_pure_producers_warm: List = [ "v2.test.parse.expression_bodied_fn_decl_parse.expression_bodied_fn_parsed", "v2.test.parse.expression_bodied_fn_decl_parse.expression_bodied_literal_fn_parsed", "v2.test.parse.expression_bodied_fn_decl_parse.braced_fn_control_parsed", + "v2.test.parse.newline_dual_role_operator_mutation.nlm_mutant_prepared", + "v2.test.parse.newline_dual_role_operator_mutation.nlm_refusal_arm_census", "v2.test.claim.body_lowering_block_body_call.body_lowering_block_call_parsed", "v2.test.claim.namespace_xl0.call_argument_mention_survival.call_argument_fixture_normalized", "v2.test.claim.namespace_xl0.body_shape_contract.body_shape_normalized_root", From 8958927d6638806c0d06b00203a31d2bf5c0e4d3 Mon Sep 17 00:00:00 2001 From: gunbc-ci-auto-heal Date: Thu, 1 Oct 2026 12:29:14 +0000 Subject: [PATCH 21/21] Newline claims: one parse per mutant claim; drop the live-grammar duplicate of the controls; smaller block source (floor budget) Co-Authored-By: Claude Opus 5.5 (1M context) --- ...wline_dual_role_operator_mutation_test.dag | 28 +++++++++---------- .../newline_dual_role_operator_parse_test.dag | 6 ++-- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/src/v2/test/claim/parse/newline_dual_role_operator_mutation_test.dag b/src/v2/test/claim/parse/newline_dual_role_operator_mutation_test.dag index 702f4b0d3fa..336e28687c0 100644 --- a/src/v2/test/claim/parse/newline_dual_role_operator_mutation_test.dag +++ b/src/v2/test/claim/parse/newline_dual_role_operator_mutation_test.dag @@ -1,7 +1,6 @@ module v2.test.parse.newline_dual_role_operator_mutation import v2.compiler.parse { PreparedGrammar, parse_module_prepared, prepare_grammar } -import v2.compiler.program_assembly { dag_prepared_grammar } import v2.std.diagnostic { Outcome } import v2.compiler.tokenize { tokenize } import v2.extdeps.languages.dag { dag_grammar, dag_lex } @@ -52,11 +51,11 @@ import v2.test.parse.newline_dual_role_operator_parse { // is a helper inlined into each operand position of the comparison and equality rows; it is one // authored row. Under the mutant the // next-line sources parse again (the silent merge into one subtraction is back); under the live -// grammar they refuse. The same-line control parses under both. +// grammar they refuse (the controls in v2.test.parse.newline_dual_role_operator_parse). The +// same-line form parses under both. // // THE GRAMMARS ARE SUPPLIED, NOT RE-PREPARED PER CLAIM (DESIGN section 3, a witness discriminates at -// one interface). The live grammar is the floor-shared dag_prepared_grammar; the mutant is prepared -// once by the nullary producer nlm_mutant_prepared and the refusal-arm counts are taken once by +// one interface). The mutant is prepared once by the nullary producer nlm_mutant_prepared and the refusal-arm counts are taken once by // nlm_refusal_arm_census, both enrolled WARM in v2.workflow.floor_pure_producer_share. Each claim pays // one tokenize and one parse of a one-line declaration. @@ -183,16 +182,17 @@ test fn witness_every_live_refusal_arm_is_the_newline_refusal() -> Bool { (census.total > 0) && (census.other == 0) } -test fn witness_the_live_grammar_refuses_both_next_line_forms() -> Bool { - let live = dag_prepared_grammar() - (accepts_prepared(text: newline_minus_after_operand_source, file: ^nlm_plain, prepared: live) == false) - && (accepts_prepared(text: newline_minus_after_block_source, file: ^nlm_block, prepared: live) == false) - && accepts_prepared(text: same_line_minus_source, file: ^nlm_same, prepared: live) +// The live grammar's refusals are the controls in v2.test.parse.newline_dual_role_operator_parse; +// this module asserts only what they cannot: that the refusal arms are what produce them. One claim +// per source, so each pays one tokenize and one parse against the supplied mutant. +test fn witness_without_the_refusal_a_next_line_minus_after_an_operand_merges_RED() -> Bool { + accepts_prepared(text: newline_minus_after_operand_source, file: ^nlm_plain, prepared: nlm_mutant_prepared()) } -test fn witness_without_the_refusal_both_next_line_forms_merge_RED() -> Bool { - let mutant = nlm_mutant_prepared() - accepts_prepared(text: newline_minus_after_operand_source, file: ^nlm_plain, prepared: mutant) - && accepts_prepared(text: newline_minus_after_block_source, file: ^nlm_block, prepared: mutant) - && accepts_prepared(text: same_line_minus_source, file: ^nlm_same, prepared: mutant) +test fn witness_without_the_refusal_a_next_line_minus_after_a_block_merges_RED() -> Bool { + accepts_prepared(text: newline_minus_after_block_source, file: ^nlm_block, prepared: nlm_mutant_prepared()) +} + +test fn witness_without_the_refusal_a_same_line_minus_still_parses() -> Bool { + accepts_prepared(text: same_line_minus_source, file: ^nlm_same, prepared: nlm_mutant_prepared()) } diff --git a/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag b/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag index 9aa73ef67ec..c5abdd10229 100644 --- a/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag +++ b/src/v2/test/claim/parse/newline_dual_role_operator_parse_test.dag @@ -91,8 +91,8 @@ fn n_refuses_at_minus(text: String, file: Symbol, start: Int, end: Int) -> Bool // `-` at bytes 39..40. data newline_minus_after_operand_source: String = "module probe.nl_plain\n\ndata d: Int = x\n- 1\n" -// `-` at bytes 74..75. -data newline_minus_after_block_source: String = "module probe.nl_block\n\ndata d: Int = match w {\n true => 1\n false => 0\n}\n- 1\n" +// `-` at bytes 58..59. +data newline_minus_after_block_source: String = "module probe.nl_block\n\ndata d: Int = match w {\n _ => 1\n}\n- 1\n" data same_line_minus_source: String = "module probe.nl_same\n\ndata d: Int = x - 1\n" @@ -101,7 +101,7 @@ test fn a_newline_minus_after_an_operand_refuses_holds() -> Bool { } test fn a_newline_minus_after_a_block_refuses_holds() -> Bool { - n_refuses_at_minus(text: newline_minus_after_block_source, file: ^probe_nl_block, start: 74, end: 75) + n_refuses_at_minus(text: newline_minus_after_block_source, file: ^probe_nl_block, start: 58, end: 59) } test fn a_same_line_minus_is_subtraction_holds() -> Bool {