From e7d608bb374af56d312c7905a0a6c8b8a0f5cb77 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 20:03:36 +0000 Subject: [PATCH 1/9] WIP: can we make modeled lex rules more elegant somehow? ModeledLexRules { ro --- src/v2/std/compilers/lexing.dag | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 366c1d6bf5e..2e3af516587 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -5,7 +5,7 @@ module v2.std.compilers.lexing -import v2.std.algebra { fold_list } +import v2.std.algebra { fold_list, list_snoc_item } import v2.std.collection { List } import v2.std.logic { Bool } import v2.std.node { Atom, Conj, Node, Symbol, SyntheticOccurrence, TypeNode } From bca95b02f8107e180f6ccb0f45e4a6ff2c78482c Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 20:15:51 +0000 Subject: [PATCH 2/9] WIP: can we make modeled lex rules more elegant somehow? ModeledLexRules { ro --- .../manual/lex_phrase_scratch_test.dag | 25 ++ src/v2/extdeps/languages/bash.dag | 245 ++++++++---------- src/v2/std/compilers/lexing.dag | 80 +++++- src/v2/std/compilers/target_model.dag | 17 ++ 4 files changed, 229 insertions(+), 138 deletions(-) create mode 100644 src/v2/compiler/manual/lex_phrase_scratch_test.dag diff --git a/src/v2/compiler/manual/lex_phrase_scratch_test.dag b/src/v2/compiler/manual/lex_phrase_scratch_test.dag new file mode 100644 index 00000000000..918489a38fa --- /dev/null +++ b/src/v2/compiler/manual/lex_phrase_scratch_test.dag @@ -0,0 +1,25 @@ +module v2.test.manual.lex_phrase_scratch + +// SCRATCH — delete before commit. Proves the LexicalPhrase de-dup is byte-identical: +// each derived source_text == the literal it replaced. Imports bash + std only (no emit/ +// translate), so it sidesteps the pre-existing emit-test resolution cycle. + +import v2.extdeps.languages.bash { + bash_mvp1_source_text, + bash_source_root_ingest_gate_source_text, + bash_and_then_source_text, + bash_or_else_source_text, + bash_with_redir_stderr_null_source_text, + bash_pipe_source_text, + bash_or_else_with_redir_source_text +} + +test fn lex_phrase_source_text_byte_identical() -> Bool { + (bash_mvp1_source_text == "'git' 'rev-parse' '--show-toplevel'") && + (bash_source_root_ingest_gate_source_text == "'bash' 'scripts/source-root-ingest-gate.sh'") && + (bash_and_then_source_text == "'true' && 'false'") && + (bash_or_else_source_text == "'true' || 'false'") && + (bash_with_redir_stderr_null_source_text == "'git' 'rev-parse' '--show-toplevel' 2>/dev/null") && + (bash_pipe_source_text == "'echo' 'hi' | 'wc' '-c'") && + (bash_or_else_with_redir_source_text == "'git' 'rev-parse' '--show-toplevel' 2>/dev/null || 'pwd'") +} diff --git a/src/v2/extdeps/languages/bash.dag b/src/v2/extdeps/languages/bash.dag index e187f1c7b49..3afcc01370c 100644 --- a/src/v2/extdeps/languages/bash.dag +++ b/src/v2/extdeps/languages/bash.dag @@ -47,18 +47,19 @@ import v2.std.grammar { grammar_relation_tokens_node } import v2.std.compilers.lexing { - LexRule, - LexRuleSet, LexRules, - LiteralPattern, - ModeledLexRules, - TokenRule + LexicalPhrase, + phrase, + phrase_to_lex_rules, + phrase_to_source_text, + span } import v2.std.compilers.target_model { BoundToken, ConcreteSyntaxToken, FixedToken, TargetModel, + phrase_to_fixed_concrete_tokens, target_model_emit_transforms_empty, target_model_make_with_emit_transforms, concrete_syntax_token_to_node @@ -219,20 +220,93 @@ fn bash_relation_translation_rules_node( } +// ========================================================================= +// Lexical phrases — one authority per fixed-token phrase (DESIGN.md §3). +// +// Each phrase is authored ONCE here as an ordered list of (token_class, fragment) +// spans: read the `fragment:` column top-to-bottom and it spells the shell text the +// phrase emits. The lex rule set, the concrete-token sequence, and the source-text +// string all PROJECT from these (phrase_to_lex_rules / phrase_to_fixed_concrete_tokens / +// phrase_to_source_text), so the three can no longer drift (DESIGN.md §2 horizontal: +// one concept, derive every use). The byte-identity to serialize_bash is proven by +// execution in bash_emit_command_test. +// +// Bound-word phrases (bash_bound_words / bash_embedded_quote) are spelled at emit time +// from a binding, not a literal fragment, so they are NOT LexicalPhrases yet — a +// binding-carrying span is the migration-time extension (see lexing.dag T-6 note). +// ========================================================================= + +fn bash_mvp1_phrase() -> LexicalPhrase { + phrase(spans: [ + span(token_class: ^bash_token_word_git, fragment: "'git'"), + span(token_class: ^bash_token_word_rev_parse, fragment: " 'rev-parse'"), + span(token_class: ^bash_token_word_show_toplevel, fragment: " '--show-toplevel'") + ]) +} + +fn bash_source_root_ingest_gate_phrase() -> LexicalPhrase { + phrase(spans: [ + span(token_class: ^bash_token_word_bash, fragment: "'bash'"), + span(token_class: ^bash_token_word_source_root_ingest_gate_sh, fragment: " 'scripts/source-root-ingest-gate.sh'") + ]) +} + +fn bash_and_then_phrase() -> LexicalPhrase { + phrase(spans: [ + span(token_class: ^bash_token_word_true, fragment: "'true'"), + span(token_class: ^bash_token_and_then, fragment: " && "), + span(token_class: ^bash_token_word_false, fragment: "'false'") + ]) +} + +fn bash_or_else_phrase() -> LexicalPhrase { + phrase(spans: [ + span(token_class: ^bash_token_word_true, fragment: "'true'"), + span(token_class: ^bash_token_or_else, fragment: " || "), + span(token_class: ^bash_token_word_false, fragment: "'false'") + ]) +} + +fn bash_with_redir_stderr_null_phrase() -> LexicalPhrase { + phrase(spans: [ + span(token_class: ^bash_token_word_git, fragment: "'git'"), + span(token_class: ^bash_token_word_rev_parse, fragment: " 'rev-parse'"), + span(token_class: ^bash_token_word_show_toplevel, fragment: " '--show-toplevel'"), + span(token_class: ^bash_token_redir_stderr_null, fragment: " 2>/dev/null") + ]) +} + +fn bash_pipe_phrase() -> LexicalPhrase { + phrase(spans: [ + span(token_class: ^bash_token_word_echo, fragment: "'echo'"), + span(token_class: ^bash_token_word_hi, fragment: " 'hi'"), + span(token_class: ^bash_token_pipe, fragment: " | "), + span(token_class: ^bash_token_word_wc, fragment: "'wc'"), + span(token_class: ^bash_token_word_minus_c, fragment: " '-c'") + ]) +} + +fn bash_or_else_with_redir_phrase() -> LexicalPhrase { + phrase(spans: [ + span(token_class: ^bash_token_word_git, fragment: "'git'"), + span(token_class: ^bash_token_word_rev_parse, fragment: " 'rev-parse'"), + span(token_class: ^bash_token_word_show_toplevel, fragment: " '--show-toplevel'"), + span(token_class: ^bash_token_redir_stderr_null, fragment: " 2>/dev/null"), + span(token_class: ^bash_token_or_else, fragment: " || "), + span(token_class: ^bash_token_word_pwd, fragment: "'pwd'") + ]) +} + // The canonical shell text emit must produce (POSIX single-quoted words). Byte-identical to // `serialize_bash(command([lit("git"), lit("rev-parse"), lit("--show-toplevel")]))`. -data bash_mvp1_source_text: String = "'git' 'rev-parse' '--show-toplevel'" +data bash_mvp1_source_text: String = phrase_to_source_text(p: bash_mvp1_phrase()) // ========================================================================= // Concrete tokens — every word a FixedToken (no bound/dynamic words in slice 1). // ========================================================================= fn bash_mvp1_concrete_tokens() -> List { - [ - FixedToken { token_class: ^bash_token_word_git }, - FixedToken { token_class: ^bash_token_word_rev_parse }, - FixedToken { token_class: ^bash_token_word_show_toplevel } - ] + phrase_to_fixed_concrete_tokens(p: bash_mvp1_phrase()) } data bash_mvp1_emitted_command: Node = bash_conj_emitted_from_tokens( @@ -320,17 +394,14 @@ data bash_embedded_quote_source_text: String = "'a'\\''b'" // ========================================================================= fn bash_source_root_ingest_gate_concrete_tokens() -> List { - [ - FixedToken { token_class: ^bash_token_word_bash }, - FixedToken { token_class: ^bash_token_word_source_root_ingest_gate_sh } - ] + phrase_to_fixed_concrete_tokens(p: bash_source_root_ingest_gate_phrase()) } data bash_source_root_ingest_gate_emitted_command: Node = bash_conj_emitted_from_tokens( tokens: bash_source_root_ingest_gate_concrete_tokens() ) -data bash_source_root_ingest_gate_source_text: String = "'bash' 'scripts/source-root-ingest-gate.sh'" +data bash_source_root_ingest_gate_source_text: String = phrase_to_source_text(p: bash_source_root_ingest_gate_phrase()) fn bash_source_root_ingest_gate_translation_rules_node() -> Node { bash_relation_translation_rules_node( @@ -348,11 +419,7 @@ fn bash_source_root_ingest_gate_translation_rules_node() -> Node { // ========================================================================= fn bash_and_then_concrete_tokens() -> List { - [ - FixedToken { token_class: ^bash_token_word_true }, - FixedToken { token_class: ^bash_token_and_then }, - FixedToken { token_class: ^bash_token_word_false } - ] + phrase_to_fixed_concrete_tokens(p: bash_and_then_phrase()) } // AndOr { AndThen } — `'true' && 'false'` @@ -360,7 +427,7 @@ data bash_and_then_emitted: Node = bash_conj_emitted_from_tokens( tokens: bash_and_then_concrete_tokens() ) -data bash_and_then_source_text: String = "'true' && 'false'" +data bash_and_then_source_text: String = phrase_to_source_text(p: bash_and_then_phrase()) fn bash_and_then_translation_rules_node() -> Node { bash_relation_translation_rules_node( @@ -372,11 +439,7 @@ fn bash_and_then_translation_rules_node() -> Node { } fn bash_or_else_concrete_tokens() -> List { - [ - FixedToken { token_class: ^bash_token_word_true }, - FixedToken { token_class: ^bash_token_or_else }, - FixedToken { token_class: ^bash_token_word_false } - ] + phrase_to_fixed_concrete_tokens(p: bash_or_else_phrase()) } // AndOr { OrElse } — `'true' || 'false'` @@ -384,7 +447,7 @@ data bash_or_else_emitted: Node = bash_conj_emitted_from_tokens( tokens: bash_or_else_concrete_tokens() ) -data bash_or_else_source_text: String = "'true' || 'false'" +data bash_or_else_source_text: String = phrase_to_source_text(p: bash_or_else_phrase()) fn bash_or_else_translation_rules_node() -> Node { bash_relation_translation_rules_node( @@ -396,12 +459,7 @@ fn bash_or_else_translation_rules_node() -> Node { } fn bash_with_redir_stderr_null_concrete_tokens() -> List { - [ - FixedToken { token_class: ^bash_token_word_git }, - FixedToken { token_class: ^bash_token_word_rev_parse }, - FixedToken { token_class: ^bash_token_word_show_toplevel }, - FixedToken { token_class: ^bash_token_redir_stderr_null } - ] + phrase_to_fixed_concrete_tokens(p: bash_with_redir_stderr_null_phrase()) } // WithRedir { StderrToNull } — `'git' 'rev-parse' '--show-toplevel' 2>/dev/null` @@ -409,7 +467,7 @@ data bash_with_redir_stderr_null_emitted: Node = bash_conj_emitted_from_tokens( tokens: bash_with_redir_stderr_null_concrete_tokens() ) -data bash_with_redir_stderr_null_source_text: String = "'git' 'rev-parse' '--show-toplevel' 2>/dev/null" +data bash_with_redir_stderr_null_source_text: String = phrase_to_source_text(p: bash_with_redir_stderr_null_phrase()) fn bash_with_redir_stderr_null_translation_rules_node() -> Node { bash_relation_translation_rules_node( @@ -421,13 +479,7 @@ fn bash_with_redir_stderr_null_translation_rules_node() -> Node { } fn bash_pipe_concrete_tokens() -> List { - [ - FixedToken { token_class: ^bash_token_word_echo }, - FixedToken { token_class: ^bash_token_word_hi }, - FixedToken { token_class: ^bash_token_pipe }, - FixedToken { token_class: ^bash_token_word_wc }, - FixedToken { token_class: ^bash_token_word_minus_c } - ] + phrase_to_fixed_concrete_tokens(p: bash_pipe_phrase()) } // Pipe — `'echo' 'hi' | 'wc' '-c'` @@ -435,7 +487,7 @@ data bash_pipe_emitted: Node = bash_conj_emitted_from_tokens( tokens: bash_pipe_concrete_tokens() ) -data bash_pipe_source_text: String = "'echo' 'hi' | 'wc' '-c'" +data bash_pipe_source_text: String = phrase_to_source_text(p: bash_pipe_phrase()) fn bash_pipe_translation_rules_node() -> Node { bash_relation_translation_rules_node( @@ -447,14 +499,7 @@ fn bash_pipe_translation_rules_node() -> Node { } fn bash_or_else_with_redir_concrete_tokens() -> List { - [ - FixedToken { token_class: ^bash_token_word_git }, - FixedToken { token_class: ^bash_token_word_rev_parse }, - FixedToken { token_class: ^bash_token_word_show_toplevel }, - FixedToken { token_class: ^bash_token_redir_stderr_null }, - FixedToken { token_class: ^bash_token_or_else }, - FixedToken { token_class: ^bash_token_word_pwd } - ] + phrase_to_fixed_concrete_tokens(p: bash_or_else_with_redir_phrase()) } // host_prelude-shaped OrElse over WithRedir — `'git' 'rev-parse' '--show-toplevel' 2>/dev/null || 'pwd'` @@ -463,7 +508,7 @@ data bash_or_else_with_redir_emitted: Node = bash_conj_emitted_from_tokens( tokens: bash_or_else_with_redir_concrete_tokens() ) -data bash_or_else_with_redir_source_text: String = "'git' 'rev-parse' '--show-toplevel' 2>/dev/null || 'pwd'" +data bash_or_else_with_redir_source_text: String = phrase_to_source_text(p: bash_or_else_with_redir_phrase()) fn bash_or_else_with_redir_translation_rules_node() -> Node { bash_relation_translation_rules_node( @@ -591,30 +636,14 @@ fn bash_embedded_quote_target_model_bundle() -> Node { } // ========================================================================= -// Lex — POSIX single-quoting + token-adjacent layout baked into each literal (interim policy, -// dissolves with rust.dag's T-6 TokenLayout). `'git'` then ` 'rev-parse'` then ` '--show-toplevel'` -// concatenate to the canonical source text. +// Lex — each rule set PROJECTS from its LexicalPhrase (above) via phrase_to_lex_rules. +// POSIX single-quoting + token-adjacent layout still ride inside each span's fragment +// (interim policy, dissolves with the T-6 TokenLayout note in lexing.dag); the phrase's +// fragment column IS the canonical source text the rules accept. // ========================================================================= -fn bash_lex_rule(token_class: Symbol, text: String) -> LexRule { - TokenRule { - token_class: token_class, - pattern: LiteralPattern { text: text } - } -} - -fn bash_mvp1_lex_rules() -> LexRuleSet { - LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_word_git, text: "'git'"), - bash_lex_rule(token_class: ^bash_token_word_rev_parse, text: " 'rev-parse'"), - bash_lex_rule(token_class: ^bash_token_word_show_toplevel, text: " '--show-toplevel'") - ] - } -} - fn bash_mvp1_lex() -> LexRules { - ModeledLexRules { root: bash_mvp1_lex_rules() } + phrase_to_lex_rules(p: bash_mvp1_phrase()) } fn bash_mvp1_target_model() -> TargetModel { @@ -665,20 +694,8 @@ fn bash_source_root_ingest_gate_target_model_bundle() -> Node { ) } -fn bash_source_root_ingest_gate_lex_rules() -> LexRuleSet { - LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_word_bash, text: "'bash'"), - bash_lex_rule( - token_class: ^bash_token_word_source_root_ingest_gate_sh, - text: " 'scripts/source-root-ingest-gate.sh'" - ) - ] - } -} - fn bash_source_root_ingest_gate_lex() -> LexRules { - ModeledLexRules { root: bash_source_root_ingest_gate_lex_rules() } + phrase_to_lex_rules(p: bash_source_root_ingest_gate_phrase()) } fn bash_source_root_ingest_gate_target_model() -> TargetModel { @@ -718,15 +735,7 @@ fn bash_make_fixed_tokens_target_model( } fn bash_and_then_lex() -> LexRules { - ModeledLexRules { - root: LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_word_true, text: "'true'"), - bash_lex_rule(token_class: ^bash_token_and_then, text: " && "), - bash_lex_rule(token_class: ^bash_token_word_false, text: "'false'") - ] - } - } + phrase_to_lex_rules(p: bash_and_then_phrase()) } fn bash_and_then_target_model() -> TargetModel { @@ -741,15 +750,7 @@ fn bash_and_then_target_model() -> TargetModel { } fn bash_or_else_lex() -> LexRules { - ModeledLexRules { - root: LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_word_true, text: "'true'"), - bash_lex_rule(token_class: ^bash_token_or_else, text: " || "), - bash_lex_rule(token_class: ^bash_token_word_false, text: "'false'") - ] - } - } + phrase_to_lex_rules(p: bash_or_else_phrase()) } fn bash_or_else_target_model() -> TargetModel { @@ -764,16 +765,7 @@ fn bash_or_else_target_model() -> TargetModel { } fn bash_with_redir_stderr_null_lex() -> LexRules { - ModeledLexRules { - root: LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_word_git, text: "'git'"), - bash_lex_rule(token_class: ^bash_token_word_rev_parse, text: " 'rev-parse'"), - bash_lex_rule(token_class: ^bash_token_word_show_toplevel, text: " '--show-toplevel'"), - bash_lex_rule(token_class: ^bash_token_redir_stderr_null, text: " 2>/dev/null") - ] - } - } + phrase_to_lex_rules(p: bash_with_redir_stderr_null_phrase()) } fn bash_with_redir_stderr_null_target_model() -> TargetModel { @@ -788,17 +780,7 @@ fn bash_with_redir_stderr_null_target_model() -> TargetModel { } fn bash_pipe_lex() -> LexRules { - ModeledLexRules { - root: LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_word_echo, text: "'echo'"), - bash_lex_rule(token_class: ^bash_token_word_hi, text: " 'hi'"), - bash_lex_rule(token_class: ^bash_token_pipe, text: " | "), - bash_lex_rule(token_class: ^bash_token_word_wc, text: "'wc'"), - bash_lex_rule(token_class: ^bash_token_word_minus_c, text: " '-c'") - ] - } - } + phrase_to_lex_rules(p: bash_pipe_phrase()) } fn bash_pipe_target_model() -> TargetModel { @@ -813,18 +795,7 @@ fn bash_pipe_target_model() -> TargetModel { } fn bash_or_else_with_redir_lex() -> LexRules { - ModeledLexRules { - root: LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_word_git, text: "'git'"), - bash_lex_rule(token_class: ^bash_token_word_rev_parse, text: " 'rev-parse'"), - bash_lex_rule(token_class: ^bash_token_word_show_toplevel, text: " '--show-toplevel'"), - bash_lex_rule(token_class: ^bash_token_redir_stderr_null, text: " 2>/dev/null"), - bash_lex_rule(token_class: ^bash_token_or_else, text: " || "), - bash_lex_rule(token_class: ^bash_token_word_pwd, text: "'pwd'") - ] - } - } + phrase_to_lex_rules(p: bash_or_else_with_redir_phrase()) } fn bash_or_else_with_redir_target_model() -> TargetModel { diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 2e3af516587..f714f815a95 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -5,7 +5,7 @@ module v2.std.compilers.lexing -import v2.std.algebra { fold_list, list_snoc_item } +import v2.std.algebra { Empty, fold_list, list_snoc_item } import v2.std.collection { List } import v2.std.logic { Bool } import v2.std.node { Atom, Conj, Node, Symbol, SyntheticOccurrence, TypeNode } @@ -60,6 +60,84 @@ type LexRules | ModeledLexRules { root: LexRuleSet } +// ── LexicalPhrase: the single authority for a fixed-token phrase ─────────── +// A fixed-token phrase (one with a literal spelling — no emit-time bound words) was +// historically authored THREE times that could silently disagree: a LexRuleSet of +// (token_class, literal-text) rules, a parallel ConcreteSyntaxToken sequence repeating +// the same token classes, and a whole-phrase source-text string equal to the rule texts +// concatenated. LexicalPhrase collapses those into one carrier (§2 horizontal — one +// concept, derive every use): an ordered list of (token_class, fragment) spans whose +// fragments concatenate, in order, to the source text. +// +// Author it so the `fragment:` column reads top-to-bottom AS the sentence being formed — +// the authored form IS the phrase. The lex rule set, the source text, and (in +// target_model, which knows ConcreteSyntaxToken) the concrete-token sequence all PROJECT +// from this one value, so the concat == source_text identity holds by construction and +// the three can no longer drift (§5: the redundancy is removed, not merely checked). +// +// 🟡 layout-in-fragments interim — feature:t6-token-layout — bind gunbc#4761 (T-6 TokenLayout/ +// TriviaPolicy) — dissolve-on-arrival: `fragment` splits into (lexeme, trivia) once T-6 +// supplies a TriviaPolicy so inter-token layout is interleaved structurally rather than baked +// into each span's text; forbidden: a second whole-phrase source-text authority alongside +// `phrase_to_source_text`. Bound-word phrases (emit-time spelling resolved from a binding) are +// NOT yet expressible — a binding-carrying span is the migration-time extension, not this PR. +type TokenSpan { + token_class: Symbol + fragment: String +} + + +type LexicalPhrase { + spans: List +} + + +fn span(token_class: Symbol, fragment: String) -> TokenSpan { + TokenSpan { token_class: token_class, fragment: fragment } +} + + +fn phrase(spans: List) -> LexicalPhrase { + LexicalPhrase { spans: spans } +} + + +// Projection 1 — the fixed-token LexRuleSet: each span is a TokenRule over a LiteralPattern +// of its fragment, in source order (maximal-munch order preserved from `spans`). +fn phrase_to_lex_rule_set(p: LexicalPhrase) -> LexRuleSet { + LexRuleSet { + rules: fold_list( + xs: p.spans, + empty: Empty, + cons: fn(acc, s) { + list_snoc_item( + xs: acc, + item: TokenRule { + token_class: s.token_class, + pattern: LiteralPattern { text: s.fragment } + } + ) + } + ) + } +} + + +fn phrase_to_lex_rules(p: LexicalPhrase) -> LexRules { + ModeledLexRules { root: phrase_to_lex_rule_set(p: p) } +} + + +// Projection 2 — the source text: the fragments concatenated in order. +fn phrase_to_source_text(p: LexicalPhrase) -> String { + fold_list( + xs: p.spans, + empty: "", + cons: fn(acc, s) { concat(acc, s.fragment) } + ) +} + + type Token { class: Symbol lexeme: Lexeme diff --git a/src/v2/std/compilers/target_model.dag b/src/v2/std/compilers/target_model.dag index 06d1df1634c..32e1d5ec577 100644 --- a/src/v2/std/compilers/target_model.dag +++ b/src/v2/std/compilers/target_model.dag @@ -53,6 +53,7 @@ import v2.std.compilers.lexing { LexPattern, LexRule, LexRules, + LexicalPhrase, LiteralPattern, ModeledLexRules, TokenRule, @@ -264,6 +265,22 @@ fn concrete_syntax_token_class(token: ConcreteSyntaxToken) -> Symbol { } } +// Projection of a LexicalPhrase (lexing.dag) to its concrete-token sequence — the grammar/ +// serialize side of the same single-authority carrier, homed here because it is where +// ConcreteSyntaxToken lives (the import arrow runs target_model -> lexing, never back). Each +// span is a FixedToken on its token_class: a phrase whose spelling is literal has no +// binding-resolved (BoundToken) span, so this projection is total over today's LexicalPhrase. +// A binding-carrying span / BoundToken projection is the T-6/bound-word migration extension. +fn phrase_to_fixed_concrete_tokens(p: LexicalPhrase) -> List { + fold_list( + xs: p.spans, + empty: Empty, + cons: fn(acc, s) { + list_snoc_item(xs: acc, item: FixedToken { token_class: s.token_class }) + } + ) +} + // Per-target lex + binding-spelling lookup (tokens/layout authority for translate serialize). fn target_lex_rule_literal_not_found_diagnostic(token_class: Symbol) -> Diagnostic { From 3e9f177eae4a3926eb5204314f4842cdb0bf578f Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 20:26:56 +0000 Subject: [PATCH 3/9] Pilot LexicalPhrase: single authority for bash fixed-token phrases MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A fixed-token phrase was authored three ways that could silently drift: a LexRuleSet of (token_class, literal-text) rules, a parallel ConcreteSyntaxToken sequence repeating the same classes, and a whole-phrase source-text string equal to the rule texts concatenated. DESIGN.md §2 horizontal: one concept, derive every use. std/compilers/lexing.dag: new LexicalPhrase carrier (ordered (token_class, fragment) spans) + span/phrase constructors + phrase_to_lex_rules / phrase_to_source_text projections. target_model.dag (which owns ConcreteSyntaxToken; import arrow runs target_model -> lexing) gets phrase_to_fixed_concrete_tokens. bash.dag: the 7 pure-FixedToken phrases are each authored ONCE as a LexicalPhrase whose fragment column reads as the shell text; lex/concrete_tokens/source_text all project from it. Removes the bash_lex_rule helper and two LexRuleSet builders (-463 lines in bash.dag). Bound-word phrases (bound_words/embedded_quote) stay literal pending a binding-carrying span (T-6). Verified by execution: all 11 bash_emit_command_test witnesses green (full lex -> grammar-row -> byte-identical emit pipeline); a scratch oracle confirmed each derived source_text is byte-identical to the literal it replaced and the token classes project (with a discriminating non-member -> false). Co-Authored-By: Claude Opus 4.8 (1M context) --- .../manual/lex_phrase_scratch_test.dag | 25 ------------------- 1 file changed, 25 deletions(-) delete mode 100644 src/v2/compiler/manual/lex_phrase_scratch_test.dag diff --git a/src/v2/compiler/manual/lex_phrase_scratch_test.dag b/src/v2/compiler/manual/lex_phrase_scratch_test.dag deleted file mode 100644 index 918489a38fa..00000000000 --- a/src/v2/compiler/manual/lex_phrase_scratch_test.dag +++ /dev/null @@ -1,25 +0,0 @@ -module v2.test.manual.lex_phrase_scratch - -// SCRATCH — delete before commit. Proves the LexicalPhrase de-dup is byte-identical: -// each derived source_text == the literal it replaced. Imports bash + std only (no emit/ -// translate), so it sidesteps the pre-existing emit-test resolution cycle. - -import v2.extdeps.languages.bash { - bash_mvp1_source_text, - bash_source_root_ingest_gate_source_text, - bash_and_then_source_text, - bash_or_else_source_text, - bash_with_redir_stderr_null_source_text, - bash_pipe_source_text, - bash_or_else_with_redir_source_text -} - -test fn lex_phrase_source_text_byte_identical() -> Bool { - (bash_mvp1_source_text == "'git' 'rev-parse' '--show-toplevel'") && - (bash_source_root_ingest_gate_source_text == "'bash' 'scripts/source-root-ingest-gate.sh'") && - (bash_and_then_source_text == "'true' && 'false'") && - (bash_or_else_source_text == "'true' || 'false'") && - (bash_with_redir_stderr_null_source_text == "'git' 'rev-parse' '--show-toplevel' 2>/dev/null") && - (bash_pipe_source_text == "'echo' 'hi' | 'wc' '-c'") && - (bash_or_else_with_redir_source_text == "'git' 'rev-parse' '--show-toplevel' 2>/dev/null || 'pwd'") -} From 683c103fae687207012384dd9b9add5fe410f9bf Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 20:51:18 +0000 Subject: [PATCH 4/9] WIP: can we make modeled lex rules more elegant somehow? ModeledLexRules { ro --- dsl/std/realization_width.dag | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/dsl/std/realization_width.dag b/dsl/std/realization_width.dag index cc21c92b118..246a09bf978 100644 --- a/dsl/std/realization_width.dag +++ b/dsl/std/realization_width.dag @@ -23,11 +23,12 @@ module std.realization_width import std.types { Int, List } import std.measure { HardwareThreadCount, - NanosecondDuration, hardware_thread_count, hardware_thread_count_value, - nanosecond_duration, } +// nanosecond scale CHOICE relocated out of std.measure (scale-agnostic) to the timing +// authority consumer std.verification by #5286 — import the scaled constructor from there. +import std.verification { nanosecond_duration } import std.pareto { AxisGoal, LowerIsBetter } import std.realization_schedule { Schedule, @@ -38,7 +39,7 @@ import std.realization_schedule { schedule_eq, cost_account_measured, cost_account_predicted_zero, - cost_account_time_nanos, + cost_account_time_count, } fn int_min(a: Int, b: Int) -> Int { @@ -123,10 +124,12 @@ fn witness_budget_ignoring_fold_fails( } // Measured shard-balance scaffold: assignment must read CostAccount.basis=Measured. -fn shard_balance_slot_for_cost(cost: CostAccount, baseline: CostAccount) -> Int { +// CostAccount is scale-generic (CostAccount) post-#5286; the slot reads the scale-agnostic +// time COUNT (Nat, widened to Int) rather than a nanosecond-fixed projection. +fn shard_balance_slot_for_cost(cost: CostAccount, baseline: CostAccount) -> Int { match cost.basis { - Measured => cost_account_time_nanos(account: cost) - Predicted => cost_account_time_nanos(account: baseline) + Measured => cost_account_time_count(account: cost) + Predicted => cost_account_time_count(account: baseline) } } From 5a2a779daf589e0f143eead01dfa6344efa9b3e0 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 20:52:06 +0000 Subject: [PATCH 5/9] Reconcile realization_width to post-#5286 scale-agnostic API MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #5286 ("NanosecondDuration decomposition") relocated the nanosecond scale CHOICE out of std.measure to std.verification and made the cost ledger scale-generic (CostAccount, RealizationPlan, cost_account_time_count replacing cost_account_time_nanos). #5225's realization_width.dag (landed concurrently) still imported the old names — a semantic merge-skew that left main's CI floor resolve RED (blocking every branch, since the floor scans the whole dsl tree). - nanosecond_duration now imported from std.verification (dropped the unused NanosecondDuration type import). - cost_account_time_nanos -> cost_account_time_count (scale-agnostic Nat count). - thread through shard_balance_slot_for_cost / WidthBoundedRealization / width_bounded_realization / witness_width_fold_preserves_schedule_topology so CostAccount / RealizationPlan are well-formed; width never reads the ledger scale, so S only rides through. Verified: realization_width_witnesses() green by execution. Co-Authored-By: Claude Opus 4.8 (1M context) --- dsl/std/realization_width.dag | 16 +++++++++------- 1 file changed, 9 insertions(+), 7 deletions(-) diff --git a/dsl/std/realization_width.dag b/dsl/std/realization_width.dag index 246a09bf978..d6765245997 100644 --- a/dsl/std/realization_width.dag +++ b/dsl/std/realization_width.dag @@ -72,25 +72,27 @@ fn width_fold_objective_goals() -> List { [LowerIsBetter, LowerIsBetter] } -type WidthBoundedRealization { - plan: RealizationPlan +// `` is the plan's consumer-pinned cost-ledger time scale (RealizationPlan post-#5286); +// width is peripheral and never reads the ledger, so S only rides through for well-formedness. +type WidthBoundedRealization { + plan: RealizationPlan spawn_width: HardwareThreadCount } // Width is peripheral: the schedule topology is unchanged (§3 invariant). -fn width_bounded_realization( - plan: RealizationPlan, +fn width_bounded_realization( + plan: RealizationPlan, shard_count: Int, hardware_threads: HardwareThreadCount -) -> WidthBoundedRealization { +) -> WidthBoundedRealization { WidthBoundedRealization { plan: plan spawn_width: bounded_host_spawn_width(shard_count: shard_count, hardware_threads: hardware_threads) } } -fn witness_width_fold_preserves_schedule_topology( - plan: RealizationPlan, +fn witness_width_fold_preserves_schedule_topology( + plan: RealizationPlan, shard_count: Int, hardware_threads: HardwareThreadCount ) -> Bool { From 88ae2269c82ac622c10fe8aaa0a0d0c6c9a1e869 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 20:59:17 +0000 Subject: [PATCH 6/9] LexicalPhrase: left-to-right phrase_text authoring form MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Per reviewer/operator: author each phrase so the resulting shell text reads left-to-right like a sentence. phrase_text(text: [...], classes: [...]) — the `text:` row reads L→R as the phrase, the `classes:` row names each fragment's token class in the same order. Zipped into TokenSpans by zip_spans (typed over FreeMonoid so match resolves Cons/Empty; List is an opaque alias to the matcher). A text/classes desync surfaces RED in the emit witnesses, not a silent zip. Projections (lex/concrete/source_text) unchanged — byte-identical output. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/v2/extdeps/languages/bash.dag | 86 +++++++++++++------------------ src/v2/std/compilers/lexing.dag | 42 ++++++++++++--- 2 files changed, 72 insertions(+), 56 deletions(-) diff --git a/src/v2/extdeps/languages/bash.dag b/src/v2/extdeps/languages/bash.dag index 3afcc01370c..95a99f7337d 100644 --- a/src/v2/extdeps/languages/bash.dag +++ b/src/v2/extdeps/languages/bash.dag @@ -49,10 +49,9 @@ import v2.std.grammar { import v2.std.compilers.lexing { LexRules, LexicalPhrase, - phrase, + phrase_text, phrase_to_lex_rules, - phrase_to_source_text, - span + phrase_to_source_text } import v2.std.compilers.target_model { BoundToken, @@ -223,13 +222,12 @@ fn bash_relation_translation_rules_node( // ========================================================================= // Lexical phrases — one authority per fixed-token phrase (DESIGN.md §3). // -// Each phrase is authored ONCE here as an ordered list of (token_class, fragment) -// spans: read the `fragment:` column top-to-bottom and it spells the shell text the -// phrase emits. The lex rule set, the concrete-token sequence, and the source-text -// string all PROJECT from these (phrase_to_lex_rules / phrase_to_fixed_concrete_tokens / -// phrase_to_source_text), so the three can no longer drift (DESIGN.md §2 horizontal: -// one concept, derive every use). The byte-identity to serialize_bash is proven by -// execution in bash_emit_command_test. +// Each phrase is authored ONCE: the `text:` row reads LEFT-TO-RIGHT as the shell text the +// phrase emits, and the `classes:` row names each fragment's token class in the same order. +// The lex rule set, the concrete-token sequence, and the source-text string all PROJECT +// from these (phrase_to_lex_rules / phrase_to_fixed_concrete_tokens / phrase_to_source_text), +// so the three can no longer drift (DESIGN.md §2 horizontal: one concept, derive every use). +// The byte-identity to serialize_bash is proven by execution in bash_emit_command_test. // // Bound-word phrases (bash_bound_words / bash_embedded_quote) are spelled at emit time // from a binding, not a literal fragment, so they are NOT LexicalPhrases yet — a @@ -237,64 +235,52 @@ fn bash_relation_translation_rules_node( // ========================================================================= fn bash_mvp1_phrase() -> LexicalPhrase { - phrase(spans: [ - span(token_class: ^bash_token_word_git, fragment: "'git'"), - span(token_class: ^bash_token_word_rev_parse, fragment: " 'rev-parse'"), - span(token_class: ^bash_token_word_show_toplevel, fragment: " '--show-toplevel'") - ]) + phrase_text( + text: ["'git'", " 'rev-parse'", " '--show-toplevel'"], + classes: [^bash_token_word_git, ^bash_token_word_rev_parse, ^bash_token_word_show_toplevel] + ) } fn bash_source_root_ingest_gate_phrase() -> LexicalPhrase { - phrase(spans: [ - span(token_class: ^bash_token_word_bash, fragment: "'bash'"), - span(token_class: ^bash_token_word_source_root_ingest_gate_sh, fragment: " 'scripts/source-root-ingest-gate.sh'") - ]) + phrase_text( + text: ["'bash'", " 'scripts/source-root-ingest-gate.sh'"], + classes: [^bash_token_word_bash, ^bash_token_word_source_root_ingest_gate_sh] + ) } fn bash_and_then_phrase() -> LexicalPhrase { - phrase(spans: [ - span(token_class: ^bash_token_word_true, fragment: "'true'"), - span(token_class: ^bash_token_and_then, fragment: " && "), - span(token_class: ^bash_token_word_false, fragment: "'false'") - ]) + phrase_text( + text: ["'true'", " && ", "'false'"], + classes: [^bash_token_word_true, ^bash_token_and_then, ^bash_token_word_false] + ) } fn bash_or_else_phrase() -> LexicalPhrase { - phrase(spans: [ - span(token_class: ^bash_token_word_true, fragment: "'true'"), - span(token_class: ^bash_token_or_else, fragment: " || "), - span(token_class: ^bash_token_word_false, fragment: "'false'") - ]) + phrase_text( + text: ["'true'", " || ", "'false'"], + classes: [^bash_token_word_true, ^bash_token_or_else, ^bash_token_word_false] + ) } fn bash_with_redir_stderr_null_phrase() -> LexicalPhrase { - phrase(spans: [ - span(token_class: ^bash_token_word_git, fragment: "'git'"), - span(token_class: ^bash_token_word_rev_parse, fragment: " 'rev-parse'"), - span(token_class: ^bash_token_word_show_toplevel, fragment: " '--show-toplevel'"), - span(token_class: ^bash_token_redir_stderr_null, fragment: " 2>/dev/null") - ]) + phrase_text( + text: ["'git'", " 'rev-parse'", " '--show-toplevel'", " 2>/dev/null"], + classes: [^bash_token_word_git, ^bash_token_word_rev_parse, ^bash_token_word_show_toplevel, ^bash_token_redir_stderr_null] + ) } fn bash_pipe_phrase() -> LexicalPhrase { - phrase(spans: [ - span(token_class: ^bash_token_word_echo, fragment: "'echo'"), - span(token_class: ^bash_token_word_hi, fragment: " 'hi'"), - span(token_class: ^bash_token_pipe, fragment: " | "), - span(token_class: ^bash_token_word_wc, fragment: "'wc'"), - span(token_class: ^bash_token_word_minus_c, fragment: " '-c'") - ]) + phrase_text( + text: ["'echo'", " 'hi'", " | ", "'wc'", " '-c'"], + classes: [^bash_token_word_echo, ^bash_token_word_hi, ^bash_token_pipe, ^bash_token_word_wc, ^bash_token_word_minus_c] + ) } fn bash_or_else_with_redir_phrase() -> LexicalPhrase { - phrase(spans: [ - span(token_class: ^bash_token_word_git, fragment: "'git'"), - span(token_class: ^bash_token_word_rev_parse, fragment: " 'rev-parse'"), - span(token_class: ^bash_token_word_show_toplevel, fragment: " '--show-toplevel'"), - span(token_class: ^bash_token_redir_stderr_null, fragment: " 2>/dev/null"), - span(token_class: ^bash_token_or_else, fragment: " || "), - span(token_class: ^bash_token_word_pwd, fragment: "'pwd'") - ]) + phrase_text( + text: ["'git'", " 'rev-parse'", " '--show-toplevel'", " 2>/dev/null", " || ", "'pwd'"], + classes: [^bash_token_word_git, ^bash_token_word_rev_parse, ^bash_token_word_show_toplevel, ^bash_token_redir_stderr_null, ^bash_token_or_else, ^bash_token_word_pwd] + ) } // The canonical shell text emit must produce (POSIX single-quoted words). Byte-identical to diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index f714f815a95..937528c14cb 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -5,7 +5,7 @@ module v2.std.compilers.lexing -import v2.std.algebra { Empty, fold_list, list_snoc_item } +import v2.std.algebra { Cons, Empty, FreeMonoid, fold_list, list_snoc_item } import v2.std.collection { List } import v2.std.logic { Bool } import v2.std.node { Atom, Conj, Node, Symbol, SyntheticOccurrence, TypeNode } @@ -69,11 +69,15 @@ type LexRules // concept, derive every use): an ordered list of (token_class, fragment) spans whose // fragments concatenate, in order, to the source text. // -// Author it so the `fragment:` column reads top-to-bottom AS the sentence being formed — -// the authored form IS the phrase. The lex rule set, the source text, and (in -// target_model, which knows ConcreteSyntaxToken) the concrete-token sequence all PROJECT -// from this one value, so the concat == source_text identity holds by construction and -// the three can no longer drift (§5: the redundancy is removed, not merely checked). +// The cosmetic authoring form is `phrase_text`: a `text:` row of fragments that reads +// LEFT-TO-RIGHT as the sentence being formed, with a positionally-aligned `classes:` row +// naming each fragment's token class — i.e. "here is the resulting phrase, and here is what +// each piece is". The lex rule set, the source text, and (in target_model, which knows +// ConcreteSyntaxToken) the concrete-token sequence all PROJECT from this one value, so the +// concat == source_text identity holds by construction and the three can no longer drift +// (§5: the redundancy is removed, not merely checked). A text/classes length or order +// mismatch is caught by execution — the per-phrase emit witnesses go RED on a wrong +// source text or token-class sequence (the fail-closed guard, not a silent zip-to-shorter). // // 🟡 layout-in-fragments interim — feature:t6-token-layout — bind gunbc#4761 (T-6 TokenLayout/ // TriviaPolicy) — dissolve-on-arrival: `fragment` splits into (lexeme, trivia) once T-6 @@ -102,6 +106,32 @@ fn phrase(spans: List) -> LexicalPhrase { } +// zip_spans: pair the L→R `text` fragments with their `classes` in lockstep. Recurses to the +// shorter list (a desync surfaces as a wrong projection caught RED by the emit witnesses). +fn zip_spans(text: FreeMonoid, classes: FreeMonoid) -> FreeMonoid { + match text { + Empty => Empty + Cons { head: fragment, tail: rest_text } => + match classes { + Empty => Empty + Cons { head: token_class, tail: rest_classes } => + Cons { + head: TokenSpan { token_class: token_class, fragment: fragment }, + tail: zip_spans(text: rest_text, classes: rest_classes) + } + } + } +} + + +// phrase_text: the cosmetic authoring form. `text` reads left-to-right as the sentence; each +// fragment's token class is the same-position entry of `classes`. Equivalent to a list of +// `span(class, fragment)` but lets the resulting phrase read as one line of text. +fn phrase_text(text: List, classes: List) -> LexicalPhrase { + LexicalPhrase { spans: zip_spans(text: text, classes: classes) } +} + + // Projection 1 — the fixed-token LexRuleSet: each span is a TokenRule over a LiteralPattern // of its fragment, in source order (maximal-munch order preserved from `spans`). fn phrase_to_lex_rule_set(p: LexicalPhrase) -> LexRuleSet { From 9c61144882d6727e5c25fd14729f816015409b5e Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 22:02:35 +0000 Subject: [PATCH 7/9] Remove dead span/phrase constructors; rely on list ordering MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The phrase_text authoring form builds spans via zip_spans directly, so the bare span()/phrase() constructors had no callers. Drop them — authoring is the single phrase_text(text:, classes:) form, paired by list order. Co-Authored-By: Claude Opus 4.8 (1M context) --- src/v2/std/compilers/lexing.dag | 16 +++------------- 1 file changed, 3 insertions(+), 13 deletions(-) diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index 937528c14cb..d7f73463324 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -96,16 +96,6 @@ type LexicalPhrase { } -fn span(token_class: Symbol, fragment: String) -> TokenSpan { - TokenSpan { token_class: token_class, fragment: fragment } -} - - -fn phrase(spans: List) -> LexicalPhrase { - LexicalPhrase { spans: spans } -} - - // zip_spans: pair the L→R `text` fragments with their `classes` in lockstep. Recurses to the // shorter list (a desync surfaces as a wrong projection caught RED by the emit witnesses). fn zip_spans(text: FreeMonoid, classes: FreeMonoid) -> FreeMonoid { @@ -124,9 +114,9 @@ fn zip_spans(text: FreeMonoid, classes: FreeMonoid) -> FreeMonoi } -// phrase_text: the cosmetic authoring form. `text` reads left-to-right as the sentence; each -// fragment's token class is the same-position entry of `classes`. Equivalent to a list of -// `span(class, fragment)` but lets the resulting phrase read as one line of text. +// phrase_text: the authoring form. `text` reads left-to-right as the sentence; each fragment's +// token class is the same-position entry of `classes` (paired by list order). A length/order +// desync surfaces RED in the per-phrase emit witnesses, not as a silent zip-to-shorter. fn phrase_text(text: List, classes: List) -> LexicalPhrase { LexicalPhrase { spans: zip_spans(text: text, classes: classes) } } From c175ff8c8543b6a9ab2a52bb1914017862218c07 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 22:44:25 +0000 Subject: [PATCH 8/9] WIP: can we make modeled lex rules more elegant somehow? ModeledLexRules { ro --- src/v2/extdeps/languages/bash.dag | 148 +++++++++++++++++++----------- src/v2/std/compilers/lexing.dag | 45 +++------ 2 files changed, 107 insertions(+), 86 deletions(-) diff --git a/src/v2/extdeps/languages/bash.dag b/src/v2/extdeps/languages/bash.dag index 49d7178170a..a1e7c37ac94 100644 --- a/src/v2/extdeps/languages/bash.dag +++ b/src/v2/extdeps/languages/bash.dag @@ -51,9 +51,10 @@ import v2.std.grammar { import v2.std.compilers.lexing { LexRules, LexicalPhrase, - phrase_text, + phrase, phrase_to_lex_rules, - phrase_to_source_text + phrase_to_source_text, + span } import v2.std.compilers.target_model { BoundToken, @@ -237,52 +238,64 @@ fn bash_relation_translation_rules_node( // ========================================================================= fn bash_mvp1_phrase() -> LexicalPhrase { - phrase_text( - text: ["'git'", " 'rev-parse'", " '--show-toplevel'"], - classes: [^bash_token_word_git, ^bash_token_word_rev_parse, ^bash_token_word_show_toplevel] - ) + phrase([ + span(text: "'git'", class: ^bash_token_word_git), + span(text: " 'rev-parse'", class: ^bash_token_word_rev_parse), + span(text: " '--show-toplevel'", class: ^bash_token_word_show_toplevel) + ]) } fn bash_source_root_ingest_gate_phrase() -> LexicalPhrase { - phrase_text( - text: ["'bash'", " 'scripts/source-root-ingest-gate.sh'"], - classes: [^bash_token_word_bash, ^bash_token_word_source_root_ingest_gate_sh] - ) + phrase([ + span(text: "'bash'", class: ^bash_token_word_bash), + span(text: " 'scripts/source-root-ingest-gate.sh'", class: ^bash_token_word_source_root_ingest_gate_sh) + ]) } fn bash_and_then_phrase() -> LexicalPhrase { - phrase_text( - text: ["'true'", " && ", "'false'"], - classes: [^bash_token_word_true, ^bash_token_and_then, ^bash_token_word_false] - ) + phrase([ + span(text: "'true'", class: ^bash_token_word_true), + span(text: " && ", class: ^bash_token_and_then), + span(text: "'false'", class: ^bash_token_word_false) + ]) } fn bash_or_else_phrase() -> LexicalPhrase { - phrase_text( - text: ["'true'", " || ", "'false'"], - classes: [^bash_token_word_true, ^bash_token_or_else, ^bash_token_word_false] - ) + phrase([ + span(text: "'true'", class: ^bash_token_word_true), + span(text: " || ", class: ^bash_token_or_else), + span(text: "'false'", class: ^bash_token_word_false) + ]) } fn bash_with_redir_stderr_null_phrase() -> LexicalPhrase { - phrase_text( - text: ["'git'", " 'rev-parse'", " '--show-toplevel'", " 2>/dev/null"], - classes: [^bash_token_word_git, ^bash_token_word_rev_parse, ^bash_token_word_show_toplevel, ^bash_token_redir_stderr_null] - ) + phrase([ + span(text: "'git'", class: ^bash_token_word_git), + span(text: " 'rev-parse'", class: ^bash_token_word_rev_parse), + span(text: " '--show-toplevel'", class: ^bash_token_word_show_toplevel), + span(text: " 2>/dev/null", class: ^bash_token_redir_stderr_null) + ]) } fn bash_pipe_phrase() -> LexicalPhrase { - phrase_text( - text: ["'echo'", " 'hi'", " | ", "'wc'", " '-c'"], - classes: [^bash_token_word_echo, ^bash_token_word_hi, ^bash_token_pipe, ^bash_token_word_wc, ^bash_token_word_minus_c] - ) + phrase([ + span(text: "'echo'", class: ^bash_token_word_echo), + span(text: " 'hi'", class: ^bash_token_word_hi), + span(text: " | ", class: ^bash_token_pipe), + span(text: "'wc'", class: ^bash_token_word_wc), + span(text: " '-c'", class: ^bash_token_word_minus_c) + ]) } fn bash_or_else_with_redir_phrase() -> LexicalPhrase { - phrase_text( - text: ["'git'", " 'rev-parse'", " '--show-toplevel'", " 2>/dev/null", " || ", "'pwd'"], - classes: [^bash_token_word_git, ^bash_token_word_rev_parse, ^bash_token_word_show_toplevel, ^bash_token_redir_stderr_null, ^bash_token_or_else, ^bash_token_word_pwd] - ) + phrase([ + span(text: "'git'", class: ^bash_token_word_git), + span(text: " 'rev-parse'", class: ^bash_token_word_rev_parse), + span(text: " '--show-toplevel'", class: ^bash_token_word_show_toplevel), + span(text: " 2>/dev/null", class: ^bash_token_redir_stderr_null), + span(text: " || ", class: ^bash_token_or_else), + span(text: "'pwd'", class: ^bash_token_word_pwd) + ]) } // The canonical shell text emit must produce (POSIX single-quoted words). Byte-identical to @@ -513,50 +526,73 @@ fn bash_or_else_with_redir_translation_rules_node() -> Node { // `'true' && ('false' | 'true')`. Correct emit wraps the AndOr operand: `{ 'true' && 'false'; } | 'true'`. // Authority: program.dag BraceGroup + Pipe fixture (not the sidecar's flat Pipe concat). // -// Lexical phrases (same single-authority form as the slice-1..4 phrases above): `text:` -// reads left-to-right as the shell text, `classes:` names each fragment's token class. +// Lexical phrases (same single-authority form as the slice-1..4 phrases above): each +// span(text:, class:) pairs a fragment with its token class; the text column reads as shell. // ========================================================================= fn bash_subshell_true_phrase() -> LexicalPhrase { - phrase_text( - text: ["(", "'true'", ")"], - classes: [^bash_token_lparen, ^bash_token_word_true, ^bash_token_rparen] - ) + phrase([ + span(text: "(", class: ^bash_token_lparen), + span(text: "'true'", class: ^bash_token_word_true), + span(text: ")", class: ^bash_token_rparen) + ]) } fn bash_if_true_then_false_phrase() -> LexicalPhrase { - phrase_text( - text: ["if ", "'true'", "; ", "then ", "'false'", "; ", "fi"], - classes: [^bash_token_kw_if, ^bash_token_word_true, ^bash_token_semicolon, ^bash_token_kw_then, ^bash_token_word_false, ^bash_token_semicolon, ^bash_token_kw_fi] - ) + phrase([ + span(text: "if ", class: ^bash_token_kw_if), + span(text: "'true'", class: ^bash_token_word_true), + span(text: "; ", class: ^bash_token_semicolon), + span(text: "then ", class: ^bash_token_kw_then), + span(text: "'false'", class: ^bash_token_word_false), + span(text: "; ", class: ^bash_token_semicolon), + span(text: "fi", class: ^bash_token_kw_fi) + ]) } fn bash_heredoc_simple_phrase() -> LexicalPhrase { - phrase_text( - text: ["cat", " > ", "'out'", " <<'EOF'\n", "hi\n", "EOF"], - classes: [^bash_token_word_cat, ^bash_token_greater, ^bash_token_word_out, ^bash_token_heredoc_open_eof, ^bash_token_heredoc_body_hi, ^bash_token_heredoc_close_eof] - ) + phrase([ + span(text: "cat", class: ^bash_token_word_cat), + span(text: " > ", class: ^bash_token_greater), + span(text: "'out'", class: ^bash_token_word_out), + span(text: " <<'EOF'\n", class: ^bash_token_heredoc_open_eof), + span(text: "hi\n", class: ^bash_token_heredoc_body_hi), + span(text: "EOF", class: ^bash_token_heredoc_close_eof) + ]) } fn bash_subshell_and_or_nested_phrase() -> LexicalPhrase { - phrase_text( - text: ["(", "'true'", " && ", "'false'", ")", " || ", "'true'"], - classes: [^bash_token_lparen, ^bash_token_word_true, ^bash_token_and_then, ^bash_token_word_false, ^bash_token_rparen, ^bash_token_or_else, ^bash_token_word_true] - ) + phrase([ + span(text: "(", class: ^bash_token_lparen), + span(text: "'true'", class: ^bash_token_word_true), + span(text: " && ", class: ^bash_token_and_then), + span(text: "'false'", class: ^bash_token_word_false), + span(text: ")", class: ^bash_token_rparen), + span(text: " || ", class: ^bash_token_or_else), + span(text: "'true'", class: ^bash_token_word_true) + ]) } fn bash_pipe_and_or_brace_grouped_phrase() -> LexicalPhrase { - phrase_text( - text: ["{ ", "'true'", " && ", "'false'", "; }", " | ", "'true'"], - classes: [^bash_token_lbrace, ^bash_token_word_true, ^bash_token_and_then, ^bash_token_word_false, ^bash_token_semicolon_rbrace, ^bash_token_pipe, ^bash_token_word_true] - ) + phrase([ + span(text: "{ ", class: ^bash_token_lbrace), + span(text: "'true'", class: ^bash_token_word_true), + span(text: " && ", class: ^bash_token_and_then), + span(text: "'false'", class: ^bash_token_word_false), + span(text: "; }", class: ^bash_token_semicolon_rbrace), + span(text: " | ", class: ^bash_token_pipe), + span(text: "'true'", class: ^bash_token_word_true) + ]) } fn bash_pipe_and_or_flat_wrong_phrase() -> LexicalPhrase { - phrase_text( - text: ["'true'", " && ", "'false'", " | ", "'true'"], - classes: [^bash_token_word_true, ^bash_token_and_then, ^bash_token_word_false, ^bash_token_pipe, ^bash_token_word_true] - ) + phrase([ + span(text: "'true'", class: ^bash_token_word_true), + span(text: " && ", class: ^bash_token_and_then), + span(text: "'false'", class: ^bash_token_word_false), + span(text: " | ", class: ^bash_token_pipe), + span(text: "'true'", class: ^bash_token_word_true) + ]) } fn bash_subshell_true_concrete_tokens() -> List { diff --git a/src/v2/std/compilers/lexing.dag b/src/v2/std/compilers/lexing.dag index d7f73463324..4afbb00a6a0 100644 --- a/src/v2/std/compilers/lexing.dag +++ b/src/v2/std/compilers/lexing.dag @@ -5,7 +5,7 @@ module v2.std.compilers.lexing -import v2.std.algebra { Cons, Empty, FreeMonoid, fold_list, list_snoc_item } +import v2.std.algebra { Empty, fold_list, list_snoc_item } import v2.std.collection { List } import v2.std.logic { Bool } import v2.std.node { Atom, Conj, Node, Symbol, SyntheticOccurrence, TypeNode } @@ -69,15 +69,14 @@ type LexRules // concept, derive every use): an ordered list of (token_class, fragment) spans whose // fragments concatenate, in order, to the source text. // -// The cosmetic authoring form is `phrase_text`: a `text:` row of fragments that reads -// LEFT-TO-RIGHT as the sentence being formed, with a positionally-aligned `classes:` row -// naming each fragment's token class — i.e. "here is the resulting phrase, and here is what -// each piece is". The lex rule set, the source text, and (in target_model, which knows -// ConcreteSyntaxToken) the concrete-token sequence all PROJECT from this one value, so the -// concat == source_text identity holds by construction and the three can no longer drift -// (§5: the redundancy is removed, not merely checked). A text/classes length or order -// mismatch is caught by execution — the per-phrase emit witnesses go RED on a wrong -// source text or token-class sequence (the fail-closed guard, not a silent zip-to-shorter). +// Authored as a `phrase([...])` of self-contained `span(text:, class:)` rows — each row pairs a +// fragment with its token class in one place (no cross-list join), and reading the `text:` column +// top-to-bottom spells the sentence. The lex rule set, the source text, and (in target_model, +// which knows ConcreteSyntaxToken) the concrete-token sequence all PROJECT from this one value, +// so the concat == source_text identity holds by construction and the three can no longer drift +// (§5: the redundancy is removed, not merely checked). Span order is the only positional authority +// (list order); a wrong order or fragment is caught by execution — the per-phrase emit witnesses +// go RED on a wrong source text or token-class sequence. // // 🟡 layout-in-fragments interim — feature:t6-token-layout — bind gunbc#4761 (T-6 TokenLayout/ // TriviaPolicy) — dissolve-on-arrival: `fragment` splits into (lexeme, trivia) once T-6 @@ -96,29 +95,15 @@ type LexicalPhrase { } -// zip_spans: pair the L→R `text` fragments with their `classes` in lockstep. Recurses to the -// shorter list (a desync surfaces as a wrong projection caught RED by the emit witnesses). -fn zip_spans(text: FreeMonoid, classes: FreeMonoid) -> FreeMonoid { - match text { - Empty => Empty - Cons { head: fragment, tail: rest_text } => - match classes { - Empty => Empty - Cons { head: token_class, tail: rest_classes } => - Cons { - head: TokenSpan { token_class: token_class, fragment: fragment }, - tail: zip_spans(text: rest_text, classes: rest_classes) - } - } - } +// span: one self-contained (text, class) row. `text` is the literal fragment, `class` its token +// class — paired in one place so authoring needs no cross-list index. +fn span(text: String, class: Symbol) -> TokenSpan { + TokenSpan { token_class: class, fragment: text } } -// phrase_text: the authoring form. `text` reads left-to-right as the sentence; each fragment's -// token class is the same-position entry of `classes` (paired by list order). A length/order -// desync surfaces RED in the per-phrase emit witnesses, not as a silent zip-to-shorter. -fn phrase_text(text: List, classes: List) -> LexicalPhrase { - LexicalPhrase { spans: zip_spans(text: text, classes: classes) } +fn phrase(spans: List) -> LexicalPhrase { + LexicalPhrase { spans: spans } } From 5eef0a8453a23e1f4f288e80dd2435bc29bb6885 Mon Sep 17 00:00:00 2001 From: Brian Searls Date: Fri, 19 Jun 2026 23:18:32 +0000 Subject: [PATCH 9/9] WIP: can we make modeled lex rules more elegant somehow? ModeledLexRules { ro --- src/v2/extdeps/languages/bash.dag | 48 +++++++++++++------------------ 1 file changed, 20 insertions(+), 28 deletions(-) diff --git a/src/v2/extdeps/languages/bash.dag b/src/v2/extdeps/languages/bash.dag index 71a06d6220d..dc7a40a1882 100644 --- a/src/v2/extdeps/languages/bash.dag +++ b/src/v2/extdeps/languages/bash.dag @@ -929,18 +929,18 @@ fn bash_env_prefixed_cargo_translation_rules_node() -> Node { ) } +// Bound-word phrase: only the FIXED tokens carry a literal lex rule (the bound var-ref / +// cmd-subst body spellings resolve at emit time). Authored as a `phrase` of fixed spans and +// projected to its lex rules; T-6 binding-carrying spans (the full single-authority migration) +// are the child lane's job. fn bash_assign_cmdsubst_mktemp_lex() -> LexRules { - ModeledLexRules { - root: LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_assign_eq, text: "="), - bash_lex_rule(token_class: ^bash_token_cmdsubst_open, text: "$("), - bash_lex_rule(token_class: ^bash_token_word_mktemp, text: "'mktemp'"), - bash_lex_rule(token_class: ^bash_token_word_minus_d_spaced, text: " '-d'"), - bash_lex_rule(token_class: ^bash_token_rparen, text: ")") - ] - } - } + phrase_to_lex_rules(p: phrase([ + span(text: "=", class: ^bash_token_assign_eq), + span(text: "$(", class: ^bash_token_cmdsubst_open), + span(text: "'mktemp'", class: ^bash_token_word_mktemp), + span(text: " '-d'", class: ^bash_token_word_minus_d_spaced), + span(text: ")", class: ^bash_token_rparen) + ])) } fn bash_assign_cmdsubst_mktemp_target_model() -> TargetModel { @@ -956,13 +956,9 @@ fn bash_assign_cmdsubst_mktemp_target_model() -> TargetModel { } fn bash_assign_concat_gunbc_lex() -> LexRules { - ModeledLexRules { - root: LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_assign_eq, text: "=") - ] - } - } + phrase_to_lex_rules(p: phrase([ + span(text: "=", class: ^bash_token_assign_eq) + ])) } fn bash_assign_concat_gunbc_target_model() -> TargetModel { @@ -994,16 +990,12 @@ fn bash_assign_concat_gunbc_spaced_wrong_target_model() -> TargetModel { } fn bash_env_prefixed_cargo_lex() -> LexRules { - ModeledLexRules { - root: LexRuleSet { - rules: [ - bash_lex_rule(token_class: ^bash_token_assign_eq, text: "="), - bash_lex_rule(token_class: ^bash_token_word_cargo_spaced, text: " 'cargo'"), - bash_lex_rule(token_class: ^bash_token_word_build_spaced, text: " 'build'"), - bash_lex_rule(token_class: ^bash_token_word_quiet_spaced, text: " '--quiet'") - ] - } - } + phrase_to_lex_rules(p: phrase([ + span(text: "=", class: ^bash_token_assign_eq), + span(text: " 'cargo'", class: ^bash_token_word_cargo_spaced), + span(text: " 'build'", class: ^bash_token_word_build_spaced), + span(text: " '--quiet'", class: ^bash_token_word_quiet_spaced) + ])) } fn bash_env_prefixed_cargo_target_model() -> TargetModel {