diff --git a/dsl/extdeps/languages/dag/syntax.dag b/dsl/extdeps/languages/dag/syntax.dag index 875726653c9..b4aa7587d6e 100644 --- a/dsl/extdeps/languages/dag/syntax.dag +++ b/dsl/extdeps/languages/dag/syntax.dag @@ -12,8 +12,8 @@ import std.syntax { ExprBody, BlockBody, TypeBody, ValueBody, NoBody, ServiceBody, ResourceBody, OperatorSpec, SyntaxSpec, - BinOp, Add, Sub, Mul, Div, - Eq, Ne, Lt, Gt, Le, Ge, And, Or, + BinOp, Add, Sub, Mul, Div, Mod, + Eq, Ne, Lt, Gt, Le, Ge, And, Or, NullCoalesce, LiteralValue, LitBool, LitNull } @@ -91,14 +91,17 @@ data dag_non_name_keywords: Map = { "acquire": true, "release": true } -// Operator precedence table. -// || < && < cmp < add < mul < |> < . +// External .dag operator precedence authority. +// This table models external .dag syntax reality, not the current v3-supported +// tokenizer/parser/operator subset. +// || < && < ?? < cmp < add < mul < |> < . // |> and . have binding power entries but their binop field is unused // (the expression loop handles them as pipe/field-access, not binary ops). data dag_operators: List = [ OperatorSpec { symbol: "||", left_bp: 5, right_bp: 6, binop: Or, algebra_field: none }, OperatorSpec { symbol: "&&", left_bp: 7, right_bp: 8, binop: And, algebra_field: none }, + OperatorSpec { symbol: "??", left_bp: 8, right_bp: 9, binop: NullCoalesce, algebra_field: none }, OperatorSpec { symbol: "==", left_bp: 9, right_bp: 10, binop: Eq, algebra_field: none }, OperatorSpec { symbol: "!=", left_bp: 9, right_bp: 10, binop: Ne, algebra_field: none }, OperatorSpec { symbol: "<", left_bp: 11, right_bp: 12, binop: Lt, algebra_field: none }, @@ -109,10 +112,25 @@ data dag_operators: List = [ OperatorSpec { symbol: "-", left_bp: 13, right_bp: 14, binop: Sub, algebra_field: none }, OperatorSpec { symbol: "*", left_bp: 15, right_bp: 16, binop: Mul, algebra_field: none }, OperatorSpec { symbol: "/", left_bp: 15, right_bp: 16, binop: Div, algebra_field: none }, + OperatorSpec { symbol: "%", left_bp: 15, right_bp: 16, binop: Mod, algebra_field: none }, OperatorSpec { symbol: "|>", left_bp: 17, right_bp: 18, binop: none, algebra_field: none }, OperatorSpec { symbol: ".", left_bp: 19, right_bp: 20, binop: none, algebra_field: none } ] +// Current v3 capability projection. +// v3 tokenizer/parser/operator validation consumes these explicit supported +// symbols, then reads precedence/binop facts from `dag_operators`. This avoids +// duplicating operator semantics while still fencing current v3 capability. +// It intentionally excludes external `dag_operators` rows that v3 does not +// parse end-to-end yet, including `??` and `%`. + +data v3_supported_dag_operators: Map = { + "||": true, "&&": true, + "==": true, "!=": true, "<": true, ">": true, "<=": true, ">=": true, + "+": true, "-": true, "*": true, "/": true, + "|>": true, ".": true +} + data dag_syntax_spec: SyntaxSpec = SyntaxSpec { item_forms: dag_item_forms, operators: dag_operators, diff --git a/src/v2/stage0/src/extdeps_languages_dag_syntax.rs b/src/v2/stage0/src/extdeps_languages_dag_syntax.rs index 7c8dc54a996..afd74804c93 100644 --- a/src/v2/stage0/src/extdeps_languages_dag_syntax.rs +++ b/src/v2/stage0/src/extdeps_languages_dag_syntax.rs @@ -1,7 +1,9 @@ // Generated by v2 compiler -- do not edit. // Source module: extdeps.languages.dag.syntax -use crate::std_syntax::BinOp::{Add, And, Div, Eq, Ge, Gt, Le, Lt, Mul, Ne, Or, Sub}; +use crate::std_syntax::BinOp::{ + Add, And, Div, Eq, Ge, Gt, Le, Lt, Mod, Mul, Ne, NullCoalesce, Or, Sub, +}; use crate::std_syntax::BodyKind::{ BlockBody, ExprBody, NoBody, ResourceBody, ServiceBody, TypeBody, ValueBody, }; @@ -193,6 +195,12 @@ pub fn dag_operators() -> Rc>> { right_bp: 8, binop: Some(BinOp::And), algebra_field: None, + }), Rc::new(OperatorSpec { + symbol: "??".to_string(), + left_bp: 8, + right_bp: 9, + binop: Some(BinOp::NullCoalesce), + algebra_field: None, }), Rc::new(OperatorSpec { symbol: "==".to_string(), left_bp: 9, @@ -253,6 +261,12 @@ pub fn dag_operators() -> Rc>> { right_bp: 16, binop: Some(BinOp::Div), algebra_field: None, + }), Rc::new(OperatorSpec { + symbol: "%".to_string(), + left_bp: 15, + right_bp: 16, + binop: Some(BinOp::Mod), + algebra_field: None, }), Rc::new(OperatorSpec { symbol: "|>".to_string(), left_bp: 17, @@ -271,6 +285,30 @@ pub fn dag_operators() -> Rc>> { CACHED.with(|c| c.clone()) } +pub fn v3_supported_dag_operators() -> Rc> { + thread_local! { + static CACHED: Rc> = { + let mut __m = HashMap::new(); + __m.insert("||".to_string(), true); + __m.insert("&&".to_string(), true); + __m.insert("==".to_string(), true); + __m.insert("!=".to_string(), true); + __m.insert("<".to_string(), true); + __m.insert(">".to_string(), true); + __m.insert("<=".to_string(), true); + __m.insert(">=".to_string(), true); + __m.insert("+".to_string(), true); + __m.insert("-".to_string(), true); + __m.insert("*".to_string(), true); + __m.insert("/".to_string(), true); + __m.insert("|>".to_string(), true); + __m.insert(".".to_string(), true); + Rc::new(__m) + }; + } + CACHED.with(|c| c.clone()) +} + pub fn dag_syntax_spec() -> Rc { thread_local! { static CACHED: Rc = { diff --git a/src/v3/compiler/src/bin/regen_tokenize.rs b/src/v3/compiler/src/bin/regen_tokenize.rs index 7156991db89..678f358e010 100644 --- a/src/v3/compiler/src/bin/regen_tokenize.rs +++ b/src/v3/compiler/src/bin/regen_tokenize.rs @@ -519,7 +519,8 @@ fn collect_punct_rows(dag: &Dag, shared_syntax: &SharedSyntaxAuthority) -> Vec<( let mut covered_kinds = BTreeSet::new(); for pattern in &shared_syntax.operators { - match classify_shared_operator_for_tokenizer(pattern) { + match classify_shared_operator_for_tokenizer(pattern, &shared_syntax.v3_supported_operators) + { SharedOperatorTokenizerBoundary::Tokenized { kind } => { assert!( punct_variants.contains(kind), @@ -610,16 +611,24 @@ fn keyword_spelling_for_token_kind(kind: &str) -> String { .to_ascii_lowercase() } -// SG-1a operator bridge: shared operators still lower through raw-source reads, -// so every `dag_operators` symbol must be classified explicitly here as either -// tokenizer punctuation or parser-only debt. Unknown symbols panic so upstream -// authority edits cannot silently disappear. +// SG-1a operator bridge: shared operators still lower through raw-source reads. +// `v3_supported_dag_operators` is the single support/exclusion projection; this +// table only maps supported symbols to their current tokenizer variant. enum SharedOperatorTokenizerBoundary { Tokenized { kind: &'static str }, ParserOnlyDebt { reason: &'static str }, } -fn classify_shared_operator_for_tokenizer(pattern: &str) -> SharedOperatorTokenizerBoundary { +fn classify_shared_operator_for_tokenizer( + pattern: &str, + v3_supported: &BTreeSet, +) -> SharedOperatorTokenizerBoundary { + if !v3_supported.contains(pattern) { + return SharedOperatorTokenizerBoundary::ParserOnlyDebt { + reason: "operator is declared in external dag_operators but excluded from \ + v3_supported_dag_operators until v3 parses it end-to-end", + }; + } match pattern { "==" => SharedOperatorTokenizerBoundary::Tokenized { kind: "EqEq" }, "!=" => SharedOperatorTokenizerBoundary::Tokenized { kind: "NotEq" }, @@ -635,18 +644,10 @@ fn classify_shared_operator_for_tokenizer(pattern: &str) -> SharedOperatorTokeni "||" => SharedOperatorTokenizerBoundary::Tokenized { kind: "PipePipe" }, "|>" => SharedOperatorTokenizerBoundary::Tokenized { kind: "PipeArrow" }, "." => SharedOperatorTokenizerBoundary::Tokenized { kind: "Dot" }, - "??" => SharedOperatorTokenizerBoundary::ParserOnlyDebt { - reason: "null-coalescing is declared in shared syntax but the v3 tokenizer/parser \ - surface does not yet admit it as punctuation", - }, - "%" => SharedOperatorTokenizerBoundary::ParserOnlyDebt { - reason: "modulo is declared in shared syntax but the v3 tokenizer/parser surface \ - does not yet admit it as punctuation", - }, other => panic!( - "shared syntax operator `{other}` has no SG-1a tokenizer bridge classification. \ - Update `classify_shared_operator_for_tokenizer` to mark it as tokenizer-owned \ - punctuation or explicit parser-only debt so the authority boundary stays fail-closed." + "`v3_supported_dag_operators` includes `{other}`, but the SG-1a tokenizer bridge has \ + no TokenKind mapping for it. Add tokenizer/parser/operator support or remove it from \ + the v3 projection." ), } } @@ -654,6 +655,7 @@ fn classify_shared_operator_for_tokenizer(pattern: &str) -> SharedOperatorTokeni struct SharedSyntaxAuthority { keywords: Vec, operators: Vec, + v3_supported_operators: BTreeSet, } impl SharedSyntaxAuthority { @@ -670,9 +672,15 @@ impl SharedSyntaxAuthority { extract_balanced_section(source, "data dag_operators", '[', ']'), "symbol", ); + let v3_supported_operators: BTreeSet<_> = parse_all_string_literals( + extract_balanced_section(source, "data v3_supported_dag_operators", '{', '}'), + ) + .into_iter() + .collect(); Self { keywords, operators, + v3_supported_operators, } } } @@ -730,6 +738,17 @@ fn parse_named_string_fields(section: &str, field_name: &str) -> Vec { out } +fn parse_all_string_literals(section: &str) -> Vec { + let mut out = Vec::new(); + let mut rest = section; + while let Some(idx) = rest.find('"') { + let (value, consumed) = parse_string_literal(&rest[idx..]); + out.push(value); + rest = &rest[idx + consumed..]; + } + out +} + fn parse_string_literal(source: &str) -> (String, usize) { assert!( source.starts_with('"'), diff --git a/src/v3/compiler/src/regen_parse_tables_emit.rs b/src/v3/compiler/src/regen_parse_tables_emit.rs index ea6a001159b..123f3a0f15f 100644 --- a/src/v3/compiler/src/regen_parse_tables_emit.rs +++ b/src/v3/compiler/src/regen_parse_tables_emit.rs @@ -17,24 +17,22 @@ //! hermetic integration snapshot test (compare in-memory only — avoids a //! `cargo run` subprocess that blows the 2s per-test ratchet on cold CI). //! -//! **Cross-validation against shared authority.** `dag_operators` in -//! `syntax.dag` owns the canonical (symbol, left_bp, right_bp, binop) for -//! every binary operator. Each `BinaryOpRow` in `parse_tables.dag` is -//! checked against that authority: the row's `operator_symbol` must -//! appear in `dag_operators`, and its declared `level` must be consistent -//! with the shared row's binding power (the coarse parser precedence -//! level covers a contiguous bp range — see `level_for_bp`). If the -//! shared authority shifts an operator's bp into a different level, or -//! drops a symbol, regen fails closed here. This closes the token ↔ -//! symbol ↔ precedence drift surface the initial SG-2c-1 landing left -//! open (`parse_tables.dag` used to carry `(symbol, level)` without any -//! structural tie to the shared authority's bp). +//! **Cross-validation against v3 projection of shared authority.** +//! `dag_operators` in `syntax.dag` owns external .dag syntax reality, including +//! operators v3 does not yet parse end-to-end. `v3_supported_dag_operators` is +//! the explicit current v3 capability projection as a symbol subset. Each +//! `BinaryOpRow` in `parse_tables.dag` is checked against that projection: the +//! row's `operator_symbol` must appear in `v3_supported_dag_operators`, while +//! binding power and binop facts are read from the corresponding +//! `dag_operators` row (the coarse parser precedence level covers a contiguous +//! bp range — see `level_for_bp`). If the v3 projection shifts, regen fails +//! closed here without forcing v3 to support every externally modeled operator. //! -//! **SG-1a scaffold extension.** `syntax.dag`'s `dag_operators` body -//! still lowers as `ValueBody::Unparsed` (same scaffold `regen_tokenize` -//! extends), so this bridge reads the raw source text. When the shared -//! authority lowers structurally, the raw-text extractor folds into a -//! typed read — same dissolution trigger as `regen_tokenize`. +//! **SG-1a scaffold extension.** `syntax.dag`'s operator-list bodies still +//! lower as `ValueBody::Unparsed` (same scaffold `regen_tokenize` extends), so +//! this bridge reads the raw source text. When the shared authority lowers +//! structurally, the raw-text extractor folds into a typed read — same +//! dissolution trigger as `regen_tokenize`. use std::collections::BTreeSet; use std::fmt; @@ -81,7 +79,7 @@ impl fmt::Display for RenderParseTablesGeneratedError { /// `parse_tables_generated.rs`, format with `rustfmt --emit stdout`. /// /// Validation: `BinaryOpRow` vs `tokenize.dag` + `operators.dag` + shared-syntax -/// `dag_operators`; `TopLevelItemKwRow` vs `tokenize.dag` + `Kw`-strip dispatch +/// `v3_supported_dag_operators`; `TopLevelItemKwRow` vs `tokenize.dag` + `Kw`-strip dispatch /// labels (SG-2c-2/SG-2c-3); `SoftKeywordIdentRow` vs `tokenize.dag` + shared keyword set /// (SG-2c-5); `PrimaryPrefixRow` / `PrimaryAtomRow` vs `tokenize.dag` + regen /// dispatch labels (SG-2c-6/SG-2c-7). @@ -95,7 +93,7 @@ pub fn render_parse_tables_generated_rs( ) -> Result { let tables_dag = compile_authority(parse_tables_source, parse_tables_file)?; let tokenize_dag = compile_authority(tokenize_source, tokenize_file)?; - let shared_operators = extract_shared_operator_bps(shared_syntax_source); + let shared_operators = extract_v3_supported_operator_bps(shared_syntax_source); let shared_keywords = extract_shared_keyword_set(shared_syntax_source); let token_variants = collect_variant_labels(&tokenize_dag, "TokenKind"); @@ -278,25 +276,28 @@ fn collect_binary_op_rows( `OperatorKind` mapping in `operators.dag::from_symbol`" ) }); - // Structural cross-validation against the shared syntax - // authority at `dsl/extdeps/languages/dag/syntax.dag`. Three + // Structural cross-validation against the v3-supported + // projection in `dsl/extdeps/languages/dag/syntax.dag`. Three // joins, closing three drift surfaces: - // (1) symbol ∈ `dag_operators` — operator is declared - // canonically; deletion there fails regen here. - // (2) shared `left_bp` implies the declared `level` via + // (1) symbol ∈ `v3_supported_dag_operators` — operator is in + // the current v3 tokenizer/parser/operator capability + // subset; capability drift fails regen here without + // shrinking external `dag_operators`. + // (2) projected `left_bp` implies the declared `level` via // `level_for_bp` — precedence drift fails closed. // (3) `operators.dag::from_symbol(symbol)` returns an - // `OperatorKind` whose leaf variant matches the shared + // `OperatorKind` whose leaf variant matches the projected // `binop` name — `symbol → OperatorKind` drift between - // `operators.dag` and `dag_operators` fails closed. + // `operators.dag` and `v3_supported_dag_operators` fails closed. // Together, these turn `parse_tables.dag` into a structural - // projection of the shared authority rather than a parallel + // projection of the v3 capability subset rather than a parallel // restatement of it. let shared = shared_operators.get(&operator_symbol).unwrap_or_else(|| { panic!( "`parse_tables.dag::{name}`: operator_symbol `{operator_symbol}` is not declared \ - in `dag_operators` at `dsl/extdeps/languages/dag/syntax.dag`. Every binary-operator \ - row must be grounded in the shared-syntax authority so drift fails closed." + in `v3_supported_dag_operators` at `dsl/extdeps/languages/dag/syntax.dag`. Every \ + v3 binary-operator row must be grounded in the v3 capability projection so drift \ + fails closed without shrinking external `dag_operators`." ) }); let expected_level = level_for_bp(shared.left_bp).unwrap_or_else(|| { @@ -310,7 +311,7 @@ fn collect_binary_op_rows( assert!( expected_level == level, "`parse_tables.dag::{name}`: declared level `{level}` disagrees with shared-syntax \ - authority. `dag_operators` assigns `{operator_symbol}` `left_bp = {}` which \ + authority. `v3_supported_dag_operators` assigns `{operator_symbol}` `left_bp = {}` which \ implies level `{expected_level}`. Either update the row to match or reclassify \ `left_bp = {}` in `level_for_bp`.", shared.left_bp, @@ -320,7 +321,7 @@ fn collect_binary_op_rows( assert!( actual_binop == shared.binop_name, "`parse_tables.dag::{name}`: `operators.dag::from_symbol(\"{operator_symbol}\")` returns \ - `OperatorKind` leaf `{actual_binop}`, but `dag_operators` in \ + `OperatorKind` leaf `{actual_binop}`, but `v3_supported_dag_operators` in \ `dsl/extdeps/languages/dag/syntax.dag` records `binop: {}` for the same symbol. One of \ the two authorities drifted — fix both to agree before shipping.", shared.binop_name @@ -1056,26 +1057,46 @@ fn rustfmt_stdout(source: &str) -> Result { String::from_utf8(output.stdout).map_err(|e| format!("rustfmt stdout utf-8: {e}")) } -/// SG-1a scaffold extension: read `data dag_operators: List` -/// directly from `syntax.dag` source text because that body still lowers -/// as `ValueBody::Unparsed` (same state `regen_tokenize` encountered). -/// A trimmed view of `OperatorSpec` sufficient for regen -/// cross-validation against `parse_tables.dag`. +/// SG-1a scaffold extension: read operator projection data directly from +/// `syntax.dag` source text because `dag_operators` still lowers as +/// `ValueBody::Unparsed` (same state `regen_tokenize` encountered). A trimmed +/// view of `OperatorSpec` is sufficient for regen cross-validation against +/// `parse_tables.dag`. +#[derive(Clone)] pub(crate) struct SharedOperatorSpec { pub left_bp: i64, - /// Variant name from the shared-authority `binop` field, e.g. + /// Variant name from the external authority's `binop` field, e.g. /// `"Or"`, `"Add"`, `"Eq"`. Matches the leaf variant label /// inside the corresponding `OperatorKind` (see /// `operator_kind_binop_name`), which is the structural join /// that closes `symbol → OperatorKind` drift between - /// `operators.dag` and `dag_operators`. + /// `operators.dag` and `v3_supported_dag_operators`. pub binop_name: String, } -/// Returns a symbol → (left_bp, binop_name) map. Fails closed on malformed input. -/// Dissolution trigger: when `dag_operators` lowers structurally, swap -/// this extractor for a typed read — same lane as `regen_tokenize`'s +/// Returns a symbol → (left_bp, binop_name) map for the current v3-supported +/// operator projection by filtering external `dag_operators` with the explicit +/// `v3_supported_dag_operators` symbol set. Fails closed on malformed input. +/// Dissolution trigger: when `dag_operators` lowers structurally, swap this +/// extractor for a typed read — same lane as `regen_tokenize`'s /// `assert_shared_syntax_raw_source_scaffold_still_required`. +fn extract_v3_supported_operator_bps( + source: &str, +) -> std::collections::BTreeMap { + let supported_symbols = extract_v3_supported_operator_symbols(source); + let external_operators = extract_shared_operator_bps(source); + let mut out = std::collections::BTreeMap::new(); + for symbol in supported_symbols { + let spec = external_operators.get(&symbol).unwrap_or_else(|| { + panic!( + "`v3_supported_dag_operators` symbol `{symbol}` is not present in external `dag_operators`" + ) + }); + out.insert(symbol, spec.clone()); + } + out +} + fn extract_shared_operator_bps( source: &str, ) -> std::collections::BTreeMap { @@ -1134,6 +1155,24 @@ fn extract_shared_operator_bps( out } +fn extract_v3_supported_operator_symbols(source: &str) -> BTreeSet { + let section = extract_balanced_section(source, "data v3_supported_dag_operators", '{', '}'); + let mut out = BTreeSet::new(); + let mut rest = section; + loop { + let Some(quote_idx) = rest.find('"') else { + break; + }; + let (symbol, consumed) = parse_string_literal(&rest[quote_idx..]); + assert!( + out.insert(symbol.clone()), + "duplicate `v3_supported_dag_operators` row for symbol `{symbol}`" + ); + rest = &rest[quote_idx + consumed..]; + } + out +} + fn extract_shared_keyword_set(source: &str) -> BTreeSet { let section = extract_balanced_section(source, "data dag_keyword_set", '{', '}'); let mut out = BTreeSet::new(); @@ -1153,10 +1192,10 @@ fn extract_shared_keyword_set(source: &str) -> BTreeSet { } /// Variant-name projection of an `OperatorKind` — the leaf label inside -/// the nested enum (e.g. `Or`, `Add`, `Eq`). The shared-syntax authority's +/// the nested enum (e.g. `Or`, `Add`, `Eq`). The v3-supported projection's /// `OperatorSpec.binop` uses the same vocabulary, so comparing on this /// label is the structural join that closes `symbol → OperatorKind` -/// drift between `operators.dag` and `dag_operators` — without a +/// drift between `operators.dag` and `v3_supported_dag_operators` — without a /// parallel name-to-name table (the projection is a pure value read). fn operator_kind_binop_name(op: OperatorKind) -> &'static str { match op { @@ -1210,3 +1249,36 @@ fn parse_string_literal(source: &str) -> (String, usize) { } panic!("unterminated string literal in shared syntax authority"); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn unsupported_external_operator_duplicates_fail_closed() { + let source = r#" +data dag_operators: List = [ + OperatorSpec { symbol: "||", left_bp: 5, right_bp: 6, binop: Or, algebra_field: none }, + OperatorSpec { symbol: "??", left_bp: 8, right_bp: 9, binop: NullCoalesce, algebra_field: none }, + OperatorSpec { symbol: "??", left_bp: 8, right_bp: 9, binop: NullCoalesce, algebra_field: none } +] + +data v3_supported_dag_operators: Map = { + "||": true +} +"#; + let err = match std::panic::catch_unwind(|| extract_v3_supported_operator_bps(source)) { + Ok(_) => panic!("duplicate unsupported external operator should panic"), + Err(err) => err, + }; + let msg = err + .downcast_ref::() + .map(String::as_str) + .or_else(|| err.downcast_ref::<&str>().copied()) + .unwrap_or(""); + assert!( + msg.contains("duplicate `dag_operators` row for symbol `??`"), + "unexpected panic: {msg}" + ); + } +} diff --git a/src/v3/compiler/tests/integration/sg1_tokenize_authority_test.rs b/src/v3/compiler/tests/integration/sg1_tokenize_authority_test.rs index 93a3ac0399e..1346139bad1 100644 --- a/src/v3/compiler/tests/integration/sg1_tokenize_authority_test.rs +++ b/src/v3/compiler/tests/integration/sg1_tokenize_authority_test.rs @@ -237,6 +237,14 @@ fn shared_operator_boundary_is_explicit_and_fail_closed() { extract_balanced_section(SHARED_SYNTAX_DAG, "data dag_operators", '[', ']'), "symbol", ); + let v3_supported_operators: BTreeSet<_> = parse_map_string_keys(extract_balanced_section( + SHARED_SYNTAX_DAG, + "data v3_supported_dag_operators", + '{', + '}', + )) + .into_iter() + .collect(); let punct_kind_decl = dag .declaration_by_name("PunctTokenKind") .expect("PunctTokenKind declaration"); @@ -253,7 +261,7 @@ fn shared_operator_boundary_is_explicit_and_fail_closed() { let mut shared_tokenized_kinds = BTreeSet::new(); for pattern in &shared_operators { - match classify_shared_operator_for_tokenizer(pattern) { + match classify_shared_operator_for_tokenizer(pattern, &v3_supported_operators) { SharedOperatorTokenizerBoundary::Tokenized { kind } => { assert!( punct_variant_labels.contains(kind), @@ -261,6 +269,12 @@ fn shared_operator_boundary_is_explicit_and_fail_closed() { ); shared_tokenized_kinds.insert(kind.to_string()); } + SharedOperatorTokenizerBoundary::ParserOnlyDebt { reason } => { + assert!( + !reason.is_empty(), + "parser-only shared operator `{pattern}` should document the v3 boundary" + ); + } } } @@ -314,9 +328,19 @@ fn keyword_spelling_for_token_kind(kind: &str) -> String { enum SharedOperatorTokenizerBoundary { Tokenized { kind: &'static str }, + ParserOnlyDebt { reason: &'static str }, } -fn classify_shared_operator_for_tokenizer(pattern: &str) -> SharedOperatorTokenizerBoundary { +fn classify_shared_operator_for_tokenizer( + pattern: &str, + v3_supported: &BTreeSet, +) -> SharedOperatorTokenizerBoundary { + if !v3_supported.contains(pattern) { + return SharedOperatorTokenizerBoundary::ParserOnlyDebt { + reason: "operator is declared in external dag_operators but excluded from \ + v3_supported_dag_operators until v3 parses it end-to-end", + }; + } match pattern { "==" => SharedOperatorTokenizerBoundary::Tokenized { kind: "EqEq" }, "!=" => SharedOperatorTokenizerBoundary::Tokenized { kind: "NotEq" }, @@ -333,8 +357,8 @@ fn classify_shared_operator_for_tokenizer(pattern: &str) -> SharedOperatorTokeni "|>" => SharedOperatorTokenizerBoundary::Tokenized { kind: "PipeArrow" }, "." => SharedOperatorTokenizerBoundary::Tokenized { kind: "Dot" }, other => panic!( - "shared syntax operator `{other}` must be classified as tokenizer punctuation or \ - removed until v3 supports it end-to-end" + "`v3_supported_dag_operators` includes `{other}`, but the SG-1 tokenizer bridge has \ + no TokenKind mapping for it" ), } } diff --git a/src/v3/compiler/tests/integration/sg2c1_parse_tables_authority_test.rs b/src/v3/compiler/tests/integration/sg2c1_parse_tables_authority_test.rs index b90eb84dd7c..8185deb174c 100644 --- a/src/v3/compiler/tests/integration/sg2c1_parse_tables_authority_test.rs +++ b/src/v3/compiler/tests/integration/sg2c1_parse_tables_authority_test.rs @@ -187,44 +187,72 @@ fn dag_binary_operators_have_token_parse_table_and_operator_mapping() { ); } - for spec in shared_dag_operator_specs(SHARED_SYNTAX_DAG) { + for spec in v3_supported_dag_operator_specs(SHARED_SYNTAX_DAG) { let Some(binop) = spec.binop else { continue; }; let expected_token = token_kind_for_shared_dag_operator(&spec.symbol); assert!( token_variant_names.contains(expected_token), - "`dag_operators` symbol `{}` expects TokenKind::{expected_token} in tokenize.dag", + "`v3_supported_dag_operators` symbol `{}` expects TokenKind::{expected_token} in tokenize.dag", spec.symbol ); let row_token = rows_by_symbol.get(&spec.symbol).unwrap_or_else(|| { panic!( - "`dag_operators` binary operator `{}` has no BinaryOpRow in parse_tables.dag", + "`v3_supported_dag_operators` binary operator `{}` has no BinaryOpRow in parse_tables.dag", spec.symbol ) }); assert_eq!( row_token, expected_token, - "`dag_operators` binary operator `{}` maps to TokenKind::{expected_token}, but \ + "`v3_supported_dag_operators` binary operator `{}` maps to TokenKind::{expected_token}, but \ parse_tables.dag routes TokenKind::{row_token}", spec.symbol ); let Some(operator_kind) = operators::from_symbol(&spec.symbol) else { panic!( - "`dag_operators` binary operator `{}` has no operators.dag::from_symbol mapping", + "`v3_supported_dag_operators` binary operator `{}` has no operators.dag::from_symbol mapping", spec.symbol ); }; let actual_binop = operator_kind_binop_name(operator_kind); assert_eq!( actual_binop, binop, - "`dag_operators` binary operator `{}` records binop `{binop}`, but \ + "`v3_supported_dag_operators` binary operator `{}` records binop `{binop}`, but \ operators.dag maps it to `{actual_binop}`", spec.symbol ); } } +#[test] +fn dag_operator_authority_keeps_external_rows_split_from_v3_projection() { + let all: std::collections::BTreeSet<_> = shared_dag_operator_specs(SHARED_SYNTAX_DAG) + .into_iter() + .map(|spec| spec.symbol) + .collect(); + let v3_supported: std::collections::BTreeSet<_> = + v3_supported_dag_operator_specs(SHARED_SYNTAX_DAG) + .into_iter() + .map(|spec| spec.symbol) + .collect(); + + for symbol in ["??", "%"] { + assert!( + all.contains(symbol), + "`dag_operators` must retain external operator row `{symbol}`" + ); + assert!( + !v3_supported.contains(symbol), + "`v3_supported_dag_operators` must exclude unsupported v3 operator `{symbol}`" + ); + } + assert!( + v3_supported.is_subset(&all), + "`v3_supported_dag_operators` should be an explicit subset of `dag_operators`" + ); +} + #[test] fn every_top_level_item_kw_row_token_variant_is_a_token_kind_variant() { let tables_dag = compile_to_dag(PARSE_TABLES_DAG, "src/v3/compiler/parse_tables.dag") @@ -739,13 +767,36 @@ fn primary_atom_rows_cover_exactly_the_tokens_parse_primary_atomic_arm() { ); } +#[derive(Clone)] struct SharedDagOperatorSpec { symbol: String, binop: Option, } fn shared_dag_operator_specs(source: &str) -> Vec { - extract_balanced_section(source, "data dag_operators", '[', ']') + dag_operator_specs(source, "data dag_operators") +} + +fn v3_supported_dag_operator_specs(source: &str) -> Vec { + let specs_by_symbol: std::collections::BTreeMap<_, _> = shared_dag_operator_specs(source) + .into_iter() + .map(|spec| (spec.symbol.clone(), spec)) + .collect(); + v3_supported_dag_operator_symbols(source) + .into_iter() + .map(|symbol| { + specs_by_symbol.get(&symbol).cloned().unwrap_or_else(|| { + panic!( + "`v3_supported_dag_operators` symbol `{symbol}` is not present in external \ + `dag_operators`" + ) + }) + }) + .collect() +} + +fn dag_operator_specs(source: &str, anchor: &str) -> Vec { + extract_balanced_section(source, anchor, '[', ']') .split("OperatorSpec") .skip(1) .filter_map(|entry| { @@ -765,6 +816,15 @@ fn shared_dag_operator_specs(source: &str) -> Vec { .collect() } +fn v3_supported_dag_operator_symbols(source: &str) -> Vec { + parse_all_string_literals(extract_balanced_section( + source, + "data v3_supported_dag_operators", + '{', + '}', + )) +} + fn token_kind_for_shared_dag_operator(symbol: &str) -> &'static str { match symbol { "==" => "EqEq", @@ -780,8 +840,8 @@ fn token_kind_for_shared_dag_operator(symbol: &str) -> &'static str { "&&" => "AmpAmp", "||" => "PipePipe", other => panic!( - "`dag_operators` binary operator `{other}` is not supported by the v3 token/parser/operator chain; \ - delete the row until the full chain lands" + "`v3_supported_dag_operators` binary operator `{other}` is not supported by the v3 token/parser/operator chain; \ + remove it from the v3 projection until the full chain lands" ), } } @@ -843,6 +903,20 @@ fn extract_balanced_section<'a>(source: &'a str, anchor: &str, open: char, close panic!("unterminated `{anchor}` section"); } +fn parse_all_string_literals(source: &str) -> Vec { + let mut out = Vec::new(); + let mut rest = source; + while let Some(start) = rest.find('"') { + let tail = &rest[start + 1..]; + let end = tail + .find('"') + .unwrap_or_else(|| panic!("unterminated string literal in `{source}`")); + out.push(tail[..end].to_string()); + rest = &tail[end + 1..]; + } + out +} + fn string_literal_after(source: &str, label: &str) -> Option { let tail = source.split_once(label)?.1.trim_start(); let tail = tail.strip_prefix('"')?;