diff --git a/.conformance-review/Rust.test.stg b/.conformance-review/Rust.test.stg index 1a31d0bc..47b9433d 100644 --- a/.conformance-review/Rust.test.stg +++ b/.conformance-review/Rust.test.stg @@ -31,10 +31,10 @@ * - Parser "members" are injected as an `impl` block on the generated parser * via `@parser::members`; helpers are called `self.foo()` / `self.pred(v)`. * - The generated listener trait is `TListener` with snake_case - * `enter_x`/`exit_x` and a defaulted `visit_terminal`; a test listener is a - * unit struct implementing it, walked by `ParseTreeWalker::walk`. - * - Downcasting a base `ParserRuleContext` to a labeled-alt / concrete - * context uses `.downcast_ref::().unwrap()`. + * `enter_x`/`exit_x` and defaulted terminal/error-node visitors; a test + * listener is a unit struct implementing it, walked by `ParseTreeWalker::walk`. + * - Casts from an active `ParserRuleContext` to a labeled-alt / concrete + * borrowing view use the generated `__active_context_view` adapter. */ writeln(s) ::= <);>> @@ -55,7 +55,7 @@ Not(v) ::= "!" // target that renders a real assertion receives uncompilable Java syntax. Assert(s) ::= "" -Cast(t,v) ::= "().downcast_ref::\<>(self.base.token_store()).unwrap()" +Cast(t,v) ::= "__active_context_view::\<>(, self.base.active_invocation_states(), self.base.parse_tree_storage(), self.base.token_store()).unwrap()" // Rust has no `&str + &str` operator, and descriptors chain Append/AppendStr // with bare string literals on the left (`"(" + $ID.text + …` in Java terms), @@ -280,7 +280,7 @@ impl TListener for LeafListener { WalkListener(s) ::= << let mut listener = LeafListener::default(); -ParseTreeWalker::walk(&mut listener, , self.base.token_store()); +ParseTreeWalker::walk_with_invocation_states(&mut listener, , self.base.active_invocation_states()); >> // Java needs a custom context superclass because its default RuleContext does @@ -379,9 +379,9 @@ impl TListener for LeafListener { DeclareContextListGettersFunction() ::= << fn foo() { - let s: Option\\> = None; - let _a: Vec\\> = s.as_ref().unwrap().a_all(); - let _b: Vec\\> = s.as_ref().unwrap().b_all(); + let s: Option\ = None; + let _a: Vec\ = s.as_ref().unwrap().a_all(); + let _b: Vec\ = s.as_ref().unwrap().b_all(); } >> diff --git a/.conformance-review/rust-test-stg-honest-reference-gap.md b/.conformance-review/rust-test-stg-honest-reference-gap.md index 31b42552..365d2468 100644 --- a/.conformance-review/rust-test-stg-honest-reference-gap.md +++ b/.conformance-review/rust-test-stg-honest-reference-gap.md @@ -12,7 +12,7 @@ > translation (`src/bin_support/embedded.rs` — the Rust analog of ANTLR's > `ActionTranslator`). The four capability axes below are now generated: > an output sink (`self.output()`), typed context views with positional -> accessors and public attribute fields (`FromRuleContext` downcast), +> accessors and public attribute fields (`FromRuleNode` conversion), > typed attrs snapshots replacing the int-only `int_return` map, and > `@members` as real struct fields / impl items. Listener traits and a > typed walker bridge cover the listener suite. Validating the reference diff --git a/CHANGELOG.md b/CHANGELOG.md index 0b988061..3db3b13f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,10 +6,16 @@ - Buffered tokens now live once in a compact `TokenStore` and are addressed by `TokenId`; public access uses borrowing `TokenView` values. -- `CommonTokenStream` owns its `TokenStore` directly, parse-tree nodes store - only `TokenId`, and token-dependent tree APIs take `&TokenStore`. -- Generated `parse()` helpers return `ParsedFile` so completed trees retain - ownership of their canonical token store. +- `CommonTokenStream` owns its `TokenStore` directly. `BaseParser` owns one flat + `ParseTreeStorage`; `NodeId` addresses compact records, rule children are + pooled ranges, and terminal/error records store only `TokenId`. +- Recursive owning `ParseTree`, `RuleNode`, `ParserRuleContext` children, and + terminal token wrappers are removed. Public tree access uses borrowing + `Node`, `RuleNodeView`, `TerminalNodeView`, and `ErrorNodeView` values. +- Generated typed contexts are borrowing views, and listener traversal runs + iteratively over flat storage without recreating recursive context objects. +- Generated `parse()` helpers return `ParsedFile`, which owns the token store, + flat CST storage, and root ID. Direct rule methods return `NodeId`. - `CommonToken`, `TokenRef`, and token factories are removed. `TokenSource` implementations write directly to `TokenSink`. - Speculative parser nodes, child sequences, recovery diagnostics, and uncommon diff --git a/README.md b/README.md index b5eb2c6d..74313f11 100644 --- a/README.md +++ b/README.md @@ -132,7 +132,7 @@ use generated::json_lexer::JsonLexer; fn main() -> Result<(), antlr4_runtime::AntlrError> { let parsed = json::parse(r#"{"a":1}"#, JsonLexer::new, Json::json)?; - println!("{}", parsed.tree().text(parsed.tokens())); + println!("{}", parsed.tree().text()); Ok(()) } ``` @@ -152,10 +152,14 @@ fn main() -> Result<(), antlr4_runtime::AntlrError> { result: tree, parser, } = output; - let tokens = parser.into_token_store(); - - println!("{} errors across {} tokens", syntax_errors, tokens.len()); - println!("{}", tree.text(&tokens)); + let parsed = parser.into_parsed_file(tree); + + println!( + "{} errors across {} tokens", + syntax_errors, + parsed.tokens().len() + ); + println!("{}", parsed.tree().text()); Ok(()) } ``` @@ -174,7 +178,7 @@ fn main() -> Result<(), antlr4_runtime::AntlrError> { let mut parser = Json::new(tokens); let tree = parser.json()?; - println!("{}", tree.text(parser.token_store())); + println!("{}", parser.node(tree).text()); Ok(()) } ``` @@ -193,11 +197,12 @@ grammar's documentation. Calling the wrong rule can still recover and return a parse tree with error nodes, so check parser diagnostics when adding a new input form. -## Token API Migration +## Token and Tree API Migration -The compact token store is a breaking runtime/generator change. Regenerate all -generated lexers and parsers with the matching `antlr4-rust-gen`; code generated -against the pointer-owned token API does not compile against this runtime. +The compact token and flat CST stores are breaking runtime/generator changes. +Regenerate all generated lexers and parsers with the matching +`antlr4-rust-gen`; code generated against the pointer-owned token or recursive +tree APIs does not compile against this runtime. `CommonToken`, `TokenRef`, and token factories are removed. Custom token sources now append a `TokenSpec` directly to the supplied `TokenSink` and return its @@ -206,12 +211,17 @@ now append a `TokenSpec` directly to the supplied `TokenSink` and return its should provide `source_text()` when the complete UTF-8 input can be shared; otherwise token text is stored explicitly in the sparse side pool. -`CommonTokenStream` owns its `TokenStore` directly. Parse-tree nodes contain -only `TokenId` values, so token-dependent tree methods take `&TokenStore`. -Generated `parse()` returns `ParsedFile`, which owns both the token store and -entry-rule result. Access them through `tokens()` and `tree()`. Direct rule calls -can resolve their returned tree through `parser.token_store()` while the parser -is alive, or consume the parser with `into_token_store()`. +`CommonTokenStream` owns its `TokenStore` directly. `BaseParser` owns one +`ParseTreeStorage`: nodes are addressed by `NodeId`, every rule child list is a +range in one shared edge pool, and terminal/error records contain only +`TokenId`. `Node`, `RuleNodeView`, and terminal/error views borrow the stores; +there is no recursive `ParserRuleContext` ownership graph or legacy materializer. + +Generated `parse()` returns `ParsedFile`, which owns the token store, flat CST, +and root ID. Access the root through `tree()`, inspect storage metrics through +`storage().stats()`, or resolve another ID through `node()`. Direct rule calls +return `NodeId`; use `parser.node(id)` while the parser is alive, or consume the +parser with `into_parsed_file(id)`. Token IDs cover indices through `u32::MAX`. Source scalar/byte offsets, line numbers, and columns are limited to `u32::MAX - 1` (4,294,967,294); @@ -452,46 +462,10 @@ group (**> 1.0** means Rust is faster than Go; **< 1.0** means slower): | Language | Fixtures | Rust vs Go (parse time) | |----------|---------:|-------------------------| -| Kotlin | 4 | ~18.6× faster | -| Java | 4 | ~2.7× faster | -| C# | 4 | ~1.7× faster | -| Trino SQL| 5 | ~2.5× faster | - -### Recognition arena results - -Issue [#83](https://github.com/ophi-dev/antlr-rust-runtime/issues/83) -replaces the interpreted parser's speculative `Rc` node lists with compact IDs -in a parser-owned arena. To isolate that path from generated-parser -optimizations, the benchmark harness routed each Kotlin entry rule directly -through ATN interpretation and built the full CST. These are same-machine -results against `c068688db` on an Apple M3 Pro: - -| Fixture | Baseline avg | Arena avg | Time | Allocations / parse | Allocated bytes / parse | -|---------|-------------:|----------:|-----:|--------------------:|------------------------:| -| `jetbrains-kotlin-lazy-bodies-test.kt` | 3.166 ms | 2.498 ms | -21.1% | 44,842 → 35,558 (-20.7%) | 20.39 MB → 18.85 MB (-7.5%) | -| `kotlinx-coroutines-flow-limit.kt` | 3.407 ms | 2.544 ms | -25.3% | 45,005 → 38,552 (-14.3%) | 19.02 MB → 18.00 MB (-5.3%) | -| `ktor-openapi-describe-route-test.kt` | 27.670 ms | 16.686 ms | -39.7% | 355,148 → 266,315 (-25.0%) | 60.22 MB → 45.82 MB (-23.9%) | -| `ktor-openapi-security-scheme-inference-test.kt` | 14.577 ms | 9.462 ms | -35.1% | 210,051 → 164,095 (-21.9%) | 38.75 MB → 30.96 MB (-20.1%) | - -Timings are averages from 100 measured parses after 10 warmups. Allocation -figures come from a separate counting-allocator run of 100 measured parses -after five warmups. On the largest fixture, maximum RSS fell from 37.29 MB to -32.83 MB (-12.0%) and macOS peak memory footprint fell from 34.87 MB to -30.44 MB (-12.7%). Arena instrumentation on the 4.8 KB lazy-bodies fixture -reported 4,876 appended nodes / 1,667 live nodes and 9,191 appended links / -1,667 live links. - -Rust is faster than Go on average in all four language groups, with -Kotlin leading dramatically (expression-ladder memoization in the generated -walker). Lexer DFAs are compiled at generation time and embedded in the -generated lexer, so tokenization needs no warmup at all; learned parser -decision DFAs are shared across parser instances, so repeated parses of the -same grammar — the common case for a CLI tool or language server — skip -relearning entirely. Shared grammar-level lookahead caches likewise amortize -left-recursive loop prediction across parses. Numbers are warm-parse minimums -from 10 measured iterations after two warmups on an Apple M3 Pro and are -indicative — re-run the benchmark on your own hardware for authoritative -figures. +| Kotlin | 4 | 24.541x | +| Java | 4 | 2.877x | +| C# | 4 | 2.092x | +| Trino SQL| 5 | 3.495x | ## Useful Information diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index 8331fa12..ca97e1f0 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -5790,7 +5790,7 @@ fn render_parser_parse_rule_fallback( if has_action_dispatch { writeln!( out, - "for action in actions {{ self.run_action(action, &tree); }}" + "for action in actions {{ self.run_action(action, tree); }}" ) .expect("writing to a string cannot fail"); } else { @@ -5825,8 +5825,11 @@ fn render_parser( const GENERATED_PARSER_RESERVED_RULE_METHODS: &[&str] = &[ "token_stream", "token_store", + "parse_tree_storage", + "node", "into_token_stream", "into_token_store", + "into_parsed_file", ]; fn parser_public_rule_method_names(rule_names: &[String]) -> Vec { @@ -6536,10 +6539,10 @@ fn embedded_rule_call_args( /// * one `Context` view per parser rule (plus one per labeled /// alternative) with positional child accessors (`ctx.e(0)` / /// `ctx.e_all()`, `ctx.INT(0)` / `ctx.INT_all()`), `child_count()`, -/// `start()`, public attribute fields, and a `FromRuleContext` impl backing -/// `$ctx.downcast_ref::(tokens)`; +/// `start()`, public attribute fields, and a `FromRuleNode` impl backing +/// `ctx.downcast_ref::()`; /// * the `Listener` trait with defaulted `enter_/exit_` (and -/// per-labeled-alternative) callbacks plus `visit_terminal`; +/// per-labeled-alternative) callbacks plus terminal/error-node visitors; /// * a module-local `ParseTreeWalker` whose bridge dispatches the runtime /// walker onto the typed listener callbacks, threading the invoking-state /// chain Java's `RuleContext.toString` renders (`[13 6]`). @@ -6567,30 +6570,80 @@ fn render_embedded_context_types( r#"#[allow(dead_code)] #[derive(Clone)] pub struct TerminalNode<'a> { - __node: RuntimeTerminalNode, - __tokens: &'a antlr4_runtime::TokenStore, + __node: RuntimeTerminalNode<'a>, } #[allow(dead_code)] impl<'a> TerminalNode<'a> { - fn new(node: RuntimeTerminalNode, tokens: &'a antlr4_runtime::TokenStore) -> Self { - Self { - __node: node, - __tokens: tokens, - } + fn new(node: RuntimeTerminalNode<'a>) -> Self { + Self { __node: node } } pub fn symbol(&self) -> antlr4_runtime::TokenView<'a> { - self.__node.symbol(self.__tokens) + self.__node.symbol() } } impl std::fmt::Display for TerminalNode<'_> { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str(self.__node.text(self.__tokens)) + f.write_str(self.__node.text()) + } +} + +#[allow(dead_code)] +#[derive(Clone)] +pub struct ErrorNode<'a> { + __node: RuntimeErrorNode<'a>, +} + +#[allow(dead_code)] +impl<'a> ErrorNode<'a> { + fn new(node: RuntimeErrorNode<'a>) -> Self { + Self { __node: node } + } + + pub fn symbol(&self) -> antlr4_runtime::TokenView<'a> { + self.__node.symbol() + } +} + +impl std::fmt::Display for ErrorNode<'_> { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str(self.__node.text()) } } +#[allow(dead_code)] +#[derive(Clone)] +enum __GeneratedRuleContext<'a> { + Stored(RuleNodeView<'a>), + Active { + context: &'a antlr4_runtime::ParserRuleContext, + storage: &'a antlr4_runtime::ParseTreeStorage, + tokens: &'a antlr4_runtime::TokenStore, + }, +} + +#[allow(dead_code)] +trait __FromActiveRuleContext<'a>: Sized { + fn __from_active( + context: &'a antlr4_runtime::ParserRuleContext, + invocation_states: Vec, + storage: &'a antlr4_runtime::ParseTreeStorage, + tokens: &'a antlr4_runtime::TokenStore, + ) -> Option; +} + +#[allow(dead_code)] +fn __active_context_view<'a, T: __FromActiveRuleContext<'a>>( + context: &'a antlr4_runtime::ParserRuleContext, + invocation_states: Vec, + storage: &'a antlr4_runtime::ParseTreeStorage, + tokens: &'a antlr4_runtime::TokenStore, +) -> Option { + T::__from_active(context, invocation_states, storage, tokens) +} + "#, ); @@ -6621,16 +6674,16 @@ impl std::fmt::Display for TerminalNode<'_> { } let _ = writeln!( out, - "#[allow(non_camel_case_types, dead_code)]\n#[derive(Clone)]\npub struct {view_name}<'a> {{\n __node: ParserRuleContext,\n __tokens: &'a antlr4_runtime::TokenStore,\n __chain: Vec,\n{fields}}}\n" + "#[allow(non_camel_case_types, dead_code)]\n#[derive(Clone)]\npub struct {view_name}<'a> {{\n __node: __GeneratedRuleContext<'a>,\n __invocation_states: Vec,\n{fields}}}\n" ); let _ = writeln!( out, - "impl<'a> FromRuleContext<'a> for {view_name}<'a> {{\n fn from_rule_context(context: &ParserRuleContext, tokens: &'a antlr4_runtime::TokenStore) -> Option {{\n if context.rule_index() != {rule_index} {{ return None; }}\n Some(Self::__from_node_with_chain(context, tokens, Vec::new()))\n }}\n}}\n" + "impl<'a> FromRuleNode<'a> for {view_name}<'a> {{\n fn from_rule_node(node: RuleNodeView<'a>) -> Option {{\n if node.rule_index() != {rule_index} {{ return None; }}\n Some(Self::__from_node(node))\n }}\n}}\n\nimpl<'a> __FromActiveRuleContext<'a> for {view_name}<'a> {{\n fn __from_active(\n context: &'a antlr4_runtime::ParserRuleContext,\n invocation_states: Vec,\n storage: &'a antlr4_runtime::ParseTreeStorage,\n tokens: &'a antlr4_runtime::TokenStore,\n ) -> Option {{\n if context.rule_index() != {rule_index} {{ return None; }}\n let __default = {attrs_struct}::default();\n let __attrs = context.generated_attrs::<{attrs_struct}>().unwrap_or(&__default);\n Some(Self {{\n __node: __GeneratedRuleContext::Active {{ context, storage, tokens }},\n __invocation_states: invocation_states,\n{field_inits} }})\n }}\n}}\n" ); let mut accessors = String::new(); let _ = writeln!( accessors, - " fn __from_node_with_chain(node: &ParserRuleContext, tokens: &'a antlr4_runtime::TokenStore, mut chain: Vec) -> Self {{\n chain.insert(0, node.invoking_state());\n let __default = {attrs_struct}::default();\n let __attrs = node.generated_attrs::<{attrs_struct}>().unwrap_or(&__default);\n Self {{\n __node: node.clone(),\n __tokens: tokens,\n __chain: chain,\n{field_inits} }}\n }}\n" + " fn __from_node(node: RuleNodeView<'a>) -> Self {{\n let invocation_states = node.invocation_states().collect();\n Self::__from_node_with_invocation_states(node, invocation_states)\n }}\n\n fn __from_child_node(node: RuleNodeView<'a>, parent_invocation_states: &[isize]) -> Self {{\n let mut invocation_states = Vec::with_capacity(parent_invocation_states.len() + 1);\n invocation_states.push(node.invoking_state());\n invocation_states.extend_from_slice(parent_invocation_states);\n Self::__from_node_with_invocation_states(node, invocation_states)\n }}\n\n fn __from_listener_node(node: RuleNodeView<'a>, invocation_states: Option<&[isize]>) -> Self {{\n invocation_states.map_or_else(\n || Self::__from_node(node),\n |states| Self::__from_node_with_invocation_states(node, states.to_vec()),\n )\n }}\n\n fn __from_node_with_invocation_states(node: RuleNodeView<'a>, invocation_states: Vec) -> Self {{\n let __default = {attrs_struct}::default();\n let __attrs = node.generated_attrs::<{attrs_struct}>().unwrap_or(&__default);\n Self {{\n __node: __GeneratedRuleContext::Stored(node),\n __invocation_states: invocation_states,\n{field_inits} }}\n }}\n" ); // A grammar rule claiming a built-in helper's name (`start`, // `child_count`) takes the accessor slot; the built-in yields. @@ -6643,13 +6696,13 @@ impl std::fmt::Display for TerminalNode<'_> { if !rule_claims("child_count") { let _ = writeln!( accessors, - " pub fn child_count(&self) -> usize {{ self.__node.child_count() }}" + " pub fn child_count(&self) -> usize {{\n match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_count(),\n __GeneratedRuleContext::Active {{ context, .. }} => context.child_count(),\n }}\n }}" ); } if !rule_claims("start") { let _ = writeln!( accessors, - " pub fn start(&self) -> __GeneratedTokenView {{ __GeneratedTokenView {{ text: self.__node.start(self.__tokens).map(|token| token.text().to_owned()).unwrap_or_default() }} }}" + " pub fn start(&self) -> __GeneratedTokenView {{\n let token = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.start(),\n __GeneratedRuleContext::Active {{ context, tokens, .. }} => context.start(tokens),\n }};\n __GeneratedTokenView {{ text: token.map(|token| token.text().to_owned()).unwrap_or_default() }}\n }}" ); } for (child_index, child) in model.rules.iter().enumerate() { @@ -6657,13 +6710,13 @@ impl std::fmt::Display for TerminalNode<'_> { let child_view = format!("{}Context", rust_type_name(&child.name)); let _ = writeln!( accessors, - " pub fn {method}(&self, index: usize) -> Rc<{child_view}<'a>> {{ let node = self.__node.child_rules({child_index}).nth(index).expect(\"missing rule child\"); Rc::new({child_view}::__from_node_with_chain(node, self.__tokens, self.__chain.clone())) }}\n pub fn {method}_all(&self) -> Vec>> {{ self.__node.child_rules({child_index}).map(|node| Rc::new({child_view}::__from_node_with_chain(node, self.__tokens, self.__chain.clone()))).collect() }}" + " pub fn {method}(&self, index: usize) -> {child_view}<'a> {{\n let node = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_rules({child_index}).nth(index),\n __GeneratedRuleContext::Active {{ context, storage, tokens, .. }} => context.child_rules(storage, tokens, {child_index}).nth(index),\n }}.expect(\"missing rule child\");\n {child_view}::__from_child_node(node, &self.__invocation_states)\n }}\n pub fn {method}_all(&self) -> Vec<{child_view}<'a>> {{\n let nodes: Vec<_> = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_rules({child_index}).collect(),\n __GeneratedRuleContext::Active {{ context, storage, tokens, .. }} => context.child_rules(storage, tokens, {child_index}).collect(),\n }};\n nodes.into_iter().map(|node| {child_view}::__from_child_node(node, &self.__invocation_states)).collect()\n }}" ); } for (token_name, token_type) in &token_accessors { let _ = writeln!( accessors, - " #[allow(non_snake_case)]\n pub fn {token_name}(&self, index: usize) -> Rc> {{ let node = self.__node.child_tokens({token_type}, self.__tokens).nth(index).expect(\"missing token child\"); Rc::new(TerminalNode::new(node.clone(), self.__tokens)) }}\n #[allow(non_snake_case)]\n pub fn {token_name}_all(&self) -> Vec>> {{ self.__node.child_tokens({token_type}, self.__tokens).map(|node| Rc::new(TerminalNode::new(node.clone(), self.__tokens))).collect() }}" + " #[allow(non_snake_case)]\n pub fn {token_name}(&self, index: usize) -> TerminalNode<'a> {{\n let node = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_tokens({token_type}).nth(index),\n __GeneratedRuleContext::Active {{ context, storage, tokens, .. }} => context.child_tokens(storage, tokens, {token_type}).nth(index),\n }}.expect(\"missing token child\");\n TerminalNode::new(node)\n }}\n #[allow(non_snake_case)]\n pub fn {token_name}_all(&self) -> Vec> {{\n let nodes: Vec<_> = match &self.__node {{\n __GeneratedRuleContext::Stored(node) => node.child_tokens({token_type}).collect(),\n __GeneratedRuleContext::Active {{ context, storage, tokens, .. }} => context.child_tokens(storage, tokens, {token_type}).collect(),\n }};\n nodes.into_iter().map(TerminalNode::new).collect()\n }}" ); } let _ = writeln!( @@ -6674,7 +6727,7 @@ impl std::fmt::Display for TerminalNode<'_> { // this context to the root, the root's sentinel excluded. let _ = writeln!( out, - "impl std::fmt::Display for {view_name}<'_> {{\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{\n let chain: Vec = self.__chain.iter().take(self.__chain.len().saturating_sub(1)).map(|state| state.to_string()).collect();\n write!(f, \"[{{}}]\", chain.join(\" \"))\n }}\n}}\n" + "impl std::fmt::Display for {view_name}<'_> {{\n fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {{\n let chain: Vec = self.__invocation_states.iter().map(|state| state.to_string()).collect();\n write!(f, \"[{{}}]\", chain.join(\" \"))\n }}\n}}\n" ); } @@ -6731,7 +6784,7 @@ impl std::fmt::Display for TerminalNode<'_> { if !has_labels { let (method, view) = &names[0]; return format!( - " self.0.{phase}_{method}(&{view}::__from_node_with_chain(context, tokens, self.1.clone()));\n" + " self.0.{phase}_{method}(&{view}::__from_listener_node(context, self.1.as_deref()));\n" ); } let mut out = String::new(); @@ -6749,7 +6802,7 @@ impl std::fmt::Display for TerminalNode<'_> { let primary_view = format!("{}Context", rust_type_name(primary)); let _ = writeln!( out, - " if context.child_rule_trees({rule_index}).next().is_some() {{ self.0.{phase}_{op_method}(&{op_view}::__from_node_with_chain(context, tokens, self.1.clone())); }} else {{ self.0.{phase}_{primary_method}(&{primary_view}::__from_node_with_chain(context, tokens, self.1.clone())); }}" + " if context.child_rule_trees({rule_index}).next().is_some() {{ self.0.{phase}_{op_method}(&{op_view}::__from_listener_node(context, self.1.as_deref())); }} else {{ self.0.{phase}_{primary_method}(&{primary_view}::__from_listener_node(context, self.1.as_deref())); }}" ); } _ => { @@ -6758,7 +6811,7 @@ impl std::fmt::Display for TerminalNode<'_> { let (method, view) = &names[0]; let _ = writeln!( out, - " self.0.{phase}_{method}(&{view}::__from_node_with_chain(context, tokens, self.1.clone()));" + " self.0.{phase}_{method}(&{view}::__from_listener_node(context, self.1.as_deref()));" ); } } @@ -6777,38 +6830,45 @@ impl std::fmt::Display for TerminalNode<'_> { } let _ = writeln!( out, - "#[allow(dead_code, unused_variables)]\npub trait {listener_trait} {{\n{trait_methods} fn visit_terminal(&mut self, _node: &TerminalNode) {{}}\n fn output(&mut self) -> std::io::Stdout {{ std::io::stdout() }}\n}}\n" + "#[allow(dead_code, unused_variables)]\npub trait {listener_trait} {{\n{trait_methods} fn visit_terminal(&mut self, _node: &TerminalNode) {{}}\n fn visit_error_node(&mut self, _node: &ErrorNode) {{}}\n fn output(&mut self) -> std::io::Stdout {{ std::io::stdout() }}\n}}\n" ); // Bridge + module-local walker (shadows the runtime walker so rendered - // `ParseTreeWalker::walk(&mut listener, tree, tokens)` dispatches typed + // `ParseTreeWalker::walk(&mut listener, tree)` dispatches typed // callbacks. let _ = writeln!( out, r#"#[allow(dead_code)] -struct __ListenerBridge<'a, T: {listener_trait}>(&'a mut T, Vec); +struct __ListenerBridge<'a, T: {listener_trait}>(&'a mut T, Option>); impl antlr4_runtime::ParseTreeListener for __ListenerBridge<'_, T> {{ - fn enter_every_rule(&mut self, context: &ParserRuleContext, tokens: &antlr4_runtime::TokenStore) -> Result<(), antlr4_runtime::AntlrError> {{ - self.1.insert(0, context.invoking_state()); + fn enter_every_rule(&mut self, context: RuleNodeView<'_>) -> Result<(), antlr4_runtime::AntlrError> {{ + if let Some(invocation_states) = &mut self.1 {{ + invocation_states.insert(0, context.invoking_state()); + }} match context.rule_index() {{ {enter_arms} _ => {{}} }} Ok(()) }} - fn exit_every_rule(&mut self, context: &ParserRuleContext, tokens: &antlr4_runtime::TokenStore) -> Result<(), antlr4_runtime::AntlrError> {{ + fn exit_every_rule(&mut self, context: RuleNodeView<'_>) -> Result<(), antlr4_runtime::AntlrError> {{ match context.rule_index() {{ {exit_arms} _ => {{}} }} - if !self.1.is_empty() {{ - self.1.remove(0); + if let Some(invocation_states) = &mut self.1 {{ + invocation_states.remove(0); }} Ok(()) }} - fn visit_terminal(&mut self, node: &RuntimeTerminalNode, tokens: &antlr4_runtime::TokenStore) -> Result<(), antlr4_runtime::AntlrError> {{ - self.0.visit_terminal(&TerminalNode::new(node.clone(), tokens)); + fn visit_terminal(&mut self, node: RuntimeTerminalNode<'_>) -> Result<(), antlr4_runtime::AntlrError> {{ + self.0.visit_terminal(&TerminalNode::new(node)); + Ok(()) + }} + + fn visit_error_node(&mut self, node: RuntimeErrorNode<'_>) -> Result<(), antlr4_runtime::AntlrError> {{ + self.0.visit_error_node(&ErrorNode::new(node)); Ok(()) }} }} @@ -6818,9 +6878,18 @@ pub struct ParseTreeWalker; #[allow(dead_code)] impl ParseTreeWalker {{ - pub fn walk(listener: &mut T, tree: &antlr4_runtime::ParseTree, tokens: &antlr4_runtime::TokenStore) {{ - let mut bridge = __ListenerBridge(listener, Vec::new()); - let _ = antlr4_runtime::ParseTreeWalker::walk(&mut bridge, tree, tokens); + pub fn walk(listener: &mut T, tree: antlr4_runtime::Node<'_>) {{ + let mut bridge = __ListenerBridge(listener, None); + let _ = antlr4_runtime::ParseTreeWalker::walk(&mut bridge, tree); + }} + + pub fn walk_with_invocation_states( + listener: &mut T, + tree: antlr4_runtime::Node<'_>, + parent_invocation_states: Vec, + ) {{ + let mut bridge = __ListenerBridge(listener, Some(parent_invocation_states)); + let _ = antlr4_runtime::ParseTreeWalker::walk(&mut bridge, tree); }} }} "# @@ -6831,7 +6900,17 @@ impl ParseTreeWalker {{ /// Post-translation cleanups shared by all embedded bodies: `TParser::NL` /// becomes `Self::NL` so token constants resolve inside the generic impl. fn post_process_embedded(_original: &str, translated: &str, type_name: &str) -> String { - replace_all(translated, &format!("{type_name}::"), "Self::") + let translated = replace_all(translated, &format!("{type_name}::"), "Self::"); + let translated = replace_all( + &translated, + "(&__ctx).to_string_tree(Some(self), self.base.token_store())", + "(&__ctx).to_string_tree(Some(self), self.base.parse_tree_storage(), self.base.token_store())", + ); + replace_all( + &translated, + "(&__ctx).to_string_tree(Some(self))", + "(&__ctx).to_string_tree(Some(self), self.base.parse_tree_storage(), self.base.token_store())", + ) } /// Plain substring replacement (`str::replace` is a disallowed method here). @@ -7242,7 +7321,7 @@ fn render_parser_with_options( let generated_footer = GENERATED_MODULE_FOOTER; let embedded_imports = if options.embedded { - "#[allow(unused_imports)]\nuse std::io::Write as _;\n#[allow(unused_imports)]\nuse std::rc::Rc;\n#[allow(unused_imports)]\nuse antlr4_runtime::{{java_style_list, PredictionMode, BailErrorStrategy, TerminalNode as RuntimeTerminalNode, ParserRuleContext, FromRuleContext, Token as _}};\n" + "#[allow(unused_imports)]\nuse std::io::Write as _;\n#[allow(unused_imports)]\nuse antlr4_runtime::{{java_style_list, PredictionMode, BailErrorStrategy, TerminalNodeView as RuntimeTerminalNode, ErrorNodeView as RuntimeErrorNode, RuleNodeView, FromRuleNode, Token as _}};\n" } else { "" }; @@ -7348,6 +7427,16 @@ where self.base.token_store() }} + #[must_use] + pub const fn parse_tree_storage(&self) -> &antlr4_runtime::ParseTreeStorage {{ + self.base.parse_tree_storage() + }} + + #[must_use] + pub fn node(&self, id: antlr4_runtime::NodeId) -> antlr4_runtime::Node<'_> {{ + self.base.node(id) + }} + #[must_use] pub fn into_token_stream(self) -> CommonTokenStream {{ self.base.into_token_stream() @@ -7358,6 +7447,11 @@ where self.base.into_token_store() }} + #[must_use] + pub fn into_parsed_file(self, root: antlr4_runtime::NodeId) -> antlr4_runtime::ParsedFile {{ + self.base.into_parsed_file(root) + }} + #[allow(dead_code)] fn simulator(&mut self) -> &mut antlr4_runtime::ParserAtnSimulator<'static> {{ self.simulator @@ -9352,16 +9446,16 @@ fn parser_action_assume_overridden( fn render_parser_action_method(has_action_states: bool, noop_states: &BTreeSet) -> String { if !has_action_states { - return " fn run_action(&mut self, _action: antlr4_runtime::ParserAction, _tree: &antlr4_runtime::ParseTree) {}\n" + return " fn run_action(&mut self, _action: antlr4_runtime::ParserAction, _tree: antlr4_runtime::ParseTree) {}\n" .to_owned(); } let mut arms = String::new(); for state in noop_states { writeln!(arms, " {state} => {{}}").expect("writing to a string cannot fail"); } - arms.push_str(" _ => { let _ = self.base.parser_action_hook(action, _tree); }\n"); + arms.push_str(" _ => { let _ = self.base.parser_action_hook(action, tree); }\n"); format!( - " fn run_action(&mut self, action: antlr4_runtime::ParserAction, _tree: &antlr4_runtime::ParseTree) {{\n match action.source_state() {{\n{arms} }}\n }}\n" + " fn run_action(&mut self, action: antlr4_runtime::ParserAction, tree: antlr4_runtime::ParseTree) {{\n match action.source_state() {{\n{arms} }}\n }}\n" ) } @@ -10312,18 +10406,18 @@ where /// Pass the generated lexer constructor and a parser entry rule, for example /// `parse(src, MyGrammarLexer::new, {type_name}::file)`. /// -/// The returned [`antlr4_runtime::ParsedFile`] owns both the entry-rule result -/// and the canonical token store referenced by any parse tree in that result. +/// The returned [`antlr4_runtime::ParsedFile`] owns the canonical token store, +/// flat CST storage, and entry-rule root. /// Use [`parse_with_parser`] instead when the caller also needs parser /// diagnostics after the entry rule runs. -pub fn parse( +pub fn parse( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, - entry: impl FnOnce(&mut {type_name}) -> Result, -) -> Result, antlr4_runtime::AntlrError> + entry: impl FnOnce(&mut {type_name}) -> Result, +) -> Result {{ let {output_type_name} {{ result, parser }} = parse_with_parser(input, lexer, entry)?; - Ok(antlr4_runtime::ParsedFile::new(parser.into_token_store(), result)) + Ok(parser.into_parsed_file(result)) }} /// Parses UTF-8 text like [`parse`] while returning the parser after the entry @@ -11619,7 +11713,7 @@ atn: assert!(fallback.contains( "parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence" )); - assert!(fallback.contains("for action in actions { self.run_action(action, &tree); }")); + assert!(fallback.contains("for action in actions { self.run_action(action, tree); }")); assert!(fallback.contains("Ok(tree)")); } @@ -11628,7 +11722,7 @@ atn: let method = render_parser_action_method(true, &BTreeSet::new()); assert!(method.contains("fn run_action")); - assert!(method.contains("self.base.parser_action_hook(action, _tree)")); + assert!(method.contains("self.base.parser_action_hook(action, tree)")); } #[test] @@ -11648,7 +11742,7 @@ atn: .find("7 => {}") .expect("assume-* action state gets an explicit no-op arm"); let hook_at = method - .find("self.base.parser_action_hook(action, _tree)") + .find("self.base.parser_action_hook(action, tree)") .expect("the hook catch-all is still emitted for hook/unknown states"); assert!( noop_at < hook_at, @@ -11710,6 +11804,53 @@ atn: ); } + #[test] + fn embedded_listener_forwards_error_nodes() { + let rendered = render_parser_with_options( + "TParser", + &minimal_parser_data(), + Some("parser grammar T; s : ;"), + ParserRenderOptions { + embedded: true, + ..ParserRenderOptions::default() + }, + ) + .expect("embedded parser should render"); + + assert!(rendered.contains("ErrorNodeView as RuntimeErrorNode")); + assert!(rendered.contains("pub struct ErrorNode<'a>")); + assert!(rendered.contains("fn visit_error_node(&mut self, _node: &ErrorNode)")); + assert!( + rendered + .contains("fn visit_error_node(&mut self, node: RuntimeErrorNode<'_>) -> Result<") + ); + assert!(rendered.contains("self.0.visit_error_node(&ErrorNode::new(node));")); + } + + #[test] + fn embedded_active_context_preserves_invocation_states() { + let rendered = render_parser_with_options( + "TParser", + &minimal_parser_data(), + Some("parser grammar T; s : ;"), + ParserRenderOptions { + embedded: true, + ..ParserRenderOptions::default() + }, + ) + .expect("embedded parser should render"); + + assert!(rendered.contains("invocation_states: Vec")); + assert!(rendered.contains("__invocation_states: invocation_states")); + assert!(rendered.contains("::__from_child_node(node, &self.__invocation_states)")); + assert!(rendered.contains("::__from_listener_node(context, self.1.as_deref())")); + assert!(rendered.contains("pub fn walk_with_invocation_states")); + assert!( + !rendered.contains("__GeneratedRuleContext::Active { .. } => Vec::new()"), + "active contexts must preserve their invoking-state chain" + ); + } + #[test] fn non_embedded_parser_action_disables_generated_rule() { let rendered = render_parser( @@ -11723,7 +11864,7 @@ atn: !rendered.contains("parse_generated_rule_0_dispatch"), "non-embedded parser action rules must stay on the interpreted path" ); - assert!(rendered.contains("self.base.parser_action_hook(action, _tree)")); + assert!(rendered.contains("self.base.parser_action_hook(action, tree)")); assert!(!rendered.contains(&format!("{}{}", "Generated", "Action"))); assert!(!rendered.contains(&format!("{}{}", "generated", "_actions"))); } @@ -11850,7 +11991,7 @@ s : ; assert!(rendered.contains("pub struct TParserParseOutput")); assert!(rendered.contains("pub result: R,")); assert!(rendered.contains("pub parser: TParser,")); - assert!(rendered.contains("pub fn parse(")); + assert!(rendered.contains("pub fn parse(")); assert!(rendered.contains("pub fn parse_with_parser(")); assert!( !rendered @@ -11867,10 +12008,7 @@ s : ; assert!(rendered.contains( "let TParserParseOutput { result, parser } = parse_with_parser(input, lexer, entry)?;" )); - assert!( - rendered - .contains("Ok(antlr4_runtime::ParsedFile::new(parser.into_token_store(), result))") - ); + assert!(rendered.contains("Ok(parser.into_parsed_file(result))")); assert!(rendered.contains("pub fn new(input: CommonTokenStream) -> Self")); assert!( rendered.contains("pub fn with_hooks(input: CommonTokenStream, hooks: H) -> Self") diff --git a/src/bin_support/embedded.rs b/src/bin_support/embedded.rs index 3fe49058..b3573cbe 100644 --- a/src/bin_support/embedded.rs +++ b/src/bin_support/embedded.rs @@ -1064,7 +1064,7 @@ fn translate_ctx_member(member: &str, ctx: &TranslationCtx<'_>, body: &str) -> i if let Some(rule_name) = member.strip_suffix("_all") { if let Some(rule_index) = ctx.rule_index_by_name(rule_name) { return Ok(format!( - "__ctx.child_rules({rule_index}).collect::>()" + "__ctx.child_rules(self.base.parse_tree_storage(), self.base.token_store(), {rule_index}).collect::>()" )); } } @@ -1093,7 +1093,7 @@ fn translate_element_read( if let Some(rule_index) = ctx.rule_index_by_name(&element.target) { return match suffix { None | Some("ctx") => Ok(format!( - "__ctx.child_rule_trees({rule_index}).collect::>()" + "__ctx.child_rule_trees(self.base.parse_tree_storage(), self.base.token_store(), {rule_index}).collect::>()" )), Some(other) => Err(io::Error::new( io::ErrorKind::InvalidData, @@ -1103,7 +1103,7 @@ fn translate_element_read( } if let Some(token_type) = ctx.token_types.get(&element.target) { return Ok(format!( - "__ctx.child_tokens({token_type}, self.base.token_store()).collect::>()" + "__ctx.child_tokens(self.base.parse_tree_storage(), self.base.token_store(), {token_type}).collect::>()" )); } } @@ -1114,11 +1114,11 @@ fn translate_element_read( // `Token.toString()`), which is the same rendering as start/stop. return match suffix { None | Some("stop" | "start") => Ok( - "__ctx.terminal_children().last().map(|__t| __t.symbol(self.base.token_store()).to_string()).unwrap_or_default()" + "__ctx.terminal_children(self.base.parse_tree_storage(), self.base.token_store()).last().map(|__t| __t.symbol().to_string()).unwrap_or_default()" .to_owned(), ), Some("text") => Ok( - "__ctx.terminal_children().last().map(|__t| __t.text(self.base.token_store()).to_owned()).unwrap_or_default()" + "__ctx.terminal_children(self.base.parse_tree_storage(), self.base.token_store()).last().map(|__t| __t.text().to_owned()).unwrap_or_default()" .to_owned(), ), _ => Err(io::Error::new( @@ -1135,16 +1135,16 @@ fn translate_element_read( }; return match suffix { Some("ctx") | None => Ok(format!( - "__ctx.child_rule_trees({rule_index}).{pick}.expect(\"labeled rule child\")" + "__ctx.child_rule_trees(self.base.parse_tree_storage(), self.base.token_store(), {rule_index}).{pick}.expect(\"labeled rule child\")" )), Some("text") => Ok(format!( - "__ctx.child_rules({rule_index}).{pick}.map(|__c| __c.text(self.base.token_store())).unwrap_or_default()" + "__ctx.child_rules(self.base.parse_tree_storage(), self.base.token_store(), {rule_index}).{pick}.map(|__c| __c.text()).unwrap_or_default()" )), Some("start") => Ok(format!( - "__ctx.child_rules({rule_index}).{pick}.and_then(|__c| __c.start(self.base.token_store())).map(|__t| __t.to_string()).unwrap_or_default()" + "__ctx.child_rules(self.base.parse_tree_storage(), self.base.token_store(), {rule_index}).{pick}.and_then(|__c| __c.start()).map(|__t| __t.to_string()).unwrap_or_default()" )), Some("stop") => Ok(format!( - "__ctx.child_rules({rule_index}).{pick}.and_then(|__c| __c.stop(self.base.token_store())).map(|__t| __t.to_string()).unwrap_or_default()" + "__ctx.child_rules(self.base.parse_tree_storage(), self.base.token_store(), {rule_index}).{pick}.and_then(|__c| __c.stop()).map(|__t| __t.to_string()).unwrap_or_default()" )), Some(attr) => { let target_rule = &ctx.model.rules[rule_index]; @@ -1160,7 +1160,7 @@ fn translate_element_read( let attrs_struct = attrs_struct_name(rule_index); let field = escape_keyword(&decl.name); Ok(format!( - "__ctx.child_rules({rule_index}).{pick}.and_then(|__c| __c.generated_attrs::<{attrs_struct}>()).map(|__a| __a.{field}.clone()).unwrap_or_default()" + "__ctx.child_rules(self.base.parse_tree_storage(), self.base.token_store(), {rule_index}).{pick}.and_then(|__c| __c.generated_attrs::<{attrs_struct}>()).map(|__a| __a.{field}.clone()).unwrap_or_default()" )) } }; @@ -1173,16 +1173,16 @@ fn translate_element_read( }; return match suffix { Some("text") => Ok(format!( - "__ctx.child_tokens({token_type}, self.base.token_store()).{pick}.map(|__t| __t.text(self.base.token_store()).to_owned()).unwrap_or_default()" + "__ctx.child_tokens(self.base.parse_tree_storage(), self.base.token_store(), {token_type}).{pick}.map(|__t| __t.text().to_owned()).unwrap_or_default()" )), Some("int") => Ok(format!( - "__ctx.child_tokens({token_type}, self.base.token_store()).{pick}.map(|__t| __t.text(self.base.token_store()).parse::().unwrap_or_default()).unwrap_or_default()" + "__ctx.child_tokens(self.base.parse_tree_storage(), self.base.token_store(), {token_type}).{pick}.map(|__t| __t.text().parse::().unwrap_or_default()).unwrap_or_default()" )), Some("line") => Ok(format!( - "__ctx.child_tokens({token_type}, self.base.token_store()).{pick}.map(|__t| __t.symbol(self.base.token_store()).line()).unwrap_or_default()" + "__ctx.child_tokens(self.base.parse_tree_storage(), self.base.token_store(), {token_type}).{pick}.map(|__t| __t.symbol().line()).unwrap_or_default()" )), None | Some("stop" | "start") => Ok(format!( - "__ctx.child_tokens({token_type}, self.base.token_store()).{pick}.map(|__t| __t.symbol(self.base.token_store()).to_string()).unwrap_or_default()" + "__ctx.child_tokens(self.base.parse_tree_storage(), self.base.token_store(), {token_type}).{pick}.map(|__t| __t.symbol().to_string()).unwrap_or_default()" )), Some(other) => Err(io::Error::new( io::ErrorKind::InvalidData, @@ -1328,7 +1328,9 @@ mod tests { let translated = translate_body("$v = $INT.int;", &ctx).expect("translates"); assert!(translated.starts_with("__attrs.v = "), "{translated}"); assert!( - translated.contains("child_tokens(1, self.base.token_store())"), + translated.contains( + "child_tokens(self.base.parse_tree_storage(), self.base.token_store(), 1)" + ), "{translated}" ); diff --git a/src/lib.rs b/src/lib.rs index a45ab737..9043befa 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -44,8 +44,9 @@ pub use token::{ }; pub use token_stream::CommonTokenStream; pub use tree::{ - ErrorNode, FromRuleContext, GeneratedAttrs, ParseTree, ParseTreeDescendants, ParseTreeListener, - ParseTreeWalker, ParsedFile, ParserRuleContext, RuleNode, TerminalNode, + ErrorNodeView, FromRuleNode, GeneratedAttrs, Node, NodeChildren, NodeId, NodeKind, ParseTree, + ParseTreeDescendants, ParseTreeListener, ParseTreeStats, ParseTreeStorage, ParseTreeWalker, + ParsedFile, ParserRuleContext, RuleNodeView, TerminalNodeView, }; pub use vocabulary::Vocabulary; diff --git a/src/parser.rs b/src/parser.rs index 5ef9211e..c2c840da 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -84,12 +84,16 @@ use crate::prediction::{EMPTY_RETURN_STATE, PredictionContext}; use crate::recognizer::{Recognizer, RecognizerData}; use crate::semir::{self, AStmt, ArithOp, CmpOp, ExprId, HookId, PExpr, SemIr, StmtId}; use crate::token::{ - TOKEN_EOF, Token, TokenId, TokenSource, TokenSourceError, TokenSpec, TokenView, + TOKEN_EOF, Token, TokenId, TokenSource, TokenSourceError, TokenSpec, TokenStore, TokenView, }; use crate::token_stream::CommonTokenStream; -use crate::tree::{ErrorNode, ParseTree, ParserRuleContext, RuleNode, TerminalNode}; +use crate::tree::{ + Node, NodeId, ParseTreeCheckpoint, ParseTreeStorage, ParsedFile, ParserRuleContext, +}; use crate::vocabulary::Vocabulary; +type ParseTree = NodeId; + /// Upper bound for the recursive metadata recognizer before it treats a path as /// non-viable. Long expression-regression descriptors legitimately walk tens /// of thousands of ATN edges. @@ -320,11 +324,12 @@ where S: TokenSource, { input: &'a mut CommonTokenStream, + tree_storage: &'a ParseTreeStorage, rule_index: usize, coordinate_index: usize, rule_name: Option, context: Option<&'a ParserRuleContext>, - tree: Option<&'a ParseTree>, + tree: Option, local_int_arg: Option<(usize, i64)>, member_values: &'a BTreeMap, action: Option, @@ -410,11 +415,30 @@ where self.context } + /// Flat tree storage containing completed children visible to this hook. + #[must_use] + pub const fn parse_tree_storage(&self) -> &'a ParseTreeStorage { + self.tree_storage + } + + /// Canonical token store used by completed flat-tree nodes. + #[must_use] + pub const fn token_store(&self) -> &TokenStore { + self.input.token_store() + } + + /// Completed parse-tree root ID passed to a replayed action hook. + #[must_use] + pub const fn tree_id(&self) -> Option { + self.tree + } + /// Completed parse tree passed to an action hook, if the action is being /// replayed after recognition. #[must_use] - pub const fn tree(&self) -> Option<&'a ParseTree> { + pub fn tree(&self) -> Option> { self.tree + .and_then(|id| self.tree_storage.node(self.input.token_store(), id)) } /// Integer local argument visible to this predicate coordinate. @@ -992,6 +1016,7 @@ const LEFT_RECURSIVE_CALLER_OVERLAP_CACHE_SIZE: usize = 16; #[derive(Debug)] pub struct BaseParser { input: CommonTokenStream, + tree: ParseTreeStorage, data: RecognizerData, semantic_hooks: H, build_parse_trees: bool, @@ -1110,6 +1135,7 @@ pub struct BaseParser { pub struct GeneratedDiagnosticsCheckpoint { diagnostics_len: usize, syntax_errors: usize, + tree: ParseTreeCheckpoint, } /// Storage and reachability counters for the most recent interpreted-rule @@ -3651,6 +3677,7 @@ where H: SemanticHooks, { input: &'a mut CommonTokenStream, + tree_storage: &'a ParseTreeStorage, semantic_hooks: &'a mut H, rule_index: usize, coordinate_index: usize, @@ -3696,12 +3723,10 @@ where fn ctx_rule_text(&self, rule_index: usize) -> Option { self.context.and_then(|context| { - context.children().iter().find_map(|child| match child { - ParseTree::Rule(rule) if rule.context().rule_index() == rule_index => { - Some(child.text(self.input.token_store())) - } - ParseTree::Rule(_) | ParseTree::Terminal(_) | ParseTree::Error(_) => None, - }) + context + .child_rules(self.tree_storage, self.input.token_store(), rule_index) + .next() + .map(crate::tree::RuleNodeView::text) }) } @@ -3728,6 +3753,7 @@ where fn hook(&mut self, _hook: HookId) -> bool { let mut ctx = ParserSemCtx { input: &mut *self.input, + tree_storage: self.tree_storage, rule_index: self.rule_index, coordinate_index: self.coordinate_index, rule_name: self.rule_name.map(str::to_owned), @@ -3915,6 +3941,7 @@ where ) -> Self { Self { input, + tree: ParseTreeStorage::new(), data, semantic_hooks, build_parse_trees: true, @@ -4009,10 +4036,24 @@ where /// Returns the canonical token store referenced by parse trees. #[must_use] - pub const fn token_store(&self) -> &crate::token::TokenStore { + pub const fn token_store(&self) -> &TokenStore { self.input.token_store() } + /// Returns the flat CST storage populated by completed rules. + #[must_use] + pub const fn parse_tree_storage(&self) -> &ParseTreeStorage { + &self.tree + } + + /// Resolves a compact parse-tree ID into a borrowing node view. + #[must_use] + pub fn node(&self, id: NodeId) -> Node<'_> { + self.tree + .node(self.input.token_store(), id) + .expect("parser-produced node ID should remain valid") + } + /// Consumes this parser and returns its token stream. #[must_use] pub fn into_token_stream(self) -> CommonTokenStream { @@ -4021,10 +4062,16 @@ where /// Consumes this parser and returns its canonical token store. #[must_use] - pub fn into_token_store(self) -> crate::token::TokenStore { + pub fn into_token_store(self) -> TokenStore { self.input.into_token_store() } + /// Consumes the parser and pairs its token store and flat CST with `root`. + #[must_use] + pub fn into_parsed_file(self, root: NodeId) -> ParsedFile { + ParsedFile::new(self.input.into_token_store(), self.tree, root) + } + /// Returns the number of parser syntax errors recorded by committed parse /// paths so far. pub const fn number_of_syntax_errors(&self) -> usize { @@ -4066,6 +4113,7 @@ where GeneratedDiagnosticsCheckpoint { diagnostics_len: self.generated_parser_diagnostics.len(), syntax_errors: self.syntax_errors, + tree: self.tree.checkpoint(), } } @@ -4075,6 +4123,7 @@ where .truncate(marker.diagnostics_len); self.syntax_errors = marker.syntax_errors; self.generated_sync_expected = None; + self.tree.rollback(marker.tree); } /// Emits diagnostics recorded by committed generated parser recovery. @@ -4261,12 +4310,20 @@ where self.input.token_store().token_type(id).unwrap_or(TOKEN_EOF) } - const fn terminal_tree(&self, id: TokenId) -> ParseTree { - ParseTree::Terminal(TerminalNode::from_id(id)) + fn terminal_tree(&mut self, id: TokenId) -> ParseTree { + if self.build_parse_trees { + self.tree.terminal(id) + } else { + NodeId::placeholder() + } } - const fn error_tree(&self, id: TokenId) -> ParseTree { - ParseTree::Error(ErrorNode::from_id(id)) + fn error_tree(&mut self, id: TokenId) -> ParseTree { + if self.build_parse_trees { + self.tree.error(id) + } else { + NodeId::placeholder() + } } const fn set_context_start(&self, context: &mut ParserRuleContext, id: TokenId) { @@ -4599,8 +4656,12 @@ where format!("{{{values}}}") } - pub const fn rule_node(&self, context: ParserRuleContext) -> ParseTree { - ParseTree::Rule(RuleNode::new(context)) + pub fn rule_node(&mut self, context: ParserRuleContext) -> ParseTree { + if self.build_parse_trees { + self.tree.finish_rule(context) + } else { + NodeId::placeholder() + } } /// Enters a generated parser rule and returns the context object the @@ -4685,22 +4746,30 @@ where /// enabled. The match is recorded on the context either way (via `add_child`, /// or `note_matched_child` when trees are off) so generated recovery can tell /// whether the rule has matched anything yet without depending on `children`. - pub fn add_parse_child(&self, context: &mut ParserRuleContext, child: ParseTree) { + pub fn add_parse_child(&mut self, context: &mut ParserRuleContext, child: ParseTree) { if self.build_parse_trees { - context.add_child(child); + self.tree.add_child(context, child); } else { context.note_matched_child(); } } + fn release_tree_scratch_if_idle(&mut self) { + if self.rule_context_stack.is_empty() { + self.tree.release_scratch(); + } + } + /// Finishes a generated parser rule and returns its parse-tree node. pub fn finish_rule(&mut self, mut context: ParserRuleContext, consumed_eof: bool) -> ParseTree { let stop_index = self.rule_stop_token_index(self.input.index(), consumed_eof); if let Some(token) = stop_index.and_then(|index| self.token_id_at(index)) { self.set_context_stop(&mut context, token); } + let node = self.rule_node(context); self.exit_rule(); - self.rule_node(context) + self.release_tree_scratch_if_idle(); + node } /// Recovers a generated rule catch block after a committed mismatch. @@ -4728,7 +4797,8 @@ where break; }; self.consume(); - self.add_parse_child(context, self.error_tree(token)); + let child = self.error_tree(token); + self.add_parse_child(context, child); } self.record_syntax_errors(1); } @@ -4786,8 +4856,10 @@ where if let Some(token) = stop_index.and_then(|index| self.token_id_at(index)) { self.set_context_stop(&mut context, token); } + let node = self.rule_node(context); self.unroll_recursion_context(); - self.rule_node(context) + self.release_tree_scratch_if_idle(); + node } /// Enters a generated left-recursive rule at `precedence`. @@ -4834,7 +4906,8 @@ where } let previous = std::mem::replace(current, replacement); if self.build_parse_trees { - current.add_child(self.rule_node(previous)); + let previous = self.rule_node(previous); + self.tree.add_child(current, previous); } } @@ -5433,20 +5506,16 @@ where /// /// Generated parsers call this for action source states that were present /// in the ATN but not translated into a built-in Rust action template. - pub fn parser_action_hook(&mut self, action: ParserAction, tree: &ParseTree) -> bool { + pub fn parser_action_hook(&mut self, action: ParserAction, tree: ParseTree) -> bool { let rule_index = action.rule_index(); let rule_name = self.rule_names().get(rule_index).cloned(); - let context = match tree { - ParseTree::Rule(rule) if rule.context().rule_index() == rule_index => { - Some(rule.context()) - } - ParseTree::Rule(_) | ParseTree::Terminal(_) | ParseTree::Error(_) => None, - }; + let context = None; let input = &mut self.input; let semantic_hooks = &mut self.semantic_hooks; let member_values = &self.int_members; let mut ctx = ParserSemCtx { input, + tree_storage: &self.tree, rule_index, coordinate_index: usize::MAX, rule_name, @@ -5485,6 +5554,7 @@ where self.clear_prediction_diagnostics(); self.reset_per_parse_caches(); self.reset_recognition_arena(); + let tree_checkpoint = self.tree.checkpoint(); let mut decision_by_state = vec![None; atn.states().len()]; for (decision, &state_number) in atn.decision_to_state().iter().enumerate() { if let Some(slot) = decision_by_state.get_mut(state_number) { @@ -5504,10 +5574,12 @@ where match result { Ok(tree) => { report_token_source_errors(&self.input.drain_source_errors()); + self.release_tree_scratch_if_idle(); Ok(tree) } Err(DirectAdaptiveParseControl::Fallback(reason)) => { let _ = reason; + self.tree.rollback(tree_checkpoint); self.input.seek(start_index); self.parse_atn_rule(atn, rule_index) } @@ -5660,7 +5732,8 @@ where { let mut cursor = live_root; while let Some(link) = self.recognition_arena.link(cursor) { - context.add_child(self.arena_recognized_node_tree(link.head, false)?); + let child = self.arena_recognized_node_tree(link.head, false)?; + self.tree.add_child(&mut context, child); cursor = link.tail; } } else { @@ -5675,7 +5748,9 @@ where self.finish_recognition_arena(live_root, outcome.diagnostics); self.input.seek(outcome.index); - Ok(self.rule_node(context)) + let tree = self.rule_node(context); + self.release_tree_scratch_if_idle(); + Ok(tree) } fn pending_invoking_follow_state(&self, atn: &Atn) -> Option { @@ -5759,11 +5834,7 @@ where } } - /// Converts one arena record into the current public recursive tree. - /// - /// The speculative representation remains flat and index-addressed; this - /// is the single compatibility boundary that #84 replaces with direct flat - /// CST handoff. + /// Converts one speculative arena record into the flat public CST. fn arena_recognized_node_tree( &mut self, node_id: RecognizedNodeId, @@ -5827,8 +5898,8 @@ where .recognition_arena .fold_left_recursive_boundaries(children); while let Some(link) = self.recognition_arena.link(cursor) { - context - .add_child(self.arena_recognized_node_tree(link.head, track_alt_numbers)?); + let child = self.arena_recognized_node_tree(link.head, track_alt_numbers)?; + self.tree.add_child(&mut context, child); cursor = link.tail; } Ok(self.rule_node(context)) @@ -5892,12 +5963,14 @@ where while let Some(link) = self.recognition_arena.link(children) { if let Some((child_start, child_stop)) = self.recognition_arena.node_span(link.head) { self.add_visible_terminals_before(context, &mut cursor, child_start)?; - context.add_child(self.arena_recognized_node_tree_with_implicit_tokens(link.head)?); + let child = self.arena_recognized_node_tree_with_implicit_tokens(link.head)?; + self.tree.add_child(context, child); if let Some(child_stop) = child_stop { cursor = self.next_visible_after_token(child_stop); } } else { - context.add_child(self.arena_recognized_node_tree_with_implicit_tokens(link.head)?); + let child = self.arena_recognized_node_tree_with_implicit_tokens(link.head)?; + self.tree.add_child(context, child); } children = link.tail; } @@ -5940,7 +6013,8 @@ where message: format!("missing token at index {index}"), })?; let is_eof = self.token_type_for_id(token) == TOKEN_EOF; - context.add_child(self.terminal_tree(token)); + let child = self.terminal_tree(token); + self.tree.add_child(context, child); if is_eof { return Ok(None); } @@ -6182,14 +6256,17 @@ where if self.build_parse_trees { let mut nodes = live_root; while let Some(link) = self.recognition_arena.link(nodes) { - context.add_child(self.arena_recognized_node_tree(link.head, track_alt_numbers)?); + let child = self.arena_recognized_node_tree(link.head, track_alt_numbers)?; + self.tree.add_child(&mut context, child); nodes = link.tail; } } self.finish_recognition_arena(live_root, outcome.diagnostics); self.input.seek(outcome.index); - Ok((self.rule_node(context), actions)) + let tree = self.rule_node(context); + self.release_tree_scratch_if_idle(); + Ok((tree, actions)) } /// Temporary parser entry used by generated parser methods while the parser @@ -6204,13 +6281,16 @@ where let token_type = self.la(1); let child = self.match_token(token_type)?; if self.build_parse_trees { - context.add_child(child); + self.tree.add_child(&mut context, child); } } if self.build_parse_trees { - context.add_child(self.match_eof()?); + let child = self.match_eof()?; + self.tree.add_child(&mut context, child); } - Ok(self.rule_node(context)) + let tree = self.rule_node(context); + self.release_tree_scratch_if_idle(); + Ok(tree) } /// Builds the parser error reported when no ATN path can reach the active @@ -8901,13 +8981,15 @@ where #[must_use] pub fn after_action_stop_index_for_tree( &mut self, - tree: &ParseTree, + tree: ParseTree, current_index: usize, ) -> Option { - if let ParseTree::Rule(rule) = tree { - if let Some(stop) = rule.context().stop_id() { - return Some(stop.index()); - } + if let Some(stop) = self + .node(tree) + .as_rule() + .and_then(crate::tree::RuleNodeView::stop_id) + { + return Some(stop.index()); } self.after_action_stop_index(current_index) } @@ -8922,15 +9004,17 @@ where /// pre-rule cursor still points at. Falls back to `fallback_index` only when /// the tree carries no rule start. #[must_use] - pub const fn after_action_start_index_for_tree( + pub fn after_action_start_index_for_tree( &self, - tree: &ParseTree, + tree: ParseTree, fallback_index: usize, ) -> usize { - if let ParseTree::Rule(rule) = tree { - if let Some(start) = rule.context().start_id() { - return start.index(); - } + if let Some(start) = self + .node(tree) + .as_rule() + .and_then(crate::tree::RuleNodeView::start_id) + { + return start.index(); } fallback_index } @@ -9022,6 +9106,7 @@ where let semantic_hooks = &mut self.semantic_hooks; let mut ctx = ParserSemCtx { input, + tree_storage: &self.tree, rule_index, coordinate_index: pred_index, rule_name, @@ -9131,6 +9216,7 @@ where let unknown_predicate_policy = self.unknown_predicate_policy; let mut ctx = ParserSemIrCtx { input: &mut self.input, + tree_storage: &self.tree, semantic_hooks: &mut self.semantic_hooks, rule_index: request.rule_index, coordinate_index: request.pred_index, @@ -9227,12 +9313,10 @@ where } ParserPredicate::ContextChildRuleTextNotEquals { rule_index, text } => context .and_then(|context| { - context.children().iter().find_map(|child| match child { - ParseTree::Rule(rule) if rule.context().rule_index() == *rule_index => { - Some(child.text(self.input.token_store())) - } - ParseTree::Rule(_) | ParseTree::Terminal(_) | ParseTree::Error(_) => None, - }) + context + .child_rules(&self.tree, self.input.token_store(), *rule_index) + .next() + .map(crate::tree::RuleNodeView::text) }) .is_none_or(|actual| actual != *text), ParserPredicate::LocalIntEquals { value } => { @@ -9559,6 +9643,18 @@ where .collect() } + /// Invoking-state chain for the active rule context, current rule first. + /// + /// The root frame is excluded, matching Java's `RuleContext.toString()`. + pub fn active_invocation_states(&self) -> Vec { + self.rule_context_stack + .iter() + .skip(1) + .rev() + .map(|frame| frame.invoking_state) + .collect() + } + /// Formats a buffered token in ANTLR's diagnostic token display form. pub fn token_display_at(&self, index: usize) -> Option { self.token_at(index).map(|token| format!("{token}")) @@ -9588,7 +9684,10 @@ where DirectAdaptiveFallback::MissingAtn, ))?; let start_index = self.parser.current_visible_index(); - let mut children = Vec::new(); + let mut context = ParserRuleContext::new(rule_index, invoking_state); + if let Some(token) = self.parser.token_id_at(start_index) { + self.parser.set_context_start(&mut context, token); + } let mut state_number = start_state; let mut consumed_eof = false; while state_number != stop_state { @@ -9626,7 +9725,7 @@ where rule_precedence, )?; if self.parser.build_parse_trees { - children.push(child); + self.parser.tree.add_child(&mut context, child); } state_number = follow_state; } @@ -9638,7 +9737,7 @@ where let (matched_eof, child) = self.consume_transition(&transition)?; consumed_eof |= matched_eof; if let Some(child) = child { - children.push(child); + self.parser.tree.add_child(&mut context, child); } state_number = transition.target(); } @@ -9655,29 +9754,12 @@ where } } - let mut context = ParserRuleContext::with_child_capacity( - rule_index, - invoking_state, - if self.parser.build_parse_trees { - children.len() - } else { - 0 - }, - ); - if let Some(token) = self.parser.token_id_at(start_index) { - self.parser.set_context_start(&mut context, token); - } let stop_index = self .parser .rule_stop_token_index(self.parser.input.index(), consumed_eof); if let Some(token) = stop_index.and_then(|index| self.parser.token_id_at(index)) { self.parser.set_context_stop(&mut context, token); } - if self.parser.build_parse_trees { - for child in children { - context.add_child(child); - } - } Ok(self.parser.rule_node(context)) } @@ -10530,6 +10612,7 @@ mod tests { use crate::atn::serialized::{AtnDeserializer, SerializedAtn}; use crate::token::{HIDDEN_CHANNEL, Token, TokenId, TokenSink, TokenSpec, TokenStoreError}; use crate::token_stream::CommonTokenStream; + use crate::tree::{NodeKind, ParseTreeStats}; use crate::vocabulary::Vocabulary; use std::mem::size_of; @@ -10651,10 +10734,6 @@ mod tests { } } - fn test_terminal(token: &TestToken) -> TerminalNode { - TerminalNode::from_id(token.id) - } - #[derive(Debug)] struct Source { tokens: Vec, @@ -11482,7 +11561,7 @@ mod tests { .sync_decision(&atn, 1, true, false) .expect("single extraneous token recovers"); assert_eq!(children.len(), 1); - assert!(matches!(children[0], ParseTree::Error(_))); + assert_eq!(single.node(children[0]).kind(), NodeKind::Error); assert_eq!(single.number_of_syntax_errors(), 1); // Exactly one token consumed (the cursor now sits on `b`). assert_eq!(single.la(1), 2); @@ -11627,7 +11706,7 @@ mod tests { .sync_decision(&atn, 5, true, false) .expect("single token before EOF recovers"); assert_eq!(children.len(), 1); - assert!(matches!(children[0], ParseTree::Error(_))); + assert_eq!(parser.node(children[0]).kind(), NodeKind::Error); assert_eq!(parser.number_of_syntax_errors(), 1); assert_eq!( parser.la(1), @@ -11689,7 +11768,11 @@ mod tests { .sync_decision(&atn, 5, false, true) .expect("loop-back multi-token deletion recovers onto EOF"); assert_eq!(children.len(), 2, "both `c`s deleted as error nodes"); - assert!(children.iter().all(|c| matches!(c, ParseTree::Error(_)))); + assert!( + children + .iter() + .all(|child| parser.node(*child).kind() == NodeKind::Error) + ); assert_eq!(parser.number_of_syntax_errors(), 1); assert_eq!(parser.la(1), TOKEN_EOF, "EOF left for the rule's EOF match"); } @@ -11857,7 +11940,7 @@ mod tests { ); let mut parser = BaseParser::new(CommonTokenStream::new(source), data); let matched = parser.match_token(1).expect("token 1 should match"); - assert_eq!(matched.text(parser.token_store()), "x"); + assert_eq!(parser.node(matched).text(), "x"); assert!(parser.match_token(1).is_err()); } @@ -11871,7 +11954,7 @@ mod tests { let matched = parser .match_set(&[(1, 1), (3, 4)]) .expect("token set should match"); - assert_eq!(matched.text(parser.token_store()), "x"); + assert_eq!(parser.node(matched).text(), "x"); assert!(parser.match_not_set(&[(1, 1)], 1, 4).is_err()); } @@ -11912,6 +11995,28 @@ mod tests { assert!(parser.rule_context_stack.is_empty()); } + #[test] + fn active_invocation_states_exclude_the_root_frame() { + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + + let _root = parser.enter_rule(0, 0); + assert!(parser.active_invocation_states().is_empty()); + + let marker = parser.push_invoking_state(6); + let _child = parser.enter_rule(2, 1); + parser.discard_invoking_state(marker); + assert_eq!(parser.active_invocation_states(), [6]); + + let marker = parser.push_invoking_state(13); + let _grandchild = parser.enter_rule(4, 2); + parser.discard_invoking_state(marker); + assert_eq!(parser.active_invocation_states(), [13, 6]); + + parser.exit_rule(); + parser.exit_rule(); + parser.exit_rule(); + } + #[test] fn parser_predicates_support_token_adjacency() { let mut parser = mini_parser(vec![ @@ -11949,10 +12054,10 @@ mod tests { ]); let mut context = ParserRuleContext::new(1, 0); let mut child_context = ParserRuleContext::new(2, 0); - child_context.add_child(ParseTree::Terminal(test_terminal( - &TestToken::new(1).with_text("var"), - ))); - context.add_child(ParseTree::Rule(RuleNode::new(child_context))); + let terminal = parser.terminal_tree(TokenId::try_from(0).expect("test token ID")); + parser.tree.add_child(&mut child_context, terminal); + let child = parser.rule_node(child_context); + parser.tree.add_child(&mut context, child); let predicates = [( 1, 0, @@ -12061,14 +12166,11 @@ mod tests { .expect("generated match should insert missing token"); assert_eq!(node.children().len(), 1); - assert_eq!( - node.children()[0].text(parser.token_store()), - "" - ); + assert_eq!(parser.node(node.children()[0]).text(), ""); assert_eq!( node.clone() .into_child_iter() - .map(|child| child.text(parser.token_store())) + .map(|child| parser.node(child).text()) .collect::>(), [""] ); @@ -12115,12 +12217,12 @@ mod tests { .expect("generated match should delete the extraneous token"); assert_eq!(node.children().len(), 2); - assert!(matches!(node.children()[0], ParseTree::Error(_))); - assert_eq!(node.children()[0].text(parser.token_store()), "z"); - assert_eq!(node.children()[1].text(parser.token_store()), "y"); + assert_eq!(parser.node(node.children()[0]).kind(), NodeKind::Error); + assert_eq!(parser.node(node.children()[0]).text(), "z"); + assert_eq!(parser.node(node.children()[1]).text(), "y"); assert_eq!( node.into_child_iter() - .map(|child| child.text(parser.token_store())) + .map(|child| parser.node(child).text()) .collect::>(), ["z", "y"] ); @@ -12155,7 +12257,7 @@ mod tests { assert_eq!( node.into_child_iter() - .map(|child| child.text(parser.token_store())) + .map(|child| parser.node(child).text()) .collect::>(), ["y"] ); @@ -12355,8 +12457,9 @@ mod tests { assert_eq!(node.children().len(), 1); assert!(!node.consumed_eof()); assert!( - node.children()[0] - .text(parser.token_store()) + parser + .node(node.children()[0]) + .text() .starts_with(""); + assert_eq!(parser.node(tree).text(), "x"); assert_eq!(parser.number_of_syntax_errors(), 0); assert_eq!( - tree.first_rule_stop(0, parser.token_store()) + parser + .node(tree) + .first_rule_stop(0) .expect("rule should stop at EOF") .token_type(), TOKEN_EOF @@ -12599,7 +12739,9 @@ mod tests { .expect("runtime-option parser rule should parse"); assert!(actions.is_empty()); assert_eq!( - tree.first_rule_stop(0, parser.token_store()) + parser + .node(tree) + .first_rule_stop(0) .expect("rule should stop at EOF") .token_type(), TOKEN_EOF @@ -12618,7 +12760,7 @@ mod tests { .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) .expect("no-op parser action should not force action replay"); - assert_eq!(tree.text(parser.token_store()), "x"); + assert_eq!(parser.node(tree).text(), "x"); assert!( actions.is_empty(), "action_index=None transitions are ANTLR metadata, not replay actions" @@ -12637,7 +12779,7 @@ mod tests { let tree = parser .parse_atn_rule(&atn, 0) .expect("artificial parser rule should parse"); - assert_eq!(tree.text(parser.token_store()), "x"); + assert_eq!(parser.node(tree).text(), "x"); let stream = parser.token_stream(); let source_index_after_parse = stream.token_source().index; @@ -12673,7 +12815,9 @@ mod tests { assert_eq!(parser.number_of_syntax_errors(), 1); assert_eq!( - tree.first_error_token(parser.token_store()) + parser + .node(tree) + .first_error_token() .expect("recovery should embed an error token") .text(), "y" @@ -12709,7 +12853,7 @@ mod tests { .parse_atn_rule_adaptive_or_fallback(&atn, &mut simulator, 0) .expect("direct adaptive rule should parse"); - assert_eq!(tree.text(parser.token_store()), "y"); + assert_eq!(parser.node(tree).text(), "y"); assert_eq!(parser.input.index(), 1); } @@ -12727,8 +12871,12 @@ mod tests { .parse_atn_rule_adaptive_or_fallback(&atn, &mut simulator, 0) .expect("fallback recognizer should parse"); - assert_eq!(tree.text(parser.token_store()), "xy"); + assert_eq!(parser.node(tree).text(), "xy"); assert_eq!(parser.input.index(), 2); + let stats = parser.parse_tree_storage().stats(); + assert_eq!(stats.nodes, parser.node(tree).descendants().count()); + assert_eq!(stats.edges, stats.nodes.saturating_sub(1)); + assert_eq!(stats.scratch_links, 0); } #[test] @@ -12744,7 +12892,7 @@ mod tests { .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) .expect("unknown predicate should pass under the default policy"); - assert_eq!(tree.text(parser.token_store()), "xy"); + assert_eq!(parser.node(tree).text(), "xy"); assert_eq!(parser.number_of_syntax_errors(), 0); } @@ -12874,6 +13022,7 @@ mod tests { struct RecordingHooks { predicates: Vec<(usize, usize, usize, Option)>, actions: Vec<(usize, String, Option)>, + action_trees: Vec>, } impl SemanticHooks for RecordingHooks { @@ -12904,6 +13053,7 @@ mod tests { ctx.action_text(), ctx.rule_name().map(str::to_owned), )); + self.action_trees.push(ctx.tree().map(Node::text)); true } } @@ -12956,7 +13106,7 @@ mod tests { ) .expect("hook supplies the missing predicate result"); - assert_eq!(tree.text(parser.token_store()), "xy"); + assert_eq!(parser.node(tree).text(), "xy"); assert_eq!( parser.semantic_hooks.predicates, vec![(1, 0, 0, Some("y".to_owned()))] @@ -13002,11 +13152,15 @@ mod tests { .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) .expect("rule parses before action hook is tested"); - assert!(parser.parser_action_hook(ParserAction::new(42, 0, 0, Some(0)), &tree)); + assert!(parser.parser_action_hook(ParserAction::new(42, 0, 0, Some(0)), tree)); assert_eq!( parser.semantic_hooks.actions, vec![(42, "x".to_owned(), Some("s".to_owned()))] ); + assert_eq!( + parser.semantic_hooks.action_trees, + [Some("x".to_owned())] + ); } #[test] @@ -13016,10 +13170,10 @@ mod tests { // Error policy, so a `hook`-disposed action is not silently dropped. let mut parser = mini_parser_with_hooks(vec![TestToken::eof("t", 0, 1, 0)], DecliningHooks); parser.set_unknown_predicate_policy(UnknownSemanticPolicy::Error); - let tree = ParseTree::Rule(RuleNode::new(ParserRuleContext::new(0, -1))); + let tree = parser.rule_node(ParserRuleContext::new(0, -1)); // DecliningHooks::action returns false (unhandled). - assert!(!parser.parser_action_hook(ParserAction::new(42, 0, 0, Some(0)), &tree)); + assert!(!parser.parser_action_hook(ParserAction::new(42, 0, 0, Some(0)), tree)); let error = parser .take_unknown_semantic_error() @@ -13035,8 +13189,8 @@ mod tests { // Under the default (assume-true) policy the same miss is not recorded. let mut lenient = mini_parser_with_hooks(vec![TestToken::eof("t", 0, 1, 0)], DecliningHooks); - let tree = ParseTree::Rule(RuleNode::new(ParserRuleContext::new(0, -1))); - assert!(!lenient.parser_action_hook(ParserAction::new(42, 0, 0, Some(0)), &tree)); + let tree = lenient.rule_node(ParserRuleContext::new(0, -1)); + assert!(!lenient.parser_action_hook(ParserAction::new(42, 0, 0, Some(0)), tree)); assert!(lenient.take_unknown_semantic_error().is_none()); } @@ -13061,7 +13215,7 @@ mod tests { ) .expect("a predicate covered by the table is not an unknown coordinate"); - assert_eq!(tree.text(parser.token_store()), "xy"); + assert_eq!(parser.node(tree).text(), "xy"); } /// Hooks that decline (`None`) must fall through to the configured policy @@ -13113,7 +13267,7 @@ mod tests { ) .expect("a declined SemIR hook must pass under assume-true"); - assert_eq!(tree.text(parser.token_store()), "xy"); + assert_eq!(parser.node(tree).text(), "xy"); } #[test] @@ -13231,12 +13385,11 @@ mod tests { let tree = parser .parse_atn_rule(&atn, 0) .expect("artificial parser rule should parse"); - let Some(ParseTree::Rule(rule)) = tree.first_rule(0) else { + let Some(rule) = parser.node(tree).first_rule(0).and_then(Node::as_rule) else { panic!("rule node should be present"); }; assert_eq!( - rule.context() - .start(parser.token_store()) + rule.start() .expect("rule should have a start token") .token_type(), 1 @@ -13282,14 +13435,14 @@ mod tests { &mut ctx, parser.token_id_at(0).expect("ID token should be buffered"), ); - let tree = ParseTree::Rule(RuleNode::new(ctx)); + let tree = parser.rule_node(ctx); let current_index = parser.input.index(); // Cursor-only inference would wrongly pick EOF (the parked cursor)... assert_eq!(parser.after_action_stop_index(current_index), Some(1)); // ...but the tree-aware helper follows the rule context stop (ID). assert_eq!( - parser.after_action_stop_index_for_tree(&tree, current_index), + parser.after_action_stop_index_for_tree(tree, current_index), Some(0) ); } @@ -13300,7 +13453,7 @@ mod tests { // start (set by `enter_rule`) is the first visible token, not the raw cursor // that may still point at the hidden prefix. The @after start must follow // the context start so `$start`/`$text` excludes the hidden prefix. - let parser = mini_parser(vec![ + let mut parser = mini_parser(vec![ TestToken::new(9) .with_text(" ") .with_channel(HIDDEN_CHANNEL), @@ -13316,15 +13469,15 @@ mod tests { &mut ctx, parser.token_id_at(2).expect("ID token should be buffered"), ); - let tree = ParseTree::Rule(RuleNode::new(ctx)); + let tree = parser.rule_node(ctx); // The raw fallback (pre-rule cursor) would be 0 (the hidden prefix)... // ...but the tree-aware helper follows the rule context start (index 2). - assert_eq!(parser.after_action_start_index_for_tree(&tree, 0), 2); + assert_eq!(parser.after_action_start_index_for_tree(tree, 0), 2); // With no rule start recorded, it falls back to the provided index. - let empty = ParseTree::Rule(RuleNode::new(ParserRuleContext::new(0, 0))); - assert_eq!(parser.after_action_start_index_for_tree(&empty, 7), 7); + let empty = parser.rule_node(ParserRuleContext::new(0, 0)); + assert_eq!(parser.after_action_start_index_for_tree(empty, 7), 7); } #[test] diff --git a/src/tree.rs b/src/tree.rs index 72257545..63192eec 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -1,320 +1,936 @@ use crate::errors::AntlrError; use crate::recognizer::Recognizer; +use crate::token::{Token, TokenId, TokenStore, TokenView}; use std::any::Any; +use std::collections::BTreeMap; use std::fmt; -use std::rc::Rc; +use std::mem::size_of; -use crate::token::{TokenId, TokenStore, TokenView}; -use std::collections::BTreeMap; +const NONE: u32 = u32::MAX; +const FLAG_MATCHED_CHILD: u8 = 1 << 0; +const FLAG_START_PRESENT: u8 = 1 << 1; +const FLAG_STOP_PRESENT: u8 = 1 << 2; + +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub struct NodeId(u32); + +impl NodeId { + pub(crate) const fn placeholder() -> Self { + Self(NONE) + } + + #[must_use] + pub const fn index(self) -> usize { + self.0 as usize + } +} + +/// Compact parser result. The tree data lives in [`ParseTreeStorage`]. +pub type ParseTree = NodeId; + +#[repr(u8)] +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub enum NodeKind { + Rule, + Terminal, + Error, +} + +#[derive(Debug)] +struct ChildLink { + node: NodeId, + next: u32, +} + +#[derive(Debug)] +struct RuleExtra { + int_returns: BTreeMap, + exception: Option, + attrs: Option, +} -#[derive(Clone, Debug, Eq, PartialEq)] -pub enum ParseTree { - Rule(RuleNode), - Terminal(TerminalNode), - Error(ErrorNode), -} - -impl ParseTree { - /// Returns this node's direct children. - /// - /// Rule nodes return their context children; terminal and error nodes - /// return an empty slice, so generic recursive walks can start from a - /// `&ParseTree` without matching on every variant first. - pub fn children(&self) -> &[Self] { - match self { - Self::Rule(rule) => rule.context().children(), - Self::Terminal(_) | Self::Error(_) => &[], +#[derive(Debug)] +enum ParseTreeExtra { + Rule(RuleExtra), +} + +/// Flat, structure-of-arrays concrete syntax tree storage. +/// +/// Every node is addressed by [`NodeId`]. Rule children occupy one range in +/// `children`; `child_links` is parser scratch used only while rule contexts +/// are open and is never exposed as part of the completed tree. +#[derive(Debug, Default)] +pub struct ParseTreeStorage { + kinds: Vec, + child_starts: Vec, + child_lens: Vec, + payload_a: Vec, + payload_b: Vec, + starts: Vec, + stops: Vec, + alt_numbers: Vec, + extra_ids: Vec, + parents: Vec, + flags: Vec, + children: Vec, + extras: Vec, + child_links: Vec, +} + +#[derive(Clone, Copy, Debug, Default, Eq, PartialEq)] +pub struct ParseTreeStats { + pub nodes: usize, + pub edges: usize, + pub extras: usize, + pub scratch_links: usize, + pub allocated_bytes: usize, +} + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +pub(crate) struct ParseTreeCheckpoint { + nodes: usize, + children: usize, + extras: usize, + child_links: usize, +} + +impl ParseTreeStorage { + #[must_use] + pub const fn new() -> Self { + Self { + kinds: Vec::new(), + child_starts: Vec::new(), + child_lens: Vec::new(), + payload_a: Vec::new(), + payload_b: Vec::new(), + starts: Vec::new(), + stops: Vec::new(), + alt_numbers: Vec::new(), + extra_ids: Vec::new(), + parents: Vec::new(), + flags: Vec::new(), + children: Vec::new(), + extras: Vec::new(), + child_links: Vec::new(), } } - /// Iterates this tree in pre-order, starting with `self`. - pub fn descendants(&self) -> ParseTreeDescendants<'_> { - ParseTreeDescendants { stack: vec![self] } + #[must_use] + pub const fn node_count(&self) -> usize { + self.kinds.len() } - /// Iterates this tree in pre-order, starting with `self`. - /// - /// This is an alias for [`Self::descendants`] for callers that prefer to - /// name the traversal order explicitly. - pub fn pre_order(&self) -> ParseTreeDescendants<'_> { - self.descendants() + #[must_use] + pub const fn edge_count(&self) -> usize { + self.children.len() } - pub fn text(&self, tokens: &TokenStore) -> String { - match self { - Self::Rule(rule) => rule.text(tokens), - Self::Terminal(node) => node.text(tokens).to_owned(), - Self::Error(node) => node.text(tokens).to_owned(), + #[must_use] + pub const fn extra_count(&self) -> usize { + self.extras.len() + } + + #[must_use] + pub const fn stats(&self) -> ParseTreeStats { + ParseTreeStats { + nodes: self.node_count(), + edges: self.edge_count(), + extras: self.extra_count(), + scratch_links: self.child_links.len(), + allocated_bytes: self.kinds.capacity() * size_of::() + + self.child_starts.capacity() * size_of::() + + self.child_lens.capacity() * size_of::() + + self.payload_a.capacity() * size_of::() + + self.payload_b.capacity() * size_of::() + + self.starts.capacity() * size_of::() + + self.stops.capacity() * size_of::() + + self.alt_numbers.capacity() * size_of::() + + self.extra_ids.capacity() * size_of::() + + self.parents.capacity() * size_of::() + + self.flags.capacity() * size_of::() + + self.children.capacity() * size_of::() + + self.extras.capacity() * size_of::() + + self.child_links.capacity() * size_of::(), } } - pub fn to_string_tree_with_names>( - &self, - rule_names: &[S], - tokens: &TokenStore, - ) -> String { - match self { - Self::Rule(rule) => rule.to_string_tree_with_names(rule_names, tokens), - Self::Terminal(node) => escape_tree_text(node.text(tokens)), - Self::Error(node) => escape_tree_text(node.text(tokens)), + pub(crate) fn release_scratch(&mut self) { + self.child_links.clear(); + } + + pub(crate) fn discard_scratch(&mut self) { + self.child_links = Vec::new(); + } + + pub(crate) const fn checkpoint(&self) -> ParseTreeCheckpoint { + ParseTreeCheckpoint { + nodes: self.kinds.len(), + children: self.children.len(), + extras: self.extras.len(), + child_links: self.child_links.len(), } } - /// Renders the LISP-style tree using rule names resolved through a - /// recognizer, matching ANTLR's `toStringTree(parser)` shape used by - /// generated test actions (`.to_string_tree(Some(self), tokens)`). - pub fn to_string_tree( - &self, - recognizer: Option<&R>, - tokens: &TokenStore, - ) -> String { - recognizer.map_or_else( - || self.to_string_tree_with_names::<&str>(&[], tokens), - |recognizer| self.to_string_tree_with_names(recognizer.data().rule_names(), tokens), - ) + pub(crate) fn rollback(&mut self, checkpoint: ParseTreeCheckpoint) { + self.kinds.truncate(checkpoint.nodes); + self.child_starts.truncate(checkpoint.nodes); + self.child_lens.truncate(checkpoint.nodes); + self.payload_a.truncate(checkpoint.nodes); + self.payload_b.truncate(checkpoint.nodes); + self.starts.truncate(checkpoint.nodes); + self.stops.truncate(checkpoint.nodes); + self.alt_numbers.truncate(checkpoint.nodes); + self.extra_ids.truncate(checkpoint.nodes); + self.parents.truncate(checkpoint.nodes); + self.flags.truncate(checkpoint.nodes); + self.children.truncate(checkpoint.children); + self.extras.truncate(checkpoint.extras); + self.child_links.truncate(checkpoint.child_links); + } + + pub(crate) fn terminal(&mut self, token: TokenId) -> NodeId { + self.push_node(NodeRecord { + kind: NodeKind::Terminal, + payload_a: token.index() as u32, + ..NodeRecord::default() + }) } - /// Finds the first rule node with `rule_index` in a depth-first walk. - pub fn first_rule(&self, rule_index: usize) -> Option<&Self> { - match self { - Self::Rule(rule) => { - if rule.context().rule_index() == rule_index { - return Some(self); - } - rule.context() - .children() - .iter() - .find_map(|child| child.first_rule(rule_index)) + pub(crate) fn error(&mut self, token: TokenId) -> NodeId { + self.push_node(NodeRecord { + kind: NodeKind::Error, + payload_a: token.index() as u32, + ..NodeRecord::default() + }) + } + + pub(crate) fn add_child(&mut self, context: &mut ParserRuleContext, child: NodeId) { + context.matched_child = true; + let link = self.child_links.len_u32("parse-tree scratch child links"); + self.child_links.push(ChildLink { + node: child, + next: NONE, + }); + if context.first_child == NONE { + context.first_child = link; + } else { + self.child_links[context.last_child as usize].next = link; + } + context.last_child = link; + context.child_count = context + .child_count + .checked_add(1) + .expect("rule child count exceeds u32"); + } + + pub(crate) fn finish_rule(&mut self, context: ParserRuleContext) -> NodeId { + let parent = NodeId(self.kinds.len_u32("parse-tree node pool")); + let child_start = self.children.len_u32("parse-tree child pool"); + let mut link = context.first_child; + while link != NONE { + let child = &self.child_links[link as usize]; + self.children.push(child.node); + self.parents[child.node.index()] = parent.0; + link = child.next; + } + + let extra_id = if context.int_returns.is_empty() + && context.exception.is_none() + && context.attrs.is_none() + { + NONE + } else { + let id = self.extras.len_u32("parse-tree extra pool"); + self.extras.push(ParseTreeExtra::Rule(RuleExtra { + int_returns: context.int_returns, + exception: context.exception, + attrs: context.attrs, + })); + id + }; + + self.push_node(NodeRecord { + kind: NodeKind::Rule, + child_start, + child_len: context.child_count, + payload_a: u32::try_from(context.rule_index).expect("rule index exceeds u32"), + payload_b: i32::try_from(context.invoking_state) + .expect("invoking state exceeds i32") + .cast_unsigned(), + start: context.start.map_or(NONE, |token| token.index() as u32), + stop: context.stop.map_or(NONE, |token| token.index() as u32), + alt_number: u32::try_from(context.alt_number).expect("alternative number exceeds u32"), + extra_id, + flags: (u8::from(context.matched_child) * FLAG_MATCHED_CHILD) + | (u8::from(context.start.is_some()) * FLAG_START_PRESENT) + | (u8::from(context.stop.is_some()) * FLAG_STOP_PRESENT), + }) + } + + fn push_node(&mut self, record: NodeRecord) -> NodeId { + let id = NodeId(self.kinds.len_u32("parse-tree node pool")); + self.kinds.push(record.kind); + self.child_starts.push(record.child_start); + self.child_lens.push(record.child_len); + self.payload_a.push(record.payload_a); + self.payload_b.push(record.payload_b); + self.starts.push(record.start); + self.stops.push(record.stop); + self.alt_numbers.push(record.alt_number); + self.extra_ids.push(record.extra_id); + self.parents.push(NONE); + self.flags.push(record.flags); + id + } + + #[must_use] + pub fn node<'tree>(&'tree self, tokens: &'tree TokenStore, id: NodeId) -> Option> { + (id.index() < self.node_count()).then_some(Node { + storage: self, + tokens, + id, + }) + } + + fn kind(&self, id: NodeId) -> NodeKind { + self.kinds[id.index()] + } + + fn child_ids(&self, id: NodeId) -> &[NodeId] { + let index = id.index(); + let start = self.child_starts[index] as usize; + let len = self.child_lens[index] as usize; + &self.children[start..start + len] + } + + const fn context_child_ids<'a>( + &'a self, + context: &'a ParserRuleContext, + ) -> ContextChildIds<'a> { + ContextChildIds { + storage: self, + next: context.first_child, + remaining: context.child_count as usize, + } + } + + fn token_id(&self, id: NodeId) -> Option { + match self.kind(id) { + NodeKind::Terminal | NodeKind::Error => { + Some(stored_token_id(self.payload_a[id.index()])) } - Self::Terminal(_) | Self::Error(_) => None, + NodeKind::Rule => None, } } - /// Finds the stop token for the first rule node with `rule_index`. - pub fn first_rule_stop<'a>( - &self, - rule_index: usize, - tokens: &'a TokenStore, - ) -> Option> { - let Self::Rule(rule) = self else { + fn rule_extra(&self, id: NodeId) -> Option<&RuleExtra> { + let extra = *self.extra_ids.get(id.index())?; + if extra == NONE { return None; - }; - if rule.context().rule_index() == rule_index { - return rule.context().stop(tokens); } - rule.context() - .children() - .iter() - .find_map(|child| child.first_rule_stop(rule_index, tokens)) + match &self.extras[extra as usize] { + ParseTreeExtra::Rule(extra) => Some(extra), + } } +} - /// Reads an integer return value from the first rule node with - /// `rule_index`, matching ANTLR's `$label.value` resolution for labeled - /// rule references in the runtime testsuite. - pub fn first_rule_int_return(&self, rule_index: usize, name: &str) -> Option { - let Self::Rule(rule) = self else { - return None; - }; - if rule.context().rule_index() == rule_index { - return rule.context().int_return(name); +#[derive(Clone, Copy)] +struct NodeRecord { + kind: NodeKind, + child_start: u32, + child_len: u32, + payload_a: u32, + payload_b: u32, + start: u32, + stop: u32, + alt_number: u32, + extra_id: u32, + flags: u8, +} + +impl Default for NodeRecord { + fn default() -> Self { + Self { + kind: NodeKind::Rule, + child_start: 0, + child_len: 0, + payload_a: 0, + payload_b: 0, + start: NONE, + stop: NONE, + alt_number: 0, + extra_id: NONE, + flags: 0, } - rule.context() - .children() - .iter() - .find_map(|child| child.first_rule_int_return(rule_index, name)) } +} - /// Reads the typed attribute snapshot from this tree's root rule node. - /// - /// Generated parsers use this for `$label.attr` / `$rule.attr` reads on a - /// child subtree returned by a rule call. - pub fn rule_attrs(&self) -> Option<&T> { - let Self::Rule(rule) = self else { - return None; - }; - rule.context().generated_attrs::() - } - - /// Finds the first recovery error token in a depth-first walk. - pub fn first_error_token<'a>(&self, tokens: &'a TokenStore) -> Option> { - match self { - Self::Rule(rule) => rule - .context() - .children() - .iter() - .find_map(|child| child.first_error_token(tokens)), - Self::Terminal(_) => None, - Self::Error(node) => Some(node.symbol(tokens)), +trait LenU32 { + fn len_u32(&self, name: &str) -> u32; +} + +impl LenU32 for Vec { + fn len_u32(&self, name: &str) -> u32 { + u32::try_from(self.len()).unwrap_or_else(|_| panic!("{name} exceeds u32")) + } +} + +#[derive(Clone, Copy, Debug)] +pub struct Node<'tree> { + storage: &'tree ParseTreeStorage, + tokens: &'tree TokenStore, + id: NodeId, +} + +impl<'tree> Node<'tree> { + #[must_use] + pub const fn id(self) -> NodeId { + self.id + } + + #[must_use] + pub fn kind(self) -> NodeKind { + self.storage.kind(self.id) + } + + #[must_use] + pub fn as_rule(self) -> Option> { + (self.kind() == NodeKind::Rule).then_some(RuleNodeView { node: self }) + } + + #[must_use] + pub fn as_terminal(self) -> Option> { + (self.kind() == NodeKind::Terminal).then_some(TerminalNodeView { node: self }) + } + + #[must_use] + pub fn as_error(self) -> Option> { + (self.kind() == NodeKind::Error).then_some(ErrorNodeView { node: self }) + } + + #[must_use] + pub fn children(self) -> NodeChildren<'tree> { + NodeChildren { + storage: self.storage, + tokens: self.tokens, + ids: self.storage.child_ids(self.id).iter(), + } + } + + #[must_use] + pub fn parent(self) -> Option { + let parent = self.storage.parents[self.id.index()]; + (parent != NONE) + .then(|| self.storage.node(self.tokens, NodeId(parent))) + .flatten() + } + + #[must_use] + pub fn descendants(self) -> ParseTreeDescendants<'tree> { + ParseTreeDescendants { + storage: self.storage, + tokens: self.tokens, + stack: vec![self.id], + } + } + + #[must_use] + pub fn pre_order(self) -> ParseTreeDescendants<'tree> { + self.descendants() + } + + #[must_use] + pub fn text(self) -> String { + match self.kind() { + NodeKind::Terminal | NodeKind::Error => self + .storage + .token_id(self.id) + .and_then(|id| self.tokens.text(id)) + .unwrap_or("") + .to_owned(), + NodeKind::Rule => { + let mut text = String::new(); + let mut stack = self + .storage + .child_ids(self.id) + .iter() + .rev() + .copied() + .collect::>(); + while let Some(id) = stack.pop() { + match self.storage.kind(id) { + NodeKind::Rule => { + stack.extend(self.storage.child_ids(id).iter().rev().copied()); + } + NodeKind::Terminal | NodeKind::Error => text.push_str( + self.storage + .token_id(id) + .and_then(|token| self.tokens.text(token)) + .unwrap_or(""), + ), + } + } + text + } + } + } + + #[must_use] + pub fn to_string_tree_with_names>(self, rule_names: &[S]) -> String { + match self.kind() { + NodeKind::Rule => self + .as_rule() + .expect("rule node kind checked") + .to_string_tree_with_names(rule_names), + NodeKind::Terminal | NodeKind::Error => escape_tree_text( + self.storage + .token_id(self.id) + .and_then(|id| self.tokens.text(id)) + .unwrap_or(""), + ), } } - /// Returns the first rule invocation stack for `rule_index`, ordered from - /// the selected rule outward to the root rule. + #[must_use] + pub fn to_string_tree( + self, + recognizer: Option<&R>, + _tokens: &TokenStore, + ) -> String { + recognizer.map_or_else( + || self.to_string_tree_with_names::<&str>(&[]), + |recognizer| self.to_string_tree_with_names(recognizer.data().rule_names()), + ) + } + + #[must_use] + pub fn first_rule(self, rule_index: usize) -> Option { + self.descendants().find(|node| { + node.as_rule() + .is_some_and(|rule| rule.rule_index() == rule_index) + }) + } + + #[must_use] + pub fn first_rule_stop(self, rule_index: usize) -> Option> { + self.first_rule(rule_index)?.as_rule()?.stop() + } + + #[must_use] + pub fn first_rule_int_return(self, rule_index: usize, name: &str) -> Option { + self.first_rule(rule_index)?.as_rule()?.int_return(name) + } + + #[must_use] + pub fn rule_attrs(self) -> Option<&'tree T> { + self.as_rule()?.generated_attrs::() + } + + #[must_use] + pub fn first_error_token(self) -> Option> { + self.descendants() + .find_map(Node::as_error) + .map(ErrorNodeView::symbol) + } + + #[must_use] pub fn rule_invocation_stack>( - &self, + self, rule_index: usize, rule_names: &[S], ) -> Option> { - let mut stack = Vec::new(); - if self.find_rule_path(rule_index, rule_names, &mut stack) { - stack.reverse(); - return Some(stack); + let mut stack = vec![(self.id, 0_usize)]; + let mut names = Vec::new(); + while let Some((id, child_index)) = stack.last_mut() { + if *child_index == 0 { + let Some(rule) = self.storage.node(self.tokens, *id).and_then(Node::as_rule) else { + stack.pop(); + continue; + }; + names.push( + rule_names + .get(rule.rule_index()) + .map_or("", |name| name.as_ref()) + .to_owned(), + ); + if rule.rule_index() == rule_index { + names.reverse(); + return Some(names); + } + } + let children = self.storage.child_ids(*id); + let next = children.get(*child_index).copied(); + *child_index += 1; + if let Some(child) = next { + if self.storage.kind(child) == NodeKind::Rule { + stack.push((child, 0)); + } + } else { + stack.pop(); + names.pop(); + } } None } +} - fn find_rule_path>( - &self, +#[derive(Clone, Debug)] +pub struct NodeChildren<'tree> { + storage: &'tree ParseTreeStorage, + tokens: &'tree TokenStore, + ids: std::slice::Iter<'tree, NodeId>, +} + +impl<'tree> Iterator for NodeChildren<'tree> { + type Item = Node<'tree>; + + fn next(&mut self) -> Option { + self.ids + .next() + .and_then(|id| self.storage.node(self.tokens, *id)) + } + + fn size_hint(&self) -> (usize, Option) { + self.ids.size_hint() + } +} + +impl DoubleEndedIterator for NodeChildren<'_> { + fn next_back(&mut self) -> Option { + self.ids + .next_back() + .and_then(|id| self.storage.node(self.tokens, *id)) + } +} + +impl ExactSizeIterator for NodeChildren<'_> {} + +#[derive(Clone, Debug)] +pub struct ParseTreeDescendants<'tree> { + storage: &'tree ParseTreeStorage, + tokens: &'tree TokenStore, + stack: Vec, +} + +impl<'tree> Iterator for ParseTreeDescendants<'tree> { + type Item = Node<'tree>; + + fn next(&mut self) -> Option { + let id = self.stack.pop()?; + self.stack + .extend(self.storage.child_ids(id).iter().rev().copied()); + Some(Node { + storage: self.storage, + tokens: self.tokens, + id, + }) + } +} + +#[derive(Clone, Copy, Debug)] +pub struct RuleNodeView<'tree> { + node: Node<'tree>, +} + +impl<'tree> RuleNodeView<'tree> { + #[must_use] + pub const fn node(self) -> Node<'tree> { + self.node + } + + #[must_use] + pub fn rule_index(self) -> usize { + self.node.storage.payload_a[self.node.id.index()] as usize + } + + #[must_use] + pub fn invoking_state(self) -> isize { + self.node.storage.payload_b[self.node.id.index()].cast_signed() as isize + } + + #[must_use] + pub fn alt_number(self) -> usize { + self.node.storage.alt_numbers[self.node.id.index()] as usize + } + + #[must_use] + pub fn start(self) -> Option> { + self.start_id().and_then(|id| self.node.tokens.view(id)) + } + + #[must_use] + pub fn start_id(self) -> Option { + let index = self.node.id.index(); + (self.node.storage.flags[index] & FLAG_START_PRESENT != 0) + .then(|| stored_token_id(self.node.storage.starts[index])) + } + + #[must_use] + pub fn stop(self) -> Option> { + self.stop_id().and_then(|id| self.node.tokens.view(id)) + } + + #[must_use] + pub fn stop_id(self) -> Option { + let index = self.node.id.index(); + (self.node.storage.flags[index] & FLAG_STOP_PRESENT != 0) + .then(|| stored_token_id(self.node.storage.stops[index])) + } + + #[must_use] + pub fn children(self) -> NodeChildren<'tree> { + self.node.children() + } + + #[must_use] + pub fn child_count(self) -> usize { + self.node.storage.child_lens[self.node.id.index()] as usize + } + + #[must_use] + pub fn child_rule(self, rule_index: usize) -> Option { + self.child_rules(rule_index).next() + } + + pub fn child_rules(self, rule_index: usize) -> impl DoubleEndedIterator + 'tree { + self.children().filter_map(move |child| { + let rule = child.as_rule()?; + (rule.rule_index() == rule_index).then_some(rule) + }) + } + + pub fn child_rule_trees( + self, rule_index: usize, - rule_names: &[S], - stack: &mut Vec, - ) -> bool { - let Self::Rule(rule) = self else { - return false; + ) -> impl DoubleEndedIterator> + 'tree { + self.child_rules(rule_index).map(Self::node) + } + + #[must_use] + pub fn child_token(self, token_type: i32) -> Option> { + self.child_tokens(token_type).next() + } + + pub fn child_tokens( + self, + token_type: i32, + ) -> impl DoubleEndedIterator> + 'tree { + self.children().filter_map(move |child| { + let terminal = match child.kind() { + NodeKind::Terminal => child.as_terminal(), + NodeKind::Error => child.as_error().map(ErrorNodeView::terminal), + NodeKind::Rule => None, + }?; + (terminal.symbol().token_type() == token_type).then_some(terminal) + }) + } + + pub fn terminal_children( + self, + ) -> impl DoubleEndedIterator> + 'tree { + self.children().filter_map(|child| match child.kind() { + NodeKind::Terminal => child.as_terminal(), + NodeKind::Error => child.as_error().map(ErrorNodeView::terminal), + NodeKind::Rule => None, + }) + } + + #[must_use] + pub fn has_token(self, token_type: i32) -> bool { + self.child_token(token_type).is_some() + } + + #[must_use] + pub fn text(self) -> String { + self.node.text() + } + + #[must_use] + pub fn int_return(self, name: &str) -> Option { + self.node + .storage + .rule_extra(self.node.id)? + .int_returns + .get(name) + .copied() + } + + #[must_use] + pub fn generated_attrs(self) -> Option<&'tree T> { + self.node + .storage + .rule_extra(self.node.id)? + .attrs + .as_ref()? + .downcast_ref::() + } + + #[must_use] + pub fn exception(self) -> Option<&'tree AntlrError> { + self.node + .storage + .rule_extra(self.node.id)? + .exception + .as_ref() + } + + #[must_use] + pub fn downcast_ref>(self) -> Option { + T::from_rule_node(self) + } + + pub fn invocation_states(self) -> impl Iterator + 'tree { + std::iter::successors(Some(self), |rule| rule.node.parent()?.as_rule()) + .take_while(|rule| rule.node.parent().is_some() && rule.invoking_state() >= 0) + .map(Self::invoking_state) + } + + #[must_use] + pub fn to_string_tree_with_names>(self, rule_names: &[S]) -> String { + let name = rule_names + .get(self.rule_index()) + .map_or("", |name| name.as_ref()); + let display_name = if self.alt_number() == 0 { + name.to_owned() + } else { + format!("{name}:{}", self.alt_number()) }; - let current_index = rule.context().rule_index(); - stack.push( - rule_names - .get(current_index) - .map_or("", |name| name.as_ref()) - .to_owned(), - ); - if current_index == rule_index - || rule - .context() - .children() - .iter() - .any(|child| child.find_rule_path(rule_index, rule_names, stack)) - { - return true; + if self.child_count() == 0 { + return display_name; } - stack.pop(); - false + let children = self + .children() + .map(|child| child.to_string_tree_with_names(rule_names)) + .collect::>() + .join(" "); + format!("({display_name} {children})") + } + + #[must_use] + pub fn to_string_tree(self, recognizer: Option<&R>) -> String { + recognizer.map_or_else( + || self.to_string_tree_with_names::<&str>(&[]), + |recognizer| self.to_string_tree_with_names(recognizer.data().rule_names()), + ) } } -#[derive(Clone, Debug)] -pub struct ParseTreeDescendants<'a> { - stack: Vec<&'a ParseTree>, +#[derive(Clone, Copy, Debug)] +pub struct TerminalNodeView<'tree> { + node: Node<'tree>, } -impl<'a> Iterator for ParseTreeDescendants<'a> { - type Item = &'a ParseTree; +impl<'tree> TerminalNodeView<'tree> { + #[must_use] + pub const fn node(self) -> Node<'tree> { + self.node + } - fn next(&mut self) -> Option { - let tree = self.stack.pop()?; - self.stack.extend(tree.children().iter().rev()); - Some(tree) + #[must_use] + pub fn token_id(self) -> TokenId { + self.node + .storage + .token_id(self.node.id) + .expect("terminal node should contain a token ID") } -} -fn escape_tree_text(text: &str) -> String { - let mut escaped = String::with_capacity(text.len()); - for ch in text.chars() { - match ch { - '\n' => escaped.push_str("\\n"), - '\r' => escaped.push_str("\\r"), - '\t' => escaped.push_str("\\t"), - _ => escaped.push(ch), - } + #[must_use] + pub fn symbol(self) -> TokenView<'tree> { + self.node + .tokens + .view(self.token_id()) + .expect("terminal node token ID should remain valid") + } + + #[must_use] + pub fn text(self) -> &'tree str { + self.node.tokens.text(self.token_id()).unwrap_or("") } - escaped } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct RuleNode { - context: ParserRuleContext, +#[derive(Clone, Copy, Debug)] +pub struct ErrorNodeView<'tree> { + node: Node<'tree>, } -impl RuleNode { - pub const fn new(context: ParserRuleContext) -> Self { - Self { context } +impl<'tree> ErrorNodeView<'tree> { + #[must_use] + pub const fn node(self) -> Node<'tree> { + self.node } - pub const fn context(&self) -> &ParserRuleContext { - &self.context + #[must_use] + pub const fn terminal(self) -> TerminalNodeView<'tree> { + TerminalNodeView { node: self.node } } - pub const fn context_mut(&mut self) -> &mut ParserRuleContext { - &mut self.context + #[must_use] + pub fn token_id(self) -> TokenId { + self.terminal().token_id() } - pub fn text(&self, tokens: &TokenStore) -> String { - self.context.text(tokens) + #[must_use] + pub fn symbol(self) -> TokenView<'tree> { + self.terminal().symbol() } - pub fn to_string_tree_with_names>( - &self, - rule_names: &[S], - tokens: &TokenStore, - ) -> String { - self.context.to_string_tree_with_names(rule_names, tokens) + #[must_use] + pub fn text(self) -> &'tree str { + self.terminal().text() } } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ParserRuleContext { - rule_index: usize, - invoking_state: isize, - alt_number: usize, - start: Option, - stop: Option, - int_returns: Option>, - children: Vec, - /// Whether any child has been offered to this context, independent of whether - /// the tree was actually built. `children` stays empty when - /// `Parser::set_build_parse_trees(false)`, so generated recovery uses this - /// flag (not `children.is_empty()`) to tell whether the rule has matched - /// anything yet. - matched_child: bool, - // Boxed: an `AntlrError` is large and only set on the rare error path, so - // keeping it behind a pointer keeps `ParserRuleContext` (and thus the - // `ParseTree::Rule` variant) compact. - exception: Option>, - /// Typed generated-rule attribute snapshot (see [`GeneratedAttrs`]). - attrs: Option, +#[derive(Debug)] +struct ContextChildIds<'a> { + storage: &'a ParseTreeStorage, + next: u32, + remaining: usize, } -#[derive(Clone, Debug, Default, Eq, PartialEq)] -struct IntReturns(BTreeMap); +impl Iterator for ContextChildIds<'_> { + type Item = NodeId; -/// Typed rule-attribute storage attached to a [`ParserRuleContext`]. -/// -/// Generated parsers keep each rule's `returns`/`locals`/argument attributes -/// in a generated per-rule struct and seal a shared snapshot onto the finished -/// context, so a parent rule (or a listener) can read `$child.attr` / -/// `ctx.attr` with its real Rust type — the analog of ANTLR's attribute -/// fields on generated context classes. -#[derive(Clone)] -pub struct GeneratedAttrs(Rc); - -impl GeneratedAttrs { - pub fn new(attrs: T) -> Self { - Self(Rc::new(attrs)) + fn next(&mut self) -> Option { + if self.next == NONE { + return None; + } + let link = &self.storage.child_links[self.next as usize]; + self.next = link.next; + self.remaining -= 1; + Some(link.node) } - pub fn downcast_ref(&self) -> Option<&T> { - self.0.downcast_ref::() + fn size_hint(&self) -> (usize, Option) { + (self.remaining, Some(self.remaining)) } } -impl fmt::Debug for GeneratedAttrs { - fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { - f.write_str("GeneratedAttrs(..)") - } -} +impl ExactSizeIterator for ContextChildIds<'_> {} -// Attribute snapshots are sealed once per finished rule; two contexts are the -// same context (and thus equal) exactly when they share the same snapshot. -impl PartialEq for GeneratedAttrs { - fn eq(&self, other: &Self) -> bool { - Rc::ptr_eq(&self.0, &other.0) - } +/// Transient builder for one open rule. +/// +/// This value owns no tree nodes or child vector. Child IDs are appended to +/// parser-owned scratch links and are copied once into the global child pool +/// when the rule completes. +#[derive(Debug)] +pub struct ParserRuleContext { + rule_index: usize, + invoking_state: isize, + alt_number: usize, + start: Option, + stop: Option, + int_returns: BTreeMap, + first_child: u32, + last_child: u32, + child_count: u32, + matched_child: bool, + exception: Option, + attrs: Option, } -impl Eq for GeneratedAttrs {} - impl ParserRuleContext { + #[must_use] pub const fn new(rule_index: usize, invoking_state: isize) -> Self { Self { rule_index, @@ -322,41 +938,35 @@ impl ParserRuleContext { alt_number: 0, start: None, stop: None, - int_returns: None, - children: Vec::new(), + int_returns: BTreeMap::new(), + first_child: NONE, + last_child: NONE, + child_count: 0, matched_child: false, exception: None, attrs: None, } } - pub(crate) fn with_child_capacity( + pub(crate) const fn with_child_capacity( rule_index: usize, invoking_state: isize, - capacity: usize, + _capacity: usize, ) -> Self { - Self { - rule_index, - invoking_state, - alt_number: 0, - start: None, - stop: None, - int_returns: None, - children: Vec::with_capacity(capacity), - matched_child: false, - exception: None, - attrs: None, - } + Self::new(rule_index, invoking_state) } + #[must_use] pub const fn rule_index(&self) -> usize { self.rule_index } + #[must_use] pub const fn invoking_state(&self) -> isize { self.invoking_state } + #[must_use] pub const fn alt_number(&self) -> usize { self.alt_number } @@ -377,10 +987,6 @@ impl ParserRuleContext { self.stop.and_then(|id| tokens.view(id)) } - pub(crate) const fn stop_id(&self) -> Option { - self.stop - } - pub(crate) const fn set_start_id(&mut self, token: TokenId) { self.start = Some(token); } @@ -393,157 +999,119 @@ impl ParserRuleContext { self.start = other.start; } - /// Stores a generated integer return value on this rule context. pub fn set_int_return(&mut self, name: impl Into, value: i64) { - self.int_returns - .get_or_insert_with(Box::default) - .0 - .insert(name.into(), value); + self.int_returns.insert(name.into(), value); } - /// Reads a generated integer return value from this rule context. + #[must_use] pub fn int_return(&self, name: &str) -> Option { - self.int_returns - .as_ref() - .and_then(|values| values.0.get(name).copied()) + self.int_returns.get(name).copied() } - /// Seals the generated rule-attribute snapshot on this context. pub fn set_generated_attrs(&mut self, attrs: GeneratedAttrs) { self.attrs = Some(attrs); } - /// Reads the typed generated rule-attribute snapshot, if sealed. + #[must_use] pub fn generated_attrs(&self) -> Option<&T> { self.attrs.as_ref().and_then(GeneratedAttrs::downcast_ref) } - pub fn exception(&self) -> Option<&AntlrError> { - self.exception.as_deref() + #[must_use] + pub const fn exception(&self) -> Option<&AntlrError> { + self.exception.as_ref() } pub fn set_exception(&mut self, error: AntlrError) { - self.exception = Some(Box::new(error)); - } - - pub fn children(&self) -> &[ParseTree] { - &self.children + self.exception = Some(error); } - /// Returns the number of direct children in this context. + #[must_use] pub const fn child_count(&self) -> usize { - self.children.len() + self.child_count as usize } - /// Finds the first direct child rule with `rule_index`. - pub fn child_rule(&self, rule_index: usize) -> Option<&Self> { - self.child_rules(rule_index).next() + #[must_use] + pub const fn has_matched_child(&self) -> bool { + self.matched_child } - /// Iterates over direct child rules with `rule_index`. - pub fn child_rules(&self, rule_index: usize) -> impl Iterator + '_ { - self.children.iter().filter_map(move |child| match child { - ParseTree::Rule(rule) if rule.context().rule_index() == rule_index => { - Some(rule.context()) - } - _ => None, - }) + pub const fn note_matched_child(&mut self) { + self.matched_child = true; } - /// Finds the first direct token child with `token_type`. - /// - /// Includes recovery error nodes, which ANTLR treats as terminal nodes for - /// token-getter helpers. - pub fn child_token<'a>( + pub fn child_nodes<'a>( &'a self, - token_type: i32, + storage: &'a ParseTreeStorage, tokens: &'a TokenStore, - ) -> Option<&'a TerminalNode> { - self.child_tokens(token_type, tokens).next() - } - - /// Iterates over direct child subtrees whose root rule has `rule_index`. - /// - /// Like [`Self::child_rules`] but yielding the full [`ParseTree`] child, - /// for generated `$label.ctx` reads and listener walks over a labeled - /// subtree. - pub fn child_rule_trees(&self, rule_index: usize) -> impl Iterator + '_ { - self.children.iter().filter(move |child| match child { - ParseTree::Rule(rule) => rule.context().rule_index() == rule_index, - ParseTree::Terminal(_) | ParseTree::Error(_) => false, - }) + ) -> impl Iterator> + 'a { + storage + .context_child_ids(self) + .filter_map(move |id| storage.node(tokens, id)) } - /// Iterates over direct token children with `token_type`, including - /// recovery error nodes (ANTLR treats those as terminals for getters). - pub fn child_tokens<'a>( + pub fn child_rules<'a>( &'a self, - token_type: i32, + storage: &'a ParseTreeStorage, tokens: &'a TokenStore, - ) -> impl Iterator + 'a { - self.children.iter().filter_map(move |child| match child { - ParseTree::Terminal(node) if tokens.token_type(node.token_id()) == Some(token_type) => { - Some(node) - } - ParseTree::Error(node) if tokens.token_type(node.token_id()) == Some(token_type) => { - Some(node.terminal()) - } - _ => None, - }) - } - - /// Iterates over all direct terminal children regardless of token type. - pub fn terminal_children(&self) -> impl Iterator + '_ { - self.children.iter().filter_map(|child| match child { - ParseTree::Terminal(node) => Some(node), - ParseTree::Error(node) => Some(node.terminal()), - ParseTree::Rule(_) => None, + rule_index: usize, + ) -> impl Iterator> + 'a { + self.child_nodes(storage, tokens).filter_map(move |child| { + let rule = child.as_rule()?; + (rule.rule_index() == rule_index).then_some(rule) }) } - /// Downcast-style conversion to a generated typed context view. - /// - /// Generated parsers implement [`FromRuleContext`] for each context type; - /// this mirrors ANTLR's `((BinaryContext) $ctx)` cast in test actions - /// (`$ctx.downcast_ref::(tokens)`). - pub fn downcast_ref<'a, T: FromRuleContext<'a>>(&self, tokens: &'a TokenStore) -> Option { - T::from_rule_context(self, tokens) - } - - /// Returns whether this context has a direct token child with `token_type`. - pub fn has_token(&self, token_type: i32, tokens: &TokenStore) -> bool { - self.child_token(token_type, tokens).is_some() - } - - pub fn add_child(&mut self, child: ParseTree) { - self.matched_child = true; - self.children.push(child); + pub fn child_rule_trees<'a>( + &'a self, + storage: &'a ParseTreeStorage, + tokens: &'a TokenStore, + rule_index: usize, + ) -> impl Iterator> + 'a { + self.child_rules(storage, tokens, rule_index) + .map(RuleNodeView::node) } - /// Records that a child was matched without storing it (used when parse-tree - /// construction is disabled). Keeps `has_matched_child` accurate even though - /// `children` stays empty. - pub const fn note_matched_child(&mut self) { - self.matched_child = true; + pub fn child_tokens<'a>( + &'a self, + storage: &'a ParseTreeStorage, + tokens: &'a TokenStore, + token_type: i32, + ) -> impl Iterator> + 'a { + self.child_nodes(storage, tokens).filter_map(move |child| { + let terminal = match child.kind() { + NodeKind::Terminal => child.as_terminal(), + NodeKind::Error => child.as_error().map(ErrorNodeView::terminal), + NodeKind::Rule => None, + }?; + (terminal.symbol().token_type() == token_type).then_some(terminal) + }) } - /// Whether this context has matched at least one child (token, rule, or error - /// node) so far, regardless of whether parse-tree construction is enabled. - pub const fn has_matched_child(&self) -> bool { - self.matched_child + pub fn terminal_children<'a>( + &'a self, + storage: &'a ParseTreeStorage, + tokens: &'a TokenStore, + ) -> impl Iterator> + 'a { + self.child_nodes(storage, tokens) + .filter_map(|child| match child.kind() { + NodeKind::Terminal => child.as_terminal(), + NodeKind::Error => child.as_error().map(ErrorNodeView::terminal), + NodeKind::Rule => None, + }) } - pub fn text(&self, tokens: &TokenStore) -> String { - self.children - .iter() - .map(|child| child.text(tokens)) - .collect() + #[must_use] + pub fn text(&self, storage: &ParseTreeStorage, tokens: &TokenStore) -> String { + self.child_nodes(storage, tokens).map(Node::text).collect() } + #[must_use] pub fn to_string_tree_with_names>( &self, - rule_names: &[S], + storage: &ParseTreeStorage, tokens: &TokenStore, + rule_names: &[S], ) -> String { let name = rule_names .get(self.rule_index) @@ -553,130 +1121,72 @@ impl ParserRuleContext { } else { format!("{name}:{}", self.alt_number) }; - if self.children.is_empty() { + if self.child_count == 0 { return display_name; } let children = self - .children - .iter() - .map(|child| child.to_string_tree_with_names(rule_names, tokens)) + .child_nodes(storage, tokens) + .map(|child| child.to_string_tree_with_names(rule_names)) .collect::>() .join(" "); format!("({display_name} {children})") } - /// Renders the LISP-style tree using rule names resolved through a - /// recognizer, matching ANTLR's `toStringTree(parser)` shape used by - /// generated test actions on a mid-rule `$ctx`. + #[must_use] pub fn to_string_tree( &self, recognizer: Option<&R>, + storage: &ParseTreeStorage, tokens: &TokenStore, ) -> String { recognizer.map_or_else( - || self.to_string_tree_with_names::<&str>(&[], tokens), - |recognizer| self.to_string_tree_with_names(recognizer.data().rule_names(), tokens), + || self.to_string_tree_with_names::<&str>(storage, tokens, &[]), + |recognizer| { + self.to_string_tree_with_names(storage, tokens, recognizer.data().rule_names()) + }, ) } } -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct TerminalNode { - token: TokenId, -} - -impl TerminalNode { - pub(crate) const fn from_id(token: TokenId) -> Self { - Self { token } - } +/// Type-erased generated-rule attributes stored only for rules that use them. +pub struct GeneratedAttrs(Box); +impl GeneratedAttrs { #[must_use] - pub const fn token_id(&self) -> TokenId { - self.token - } - - pub fn symbol<'a>(&self, tokens: &'a TokenStore) -> TokenView<'a> { - tokens - .view(self.token) - .expect("terminal node token ID should remain valid") - } - - pub fn text<'a>(&self, tokens: &'a TokenStore) -> &'a str { - tokens.text(self.token).unwrap_or("") - } -} - -#[derive(Clone, Debug, Eq, PartialEq)] -pub struct ErrorNode { - terminal: TerminalNode, -} - -impl ErrorNode { - pub(crate) const fn from_id(token: TokenId) -> Self { - Self { - terminal: TerminalNode::from_id(token), - } - } - - const fn terminal(&self) -> &TerminalNode { - &self.terminal + pub fn new(attrs: T) -> Self { + Self(Box::new(attrs)) } #[must_use] - pub const fn token_id(&self) -> TokenId { - self.terminal.token_id() - } - - pub fn symbol<'a>(&self, tokens: &'a TokenStore) -> TokenView<'a> { - self.terminal.symbol(tokens) + pub fn downcast_ref(&self) -> Option<&T> { + self.0.downcast_ref::() } +} - pub fn text<'a>(&self, tokens: &'a TokenStore) -> &'a str { - tokens.text(self.token_id()).unwrap_or("") +impl fmt::Debug for GeneratedAttrs { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.write_str("GeneratedAttrs(..)") } } -/// Conversion from a dynamic [`ParserRuleContext`] into a generated typed -/// context view. -/// -/// Implemented by generated per-rule / per-labeled-alternative context types -/// so `ctx.downcast_ref::(tokens)` can check the rule shape and -/// materialize the typed view with a token resolver, mirroring ANTLR's -/// context-class casts. -pub trait FromRuleContext<'a>: Sized { - fn from_rule_context(context: &ParserRuleContext, tokens: &'a TokenStore) -> Option; +pub trait FromRuleNode<'tree>: Sized { + fn from_rule_node(node: RuleNodeView<'tree>) -> Option; } pub trait ParseTreeListener { - fn enter_every_rule( - &mut self, - _ctx: &ParserRuleContext, - _tokens: &TokenStore, - ) -> Result<(), AntlrError> { + fn enter_every_rule(&mut self, _ctx: RuleNodeView<'_>) -> Result<(), AntlrError> { Ok(()) } - fn exit_every_rule( - &mut self, - _ctx: &ParserRuleContext, - _tokens: &TokenStore, - ) -> Result<(), AntlrError> { + fn exit_every_rule(&mut self, _ctx: RuleNodeView<'_>) -> Result<(), AntlrError> { Ok(()) } - fn visit_terminal( - &mut self, - _node: &TerminalNode, - _tokens: &TokenStore, - ) -> Result<(), AntlrError> { + fn visit_terminal(&mut self, _node: TerminalNodeView<'_>) -> Result<(), AntlrError> { Ok(()) } - fn visit_error_node( - &mut self, - _node: &ErrorNode, - _tokens: &TokenStore, - ) -> Result<(), AntlrError> { + fn visit_error_node(&mut self, _node: ErrorNodeView<'_>) -> Result<(), AntlrError> { Ok(()) } } @@ -685,38 +1195,70 @@ pub trait ParseTreeListener { pub struct ParseTreeWalker; impl ParseTreeWalker { - /// Walks a parse tree depth-first, invoking listener callbacks in ANTLR's - /// enter/child/exit order for rule nodes. - pub fn walk( - listener: &mut L, - tree: &ParseTree, - tokens: &TokenStore, - ) -> Result<(), AntlrError> { - match tree { - ParseTree::Rule(rule) => { - listener.enter_every_rule(rule.context(), tokens)?; - for child in rule.context().children() { - Self::walk(listener, child, tokens)?; + pub fn walk(listener: &mut L, tree: Node<'_>) -> Result<(), AntlrError> { + enum Event { + Enter(NodeId), + Exit(NodeId), + } + + let storage = tree.storage; + let tokens = tree.tokens; + let mut stack = vec![Event::Enter(tree.id)]; + while let Some(event) = stack.pop() { + match event { + Event::Enter(id) => { + let node = storage + .node(tokens, id) + .expect("walker node ID should remain valid"); + match node.kind() { + NodeKind::Rule => { + let rule = node.as_rule().expect("rule node kind checked"); + listener.enter_every_rule(rule)?; + stack.push(Event::Exit(id)); + stack.extend( + storage + .child_ids(id) + .iter() + .rev() + .copied() + .map(Event::Enter), + ); + } + NodeKind::Terminal => listener.visit_terminal( + node.as_terminal().expect("terminal node kind checked"), + )?, + NodeKind::Error => { + listener.visit_error_node( + node.as_error().expect("error node kind checked"), + )?; + } + } + } + Event::Exit(id) => { + let rule = storage + .node(tokens, id) + .and_then(Node::as_rule) + .expect("walker exit node should remain a rule"); + listener.exit_every_rule(rule)?; } - listener.exit_every_rule(rule.context(), tokens) } - ParseTree::Terminal(node) => listener.visit_terminal(node, tokens), - ParseTree::Error(node) => listener.visit_error_node(node, tokens), } + Ok(()) } } -/// A completed parse result paired with the canonical tokens referenced by it. #[derive(Debug)] -pub struct ParsedFile { +pub struct ParsedFile { tokens: TokenStore, - tree: R, + tree: ParseTreeStorage, + root: NodeId, } -impl ParsedFile { +impl ParsedFile { #[must_use] - pub const fn new(tokens: TokenStore, tree: R) -> Self { - Self { tokens, tree } + pub fn new(tokens: TokenStore, mut tree: ParseTreeStorage, root: NodeId) -> Self { + tree.discard_scratch(); + Self { tokens, tree, root } } #[must_use] @@ -725,276 +1267,227 @@ impl ParsedFile { } #[must_use] - pub const fn tree(&self) -> &R { + pub const fn storage(&self) -> &ParseTreeStorage { &self.tree } #[must_use] - pub fn into_parts(self) -> (TokenStore, R) { - (self.tokens, self.tree) + pub const fn root_id(&self) -> NodeId { + self.root } -} -#[cfg(test)] -mod tests { - use super::*; - use crate::token::{TokenSpec, TokenStore}; - - struct TreeToken(TokenSpec); - - impl TreeToken { - fn new(token_type: i32) -> Self { - Self(TokenSpec::explicit(token_type, "")) - } - - fn with_text(mut self, text: impl Into) -> Self { - self.0.text = Some(text.into()); - self - } + #[must_use] + pub fn tree(&self) -> Node<'_> { + self.tree + .node(&self.tokens, self.root) + .expect("parsed file root ID should remain valid") } - fn terminal(store: &mut TokenStore, token: TreeToken) -> TerminalNode { - let id = store.push(token.0).expect("test token should fit"); - TerminalNode::from_id(id) + #[must_use] + pub fn node(&self, id: NodeId) -> Option> { + self.tree.node(&self.tokens, id) } - fn error(store: &mut TokenStore, token: TreeToken) -> ErrorNode { - let id = store.push(token.0).expect("test token should fit"); - ErrorNode::from_id(id) + #[must_use] + pub fn into_parts(self) -> (TokenStore, ParseTreeStorage, NodeId) { + (self.tokens, self.tree, self.root) } +} - #[test] - fn renders_rule_tree() { - let mut tokens = TokenStore::new(None, ""); - let mut ctx = ParserRuleContext::new(0, -1); - ctx.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(1).with_text("x"), - ))); - let tree = ParseTree::Rule(RuleNode::new(ctx)); - assert_eq!( - tree.to_string_tree_with_names(&["expr"], &tokens), - "(expr x)" - ); +fn escape_tree_text(text: &str) -> String { + let mut escaped = String::with_capacity(text.len()); + for ch in text.chars() { + match ch { + '\n' => escaped.push_str("\\n"), + '\r' => escaped.push_str("\\r"), + '\t' => escaped.push_str("\\t"), + _ => escaped.push(ch), + } } + escaped +} - #[test] - fn finds_first_rule_depth_first() { - let mut tokens = TokenStore::new(None, ""); - let mut nested = ParserRuleContext::new(1, -1); - nested.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(1).with_text("x"), - ))); +fn stored_token_id(raw: u32) -> TokenId { + TokenId::try_from(raw as usize).expect("stored token ID should fit in u32") +} - let mut root = ParserRuleContext::new(0, -1); - root.add_child(ParseTree::Rule(RuleNode::new(nested))); - let tree = ParseTree::Rule(RuleNode::new(root)); +#[cfg(test)] +mod tests { + use super::*; + use crate::token::TokenSpec; - let rule = tree.first_rule(1).expect("nested rule should be found"); - assert_eq!( - rule.to_string_tree_with_names(&["root".to_owned(), "child".to_owned()], &tokens), - "(child x)" - ); - assert!(tree.first_rule(2).is_none()); + fn token(store: &mut TokenStore, token_type: i32, text: &str) -> TokenId { + store + .push(TokenSpec::explicit(token_type, text)) + .expect("test token should fit") } #[test] - fn reports_rule_invocation_stack_from_leaf_to_root() { + fn stores_rule_children_in_one_pooled_range() { let mut tokens = TokenStore::new(None, ""); - let mut nested = ParserRuleContext::new(1, -1); - nested.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(1).with_text("x"), - ))); - - let mut root = ParserRuleContext::new(0, -1); - root.add_child(ParseTree::Rule(RuleNode::new(nested))); - let tree = ParseTree::Rule(RuleNode::new(root)); - + let first = token(&mut tokens, 1, "a"); + let second = token(&mut tokens, 2, "b"); + let mut storage = ParseTreeStorage::new(); + let first = storage.terminal(first); + let second = storage.error(second); + let mut context = ParserRuleContext::new(0, -1); + storage.add_child(&mut context, first); + storage.add_child(&mut context, second); + let root = storage.finish_rule(context); + let parsed = ParsedFile::new(tokens, storage, root); + + assert_eq!(parsed.tree().text(), "ab"); + assert_eq!(parsed.tree().children().count(), 2); + assert_eq!(parsed.storage().stats().edges, 2); + assert_eq!(parsed.storage().stats().scratch_links, 0); assert_eq!( - tree.rule_invocation_stack(1, &["s", "a"]), - Some(vec!["a".to_owned(), "s".to_owned()]) + parsed.tree().to_string_tree_with_names(&["root"]), + "(root a b)" ); } #[test] - fn parse_tree_children_returns_rule_children_and_empty_leaf_slices() { + fn descendants_and_walker_preserve_antlr_order() { let mut tokens = TokenStore::new(None, ""); - let terminal = ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(1).with_text("terminal"), - )); - let error = ParseTree::Error(error(&mut tokens, TreeToken::new(2).with_text("error"))); - + let a = token(&mut tokens, 1, "a"); + let b = token(&mut tokens, 2, "b"); + let mut storage = ParseTreeStorage::new(); + let a = storage.terminal(a); + let b = storage.terminal(b); + let mut child = ParserRuleContext::new(1, 7); + storage.add_child(&mut child, b); + let child = storage.finish_rule(child); let mut root = ParserRuleContext::new(0, -1); - root.add_child(terminal); - root.add_child(error); - let tree = ParseTree::Rule(RuleNode::new(root)); + storage.add_child(&mut root, a); + storage.add_child(&mut root, child); + let root = storage.finish_rule(root); + let parsed = ParsedFile::new(tokens, storage, root); - let children = tree.children(); - let [first, second] = children else { - panic!("expected exactly 2 children"); - }; - assert_eq!(first.text(&tokens), "terminal"); - assert_eq!(second.text(&tokens), "error"); - assert!(first.children().is_empty()); - assert!(second.children().is_empty()); - } - - #[test] - fn iterates_descendants_in_pre_order() { - let mut tokens = TokenStore::new(None, ""); - let mut nested = ParserRuleContext::new(1, -1); - nested.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(10).with_text("child"), - ))); - - let mut root = ParserRuleContext::new(0, -1); - root.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(11).with_text("prefix"), - ))); - root.add_child(ParseTree::Rule(RuleNode::new(nested))); - root.add_child(ParseTree::Error(error( - &mut tokens, - TreeToken::new(12).with_text("error"), - ))); - let tree = ParseTree::Rule(RuleNode::new(root)); - - let visited = tree + let visited = parsed + .tree() .descendants() - .map(|node| match node { - ParseTree::Rule(rule) => format!("rule:{}", rule.context().rule_index()), - ParseTree::Terminal(terminal) => { - format!("terminal:{}", terminal.text(&tokens)) - } - ParseTree::Error(error) => format!("error:{}", error.text(&tokens)), + .map(|node| match node.kind() { + NodeKind::Rule => format!( + "r{}", + node.as_rule().expect("rule node kind checked").rule_index() + ), + NodeKind::Terminal => node + .as_terminal() + .expect("terminal node kind checked") + .text() + .to_owned(), + NodeKind::Error => node + .as_error() + .expect("error node kind checked") + .text() + .to_owned(), }) .collect::>(); + assert_eq!(visited, ["r0", "a", "r1", "b"]); + + #[derive(Default)] + struct Listener(Vec); + impl ParseTreeListener for Listener { + fn enter_every_rule(&mut self, ctx: RuleNodeView<'_>) -> Result<(), AntlrError> { + self.0.push(format!("enter{}", ctx.rule_index())); + Ok(()) + } - assert_eq!( - visited, - vec![ - "rule:0".to_owned(), - "terminal:prefix".to_owned(), - "rule:1".to_owned(), - "terminal:child".to_owned(), - "error:error".to_owned(), - ] - ); + fn exit_every_rule(&mut self, ctx: RuleNodeView<'_>) -> Result<(), AntlrError> { + self.0.push(format!("exit{}", ctx.rule_index())); + Ok(()) + } - let preorder = tree - .pre_order() - .map(|node| node.text(&tokens)) - .collect::>(); - assert_eq!( - preorder, - vec!["prefixchilderror", "prefix", "child", "child", "error"] - ); + fn visit_terminal(&mut self, node: TerminalNodeView<'_>) -> Result<(), AntlrError> { + self.0.push(node.text().to_owned()); + Ok(()) + } + } + let mut listener = Listener::default(); + ParseTreeWalker::walk(&mut listener, parsed.tree()) + .expect("test listener should accept every node"); + assert_eq!(listener.0, ["enter0", "a", "enter1", "b", "exit1", "exit0"]); } #[test] - fn parsed_file_resolves_tokens_while_traversing() { - let mut tokens = TokenStore::new(None, ""); - let mut root = ParserRuleContext::new(0, -1); - root.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(10).with_text("first"), - ))); - root.add_child(ParseTree::Error(error( - &mut tokens, - TreeToken::new(11).with_text("second"), - ))); - let parsed = ParsedFile::new(tokens, ParseTree::Rule(RuleNode::new(root))); - - let visited = parsed - .tree() - .descendants() - .map(|node| node.text(parsed.tokens())) - .collect::>(); - - assert_eq!(visited, vec!["firstsecond", "first", "second"]); + fn invocation_states_exclude_a_nonnegative_root_frame() { + let tokens = TokenStore::new(None, ""); + let mut storage = ParseTreeStorage::new(); + let grandchild = storage.finish_rule(ParserRuleContext::new(2, 13)); + let mut child = ParserRuleContext::new(1, 7); + storage.add_child(&mut child, grandchild); + let child = storage.finish_rule(child); + let mut root = ParserRuleContext::new(0, 4); + storage.add_child(&mut root, child); + let root = storage.finish_rule(root); + let parsed = ParsedFile::new(tokens, storage, root); + + let root = parsed + .node(root) + .and_then(Node::as_rule) + .expect("root rule should be stored"); + let child = parsed + .node(child) + .and_then(Node::as_rule) + .expect("child rule should be stored"); + let grandchild = parsed + .node(grandchild) + .and_then(Node::as_rule) + .expect("grandchild rule should be stored"); + + assert_eq!(root.invocation_states().collect::>(), []); + assert_eq!(child.invocation_states().collect::>(), [7]); + assert_eq!(grandchild.invocation_states().collect::>(), [13, 7]); } #[test] - fn finds_direct_child_rules_by_index() { - let mut tokens = TokenStore::new(None, ""); - let mut direct = ParserRuleContext::new(1, -1); - direct.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(10).with_text("direct"), - ))); - - let mut nested_match = ParserRuleContext::new(1, -1); - nested_match.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(11).with_text("nested"), - ))); - let mut wrapper = ParserRuleContext::new(2, -1); - wrapper.add_child(ParseTree::Rule(RuleNode::new(nested_match))); - - let mut root = ParserRuleContext::new(0, -1); - root.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(12).with_text("prefix"), - ))); - root.add_child(ParseTree::Rule(RuleNode::new(direct))); - root.add_child(ParseTree::Rule(RuleNode::new(wrapper))); - - assert_eq!(root.child_count(), 3); - assert_eq!(root.child_rules(1).count(), 1); + fn uncommon_rule_payloads_live_in_sparse_extras() { + let tokens = TokenStore::new(None, ""); + let mut storage = ParseTreeStorage::new(); + let plain = storage.finish_rule(ParserRuleContext::new(0, -1)); + let mut rich = ParserRuleContext::new(1, 3); + rich.set_int_return("value", 42); + let rich = storage.finish_rule(rich); + let parsed = ParsedFile::new(tokens, storage, rich); + + assert_eq!(parsed.storage().extra_count(), 1); assert_eq!( - root.child_rule(1).map(|context| context.text(&tokens)), - Some("direct".to_owned()) + parsed + .node(rich) + .expect("rich rule should be stored") + .as_rule() + .expect("rich node should be a rule") + .int_return("value"), + Some(42) ); - assert_eq!( - root.child_rule(2).map(ParserRuleContext::rule_index), - Some(2) + assert!( + parsed + .node(plain) + .expect("plain rule should be stored") + .as_rule() + .expect("plain node should be a rule") + .int_return("value") + .is_none() ); - assert!(root.child_rule(3).is_none()); } #[test] - fn finds_direct_terminal_children_by_token_type() { - let mut tokens = TokenStore::new(None, ""); - let mut nested = ParserRuleContext::new(1, -1); - nested.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(13).with_text("nested"), - ))); - - let mut root = ParserRuleContext::new(0, -1); - root.add_child(ParseTree::Error(error( - &mut tokens, - TreeToken::new(12).with_text("error"), - ))); - root.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(10).with_text("direct"), - ))); - root.add_child(ParseTree::Terminal(terminal( - &mut tokens, - TreeToken::new(11).with_text("other"), - ))); - root.add_child(ParseTree::Rule(RuleNode::new(nested))); - - assert_eq!(root.child_count(), 4); - assert!(root.has_token(10, &tokens)); - assert!(root.has_token(12, &tokens)); - assert_eq!( - root.child_token(10, &tokens).map(|node| node.text(&tokens)), - Some("direct") - ); - assert_eq!( - root.child_token(11, &tokens).map(|node| node.text(&tokens)), - Some("other") - ); - assert_eq!( - root.child_token(12, &tokens).map(|node| node.text(&tokens)), - Some("error") - ); - assert!(root.child_token(13, &tokens).is_none()); + fn preserves_maximum_token_id_in_nodes_and_rule_spans() { + let max = TokenId::try_from(u32::MAX as usize).expect("maximum token ID should fit"); + let tokens = TokenStore::new(None, ""); + let mut storage = ParseTreeStorage::new(); + let terminal = storage.terminal(max); + assert_eq!(storage.token_id(terminal), Some(max)); + + let mut context = ParserRuleContext::new(0, -1); + context.set_start_id(max); + context.set_stop_id(max); + let rule = storage.finish_rule(context); + let rule = storage + .node(&tokens, rule) + .and_then(Node::as_rule) + .expect("rule should be stored"); + assert_eq!(rule.start_id(), Some(max)); + assert_eq!(rule.stop_id(), Some(max)); } } diff --git a/tests/javascript-parity/dumper/src/main.rs b/tests/javascript-parity/dumper/src/main.rs index dc68c85c..ea2bc9dc 100644 --- a/tests/javascript-parity/dumper/src/main.rs +++ b/tests/javascript-parity/dumper/src/main.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; use std::process::ExitCode; use antlr4_runtime::{ - CommonTokenStream, InputStream, ParseTree, Parser, TOKEN_EOF, Token, TokenStore, + CommonTokenStream, InputStream, Node, NodeKind, Parser, TOKEN_EOF, Token, }; #[allow(dead_code, unused_imports, unreachable_pub, unused_qualifications)] @@ -25,28 +25,36 @@ use javascript_parser_base::JavaScriptParserBase; fn dump_tree>( out: &mut dyn Write, - tree: &ParseTree, - tokens: &TokenStore, + tree: Node<'_>, rule_names: &[S], depth: usize, ) -> io::Result<()> { let pad = " ".repeat(depth); - match tree { - ParseTree::Rule(rule) => { + match tree.kind() { + NodeKind::Rule => { + let rule = tree.as_rule().expect("rule node kind checked"); let name = rule_names - .get(rule.context().rule_index()) + .get(rule.rule_index()) .map_or("", AsRef::as_ref); writeln!( out, "{pad}Rule({name}, children={})", - rule.context().children().len() + rule.child_count() )?; - for child in rule.context().children() { - dump_tree(out, child, tokens, rule_names, depth + 1)?; + for child in rule.children() { + dump_tree(out, child, rule_names, depth + 1)?; } } - ParseTree::Terminal(token) => writeln!(out, "{pad}Term({:?})", token.text(tokens))?, - ParseTree::Error(token) => writeln!(out, "{pad}Err({:?})", token.text(tokens))?, + NodeKind::Terminal => writeln!( + out, + "{pad}Term({:?})", + tree.as_terminal().expect("terminal node kind checked").text() + )?, + NodeKind::Error => writeln!( + out, + "{pad}Err({:?})", + tree.as_error().expect("error node kind checked").text() + )?, } Ok(()) } @@ -126,8 +134,7 @@ fn main() -> ExitCode { } if let Err(error) = dump_tree( &mut io::stdout().lock(), - &tree, - parser.token_store(), + parser.node(tree), java_script_parser::rule_names(), 0, ) { diff --git a/tests/kotlin-parity/dumper/src/main.rs b/tests/kotlin-parity/dumper/src/main.rs index 628f0a56..1f3e0ce4 100644 --- a/tests/kotlin-parity/dumper/src/main.rs +++ b/tests/kotlin-parity/dumper/src/main.rs @@ -15,7 +15,7 @@ use std::path::PathBuf; use std::process::ExitCode; use std::time::Instant; -use antlr4_runtime::{ParseTree, TokenStore}; +use antlr4_runtime::{Node, NodeKind}; mod generated { #![allow(dead_code, unused_imports, unreachable_pub, unused_qualifications)] @@ -44,28 +44,36 @@ impl EntryRule { fn dump>( out: &mut dyn Write, - tree: &ParseTree, - tokens: &TokenStore, + tree: Node<'_>, rule_names: &[S], depth: usize, ) -> io::Result<()> { let pad = " ".repeat(depth); - match tree { - ParseTree::Rule(rl) => { + match tree.kind() { + NodeKind::Rule => { + let rule = tree.as_rule().expect("rule node kind checked"); let name = rule_names - .get(rl.context().rule_index()) + .get(rule.rule_index()) .map_or("", |name| name.as_ref()); writeln!( out, "{pad}Rule({name}, children={})", - rl.context().children().len() + rule.child_count() )?; - for c in rl.context().children() { - dump(out, c, tokens, rule_names, depth + 1)?; + for child in rule.children() { + dump(out, child, rule_names, depth + 1)?; } } - ParseTree::Terminal(t) => writeln!(out, "{pad}Term({:?})", t.text(tokens))?, - ParseTree::Error(e) => writeln!(out, "{pad}Err({:?})", e.text(tokens))?, + NodeKind::Terminal => writeln!( + out, + "{pad}Term({:?})", + tree.as_terminal().expect("terminal node kind checked").text() + )?, + NodeKind::Error => writeln!( + out, + "{pad}Err({:?})", + tree.as_error().expect("error node kind checked").text() + )?, } Ok(()) } @@ -178,7 +186,6 @@ fn main() -> ExitCode { if let Err(err) = dump( sink.as_mut(), parsed.tree(), - parsed.tokens(), &rule_names, 0, ) { diff --git a/tests/typescript-parity/dumper/src/main.rs b/tests/typescript-parity/dumper/src/main.rs index 301a803c..5d92544c 100644 --- a/tests/typescript-parity/dumper/src/main.rs +++ b/tests/typescript-parity/dumper/src/main.rs @@ -7,7 +7,7 @@ use std::path::PathBuf; use std::process::ExitCode; use antlr4_runtime::{ - CommonTokenStream, InputStream, ParseTree, Parser, TOKEN_EOF, Token, TokenStore, + CommonTokenStream, InputStream, Node, NodeKind, Parser, TOKEN_EOF, Token, }; #[allow(dead_code, unused_imports, unreachable_pub, unused_qualifications)] @@ -25,28 +25,36 @@ use typescript_parser_base::TypeScriptParserBase; fn dump_tree>( out: &mut dyn Write, - tree: &ParseTree, - tokens: &TokenStore, + tree: Node<'_>, rule_names: &[S], depth: usize, ) -> io::Result<()> { let pad = " ".repeat(depth); - match tree { - ParseTree::Rule(rule) => { + match tree.kind() { + NodeKind::Rule => { + let rule = tree.as_rule().expect("rule node kind checked"); let name = rule_names - .get(rule.context().rule_index()) + .get(rule.rule_index()) .map_or("", AsRef::as_ref); writeln!( out, "{pad}Rule({name}, children={})", - rule.context().children().len() + rule.child_count() )?; - for child in rule.context().children() { - dump_tree(out, child, tokens, rule_names, depth + 1)?; + for child in rule.children() { + dump_tree(out, child, rule_names, depth + 1)?; } } - ParseTree::Terminal(token) => writeln!(out, "{pad}Term({:?})", token.text(tokens))?, - ParseTree::Error(token) => writeln!(out, "{pad}Err({:?})", token.text(tokens))?, + NodeKind::Terminal => writeln!( + out, + "{pad}Term({:?})", + tree.as_terminal().expect("terminal node kind checked").text() + )?, + NodeKind::Error => writeln!( + out, + "{pad}Err({:?})", + tree.as_error().expect("error node kind checked").text() + )?, } Ok(()) } @@ -126,8 +134,7 @@ fn main() -> ExitCode { } if let Err(error) = dump_tree( &mut io::stdout().lock(), - &tree, - parser.token_store(), + parser.node(tree), type_script_parser::rule_names(), 0, ) {