diff --git a/README.md b/README.md index 79479d52..35955a12 100644 --- a/README.md +++ b/README.md @@ -246,6 +246,33 @@ grammar's documentation. Calling the wrong rule can still recover and return a parse tree with error nodes, so check parser diagnostics when adding a new input form. +### Parse-Tree Pattern Matching + +Generated parsers expose `compile_parse_tree_pattern`, the analog of ANTLR's +`Parser.compileParseTreePattern`. A *tree pattern* is grammar input with `` +placeholders: literals must match exactly, `` matches any `expr` subtree, +`` matches any `ID` token, and `` binds the match to a label. + +```rust +let pattern = parser.compile_parse_tree_pattern( + " = ;", + RULE_STAT, + MyGrammarLexer::new, // lexes the pattern's literal chunks +)?; + +let m = pattern.match_tree(subtree); +if m.succeeded() { + println!("assigns to {}", m.get("ID").unwrap().text()); +} +``` + +Compilation interprets the pattern over a rule-bypass ATN +(`ParserAtn::with_bypass_alternatives`), the same mechanism the reference +runtimes use; matching walks the subject and pattern trees in lockstep. The +generated method caches the compiler per process, so compiling many patterns is +cheap; to change the `<`/`>`/`\` delimiters, use `ParseTreePatternMatcher` +directly. + ## Technical Notes - Pure Rust runtime implementation. @@ -270,6 +297,8 @@ The runtime contains: - recognizer metadata and error listener plumbing - parse tree node types, rule contexts, terminal nodes, error nodes, and walkers - parse-tree XPath queries on par with the official ANTLR runtimes +- parse-tree pattern matching (`compileParseTreePattern` / `ParseTreePattern` / + `ParseTreeMatch`) with rule/token tags, labels, and rule-bypass ATNs - ANTLR v4 serialized lexer ATN deserialization - lexer ATN recognition with longest-match/rule-priority behavior and lexer actions diff --git a/src/atn/bypass.rs b/src/atn/bypass.rs new file mode 100644 index 00000000..070d8cb7 --- /dev/null +++ b/src/atn/bypass.rs @@ -0,0 +1,864 @@ +//! Rule-bypass alternatives for parse-tree pattern matching. +//! +//! ANTLR's parse-tree pattern matcher needs to interpret a hybrid token stream +//! in which an *imaginary* token can stand in for an entire parser rule (the +//! `` tag in a pattern like `x = ;`). Upstream implements this by +//! re-deserializing the grammar ATN with `generateRuleBypassTransitions` +//! enabled (`ATNDeserializer`): each rule gains a **bypass alternative** so the +//! ordinary parser interpreter can match one imaginary token as if it were the +//! whole rule. See `ATNDeserializer.java` (the `isGenerateRuleBypassTransitions` +//! block) for the reference algorithm this mirrors. +//! +//! This runtime stores parser ATNs as a packed, immutable word stream rather +//! than a mutable object graph, so the transform reads the source ATN through +//! its borrowing views into an out-adjacency model, applies the rewrite there, +//! and emits a fresh [`ParserAtn`] through [`ParserAtnBuilder`]. The existing +//! ATN interpreter then runs over the result unchanged — no hot-path edits. +//! +//! ### Why `max_token_type` is left unchanged +//! +//! Upstream assigns each rule the imaginary token type `maxTokenType + i + 1` +//! but never raises `maxTokenType` itself. We keep the same invariant on +//! purpose: an [`Atom`](ParserTransitionSpec::Atom) transition matches by exact +//! label equality (no range check), so the bypass edge matches its imaginary +//! type fine, while grammar wildcards and `~x` negated sets — which the runtime +//! bounds by `min..=max_token_type` — can never accidentally match an imaginary +//! token that lives *above* the unchanged maximum. + +use std::collections::BTreeSet; + +use super::AtnStateKind; +use super::parser_atn::{ + ParserAtn, ParserAtnBuilder, ParserAtnError, ParserIntervalSetId, ParserTransitionData, + ParserTransitionSpec, +}; + +impl ParserAtn { + /// Builds a copy of this parser ATN with rule-bypass alternatives added. + /// + /// Every rule gains an imaginary token type (`max_token_type + rule + 1`) + /// and a bypass block so the ATN interpreter can match that single + /// imaginary token in place of the whole rule. `max_token_type` is + /// unchanged (see the module docs). The returned ATN is otherwise a faithful + /// copy: state kinds, transitions, interval sets, decisions, and + /// rule/precedence metadata are all preserved. + /// + /// # Errors + /// + /// Returns [`ParserAtnError`] if the state/transition/token counts overflow + /// the packed compact-index range, if a left-recursive rule's precedence + /// prefix cannot be identified, or if the re-emitted stream fails + /// validation. + pub fn with_bypass_alternatives(&self) -> Result { + BypassBuilder::new(self)?.build() + } + + /// The imaginary token type reserved for a rule's bypass alternative: + /// `max_token_type + rule_index + 1`. + /// + /// This is the single source of the formula shared by + /// [`Self::with_bypass_alternatives`] (which labels the bypass `Atom` edge + /// with it) and the pattern matcher (which stamps rule-tag tokens with it), + /// so the two can never disagree about a tag's token type. + /// + /// # Errors + /// + /// Returns [`ParserAtnError::Overflow`] when the type would exceed `i32`. + pub fn bypass_token_type(&self, rule_index: usize) -> Result { + imaginary_token_type(self.max_token_type(), rule_index) + } +} + +/// Shared formula for a rule's imaginary bypass token type. +fn imaginary_token_type(max_token_type: i32, rule: usize) -> Result { + let overflow = || ParserAtnError::Overflow { + field: "bypass imaginary token type", + value: rule, + }; + let rule = i32::try_from(rule).map_err(|_| overflow())?; + max_token_type + .checked_add(rule) + .and_then(|value| value.checked_add(1)) + .ok_or_else(overflow) +} + +/// Mutable working copy of a parser ATN used to apply the bypass rewrite. +struct BypassBuilder { + max_token_type: i32, + /// Per-state kind, parallel to state number. + kinds: Vec, + /// Per-state grammar rule index (`None` for rule-agnostic states). + rule_indices: Vec>, + /// Block-start end state, if any. + end_states: Vec>, + /// Loop-end loop-back state, if any. + loop_back_states: Vec>, + /// Non-greedy decision flag, preserved verbatim. + non_greedy: Vec, + /// Left-recursive rule flag, preserved on rule-start states. + left_recursive: Vec, + /// Out-adjacency: `out[source]` holds that state's transitions in order. + out: Vec>, + /// Interval sets copied verbatim; identities stay valid because order is + /// preserved. + interval_sets: Vec>, + decisions: Vec, + rule_starts: Vec, + rule_stops: Vec, +} + +impl BypassBuilder { + /// Reads the packed source ATN into the mutable model. + fn new(atn: &ParserAtn) -> Result { + let state_count = atn.state_count(); + let mut kinds = Vec::with_capacity(state_count); + let mut rule_indices = Vec::with_capacity(state_count); + let mut end_states = Vec::with_capacity(state_count); + let mut loop_back_states = Vec::with_capacity(state_count); + let mut non_greedy = Vec::with_capacity(state_count); + let mut left_recursive = Vec::with_capacity(state_count); + let mut out = Vec::with_capacity(state_count); + + for number in 0..state_count { + let state = atn + .state(number) + .expect("state index below state_count is in bounds"); + kinds.push(state.kind()); + rule_indices.push(state.rule_index()); + end_states.push(state.end_state()); + loop_back_states.push(state.loop_back_state()); + non_greedy.push(state.non_greedy()); + left_recursive.push(state.left_recursive_rule()); + out.push( + state + .transitions() + .iter() + .map(|transition| data_to_spec(transition.data())) + .collect::, _>>()?, + ); + } + + let interval_sets = (0..atn.set_count()) + .map(|index| { + atn.token_set(index) + .expect("set index below set_count is in bounds") + .ranges() + .collect::>() + }) + .collect(); + + let decisions = atn.decision_to_state().into_iter().collect(); + let rule_starts = atn.rule_to_start_state().into_iter().collect(); + let rule_stops = atn.rule_to_stop_state().into_iter().collect(); + + Ok(Self { + max_token_type: atn.max_token_type(), + kinds, + rule_indices, + end_states, + loop_back_states, + non_greedy, + left_recursive, + out, + interval_sets, + decisions, + rule_starts, + rule_stops, + }) + } + + /// Applies the rewrite and emits the packed result. + fn build(mut self) -> Result { + let rule_count = self.rule_starts.len(); + let original_state_count = self.kinds.len(); + + // Reserve the three new states per rule up front so their numbers are + // known before any edge is rewired. Layout: for rule `i`, + // `bypass_start = base`, `bypass_stop = base + 1`, `match = base + 2`. + let new_state_base = original_state_count; + for rule in 0..rule_count { + let bypass_start = new_state_base + rule * 3; + let bypass_stop = bypass_start + 1; + let match_state = bypass_start + 2; + // Bypass start is a decision block start whose end is the stop. + self.push_state( + AtnStateKind::BlockStart, + Some(rule), + Some(bypass_stop), + None, + ); + self.push_state(AtnStateKind::BlockEnd, Some(rule), None, None); + self.push_state(AtnStateKind::Basic, None, None, None); + debug_assert_eq!(self.out.len(), match_state + 1); + } + + // Retarget/move plan per rule, computed against the *pre-move* graph so + // the left-recursive exclude transition is identified correctly. End + // states are captured here and reused below — recomputing them after + // the move/retarget passes would scan a mutated graph. + let mut end_states: Vec = Vec::with_capacity(rule_count); + let mut bypass_stop_for_end: Vec> = vec![None; self.kinds.len()]; + let mut excluded: BTreeSet<(usize, usize)> = BTreeSet::new(); + for rule in 0..rule_count { + let bypass_stop = new_state_base + rule * 3 + 1; + let (end_state, exclude) = self.rule_end_state(rule)?; + end_states.push(end_state); + bypass_stop_for_end[end_state] = Some(bypass_stop); + if let Some(exclude) = exclude { + excluded.insert(exclude); + } + } + + // Move each rule-start's transitions onto its bypass-start block. + for rule in 0..rule_count { + let rule_start = self.rule_starts[rule]; + let bypass_start = new_state_base + rule * 3; + let moved = std::mem::take(&mut self.out[rule_start]); + self.out[bypass_start] = moved; + } + + // Retarget every edge that targeted a rule's end state onto that rule's + // bypass stop, skipping the left-recursive exclude edge(s). + for (source, transitions) in self.out.iter_mut().enumerate() { + for (index, spec) in transitions.iter_mut().enumerate() { + if excluded.contains(&(source, index)) { + continue; + } + if let Some(bypass_stop) = bypass_stop_for_end[spec.target()] { + *spec = spec.with_target(bypass_stop); + } + } + } + + // Add the bypass edges last so the `bypass_stop -> end_state` link + // (which targets an end state) is never caught by the retarget pass. + for (rule, &end_state) in end_states.iter().enumerate() { + let rule_start = self.rule_starts[rule]; + let bypass_start = new_state_base + rule * 3; + let bypass_stop = bypass_start + 1; + let match_state = bypass_start + 2; + let imaginary = self.imaginary_token_type(rule)?; + + self.out[rule_start].push(ParserTransitionSpec::Epsilon { + target: bypass_start, + }); + self.out[bypass_start].push(ParserTransitionSpec::Epsilon { + target: match_state, + }); + self.out[bypass_stop].push(ParserTransitionSpec::Epsilon { target: end_state }); + self.out[match_state].push(ParserTransitionSpec::Atom { + target: bypass_stop, + label: imaginary, + }); + self.decisions.push(bypass_start); + } + + self.emit() + } + + /// Appends one new state to the model, keeping every parallel array aligned. + fn push_state( + &mut self, + kind: AtnStateKind, + rule_index: Option, + end_state: Option, + loop_back_state: Option, + ) { + self.kinds.push(kind); + self.rule_indices.push(rule_index); + self.end_states.push(end_state); + self.loop_back_states.push(loop_back_state); + self.non_greedy.push(false); + self.left_recursive.push(false); + self.out.push(Vec::new()); + } + + /// Returns `(end_state, exclude_edge)` for a rule. + /// + /// For an ordinary rule the end state is the rule stop and there is no + /// excluded edge. For a left-recursive rule the end state is the + /// `StarLoopEntry` that begins the precedence-climbing loop, and the excluded + /// edge is the loop-back edge into it (which must keep pointing at the entry + /// rather than being diverted to the bypass block). Mirrors the + /// `isLeftRecursiveRule` branch in `ATNDeserializer`. + fn rule_end_state( + &self, + rule: usize, + ) -> Result<(usize, Option<(usize, usize)>), ParserAtnError> { + let rule_start = self.rule_starts[rule]; + if !self.left_recursive[rule_start] { + return Ok((self.rule_stops[rule], None)); + } + + // Find the StarLoopEntry whose last edge reaches a LoopEnd that + // epsilon-transitions to the rule stop: that entry is the precedence + // prefix boundary ANTLR wraps. + for state in 0..self.kinds.len() { + if self.rule_indices[state] != Some(rule) + || self.kinds[state] != AtnStateKind::StarLoopEntry + { + continue; + } + let Some(last) = self.out[state].last() else { + continue; + }; + let loop_end = last.target(); + if self.kinds.get(loop_end).copied() != Some(AtnStateKind::LoopEnd) { + continue; + } + // Upstream additionally requires the loop end to be epsilon-only + // before trusting its first edge (`maybeLoopEndState + // .epsilonOnlyTransitions && ... instanceof RuleStopState`). + let epsilon_only = self.out[loop_end] + .iter() + .all(|edge| matches!(edge, ParserTransitionSpec::Epsilon { .. })); + let reaches_stop = self.out[loop_end] + .first() + .is_some_and(|edge| self.kinds.get(edge.target()) == Some(&AtnStateKind::RuleStop)); + if !epsilon_only || !reaches_stop { + continue; + } + let Some(loop_back) = self.loop_back_states[loop_end] else { + continue; + }; + // The loop-back state's first edge is the excluded loop-back into + // the entry; verify the structure before trusting it. + if self.out[loop_back] + .first() + .is_some_and(|edge| edge.target() == state) + { + return Ok((state, Some((loop_back, 0)))); + } + } + + Err(ParserAtnError::InvalidData(format!( + "could not identify precedence prefix boundary for left-recursive rule {rule}" + ))) + } + + /// Imaginary token type reserved for a rule's bypass alternative. + fn imaginary_token_type(&self, rule: usize) -> Result { + imaginary_token_type(self.max_token_type, rule) + } + + /// Emits the mutable model as a validated packed [`ParserAtn`]. + fn emit(self) -> Result { + let mut builder = ParserAtnBuilder::new(self.max_token_type); + + for (index, &kind) in self.kinds.iter().enumerate() { + builder.add_state(kind, self.rule_indices[index])?; + } + for (index, end_state) in self.end_states.iter().enumerate() { + if let Some(end_state) = end_state { + builder.set_end_state(index, *end_state)?; + } + } + for (index, loop_back) in self.loop_back_states.iter().enumerate() { + if let Some(loop_back) = loop_back { + builder.set_loop_back_state(index, *loop_back)?; + } + } + for (index, &flag) in self.non_greedy.iter().enumerate() { + if flag { + builder.set_non_greedy(index)?; + } + } + for (index, &flag) in self.left_recursive.iter().enumerate() { + if flag { + builder.set_left_recursive_rule(index)?; + } + } + for ranges in &self.interval_sets { + builder.add_interval_set(ranges.iter().copied())?; + } + for (source, transitions) in self.out.iter().enumerate() { + for spec in transitions { + builder.add_transition(source, *spec)?; + } + } + for &state in &self.decisions { + builder.add_decision_state(state)?; + } + builder.set_rule_to_start_state(self.rule_starts)?; + builder.set_rule_to_stop_state(self.rule_stops)?; + + builder.finish() + } +} + +/// Converts a borrowing transition view into a builder spec, translating the +/// borrowed interval set back into its stable identity. +fn data_to_spec(data: ParserTransitionData<'_>) -> Result { + Ok(match data { + ParserTransitionData::Epsilon { target } => ParserTransitionSpec::Epsilon { target }, + ParserTransitionData::Atom { target, label } => { + ParserTransitionSpec::Atom { target, label } + } + ParserTransitionData::Range { + target, + start, + stop, + } => ParserTransitionSpec::Range { + target, + start, + stop, + }, + ParserTransitionData::Set { target, set } => ParserTransitionSpec::Set { + target, + set: ParserIntervalSetId::try_from(set.index())?, + }, + ParserTransitionData::NotSet { target, set } => ParserTransitionSpec::NotSet { + target, + set: ParserIntervalSetId::try_from(set.index())?, + }, + ParserTransitionData::Wildcard { target } => ParserTransitionSpec::Wildcard { target }, + ParserTransitionData::Rule { + target, + rule_index, + follow_state, + precedence, + } => ParserTransitionSpec::Rule { + target, + rule_index, + follow_state, + precedence, + }, + ParserTransitionData::Predicate { + target, + rule_index, + pred_index, + context_dependent, + } => ParserTransitionSpec::Predicate { + target, + rule_index, + pred_index, + context_dependent, + }, + ParserTransitionData::Action { + target, + rule_index, + action_index, + context_dependent, + } => ParserTransitionSpec::Action { + target, + rule_index, + action_index, + context_dependent, + }, + ParserTransitionData::Precedence { target, precedence } => { + ParserTransitionSpec::Precedence { target, precedence } + } + }) +} + +#[cfg(test)] +#[allow(clippy::disallowed_methods)] // `insta` assertion macros unwrap internal I/O. +mod tests { + use super::*; + use crate::atn::parser_atn::{ParserTransition, ParserTransitionKind}; + use crate::token::{Token, TokenId, TokenSink, TokenSource, TokenSpec, TokenStoreError}; + use crate::{BaseParser, CommonTokenStream, NodeKind, RecognizerData, Vocabulary}; + + /// A token source over a fixed list of specs, ending in EOF — the runtime + /// analog of ANTLR's `ListTokenSource` used to feed a hybrid pattern stream. + #[derive(Debug)] + struct ListSource { + specs: Vec, + index: usize, + } + + impl TokenSource for ListSource { + fn next_token(&mut self, sink: &mut TokenSink<'_>) -> Result { + let spec = self + .specs + .get(self.index) + .cloned() + .unwrap_or_else(|| TokenSpec::eof(self.index, self.index, 1, self.index)); + self.index += 1; + sink.push(spec) + } + + fn line(&self) -> usize { + 1 + } + + fn column(&self) -> usize { + self.index + } + + fn source_name(&self) -> &'static str { + "bypass-test" + } + } + + /// Two-rule grammar `a : X b ; b : Y ;` (tokens `X=1`, `Y=2`). + /// + /// Deliberately plain (no loops, no left recursion) so the bypass rewrite's + /// state growth and edge rewiring are easy to reason about. + fn two_rule_atn() -> ParserAtn { + let mut atn = ParserAtnBuilder::new(2); + for (number, kind, rule) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::Basic, 0), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::RuleStop, 0), + (4, AtnStateKind::RuleStart, 1), + (5, AtnStateKind::Basic, 1), + (6, AtnStateKind::RuleStop, 1), + ] { + assert_eq!( + atn.add_state(kind, Some(rule)).expect("state").index(), + number + ); + } + atn.set_rule_to_start_state(vec![0, 4]).expect("starts"); + atn.set_rule_to_stop_state(vec![3, 6]).expect("stops"); + // a : X b ; + atn.add_transition( + 0, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("edge"); + atn.add_transition( + 1, + ParserTransitionSpec::Rule { + target: 4, + rule_index: 1, + follow_state: 2, + precedence: 0, + }, + ) + .expect("edge"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("edge"); + // Synthetic rule-return edge already present in a packed ATN. + atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("edge"); + // b : Y ; + atn.add_transition( + 4, + ParserTransitionSpec::Atom { + target: 5, + label: 2, + }, + ) + .expect("edge"); + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("edge"); + atn.finish().expect("valid base ATN") + } + + /// One left-recursive rule shaped like ANTLR's transformed `e : e '+' e | X`: + /// `RuleStart(LR) -> Basic(prefix) -> StarLoopEntry -> {StarBlockStart(body), + /// LoopEnd -> RuleStop}`, with the loop body returning through a + /// `StarLoopBack` whose sole edge re-enters the entry. + fn left_recursive_atn() -> ParserAtn { + let mut atn = ParserAtnBuilder::new(2); + for (number, kind, rule) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::Basic, 0), // primary/prefix matcher + (2, AtnStateKind::StarLoopEntry, 0), + (3, AtnStateKind::Basic, 0), // loop body: '+' e + (4, AtnStateKind::StarLoopBack, 0), + (5, AtnStateKind::LoopEnd, 0), + (6, AtnStateKind::RuleStop, 0), + ] { + assert_eq!( + atn.add_state(kind, Some(rule)).expect("state").index(), + number + ); + } + atn.set_left_recursive_rule(0).expect("LR flag"); + atn.set_rule_to_start_state(vec![0]).expect("starts"); + atn.set_rule_to_stop_state(vec![6]).expect("stops"); + atn.set_loop_back_state(5, 4).expect("loop back"); + atn.add_decision_state(2).expect("decision"); + atn.add_transition( + 0, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("edge"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("edge"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("edge"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("edge"); + atn.add_transition( + 3, + ParserTransitionSpec::Atom { + target: 4, + label: 2, + }, + ) + .expect("edge"); + // The loop-back edge that must stay pointed at the StarLoopEntry. + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("edge"); + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("edge"); + atn.finish().expect("valid left-recursive ATN") + } + + /// Pins the left-recursive branch of `rule_end_state`: the bypass block + /// wraps the precedence prefix (ending at the `StarLoopEntry`, not the rule + /// stop) and the loop-back edge is excluded from retargeting. + #[test] + fn bypass_wraps_left_recursive_prefix_and_preserves_loop_back() { + let base = left_recursive_atn(); + let bypass = base.with_bypass_alternatives().expect("bypass ATN"); + + let star_loop_entry = 2; + let star_loop_back = 4; + let bypass_start = base.state_count(); + let bypass_stop = bypass_start + 1; + + // The prefix edge into the StarLoopEntry is retargeted to bypass stop… + let prefix_targets: Vec<_> = bypass + .state(1) + .expect("prefix state") + .transitions() + .iter() + .map(ParserTransition::target) + .collect(); + assert_eq!(prefix_targets, vec![bypass_stop]); + // …while the loop-back edge still re-enters the StarLoopEntry. + let loop_back_targets: Vec<_> = bypass + .state(star_loop_back) + .expect("loop-back state") + .transitions() + .iter() + .map(ParserTransition::target) + .collect(); + assert_eq!(loop_back_targets, vec![star_loop_entry]); + // The bypass stop rejoins the graph at the StarLoopEntry (the + // left-recursive "end state"), not at the rule stop. + let stop_targets: Vec<_> = bypass + .state(bypass_stop) + .expect("bypass stop") + .transitions() + .iter() + .map(ParserTransition::target) + .collect(); + assert_eq!(stop_targets, vec![star_loop_entry]); + } + + /// A rule flagged left-recursive whose precedence prefix boundary cannot be + /// identified must fail loudly instead of producing a broken bypass ATN. + #[test] + fn bypass_reports_unrecognizable_left_recursive_structure() { + let mut atn = ParserAtnBuilder::new(1); + for (number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::Basic), + (2, AtnStateKind::RuleStop), + ] { + assert_eq!(atn.add_state(kind, Some(0)).expect("state").index(), number); + } + // Flag the rule left-recursive without any StarLoopEntry structure. + atn.set_left_recursive_rule(0).expect("LR flag"); + atn.set_rule_to_start_state(vec![0]).expect("starts"); + atn.set_rule_to_stop_state(vec![2]).expect("stops"); + atn.add_transition( + 0, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("edge"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("edge"); + let base = atn.finish().expect("valid base ATN"); + + let error = base + .with_bypass_alternatives() + .expect_err("missing precedence prefix must be reported"); + assert!( + error.to_string().contains("left-recursive rule 0"), + "unexpected error: {error}" + ); + } + + #[test] + fn bypass_grows_three_states_per_rule_and_keeps_max_token_type() { + let base = two_rule_atn(); + let bypass = base.with_bypass_alternatives().expect("bypass ATN"); + + // Three new states per rule, appended after the originals. + assert_eq!( + bypass.state_count(), + base.state_count() + 3 * base.rule_count() + ); + // Imaginary token types live above the *unchanged* maximum. + assert_eq!(bypass.max_token_type(), base.max_token_type()); + } + + #[test] + fn bypass_adds_one_imaginary_atom_per_rule() { + let base = two_rule_atn(); + let bypass = base.with_bypass_alternatives().expect("bypass ATN"); + + // Rule i's imaginary token type is max_token_type + i + 1. + let mut imaginary_atoms = Vec::new(); + for state in 0..bypass.state_count() { + for transition in bypass.state(state).expect("state").transitions() { + if transition.kind() == ParserTransitionKind::Atom { + if let ParserTransitionData::Atom { label, .. } = transition.data() { + if label > base.max_token_type() { + imaginary_atoms.push(label); + } + } + } + } + } + imaginary_atoms.sort_unstable(); + assert_eq!(imaginary_atoms, vec![3, 4]); // max(2) + {1, 2} + } + + #[test] + fn bypass_preserves_rule_start_and_stop_tables() { + let base = two_rule_atn(); + let bypass = base.with_bypass_alternatives().expect("bypass ATN"); + + let base_starts: Vec<_> = base.rule_to_start_state().into_iter().collect(); + let bypass_starts: Vec<_> = bypass.rule_to_start_state().into_iter().collect(); + assert_eq!(base_starts, bypass_starts); + + let base_stops: Vec<_> = base.rule_to_stop_state().into_iter().collect(); + let bypass_stops: Vec<_> = bypass.rule_to_stop_state().into_iter().collect(); + assert_eq!(base_stops, bypass_stops); + } + + #[test] + fn rule_start_gains_epsilon_into_bypass_block() { + let base = two_rule_atn(); + let bypass = base.with_bypass_alternatives().expect("bypass ATN"); + + // Rule 0's start state (0) should now epsilon into its bypass-start + // block (the first appended state), added as the last outgoing edge. + let bypass_start_for_rule_0 = base.state_count(); + let rule_start = bypass.state(0).expect("rule start"); + let epsilon_targets: Vec<_> = rule_start + .transitions() + .iter() + .filter(|t| t.kind() == ParserTransitionKind::Epsilon) + .map(ParserTransition::target) + .collect(); + assert!( + epsilon_targets.contains(&bypass_start_for_rule_0), + "rule start must epsilon into its bypass block; got {epsilon_targets:?}" + ); + } + + fn two_rule_recognizer_data() -> RecognizerData { + RecognizerData::new( + "Bypass.g4", + Vocabulary::new( + [None, Some("'x'"), Some("'y'")], + [None, Some("X"), Some("Y")], + [None::<&str>, None], + ), + ) + .with_rule_names(["a", "b"]) + } + + /// The load-bearing end-to-end proof: the *unchanged* ATN interpreter, run + /// over the bypass ATN, matches a single imaginary token in place of a whole + /// rule and renders it as a one-terminal rule subtree — exactly the shape + /// ANTLR's `getRuleTagToken` detects. + #[test] + fn interpreter_matches_imaginary_token_as_whole_rule() { + let bypass = two_rule_atn() + .with_bypass_alternatives() + .expect("bypass ATN"); + // Rule 1 ("b")'s imaginary token type = max_token_type(2) + 1 + 1 = 4. + let imaginary_b = 4; + let source = ListSource { + specs: vec![ + TokenSpec::explicit(1, "x"), // real X token + TokenSpec::explicit(imaginary_b, ""), // imaginary rule-b tag + ], + index: 0, + }; + let mut parser = + BaseParser::new(CommonTokenStream::new(source), two_rule_recognizer_data()); + + let tree = parser + .parse_atn_rule(&bypass, 0) + .expect("bypass interpret of `a : X b` with imaginary b"); + + let root = parser.node(tree).as_rule().expect("root is rule a"); + assert_eq!(root.rule_index(), 0); + let children: Vec<_> = root.node().children().collect(); + assert_eq!(children.len(), 2, "a has children [X, b]"); + + // First child: the real X terminal. + let x = children[0].as_terminal().expect("first child terminal X"); + assert_eq!(x.symbol().token_type(), 1); + + // Second child: rule b rendered as a single-terminal subtree whose lone + // leaf carries the imaginary token type. This is the `(b )` shape. + let b = children[1].as_rule().expect("second child rule b"); + assert_eq!(b.rule_index(), 1); + let b_children: Vec<_> = b.node().children().collect(); + assert_eq!(b_children.len(), 1, "bypassed rule b has exactly one child"); + assert_eq!(b_children[0].kind(), NodeKind::Terminal); + assert_eq!( + b_children[0] + .as_terminal() + .expect("b's lone child is a terminal") + .symbol() + .token_type(), + imaginary_b, + "the lone child carries the imaginary bypass token type" + ); + } + + /// The bypass ATN must still parse ordinary input identically: feeding the + /// real tokens `X Y` reconstructs `(a X (b Y))` with no imaginary tokens. + #[test] + fn bypass_atn_still_parses_ordinary_input() { + let bypass = two_rule_atn() + .with_bypass_alternatives() + .expect("bypass ATN"); + let source = ListSource { + specs: vec![TokenSpec::explicit(1, "x"), TokenSpec::explicit(2, "y")], + index: 0, + }; + let mut parser = + BaseParser::new(CommonTokenStream::new(source), two_rule_recognizer_data()); + + let tree = parser + .parse_atn_rule(&bypass, 0) + .expect("bypass interpret of ordinary `X Y`"); + + let root = parser.node(tree).as_rule().expect("root is rule a"); + let children: Vec<_> = root.node().children().collect(); + assert_eq!(children.len(), 2); + assert_eq!( + children[0] + .as_terminal() + .expect("X terminal") + .symbol() + .token_type(), + 1 + ); + let b = children[1].as_rule().expect("rule b"); + let y = b + .node() + .children() + .next() + .expect("b child") + .as_terminal() + .expect("Y terminal"); + assert_eq!(y.symbol().token_type(), 2); + assert_eq!(parser.number_of_syntax_errors(), 0); + } +} diff --git a/src/atn/mod.rs b/src/atn/mod.rs index 99629ccf..1d385ddd 100644 --- a/src/atn/mod.rs +++ b/src/atn/mod.rs @@ -6,6 +6,7 @@ //! index-addressed [`parser_atn::ParserAtn`] representation instead. pub(crate) mod ascii_range; +mod bypass; pub mod lexer; pub mod lexer_dfa; pub mod parser; diff --git a/src/atn/parser_atn.rs b/src/atn/parser_atn.rs index f3dc0cf0..73954429 100644 --- a/src/atn/parser_atn.rs +++ b/src/atn/parser_atn.rs @@ -10,6 +10,7 @@ #![allow(clippy::inline_always)] use std::borrow::Cow; +use std::collections::BTreeMap; use std::fmt; use std::iter::FusedIterator; @@ -371,7 +372,7 @@ impl ParserAtn { } } - const fn set_count(&self) -> usize { + pub(crate) const fn set_count(&self) -> usize { self.layout.sets.len / self.layout.set_words } @@ -1188,6 +1189,10 @@ pub struct ParserAtnBuilder { max_token_type: i32, states: Vec, transitions: Vec, + /// Per-source transition indices in insertion order, so duplicate + /// detection scans one state's out-edges instead of every transition + /// added so far (which made re-emitting a whole ATN quadratic). + transitions_by_source: BTreeMap>, interval_sets: Vec, interval_ranges: Vec<(i32, i32)>, token_bit_words: Vec, @@ -1202,6 +1207,7 @@ impl ParserAtnBuilder { max_token_type, states: Vec::new(), transitions: Vec::new(), + transitions_by_source: BTreeMap::new(), interval_sets: Vec::new(), interval_ranges: Vec::new(), token_bit_words: Vec::new(), @@ -1288,17 +1294,22 @@ impl ParserAtnBuilder { transition: ParserTransitionSpec, ) -> Result { let source = self.checked_state(source, "transition source")?; - if let Some((index, _)) = self - .transitions - .iter() - .enumerate() - .find(|(_, existing)| existing.source == source && existing.spec() == transition) - { - return TransitionId::try_from(index); + if let Some(existing) = self.transitions_by_source.get(&source) { + if let Some(&index) = existing + .iter() + .find(|&&index| self.transitions[index].spec() == transition) + { + return TransitionId::try_from(index); + } } let record = self.transition_record(source, transition)?; - let id = TransitionId::try_from(self.transitions.len())?; + let index = self.transitions.len(); + let id = TransitionId::try_from(index)?; self.transitions.push(record); + self.transitions_by_source + .entry(source) + .or_default() + .push(index); Ok(id) } @@ -1713,6 +1724,58 @@ impl ParserTransitionSpec { | Self::Precedence { target, .. } => target, } } + + /// Returns this spec with its target redirected, preserving every other + /// field. + #[must_use] + pub(crate) const fn with_target(self, target: usize) -> Self { + match self { + Self::Epsilon { .. } => Self::Epsilon { target }, + Self::Atom { label, .. } => Self::Atom { target, label }, + Self::Range { start, stop, .. } => Self::Range { + target, + start, + stop, + }, + Self::Set { set, .. } => Self::Set { target, set }, + Self::NotSet { set, .. } => Self::NotSet { target, set }, + Self::Wildcard { .. } => Self::Wildcard { target }, + Self::Rule { + rule_index, + follow_state, + precedence, + .. + } => Self::Rule { + target, + rule_index, + follow_state, + precedence, + }, + Self::Predicate { + rule_index, + pred_index, + context_dependent, + .. + } => Self::Predicate { + target, + rule_index, + pred_index, + context_dependent, + }, + Self::Action { + rule_index, + action_index, + context_dependent, + .. + } => Self::Action { + target, + rule_index, + action_index, + context_dependent, + }, + Self::Precedence { precedence, .. } => Self::Precedence { target, precedence }, + } + } } #[derive(Clone, Copy, Debug, Eq, PartialEq)] diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index abde239d..ac532dac 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -7090,6 +7090,7 @@ const GENERATED_PARSER_RESERVED_RULE_METHODS: &[&str] = &[ "into_token_stream", "into_token_store", "into_parsed_file", + "compile_parse_tree_pattern", ]; fn parser_public_rule_method_names(rule_names: &[String]) -> Vec { @@ -9220,6 +9221,71 @@ fn embedded_render_slots( ) } +/// The `compile_parse_tree_pattern` method spliced into every generated parser. +/// +/// Kept as a standalone literal (single braces, no format placeholders) so the +/// large parser template stays under the line-length lint and this ANTLR +/// `Parser.compileParseTreePattern` analog reads as ordinary code. +const fn render_compile_parse_tree_pattern_method() -> &'static str { + r#" + /// Compiles a tree pattern rooted at parser rule `rule_index`. + /// + /// Mirrors ANTLR's `Parser.compileParseTreePattern`. Literal chunks of + /// `pattern` are lexed with a fresh lexer built by `make_lexer` (pass this + /// grammar's generated lexer constructor, e.g. `MyGrammarLexer::new`); + /// `` placeholders become rule/token references matched over a + /// rule-bypass ATN. The returned [`antlr4_runtime::ParseTreePattern`] can + /// then match subtrees. + /// + /// Takes `&self` only to mirror ANTLR's instance method; the ATN and + /// grammar metadata come from this module, so the parser's own state is + /// untouched. The pattern compiler (and its rule-bypass ATN) is built once + /// per process and shared by every call. + /// + /// # Errors + /// + /// Returns a [`antlr4_runtime::ParseTreePatternError`] for a malformed + /// pattern, an unknown tag, a lexer failure, or a pattern the start rule + /// does not parse cleanly and fully consume. + pub fn compile_parse_tree_pattern( + &self, + pattern: &str, + rule_index: usize, + mut make_lexer: impl FnMut(antlr4_runtime::InputStream) -> PL, + ) -> Result + where + PL: antlr4_runtime::TokenSource, + { + // The rule-bypass ATN derivation inside `ParseTreePatternMatcher::new` + // is O(states + transitions), so — like ANTLR's + // `Parser.bypassAltsAtnCache` — the matcher is built once per process + // and shared by every subsequent compile. A failed build is not cached + // and is retried (and re-reported) on the next call. + static PATTERN_DATA: OnceLock = OnceLock::new(); + static PATTERN_MATCHER: OnceLock> = + OnceLock::new(); + let matcher = match PATTERN_MATCHER.get() { + Some(matcher) => matcher, + None => { + let data = PATTERN_DATA.get_or_init(|| { + let grammar_metadata = metadata(); + RecognizerData::new( + grammar_metadata.grammar_file_name(), + grammar_metadata.vocabulary(), + ) + .with_rule_names(grammar_metadata.rule_names().iter().copied()) + }); + let matcher = antlr4_runtime::ParseTreePatternMatcher::new(parser_atn(), data)?; + PATTERN_MATCHER.get_or_init(|| matcher) + } + }; + matcher.compile(pattern, rule_index, move |text: &str| { + antlr4_runtime::lex_pattern_chunk(text, &mut make_lexer) + }) + } +"# +} + fn render_parser_with_options( grammar_name: &str, data: &CodegenData<'_>, @@ -9228,6 +9294,7 @@ fn render_parser_with_options( let empty_patterns = SemPatternFile::default(); let patterns = options.patterns.unwrap_or(&empty_patterns); let type_name = rust_type_name(grammar_name); + let compile_pattern_method = render_compile_parse_tree_pattern_method(); let metadata = render_parser_metadata(grammar_name, data); let parser_atn = data.parser_atn()?; let parser_atn_data = render_u32_slice(parser_atn.packed_words()); @@ -9642,7 +9709,7 @@ where pub fn into_parsed_file(self, root: antlr4_runtime::NodeId) -> antlr4_runtime::ParsedFile {{ self.base.into_parsed_file(root) }} - +{compile_pattern_method} #[allow(dead_code)] fn simulator(&mut self) -> &mut antlr4_runtime::ParserAtnSimulator<'static> {{ self.simulator @@ -11921,6 +11988,7 @@ mod tests { "clearDfa".to_owned(), "addErrorListener".to_owned(), "removeErrorListeners".to_owned(), + "compileParseTreePattern".to_owned(), "regularRule".to_owned(), ]; @@ -11935,6 +12003,7 @@ mod tests { "clear_dfa_rule", "add_error_listener_rule", "remove_error_listeners_rule", + "compile_parse_tree_pattern_rule", "regular_rule" ] ); diff --git a/src/byte_stream.rs b/src/byte_stream.rs index 759eb2c4..7bd02cc1 100644 --- a/src/byte_stream.rs +++ b/src/byte_stream.rs @@ -54,8 +54,8 @@ //! Because the bytes are not text, [`CharStream::text`] renders the matched //! span as a lowercase hex string with no separators (`[0xDE, 0xAD]` becomes //! `"dead"`). Token *positions* are still exact byte offsets; use -//! [`IntStream::index`](crate::IntStream::index) or a token's byte span when you -//! need to slice the original bytes. +//! [`crate::IntStream::index`] or a token's byte span when you need to slice +//! the original bytes. use std::io; diff --git a/src/lib.rs b/src/lib.rs index cbbce5d0..aec9215a 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -19,6 +19,7 @@ pub mod semir; pub mod token; pub mod token_stream; pub mod tree; +pub mod tree_pattern; pub mod vocabulary; pub mod xpath; @@ -54,6 +55,10 @@ pub use tree::{ ParseTreeStorage, ParseTreeVisitor, ParseTreeWalker, ParsedFile, ParserRuleContext, RuleNodeView, TerminalNodeView, }; +pub use tree_pattern::{ + ParseTreeMatch, ParseTreePattern, ParseTreePatternError, ParseTreePatternMatcher, PatternLexer, + lex_pattern_chunk, +}; pub use vocabulary::Vocabulary; pub use xpath::{XPath, XPathError}; diff --git a/src/recognizer.rs b/src/recognizer.rs index 0209c47f..521f848a 100644 --- a/src/recognizer.rs +++ b/src/recognizer.rs @@ -97,6 +97,12 @@ impl RecognizerData { &self.rule_names } + /// The token vocabulary for literal/symbolic name resolution. + #[must_use] + pub const fn vocabulary(&self) -> &Vocabulary { + &self.vocabulary + } + pub const fn state(&self) -> isize { self.state } diff --git a/src/snapshots/antlr4_runtime__tree_pattern__tests__split_custom_delimiters.snap b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_custom_delimiters.snap new file mode 100644 index 00000000..2507b692 --- /dev/null +++ b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_custom_delimiters.snap @@ -0,0 +1,16 @@ +--- +source: src/tree_pattern.rs +expression: chunks +--- +[ + Text( + "x ", + ), + Tag { + name: "expr", + label: None, + }, + Text( + " y", + ), +] diff --git a/src/snapshots/antlr4_runtime__tree_pattern__tests__split_interleaves_text_and_tags.snap b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_interleaves_text_and_tags.snap new file mode 100644 index 00000000..f352dffc --- /dev/null +++ b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_interleaves_text_and_tags.snap @@ -0,0 +1,20 @@ +--- +source: src/tree_pattern.rs +expression: chunks +--- +[ + Tag { + name: "ID", + label: None, + }, + Text( + " = ", + ), + Tag { + name: "expr", + label: None, + }, + Text( + " ;", + ), +] diff --git a/src/snapshots/antlr4_runtime__tree_pattern__tests__split_no_tags.snap b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_no_tags.snap new file mode 100644 index 00000000..d7a1af0f --- /dev/null +++ b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_no_tags.snap @@ -0,0 +1,9 @@ +--- +source: src/tree_pattern.rs +expression: chunks +--- +[ + Text( + "a = 3 ;", + ), +] diff --git a/src/snapshots/antlr4_runtime__tree_pattern__tests__split_parses_labeled_tags.snap b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_parses_labeled_tags.snap new file mode 100644 index 00000000..f75a97b1 --- /dev/null +++ b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_parses_labeled_tags.snap @@ -0,0 +1,21 @@ +--- +source: src/tree_pattern.rs +expression: chunks +--- +[ + Tag { + name: "ID", + label: Some( + "lhs", + ), + }, + Text( + " = ", + ), + Tag { + name: "expr", + label: Some( + "e", + ), + }, +] diff --git a/src/snapshots/antlr4_runtime__tree_pattern__tests__split_rejects_malformed.snap b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_rejects_malformed.snap new file mode 100644 index 00000000..56fe91cd --- /dev/null +++ b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_rejects_malformed.snap @@ -0,0 +1,30 @@ +--- +source: src/tree_pattern.rs +expression: errors +--- +[ + ( + "", + "missing start tag in pattern: ID>", + ), + ( + "><", + "tag delimiters out of order in pattern: ><", + ), + ( + "<>", + "empty tag in pattern: <>", + ), + ( + "", + "empty tag in pattern: ", + ), + ( + ">", + "tag delimiters out of order in pattern: >", + ), +] diff --git a/src/snapshots/antlr4_runtime__tree_pattern__tests__split_strips_escapes.snap b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_strips_escapes.snap new file mode 100644 index 00000000..b822261d --- /dev/null +++ b/src/snapshots/antlr4_runtime__tree_pattern__tests__split_strips_escapes.snap @@ -0,0 +1,16 @@ +--- +source: src/tree_pattern.rs +expression: chunks +--- +[ + Text( + "a < b ", + ), + Tag { + name: "ID", + label: None, + }, + Text( + " c > d", + ), +] diff --git a/src/tree_pattern.rs b/src/tree_pattern.rs new file mode 100644 index 00000000..113eaac9 --- /dev/null +++ b/src/tree_pattern.rs @@ -0,0 +1,1735 @@ +//! ANTLR parse-tree pattern matching. +//! +//! A *tree pattern* is a string of ordinary grammar input with embedded +//! `` placeholders — for example ` = ;` matched as the rule +//! `stat`. Literals must match exactly; a `` tag matches any subtree of +//! that parser rule and a `` tag matches any token of that type. Tags +//! may carry a label, ``, so matched nodes can be looked up by name. +//! +//! This mirrors ANTLR's `org.antlr.v4.runtime.tree.pattern` package. Compiling +//! a pattern lexes its literal chunks with the real lexer, converts each tag +//! into a synthetic rule/token tag token (ANTLR's `RuleTagToken` / +//! `TokenTagToken`), and interprets that hybrid token stream over a rule-bypass +//! ATN (see [`crate::atn::parser_atn::ParserAtn::with_bypass_alternatives`]) to +//! build a *pattern tree*. [`ParseTreePattern::match_tree`] then walks a +//! subject tree and the pattern tree in lockstep, binding tag labels. +//! +//! The ergonomic entry point is the `compile_parse_tree_pattern` method every +//! generated parser exposes; [`ParseTreePatternMatcher`] is the lower-level, +//! reusable compiler behind it. + +use std::collections::BTreeMap; + +use thiserror::Error; + +use crate::atn::parser_atn::ParserAtn; +use crate::recognizer::{Recognizer, RecognizerData}; +use crate::token::{Token, TokenId, TokenSink, TokenSource, TokenSpec, TokenStoreError}; +use crate::tree::{Node, NodeKind}; +use crate::{BaseParser, CommonTokenStream, TOKEN_EOF}; + +const MATCH_STACK_RED_ZONE: usize = 1024 * 1024; +const MATCH_STACK_SIZE: usize = 4 * 1024 * 1024; + +/// A tag's disposition: a reference to a parser rule or to a token type. +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum TagKind { + /// `` — matches an entire subtree produced by the named parser rule. + /// Carries the rule index and the imaginary bypass token type used to drive + /// the interpreter. + Rule { rule_index: usize, bypass_type: i32 }, + /// `` — matches a single token of the named type. + Token { token_type: i32 }, +} + +/// Identity of a synthetic tag token, tracked out of band keyed by the +/// [`TokenId`] the tag occupies in the pattern token store. +/// +/// The compact [`crate::token::TokenStore`] has no room for the rule/token +/// name and label a tag carries, so — like the store's own sparse +/// `explicit_text` side table — the matcher keeps this data beside the store +/// rather than inside it. +#[derive(Clone, Debug, Eq, PartialEq)] +struct TagInfo { + kind: TagKind, + /// The rule or token name the tag references (e.g. `"expr"`, `"ID"`). + name: String, + /// The explicit label, if the tag was written ``. + label: Option, +} + +impl TagInfo { + /// The names a matched node is filed under: always the referenced rule or + /// token name, plus the explicit label when present. Mirrors the dual + /// `labels.map(name, ...)` / `labels.map(label, ...)` calls in ANTLR's + /// `matchImpl`. + fn label_keys(&self) -> impl Iterator { + std::iter::once(self.name.as_str()).chain(self.label.as_deref()) + } +} + +/// A raw fragment of a split pattern: either literal text or a ``. +#[derive(Clone, Debug, Eq, PartialEq)] +enum Chunk { + /// Literal input text, with escape sequences already stripped. + Text(String), + /// A `` island: the referenced rule/token name and optional label. + Tag { name: String, label: Option }, +} + +/// An invalid tree pattern or a failure while compiling one. +#[derive(Clone, Debug, Eq, Error, PartialEq)] +pub enum ParseTreePatternError { + /// A start delimiter was seen without a matching stop delimiter. + #[error("unterminated tag in pattern: {pattern}")] + UnterminatedTag { pattern: String }, + /// A stop delimiter was seen without a preceding start delimiter. + #[error("missing start tag in pattern: {pattern}")] + MissingStartTag { pattern: String }, + /// A stop delimiter appeared at or before its start delimiter. + #[error("tag delimiters out of order in pattern: {pattern}")] + DelimitersOutOfOrder { pattern: String }, + /// A tag was empty (`<>` or ``). + #[error("empty tag in pattern: {pattern}")] + EmptyTag { pattern: String }, + /// A `` tag named a token the grammar does not define. + #[error("unknown token {name} in pattern: {pattern}")] + UnknownToken { name: String, pattern: String }, + /// A `` tag named a parser rule the grammar does not define. + #[error("unknown rule {name} in pattern: {pattern}")] + UnknownRule { name: String, pattern: String }, + /// A tag started with neither an upper- nor lower-case letter, so it could + /// not be classified as a token or rule reference. + #[error("invalid tag {tag} in pattern: {pattern}")] + InvalidTag { tag: String, pattern: String }, + /// Lexing a literal chunk with the real lexer failed. + #[error("could not tokenize pattern chunk: {message}")] + Tokenization { message: String }, + /// The start rule did not consume the whole pattern (ANTLR issue #413). + #[error("start rule did not consume the full pattern: {pattern}")] + StartRuleDoesNotConsumeFullPattern { pattern: String }, + /// Interpreting the pattern token stream failed. + #[error("could not interpret pattern as rule {rule_index}: {message}")] + CannotInvokeStartRule { rule_index: usize, message: String }, + /// The rule-bypass transform of the grammar ATN failed. + #[error("could not build rule-bypass ATN: {message}")] + BypassAtn { message: String }, + /// [`ParseTreePatternMatcher::set_delimiters`] was given an empty start or + /// stop delimiter. + #[error("{which} delimiter cannot be empty")] + EmptyDelimiter { which: &'static str }, +} + +/// Tag delimiters and escape string used to split a pattern. +/// +/// Defaults to `<`, `>`, and `\`, matching ANTLR. Grammars that use `<...>` in +/// their own concrete syntax (e.g. Java generics) can pick different delimiters +/// via [`ParseTreePatternMatcher::set_delimiters`]. +#[derive(Clone, Debug, Eq, PartialEq)] +struct Delimiters { + start: String, + stop: String, + escape: String, +} + +impl Default for Delimiters { + fn default() -> Self { + Self { + start: "<".to_owned(), + stop: ">".to_owned(), + escape: "\\".to_owned(), + } + } +} + +/// Splits a pattern into interleaved literal text and `` chunks. +/// +/// Faithful port of ANTLR's `ParseTreePatternMatcher.split`: it scans for the +/// escaped and unescaped delimiters, validates that starts and stops are +/// balanced and ordered, slices the chunks, then strips escape sequences from +/// the text chunks only (never from tags). Operates on `char` boundaries so +/// multi-byte delimiters and Unicode input are handled correctly. +fn split(pattern: &str, delimiters: &Delimiters) -> Result, ParseTreePatternError> { + let chars: Vec = pattern.chars().collect(); + let start: Vec = delimiters.start.chars().collect(); + let stop: Vec = delimiters.stop.chars().collect(); + let escape: Vec = delimiters.escape.chars().collect(); + + let matches_at = |at: usize, needle: &[char]| -> bool { + !needle.is_empty() && chars[at..].starts_with(needle) + }; + + // Pass 1: locate every unescaped start/stop delimiter, by char index. + let mut starts = Vec::new(); + let mut stops = Vec::new(); + let mut position = 0; + while position < chars.len() { + if matches_at(position, &escape) && matches_at(position + escape.len(), &start) { + position += escape.len() + start.len(); + } else if matches_at(position, &escape) && matches_at(position + escape.len(), &stop) { + position += escape.len() + stop.len(); + } else if matches_at(position, &start) { + starts.push(position); + position += start.len(); + } else if matches_at(position, &stop) { + stops.push(position); + position += stop.len(); + } else { + position += 1; + } + } + + if starts.len() > stops.len() { + return Err(ParseTreePatternError::UnterminatedTag { + pattern: pattern.to_owned(), + }); + } + if starts.len() < stops.len() { + return Err(ParseTreePatternError::MissingStartTag { + pattern: pattern.to_owned(), + }); + } + for (open, close) in starts.iter().zip(&stops) { + if open >= close { + return Err(ParseTreePatternError::DelimitersOutOfOrder { + pattern: pattern.to_owned(), + }); + } + } + // Tags must also not overlap each other (e.g. `>` pairs 0/4 and 2/5): + // each close must come before the next open, or the inter-tag text slice + // below would be an inverted range. Upstream reaches the same shape and + // throws from `String.substring`; returning the structured error is safer. + for (close, next_open) in stops.iter().zip(starts.iter().skip(1)) { + if close + stop.len() > *next_open { + return Err(ParseTreePatternError::DelimitersOutOfOrder { + pattern: pattern.to_owned(), + }); + } + } + + let slice = |from: usize, to: usize| -> String { chars[from..to].iter().collect() }; + + // Pass 2: collect chunks between the located delimiters. + let ntags = starts.len(); + let mut chunks = Vec::new(); + if ntags == 0 { + chunks.push(Chunk::Text(slice(0, chars.len()))); + } else if starts[0] > 0 { + chunks.push(Chunk::Text(slice(0, starts[0]))); + } + for index in 0..ntags { + let tag = slice(starts[index] + start.len(), stops[index]); + chunks.push(parse_tag(&tag, pattern)?); + if index + 1 < ntags { + chunks.push(Chunk::Text(slice( + stops[index] + stop.len(), + starts[index + 1], + ))); + } + } + if ntags > 0 { + let after_last = stops[ntags - 1] + stop.len(); + if after_last < chars.len() { + chunks.push(Chunk::Text(slice(after_last, chars.len()))); + } + } + + // Strip escape sequences from text chunks (tags are left untouched). + if !delimiters.escape.is_empty() { + for chunk in &mut chunks { + if let Chunk::Text(text) = chunk { + *text = strip_escape(text, &delimiters.escape); + } + } + } + + Ok(chunks) +} + +/// Removes every occurrence of `escape` from `text`, non-overlapping and +/// left to right — the escape-stripping ANTLR does with `String.replace`. +fn strip_escape(text: &str, escape: &str) -> String { + let mut out = String::with_capacity(text.len()); + let mut rest = text; + while let Some(at) = rest.find(escape) { + out.push_str(&rest[..at]); + rest = &rest[at + escape.len()..]; + } + out.push_str(rest); + out +} + +/// Parses the inside of a `<...>` into a tag chunk, splitting an optional +/// `label:` prefix. Empty tags (`<>`, ``) are rejected. +fn parse_tag(tag: &str, pattern: &str) -> Result { + let (label, name) = tag.find(':').map_or((None, tag), |colon| { + (Some(tag[..colon].to_owned()), &tag[colon + 1..]) + }); + if name.is_empty() { + return Err(ParseTreePatternError::EmptyTag { + pattern: pattern.to_owned(), + }); + } + Ok(Chunk::Tag { + name: name.to_owned(), + label, + }) +} + +/// Lexes one literal pattern chunk into the token specs it produces. +/// +/// The matcher owns all split/tag/interpret logic; this trait is the single +/// grammar-specific hook, supplying the real lexer's output for a run of +/// concrete input text. The trailing EOF must be excluded; off-default-channel +/// tokens (whitespace, comments) may be returned on their own channel and are +/// skipped by the interpreter exactly as in a normal parse, matching ANTLR's +/// `tokenize`. Implemented for +/// `FnMut(&str) -> Result, ParseTreePatternError>` so a closure +/// suffices. +pub trait PatternLexer { + /// Tokenizes `text` into token specs (no EOF), each on its original channel. + /// + /// # Errors + /// + /// Returns a [`ParseTreePatternError::Tokenization`] if the lexer rejects + /// the chunk. + fn tokenize_chunk(&mut self, text: &str) -> Result, ParseTreePatternError>; +} + +impl PatternLexer for F +where + F: FnMut(&str) -> Result, ParseTreePatternError>, +{ + fn tokenize_chunk(&mut self, text: &str) -> Result, ParseTreePatternError> { + self(text) + } +} + +/// Runs a lexer over one chunk of pattern text and returns its token specs +/// (every non-EOF token, on its original channel), suitable for a +/// [`PatternLexer`]. +/// +/// This is the bridge generated parsers use to satisfy [`PatternLexer`] from +/// their concrete lexer: `make_lexer` builds a fresh lexer over the chunk's +/// [`InputStream`](crate::InputStream), the tokens are buffered, and each +/// non-EOF token becomes a `TokenSpec::explicit(type, text)` carrying its +/// channel. Like ANTLR's `tokenize`, hidden-channel tokens (whitespace, +/// comments) are preserved on their channel so the interpreter skips them the +/// same way it does during a normal parse. Positions are dropped because +/// pattern trees compare by type and text, not span. +/// +/// # Errors +/// +/// Returns [`ParseTreePatternError::Tokenization`] if the lexer reports a +/// tokenization error for the chunk. +pub fn lex_pattern_chunk( + text: &str, + make_lexer: impl FnOnce(crate::InputStream) -> L, +) -> Result, ParseTreePatternError> +where + L: TokenSource, +{ + let lexer = make_lexer(crate::InputStream::new(text)); + let mut stream = + CommonTokenStream::try_new(lexer).map_err(|error| ParseTreePatternError::Tokenization { + message: error.to_string(), + })?; + stream.fill(); + if let Some(error) = stream.drain_source_errors().into_iter().next() { + return Err(ParseTreePatternError::Tokenization { + message: format!("lexer error at {}:{}", error.line, error.column), + }); + } + Ok(stream + .tokens() + .filter(|token| token.token_type() != TOKEN_EOF) + .map(|token| { + TokenSpec::explicit(token.token_type(), token.text_or_empty()) + .with_channel(token.channel()) + }) + .collect()) +} + +/// Compiles tree patterns for one grammar. +/// +/// Holds the grammar's rule-bypass ATN and recognizer metadata; each +/// [`Self::compile`] call lexes the pattern's literal chunks (via the supplied +/// [`PatternLexer`]), converts tags into synthetic tokens, and interprets the +/// hybrid stream to build a reusable [`ParseTreePattern`]. +/// +/// Most callers reach this through +/// the `compile_parse_tree_pattern` method on generated parsers; construct one directly to +/// reuse the bypass ATN across many patterns or to customize delimiters. +#[derive(Debug)] +pub struct ParseTreePatternMatcher<'a> { + bypass_atn: ParserAtn, + data: &'a RecognizerData, + delimiters: Delimiters, +} + +impl<'a> ParseTreePatternMatcher<'a> { + /// Creates a matcher for a grammar's parser ATN and recognizer metadata. + /// + /// The bypass ATN is derived from `atn` once here and reused by every + /// compile. `data` supplies rule and token names for resolving tags. + /// + /// # Errors + /// + /// Returns [`ParseTreePatternError::BypassAtn`] if the rule-bypass + /// transform of `atn` fails (e.g. an unrecognizable left-recursive + /// precedence prefix). + pub fn new(atn: &ParserAtn, data: &'a RecognizerData) -> Result { + let bypass_atn = + atn.with_bypass_alternatives() + .map_err(|error| ParseTreePatternError::BypassAtn { + message: error.to_string(), + })?; + Ok(Self { + bypass_atn, + data, + delimiters: Delimiters::default(), + }) + } + + /// Overrides the tag delimiters and escape string (defaults `<`, `>`, `\`). + /// + /// Useful for grammars whose concrete syntax already uses `<...>`. Unlike + /// upstream, an empty `escape` is accepted and simply disables escaping + /// (Java's `indexOf`-based scan misbehaves on an empty escape string). + /// + /// # Errors + /// + /// Returns [`ParseTreePatternError::EmptyDelimiter`] when `start` or `stop` + /// is empty, mirroring upstream's `IllegalArgumentException` — an empty + /// delimiter would silently collapse every pattern into one text chunk. + pub fn set_delimiters( + &mut self, + start: impl Into, + stop: impl Into, + escape: impl Into, + ) -> Result<(), ParseTreePatternError> { + let start = start.into(); + let stop = stop.into(); + if start.is_empty() { + return Err(ParseTreePatternError::EmptyDelimiter { which: "start" }); + } + if stop.is_empty() { + return Err(ParseTreePatternError::EmptyDelimiter { which: "stop" }); + } + self.delimiters = Delimiters { + start, + stop, + escape: escape.into(), + }; + Ok(()) + } + + /// Compiles `pattern`, rooted at parser rule `rule_index`, into a reusable + /// [`ParseTreePattern`]. + /// + /// `lexer` tokenizes the pattern's literal chunks; tags become synthetic + /// rule/token tokens, and the hybrid stream is interpreted over the bypass + /// ATN starting at `rule_index`. + /// + /// # Errors + /// + /// Returns a [`ParseTreePatternError`] for a malformed pattern, an unknown + /// rule/token tag, a lexer failure, an interpretation failure, or a pattern + /// the start rule does not fully consume. + pub fn compile( + &self, + pattern: &str, + rule_index: usize, + lexer: impl PatternLexer, + ) -> Result { + let chunks = split(pattern, &self.delimiters)?; + let (specs, tags_by_index) = self.tokenize(&chunks, pattern, lexer)?; + let tree = self.interpret(specs, &tags_by_index, rule_index, pattern)?; + Ok(ParseTreePattern { + pattern: pattern.to_owned(), + pattern_rule_index: rule_index, + tree, + }) + } + + /// Converts chunks into a flat token-spec list, recording which flat indices + /// are tags. Mirrors ANTLR's `tokenize`: upper-case tags are token + /// references, lower-case tags are rule references, literals are lexed. + fn tokenize( + &self, + chunks: &[Chunk], + pattern: &str, + mut lexer: impl PatternLexer, + ) -> Result<(Vec, BTreeMap), ParseTreePatternError> { + let mut specs = Vec::new(); + let mut tags_by_index = BTreeMap::new(); + for chunk in chunks { + match chunk { + Chunk::Tag { name, label } => { + let (spec, tag) = self.tag_token(name, label.clone(), pattern)?; + tags_by_index.insert(specs.len(), tag); + specs.push(spec); + } + Chunk::Text(text) => { + specs.extend(lexer.tokenize_chunk(text)?); + } + } + } + // An EOF-typed token (an `` tag, or a stray EOF from a custom + // lexer) terminates the buffered token stream, so anything after it + // would be dropped before the full-consumption check could see it. + // A trailing EOF is legitimate for rules that end in `EOF`; anywhere + // else the pattern is broken and must fail loudly instead of silently + // truncating (upstream ANTLR silently ignores the suffix here). + if let Some(at) = specs + .iter() + .position(|spec| spec.token_type == TOKEN_EOF) + .filter(|at| at + 1 < specs.len()) + { + return Err(ParseTreePatternError::Tokenization { + message: format!( + "EOF at pattern token {at} terminates the stream; {} following token(s) \ + would be ignored", + specs.len() - at - 1 + ), + }); + } + Ok((specs, tags_by_index)) + } + + /// Builds the synthetic token and tag record for one ``. + /// + /// An upper-case initial classifies a token reference (``), a lower-case + /// initial a rule reference (``). Names resolve to token types via the + /// vocabulary and to rule indices via the rule-name list. + fn tag_token( + &self, + name: &str, + label: Option, + pattern: &str, + ) -> Result<(TokenSpec, TagInfo), ParseTreePatternError> { + let display = tag_display(name, label.as_deref()); + let first = name + .chars() + .next() + .ok_or_else(|| ParseTreePatternError::InvalidTag { + tag: name.to_owned(), + pattern: pattern.to_owned(), + })?; + if first.is_uppercase() { + let token_type = self.data.vocabulary().token_type(name).ok_or_else(|| { + ParseTreePatternError::UnknownToken { + name: name.to_owned(), + pattern: pattern.to_owned(), + } + })?; + let spec = TokenSpec::explicit(token_type, display); + let tag = TagInfo { + kind: TagKind::Token { token_type }, + name: name.to_owned(), + label, + }; + Ok((spec, tag)) + } else if first.is_lowercase() { + let rule_index = + self.rule_index(name) + .ok_or_else(|| ParseTreePatternError::UnknownRule { + name: name.to_owned(), + pattern: pattern.to_owned(), + })?; + // The bypass ATN owns the imaginary-type formula, so the matcher + // and the ATN's bypass `Atom` edges can never disagree. + let bypass_type = self + .bypass_atn + .bypass_token_type(rule_index) + .map_err(|error| ParseTreePatternError::BypassAtn { + message: error.to_string(), + })?; + let spec = TokenSpec::explicit(bypass_type, display); + let tag = TagInfo { + kind: TagKind::Rule { + rule_index, + bypass_type, + }, + name: name.to_owned(), + label, + }; + Ok((spec, tag)) + } else { + Err(ParseTreePatternError::InvalidTag { + tag: name.to_owned(), + pattern: pattern.to_owned(), + }) + } + } + + /// Resolves a parser rule name to its index (last wins, like the runtime's + /// other name lookups). + fn rule_index(&self, name: &str) -> Option { + self.data.rule_names().iter().rposition(|rule| rule == name) + } + + /// Interprets the hybrid token specs over the bypass ATN, producing the + /// pattern tree and re-keying the tag table by the tokens' final store IDs. + fn interpret( + &self, + specs: Vec, + tags_by_index: &BTreeMap, + rule_index: usize, + pattern: &str, + ) -> Result { + let trailing_eof = specs + .last() + .is_some_and(|spec| spec.token_type == TOKEN_EOF); + let source = PatternTokenSource { specs, index: 0 }; + let mut parser = BaseParser::new(CommonTokenStream::new(source), self.data.clone()); + // ANTLR installs a BailErrorStrategy for the pattern parse: a pattern + // the grammar only accepts through error recovery must fail loudly, not + // bake `` error nodes into the pattern tree (which would + // then match nothing). Recovery diagnostics are checked below; the + // default console listener is removed so a rejected pattern does not + // also print to stderr. + parser.remove_error_listeners(); + let root = parser + .parse_atn_rule(&self.bypass_atn, rule_index) + .map_err(|error| ParseTreePatternError::CannotInvokeStartRule { + rule_index, + message: error.to_string(), + })?; + if parser.number_of_syntax_errors() > 0 { + return Err(ParseTreePatternError::CannotInvokeStartRule { + rule_index, + message: format!( + "pattern is not valid for the rule: {} syntax error(s) during pattern parse", + parser.number_of_syntax_errors() + ), + }); + } + + // The start rule must consume the whole pattern (ANTLR issue #413): + // the next visible token after the parse must be EOF. + if parser.token_stream().la_token(1) != TOKEN_EOF { + return Err(ParseTreePatternError::StartRuleDoesNotConsumeFullPattern { + pattern: pattern.to_owned(), + }); + } + + let file = parser.into_parsed_file(root); + // A trailing `` tag doubles as the stream terminator, so the + // lookahead check above cannot tell "the rule matched EOF" from "the + // tag was silently ignored". Rules that end in `EOF` put an EOF + // terminal in the tree; require it, and reject the pattern otherwise + // (upstream silently drops the tag here). + if trailing_eof + && !file.tree().descendants().any(|node| { + node.as_terminal() + .is_some_and(|terminal| terminal.symbol().token_type() == TOKEN_EOF) + }) + { + return Err(ParseTreePatternError::StartRuleDoesNotConsumeFullPattern { + pattern: pattern.to_owned(), + }); + } + let tags = rekey_tags_by_token_id(tags_by_index); + Ok(PatternTree { file, tags }) + } +} + +/// A token source over pre-lexed pattern specs, terminated by EOF — the runtime +/// analog of ANTLR's `ListTokenSource`. +#[derive(Debug)] +struct PatternTokenSource { + specs: Vec, + index: usize, +} + +impl TokenSource for PatternTokenSource { + fn next_token(&mut self, sink: &mut TokenSink<'_>) -> Result { + let spec = self + .specs + .get(self.index) + .cloned() + .unwrap_or_else(|| TokenSpec::eof(self.index, self.index, 1, self.index)); + self.index += 1; + sink.push(spec) + } + + fn line(&self) -> usize { + 1 + } + + fn column(&self) -> usize { + self.index + } + + fn source_name(&self) -> &'static str { + "tree-pattern" + } +} + +/// Re-keys the tag table from flat spec indices to the [`TokenId`]s the tokens +/// occupy in the finished store. +/// +/// [`PatternTokenSource`] pushes specs in order from index 0, and +/// `buffer_token_source` asserts each pushed token lands at its expected index, +/// so a tag recorded at flat index `i` occupies exactly `TokenId(i)`. +fn rekey_tags_by_token_id(tags_by_index: &BTreeMap) -> BTreeMap { + tags_by_index + .iter() + .filter_map(|(&index, tag)| Some((TokenId::try_from(index).ok()?, tag.clone()))) + .collect() +} + +/// Formats a tag for a synthetic token's text, e.g. `` or ``. +fn tag_display(name: &str, label: Option<&str>) -> String { + label.map_or_else(|| format!("<{name}>"), |label| format!("<{label}:{name}>")) +} + +/// The compiled pattern tree plus the tag side table describing its tag leaves. +/// +/// The tree is owned as a [`ParsedFile`](crate::tree::ParsedFile); `tags` maps +/// each tag leaf's [`TokenId`] to the rule/token it stands in for. Both are +/// produced once by [`ParseTreePatternMatcher::compile`] and shared by every +/// match. +#[derive(Debug)] +struct PatternTree { + file: crate::tree::ParsedFile, + tags: BTreeMap, +} + +/// A pattern like ` = ;` compiled to a reusable tree. +/// +/// Created by [`ParseTreePatternMatcher::compile`] or +/// a generated parser's `compile_parse_tree_pattern`. Match a subject tree with +/// [`Self::match_tree`] (full result) or [`Self::matches`] (boolean). +#[derive(Debug)] +pub struct ParseTreePattern { + pattern: String, + pattern_rule_index: usize, + tree: PatternTree, +} + +impl ParseTreePattern { + /// The tree-pattern source string this was compiled from. + #[must_use] + pub fn pattern(&self) -> &str { + &self.pattern + } + + /// The parser rule index that roots the pattern. + #[must_use] + pub const fn pattern_rule_index(&self) -> usize { + self.pattern_rule_index + } + + /// The compiled pattern as a parse tree, with tags present as terminal + /// leaves (a rule tag is a rule node whose single child carries the + /// imaginary bypass token). + /// + /// Mirrors ANTLR's `ParseTreePattern.getPatternTree`; useful for inspecting + /// why a pattern that compiled does not match a subject — + /// `pattern_tree().text()` renders the tag placeholders inline. + #[must_use] + pub fn pattern_tree(&self) -> Node<'_> { + self.tree.file.tree() + } + + /// Matches `tree` against this pattern, returning the full result including + /// bound labels and the first mismatched node (if any). + #[must_use] + pub fn match_tree<'subject>(&self, tree: Node<'subject>) -> ParseTreeMatch<'subject> { + let mut labels: BTreeMap>> = BTreeMap::new(); + let pattern_root = self.tree.file.tree(); + let mismatched = match_impl(tree, pattern_root, &self.tree.tags, &mut labels); + ParseTreeMatch { + tree, + labels, + mismatched_node: mismatched, + } + } + + /// Returns whether `tree` matches this pattern. + #[must_use] + pub fn matches(&self, tree: Node<'_>) -> bool { + self.match_tree(tree).succeeded() + } + + /// Finds nodes under `tree` with an `XPath` expression, then returns the + /// successful matches of this pattern against those subtrees. + /// + /// Mirrors ANTLR's `ParseTreePattern.findAll`: unsuccessful matches are + /// omitted, whatever the reason for the failure. `recognizer` resolves the + /// rule and token names in `xpath`, exactly as in + /// [`XPath::find_all`](crate::XPath::find_all). + /// + /// # Errors + /// + /// Returns [`XPathError`](crate::XPathError) when `xpath` is not a valid + /// parse-tree path expression. + pub fn find_all<'subject, R>( + &self, + tree: Node<'subject>, + xpath: &str, + recognizer: &R, + ) -> Result>, crate::XPathError> + where + R: Recognizer + ?Sized, + { + Ok(crate::XPath::find_all(tree, xpath, recognizer)? + .into_iter() + .map(|subtree| self.match_tree(subtree)) + .filter(ParseTreeMatch::succeeded) + .collect()) + } +} + +/// The result of matching a subject tree against a [`ParseTreePattern`]. +/// +/// Holds the label bindings discovered during the match and, on failure, the +/// first subject node that did not match. Borrows the subject tree. +#[derive(Clone, Debug)] +pub struct ParseTreeMatch<'subject> { + tree: Node<'subject>, + labels: BTreeMap>>, + mismatched_node: Option>, +} + +impl<'subject> ParseTreeMatch<'subject> { + /// Returns whether the match succeeded (no node mismatched). + #[must_use] + pub const fn succeeded(&self) -> bool { + self.mismatched_node.is_none() + } + + /// The subject tree this match was computed against. + #[must_use] + pub const fn tree(&self) -> Node<'subject> { + self.tree + } + + /// The first subject node that failed to match, or `None` on success. + #[must_use] + pub const fn mismatched_node(&self) -> Option> { + self.mismatched_node + } + + /// The last node bound to `label`, or `None` if nothing matched it. + /// + /// Unlabeled tags ``/`` are filed under their rule/token name, so + /// `get("expr")` returns a node matched by ``. + #[must_use] + pub fn get(&self, label: &str) -> Option> { + self.labels + .get(label) + .and_then(|nodes| nodes.last().copied()) + } + + /// Every node bound to `label`, in match order. + #[must_use] + pub fn get_all(&self, label: &str) -> &[Node<'subject>] { + self.labels.get(label).map_or(&[], Vec::as_slice) + } + + /// All label bindings, keyed by label name. + #[must_use] + pub const fn labels(&self) -> &BTreeMap>> { + &self.labels + } +} + +/// Walks a subject node and a pattern node in lockstep, recording label +/// bindings and returning the first subject node that failed to match. +/// +/// Faithful port of ANTLR's `ParseTreePatternMatcher.matchImpl`: +/// - two terminals match when their token types agree; if the pattern terminal +/// is a token tag it binds, else the texts must be equal; +/// - a rule node paired with a single-terminal rule-tag subtree binds if the +/// rule indices agree; +/// - otherwise two rule nodes must have equal child counts and matching +/// children; +/// - a shape mismatch (terminal vs rule) fails at the subject node. +fn match_impl<'subject>( + tree: Node<'subject>, + pattern: Node<'_>, + tags: &BTreeMap, + labels: &mut BTreeMap>>, +) -> Option> { + // Grown like the runtime's other recursive tree descents + // (`ParseTreeVisitor::visit_children`) so a deep subject/pattern pair + // cannot overflow the native stack. + stacker::maybe_grow(MATCH_STACK_RED_ZONE, MATCH_STACK_SIZE, || { + match (leaf_kind(tree), leaf_kind(pattern)) { + (Some(_), Some(_)) => match_terminals(tree, pattern, tags, labels), + (None, None) => match_rules(tree, pattern, tags, labels), + // One is a leaf and the other a rule: shape mismatch. + _ => Some(tree), + } + }) +} + +/// Returns the token type of a leaf (terminal or error node), or `None` for a +/// rule node. Error nodes carry a symbol just like terminals, so they compare +/// by token type too. +fn leaf_kind(node: Node<'_>) -> Option { + match node.kind() { + NodeKind::Terminal => node.as_terminal().map(|t| t.symbol().token_type()), + NodeKind::Error => node.as_error().map(|e| e.symbol().token_type()), + NodeKind::Rule => None, + } +} + +fn match_terminals<'subject>( + tree: Node<'subject>, + pattern: Node<'_>, + tags: &BTreeMap, + labels: &mut BTreeMap>>, +) -> Option> { + let tree_type = leaf_kind(tree); + let pattern_type = leaf_kind(pattern); + if tree_type != pattern_type { + return Some(tree); + } + // A token tag binds; otherwise the concrete texts must be equal. + match pattern_token_tag(pattern, tags) { + Some(tag) => { + bind(labels, tag, tree); + None + } + None if leaf_text(tree) == leaf_text(pattern) => None, + None => Some(tree), + } +} + +fn match_rules<'subject>( + tree: Node<'subject>, + pattern: Node<'_>, + tags: &BTreeMap, + labels: &mut BTreeMap>>, +) -> Option> { + // `match_impl` only routes rule-kinded nodes here, so a failed view is a + // structural inconsistency; fail the match rather than fail open. + let (Some(tree_rule), Some(pattern_rule)) = (tree.as_rule(), pattern.as_rule()) else { + return Some(tree); + }; + + // (expr ...) matched against a `` rule-tag subtree. + if let Some((tag_rule_index, tag)) = rule_tag_of(pattern, tags) { + return if tree_rule.rule_index() == tag_rule_index { + bind(labels, tag, tree); + None + } else { + Some(tree) + }; + } + + if tree_rule.child_count() != pattern_rule.child_count() { + return Some(tree); + } + for (tree_child, pattern_child) in tree.children().zip(pattern.children()) { + if let Some(mismatch) = match_impl(tree_child, pattern_child, tags, labels) { + return Some(mismatch); + } + } + None +} + +/// Returns the tag for a `` terminal leaf, if this pattern leaf is one. +fn pattern_token_tag<'a>( + pattern: Node<'_>, + tags: &'a BTreeMap, +) -> Option<&'a TagInfo> { + let token_id = pattern.as_terminal()?.token_id(); + let tag = tags.get(&token_id)?; + matches!(tag.kind, TagKind::Token { .. }).then_some(tag) +} + +/// Detects a `` tag subtree — a rule node with exactly one terminal child +/// whose symbol is a rule tag — returning the referenced rule index alongside +/// the tag. Mirrors ANTLR's `getRuleTagToken`. +fn rule_tag_of<'a>( + pattern: Node<'_>, + tags: &'a BTreeMap, +) -> Option<(usize, &'a TagInfo)> { + let rule = pattern.as_rule()?; + if rule.child_count() != 1 { + return None; + } + let child = pattern.children().next()?; + let token_id = child.as_terminal()?.token_id(); + let tag = tags.get(&token_id)?; + match tag.kind { + TagKind::Rule { rule_index, .. } => Some((rule_index, tag)), + TagKind::Token { .. } => None, + } +} + +/// Files `node` under every label key the tag contributes. +fn bind<'subject>( + labels: &mut BTreeMap>>, + tag: &TagInfo, + node: Node<'subject>, +) { + for key in tag.label_keys() { + labels.entry(key.to_owned()).or_default().push(node); + } +} + +/// Borrowed text of a leaf node (terminal or error), for literal comparison. +fn leaf_text(node: Node<'_>) -> &str { + node.as_terminal() + .map(crate::tree::TerminalNodeView::text) + .or_else(|| node.as_error().map(crate::tree::ErrorNodeView::text)) + .unwrap_or("") +} + +#[cfg(test)] +#[allow(clippy::disallowed_methods)] // `insta` assertion macros unwrap internal I/O. +mod tests { + use super::*; + use crate::token::{TokenSpec, TokenStore}; + use crate::tree::{NodeId, ParseTreeStorage, ParsedFile, ParserRuleContext}; + + // Fixture grammar: `stat : ID '=' expr ';' ; expr : INT | ID ;` + const RULE_STAT: usize = 0; + const RULE_EXPR: usize = 1; + const ASSIGN: i32 = 1; + const SEMI: i32 = 2; + const ID: i32 = 3; + const INT: i32 = 4; + // Imaginary bypass token types live above max_token_type (=4); rule 0's + // would be 5, rule 1's is 6. + const BYPASS_EXPR: i32 = 6; + + // ---- chunk splitting ------------------------------------------------- + + fn split_default(pattern: &str) -> Result, ParseTreePatternError> { + split(pattern, &Delimiters::default()) + } + + #[test] + fn split_interleaves_text_and_tags() { + let chunks = split_default(" = ;").expect("valid pattern"); + insta::assert_debug_snapshot!("split_interleaves_text_and_tags", chunks); + } + + #[test] + fn split_parses_labeled_tags() { + let chunks = split_default(" = ").expect("valid pattern"); + insta::assert_debug_snapshot!("split_parses_labeled_tags", chunks); + } + + #[test] + fn split_strips_escapes_from_text_only() { + // Escaped delimiters are literal text; the tag survives. + let chunks = split_default(r"a \< b c \> d").expect("valid pattern"); + insta::assert_debug_snapshot!("split_strips_escapes", chunks); + } + + #[test] + fn split_no_tags_is_single_text_chunk() { + let chunks = split_default("a = 3 ;").expect("valid pattern"); + insta::assert_debug_snapshot!("split_no_tags", chunks); + } + + #[test] + fn split_rejects_malformed_patterns() { + let cases = ["", "><", "<>", "", ">"]; + let errors: Vec<_> = cases + .into_iter() + .map(|pattern| { + ( + pattern, + split_default(pattern).expect_err("invalid").to_string(), + ) + }) + .collect(); + insta::assert_debug_snapshot!("split_rejects_malformed", errors); + } + + #[test] + fn split_honors_custom_delimiters() { + let delimiters = Delimiters { + start: "[[".to_owned(), + stop: "]]".to_owned(), + escape: "%".to_owned(), + }; + let chunks = split("x [[expr]] y", &delimiters).expect("valid pattern"); + insta::assert_debug_snapshot!("split_custom_delimiters", chunks); + } + + // ---- lockstep matching against hand-built pattern trees -------------- + + /// Builds one tree node recursively, recording any tag leaves. + enum Build { + Rule(usize, Vec), + /// A concrete terminal: token type + text. + Token(i32, &'static str), + /// A token tag `` occupying an imaginary/real slot. + TokenTag { + token_type: i32, + name: &'static str, + label: Option<&'static str>, + }, + /// A rule tag ``: a single-terminal rule subtree. + RuleTag { + rule_index: usize, + bypass_type: i32, + name: &'static str, + label: Option<&'static str>, + }, + } + + struct TreeFactory { + tokens: TokenStore, + storage: ParseTreeStorage, + tags: BTreeMap, + } + + impl TreeFactory { + fn new() -> Self { + Self { + tokens: TokenStore::new(None, "TreePattern"), + storage: ParseTreeStorage::new(), + tags: BTreeMap::new(), + } + } + + fn push_token(&mut self, token_type: i32, text: &str) -> TokenId { + self.tokens + .push(TokenSpec::explicit(token_type, text)) + .expect("test token fits") + } + + fn build(&mut self, spec: &Build) -> NodeId { + match spec { + Build::Token(token_type, text) => { + let id = self.push_token(*token_type, text); + self.storage.terminal(id) + } + Build::TokenTag { + token_type, + name, + label, + } => { + let id = self.push_token(*token_type, &format!("<{name}>")); + self.tags.insert( + id, + TagInfo { + kind: TagKind::Token { + token_type: *token_type, + }, + name: (*name).to_owned(), + label: label.map(str::to_owned), + }, + ); + self.storage.terminal(id) + } + Build::RuleTag { + rule_index, + bypass_type, + name, + label, + } => { + let id = self.push_token(*bypass_type, &format!("<{name}>")); + self.tags.insert( + id, + TagInfo { + kind: TagKind::Rule { + rule_index: *rule_index, + bypass_type: *bypass_type, + }, + name: (*name).to_owned(), + label: label.map(str::to_owned), + }, + ); + // Rule tag renders as a single-terminal rule subtree. + let leaf = self.storage.terminal(id); + let mut context = ParserRuleContext::new(*rule_index, -1); + self.storage.add_child(&mut context, leaf); + self.storage.finish_rule(context) + } + Build::Rule(rule_index, children) => { + let child_ids: Vec<_> = children.iter().map(|c| self.build(c)).collect(); + let mut context = ParserRuleContext::new(*rule_index, -1); + for child in child_ids { + self.storage.add_child(&mut context, child); + } + self.storage.finish_rule(context) + } + } + } + + fn into_file(self, root: NodeId) -> (ParsedFile, BTreeMap) { + (ParsedFile::new(self.tokens, self.storage, root), self.tags) + } + } + + /// Builds a subject tree (no tags expected). + fn subject_tree(spec: &Build) -> ParsedFile { + let mut factory = TreeFactory::new(); + let root = factory.build(spec); + factory.into_file(root).0 + } + + /// Builds a pattern from a spec, wrapping it as a `ParseTreePattern`. + fn pattern_from(rule_index: usize, spec: &Build) -> ParseTreePattern { + let mut factory = TreeFactory::new(); + let root = factory.build(spec); + let (file, tags) = factory.into_file(root); + ParseTreePattern { + pattern: "".to_owned(), + pattern_rule_index: rule_index, + tree: PatternTree { file, tags }, + } + } + + /// Subject `x = 3 ;` as `stat`. + fn subject_x_eq_3() -> ParsedFile { + subject_tree(&Build::Rule( + RULE_STAT, + vec![ + Build::Token(ID, "x"), + Build::Token(ASSIGN, "="), + Build::Rule(RULE_EXPR, vec![Build::Token(INT, "3")]), + Build::Token(SEMI, ";"), + ], + )) + } + + #[test] + fn matches_rule_tag_and_binds_label() { + // Pattern: ` = ;` + let pattern = pattern_from( + RULE_STAT, + &Build::Rule( + RULE_STAT, + vec![ + Build::TokenTag { + token_type: ID, + name: "ID", + label: None, + }, + Build::Token(ASSIGN, "="), + Build::RuleTag { + rule_index: RULE_EXPR, + bypass_type: BYPASS_EXPR, + name: "expr", + label: Some("e"), + }, + Build::Token(SEMI, ";"), + ], + ), + ); + let subject = subject_x_eq_3(); + let result = pattern.match_tree(subject.tree()); + + assert!(result.succeeded(), "pattern should match"); + // Unlabeled files under "ID"; labeled under both "e" and "expr". + assert_eq!(result.get("ID").map(Node::text), Some("x".to_owned())); + assert_eq!(result.get("e").map(Node::text), Some("3".to_owned())); + assert_eq!(result.get("expr").map(Node::text), Some("3".to_owned())); + assert!(result.get("absent").is_none()); + } + + #[test] + fn literal_mismatch_reports_first_bad_node() { + // Pattern requires the identifier to be exactly `y`, subject has `x`. + let pattern = pattern_from( + RULE_STAT, + &Build::Rule( + RULE_STAT, + vec![ + Build::Token(ID, "y"), + Build::Token(ASSIGN, "="), + Build::RuleTag { + rule_index: RULE_EXPR, + bypass_type: BYPASS_EXPR, + name: "expr", + label: None, + }, + Build::Token(SEMI, ";"), + ], + ), + ); + let subject = subject_x_eq_3(); + let result = pattern.match_tree(subject.tree()); + + assert!(!result.succeeded()); + assert_eq!( + result.mismatched_node().map(Node::text), + Some("x".to_owned()) + ); + } + + #[test] + fn child_count_mismatch_fails_at_rule() { + // Pattern `stat` with only 3 children vs subject's 4. + let pattern = pattern_from( + RULE_STAT, + &Build::Rule( + RULE_STAT, + vec![ + Build::TokenTag { + token_type: ID, + name: "ID", + label: None, + }, + Build::Token(ASSIGN, "="), + Build::RuleTag { + rule_index: RULE_EXPR, + bypass_type: BYPASS_EXPR, + name: "expr", + label: None, + }, + ], + ), + ); + let subject = subject_x_eq_3(); + let result = pattern.match_tree(subject.tree()); + + assert!(!result.succeeded()); + // The whole stat node mismatches on arity. + assert!(result.mismatched_node().and_then(Node::as_rule).is_some()); + } + + #[test] + fn rule_tag_type_mismatch_fails() { + // A `` rule tag positioned where the subject has a `stat`. + let pattern = pattern_from( + RULE_STAT, + &Build::RuleTag { + rule_index: RULE_EXPR, + bypass_type: BYPASS_EXPR, + name: "expr", + label: None, + }, + ); + let subject = subject_x_eq_3(); // root is stat, not expr + let result = pattern.match_tree(subject.tree()); + assert!(!result.succeeded()); + } + + #[test] + fn get_all_returns_every_binding_in_order() { + // Pattern `expr expr` (two INT tags) against subject with two exprs. + let pattern = pattern_from( + RULE_STAT, + &Build::Rule( + RULE_STAT, + vec![ + Build::RuleTag { + rule_index: RULE_EXPR, + bypass_type: BYPASS_EXPR, + name: "expr", + label: Some("operand"), + }, + Build::RuleTag { + rule_index: RULE_EXPR, + bypass_type: BYPASS_EXPR, + name: "expr", + label: Some("operand"), + }, + ], + ), + ); + let subject = subject_tree(&Build::Rule( + RULE_STAT, + vec![ + Build::Rule(RULE_EXPR, vec![Build::Token(INT, "1")]), + Build::Rule(RULE_EXPR, vec![Build::Token(INT, "2")]), + ], + )); + let result = pattern.match_tree(subject.tree()); + + assert!(result.succeeded()); + let operands: Vec<_> = result.get_all("operand").iter().map(|n| n.text()).collect(); + assert_eq!(operands, vec!["1".to_owned(), "2".to_owned()]); + // Unlabeled rule name "expr" also collects both. + assert_eq!(result.get_all("expr").len(), 2); + } + + // ---- end-to-end compile() against a real ATN ------------------------ + + use crate::atn::AtnStateKind; + use crate::atn::parser_atn::{ParserAtn, ParserAtnBuilder, ParserTransitionSpec}; + use crate::vocabulary::Vocabulary; + + /// Real parser ATN for `stat : ID '=' expr ';' ; expr : INT | ID ;`. + /// + /// Hand-built rather than generated so the test stays self-contained; the + /// shape (rule start/stop, a two-alt block in `expr`, a rule-call from + /// `stat`) exercises the bypass transform on genuine grammar structure. + fn stat_expr_atn() -> ParserAtn { + let mut atn = ParserAtnBuilder::new(4); + // States: stat = 0..=6, expr = 7..=12. + for (number, kind, rule) in [ + (0, AtnStateKind::RuleStart, 0), // stat start + (1, AtnStateKind::Basic, 0), // after ID + (2, AtnStateKind::Basic, 0), // after '=' + (3, AtnStateKind::Basic, 0), // after expr + (4, AtnStateKind::RuleStop, 0), // stat stop + (5, AtnStateKind::RuleStart, 1), // expr start + (6, AtnStateKind::BlockStart, 1), // expr decision + (7, AtnStateKind::Basic, 1), // INT alt + (8, AtnStateKind::Basic, 1), // ID alt + (9, AtnStateKind::BlockEnd, 1), // expr block end + (10, AtnStateKind::RuleStop, 1), // expr stop + ] { + assert_eq!( + atn.add_state(kind, Some(rule)).expect("state").index(), + number + ); + } + atn.set_rule_to_start_state(vec![0, 5]).expect("starts"); + atn.set_rule_to_stop_state(vec![4, 10]).expect("stops"); + atn.set_end_state(6, 9).expect("expr block end"); + atn.add_decision_state(6).expect("decision"); + + // stat : ID '=' expr ';' ; + atn.add_transition( + 0, + ParserTransitionSpec::Atom { + target: 1, + label: ID, + }, + ) + .expect("edge"); + atn.add_transition( + 1, + ParserTransitionSpec::Atom { + target: 2, + label: ASSIGN, + }, + ) + .expect("edge"); + atn.add_transition( + 2, + ParserTransitionSpec::Rule { + target: 5, + rule_index: 1, + follow_state: 3, + precedence: 0, + }, + ) + .expect("edge"); + atn.add_transition( + 3, + ParserTransitionSpec::Atom { + target: 4, + label: SEMI, + }, + ) + .expect("edge"); + // Synthetic rule-return edge (expr stop -> stat follow), as a packed ATN + // would already contain. + atn.add_transition(10, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("edge"); + + // expr : INT | ID ; + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("edge"); + atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) + .expect("edge"); + atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 8 }) + .expect("edge"); + atn.add_transition( + 7, + ParserTransitionSpec::Atom { + target: 9, + label: INT, + }, + ) + .expect("edge"); + atn.add_transition( + 8, + ParserTransitionSpec::Atom { + target: 9, + label: ID, + }, + ) + .expect("edge"); + atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 10 }) + .expect("edge"); + + atn.finish().expect("valid stat/expr ATN") + } + + fn stat_expr_data() -> RecognizerData { + RecognizerData::new( + "StatExpr.g4", + Vocabulary::new( + [None, Some("'='"), Some("';'"), None, None], + [None, Some("ASSIGN"), Some("SEMI"), Some("ID"), Some("INT")], + [None::<&str>, None], + ), + ) + .with_rule_names(["stat", "expr"]) + } + + /// A minimal whitespace-splitting chunk lexer for the fixture grammar. + /// + /// Stands in for a real generated lexer: it turns each whitespace-delimited + /// word of a literal chunk into a token spec, classifying identifiers, + /// integers, and the two punctuation tokens. + fn stat_expr_chunk_lexer(text: &str) -> Result, ParseTreePatternError> { + let mut specs = Vec::new(); + for word in text.split_whitespace() { + let token_type = match word { + "=" => ASSIGN, + ";" => SEMI, + _ if word.chars().all(|c| c.is_ascii_digit()) => INT, + _ if word.chars().all(|c| c.is_ascii_alphanumeric()) => ID, + other => { + return Err(ParseTreePatternError::Tokenization { + message: format!("unexpected chunk word {other:?}"), + }); + } + }; + specs.push(TokenSpec::explicit(token_type, word)); + } + Ok(specs) + } + + fn stat_expr_matcher_and_data() -> (ParserAtn, RecognizerData) { + (stat_expr_atn(), stat_expr_data()) + } + + #[test] + fn compile_and_match_full_pattern() { + let (atn, data) = stat_expr_matcher_and_data(); + let matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + let pattern = matcher + .compile(" = ;", RULE_STAT, stat_expr_chunk_lexer) + .expect("compiles"); + + // Subject `x = 3 ;` parsed by the same ATN (no tags). + let mut parser = BaseParser::new( + CommonTokenStream::new(stat_expr_subject("x = 3 ;")), + data.clone(), + ); + let root = parser + .parse_atn_rule(&atn, RULE_STAT) + .expect("subject parse"); + let subject = parser.into_parsed_file(root); + + let result = pattern.match_tree(subject.tree()); + assert!(result.succeeded(), "pattern should match `x = 3 ;`"); + assert_eq!(result.get("ID").map(Node::text), Some("x".to_owned())); + assert_eq!(result.get("e").map(Node::text), Some("3".to_owned())); + } + + #[test] + fn compile_rejects_patterns_that_only_parse_via_recovery() { + // Upstream installs a BailErrorStrategy for the pattern parse. Without + // the syntax-error gate these all "compile" by error recovery, baking + // `` error nodes into pattern trees that then match + // nothing: missing '=', missing expr, missing leading ID. + let (atn, data) = stat_expr_matcher_and_data(); + let matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + for pattern in [" ;", " = ;", "= ;", "x 3 ;"] { + let error = matcher + .compile(pattern, RULE_STAT, stat_expr_chunk_lexer) + .expect_err("recovered pattern parse must be rejected"); + assert!( + matches!(error, ParseTreePatternError::CannotInvokeStartRule { .. }), + "unexpected error for {pattern:?}: {error}" + ); + } + } + + #[test] + fn split_rejects_overlapping_tags_without_panicking() { + // `>` pairs starts [0, 2] with stops [4, 5]; the inter-tag text + // slice would be inverted (5..2). Must surface as an error, not a panic. + let error = split_default(">").expect_err("overlapping tags"); + assert!(matches!( + error, + ParseTreePatternError::DelimitersOutOfOrder { .. } + )); + } + + #[test] + fn compile_rejects_tokens_after_an_eof_tag() { + // An EOF-typed token terminates the buffered stream, so a suffix after + // `` would silently vanish before the full-consumption check. + let (atn, data) = stat_expr_matcher_and_data(); + let matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + let error = matcher + .compile( + " = ; garbage", + RULE_STAT, + stat_expr_chunk_lexer, + ) + .expect_err("tokens after an EOF tag must be rejected"); + assert!( + matches!(error, ParseTreePatternError::Tokenization { .. }), + "unexpected error: {error}" + ); + } + + #[test] + fn compile_rejects_unconsumed_trailing_eof_tag() { + // `stat` does not end in EOF, so a trailing `` tag can never be + // consumed by the rule — it only terminates the token stream, and the + // resulting tree is identical to the pattern without the tag. That + // must be an error, not a silently dropped tag. + let (atn, data) = stat_expr_matcher_and_data(); + let matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + let error = matcher + .compile(" = ; ", RULE_STAT, stat_expr_chunk_lexer) + .expect_err("unconsumed trailing EOF tag must be rejected"); + assert!( + matches!( + error, + ParseTreePatternError::StartRuleDoesNotConsumeFullPattern { .. } + ), + "unexpected error: {error}" + ); + } + + #[test] + fn compile_rejects_partial_pattern() { + // ANTLR issue #413: the start rule must consume the whole pattern. A + // pattern that stops short of `;` leaves an unconsumed token. + let (atn, data) = stat_expr_matcher_and_data(); + let matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + let error = matcher + .compile(" = ; extra", RULE_STAT, stat_expr_chunk_lexer) + .expect_err("trailing token should be rejected"); + assert!( + matches!( + error, + ParseTreePatternError::StartRuleDoesNotConsumeFullPattern { .. } + ), + "unexpected error: {error}" + ); + } + + #[test] + fn compile_rejects_unknown_tag_names() { + let (atn, data) = stat_expr_matcher_and_data(); + let matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + let unknown_token = matcher + .compile(" = ;", RULE_STAT, stat_expr_chunk_lexer) + .expect_err("unknown token tag"); + assert!(matches!( + unknown_token, + ParseTreePatternError::UnknownToken { .. } + )); + let unknown_rule = matcher + .compile(" = ;", RULE_STAT, stat_expr_chunk_lexer) + .expect_err("unknown rule tag"); + assert!(matches!( + unknown_rule, + ParseTreePatternError::UnknownRule { .. } + )); + } + + #[test] + fn set_delimiters_validates_and_switches_tag_syntax() { + let (atn, data) = stat_expr_matcher_and_data(); + let mut matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + + // Empty start/stop are rejected like upstream's IllegalArgumentException. + assert!(matches!( + matcher.set_delimiters("", ">", "\\"), + Err(ParseTreePatternError::EmptyDelimiter { which: "start" }) + )); + assert!(matches!( + matcher.set_delimiters("<", "", "\\"), + Err(ParseTreePatternError::EmptyDelimiter { which: "stop" }) + )); + + // Custom delimiters compile end-to-end; the old `<...>` is now literal + // text the chunk lexer rejects. + matcher + .set_delimiters("[[", "]]", "%") + .expect("valid delimiters"); + matcher + .compile("[[ID]] = [[e:expr]] ;", RULE_STAT, stat_expr_chunk_lexer) + .expect("custom-delimiter pattern compiles"); + matcher + .compile(" = ;", RULE_STAT, stat_expr_chunk_lexer) + .expect_err("old delimiters are literal text now"); + } + + #[test] + fn compiled_pattern_does_not_match_different_structure() { + let (atn, data) = stat_expr_matcher_and_data(); + let matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + // Pattern requires the literal identifier `y`. + let pattern = matcher + .compile("y = ;", RULE_STAT, stat_expr_chunk_lexer) + .expect("compiles"); + + let mut parser = BaseParser::new( + CommonTokenStream::new(stat_expr_subject("x = 3 ;")), + data.clone(), + ); + let root = parser + .parse_atn_rule(&atn, RULE_STAT) + .expect("subject parse"); + let subject = parser.into_parsed_file(root); + + let result = pattern.match_tree(subject.tree()); + assert!( + !result.succeeded(), + "identifier `x` should not match literal `y`" + ); + } + + /// Subject-side token source: lexes a whole input string like the chunk + /// lexer, then appends EOF. + fn stat_expr_subject(input: &str) -> PatternTokenSource { + let specs = stat_expr_chunk_lexer(input).expect("valid subject input"); + PatternTokenSource { specs, index: 0 } + } + + #[derive(Debug)] + struct StatExprRecognizer { + data: RecognizerData, + } + + impl Recognizer for StatExprRecognizer { + fn data(&self) -> &RecognizerData { + &self.data + } + + fn data_mut(&mut self) -> &mut RecognizerData { + &mut self.data + } + } + + #[test] + fn find_all_pairs_xpath_selection_with_pattern_matching() { + let (atn, data) = stat_expr_matcher_and_data(); + let matcher = ParseTreePatternMatcher::new(&atn, &data).expect("matcher"); + // Matches only integer expressions. + let pattern = matcher + .compile("", RULE_EXPR, stat_expr_chunk_lexer) + .expect("compiles"); + + let mut parser = BaseParser::new( + CommonTokenStream::new(stat_expr_subject("x = 3 ;")), + data.clone(), + ); + let root = parser + .parse_atn_rule(&atn, RULE_STAT) + .expect("subject parse"); + let subject = parser.into_parsed_file(root); + let recognizer = StatExprRecognizer { data }; + + // `//expr` selects the one expr subtree; the `` pattern matches it. + let matches = pattern + .find_all(subject.tree(), "//expr", &recognizer) + .expect("valid xpath"); + assert_eq!(matches.len(), 1); + assert_eq!(matches[0].tree().text(), "3"); + // A path selecting nothing that matches yields no results. + let none = pattern + .find_all(subject.tree(), "//stat", &recognizer) + .expect("valid xpath"); + assert!(none.is_empty()); + } +} diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index f14d287f..e5b2235b 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -458,6 +458,141 @@ mod grouped_token_tests { ); } +#[test] +fn compile_parse_tree_pattern_matches_and_binds_against_generated_parser() { + let temp = temporary_directory("tree-pattern-compile"); + let grammar = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/antlr4-rust-gen/typed-tree-walkers/Calculator.g4"); + let out = temp.path().join("generated"); + + let output = run_antlr4_rust_gen(&[ + grammar.as_os_str(), + OsStr::new("--out-dir"), + out.as_os_str(), + ]); + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + utf8(&output.stdout), + utf8(&output.stderr) + ); + + let parser = + fs::read_to_string(out.join("calculator_parser.rs")).expect("parser should be emitted"); + assert!( + parser.contains("pub fn compile_parse_tree_pattern("), + "generated parser must expose compile_parse_tree_pattern\n{parser}" + ); + + // End-to-end: compile a pattern rooted at the (left-recursive) `expression` + // rule and match it against a real parse. Exercises the rule-bypass ATN's + // left-recursive path through a genuinely generated grammar. + assert_generated_project( + temp.path(), + &["calculator_lexer.rs", "calculator_parser.rs"], + r#" +#[cfg(test)] +mod tree_pattern_tests { + use super::calculator_lexer::CalculatorLexer; + use super::calculator_parser::*; + use antlr4_runtime::{CommonTokenStream, InputStream, Node, Parser as _}; + + /// Parses `input` and returns the owned tree plus the top-level expression + /// node id, for matching against a pattern. + fn parse_top_expression(input: &'static str) -> antlr4_runtime::ParsedFile { + let lexer = CalculatorLexer::new(InputStream::new(input)); + let mut parser = CalculatorParser::new(CommonTokenStream::new(lexer)); + let root = parser.start().expect("input parses"); + assert_eq!(parser.number_of_syntax_errors(), 0); + parser.into_parsed_file(root) + } + + fn top_expression(parsed: &antlr4_runtime::ParsedFile) -> Node<'_> { + parsed + .tree() + .as_rule() + .expect("start rule") + .child_rule(RULE_EXPRESSION) + .expect("top-level expression") + .node() + } + + #[test] + fn compiles_and_matches_expression_pattern() { + let lexer = CalculatorLexer::new(InputStream::new("")); + let parser = CalculatorParser::new(CommonTokenStream::new(lexer)); + // ` + ` rooted at the expression rule. + let pattern = parser + .compile_parse_tree_pattern( + " + ", + RULE_EXPRESSION, + CalculatorLexer::new, + ) + .expect("pattern compiles"); + + let parsed = parse_top_expression("2 + 8"); + let result = pattern.match_tree(top_expression(&parsed)); + assert!(result.succeeded(), "2 + 8 should match ` + `"); + // Both operands bind under the rule name `expression`. + let operands: Vec<_> = result + .get_all("expression") + .iter() + .map(|node| node.text()) + .collect(); + assert_eq!(operands, vec!["2".to_owned(), "8".to_owned()]); + } + + #[test] + fn rejects_non_matching_structure() { + let lexer = CalculatorLexer::new(InputStream::new("")); + let parser = CalculatorParser::new(CommonTokenStream::new(lexer)); + let pattern = parser + .compile_parse_tree_pattern( + " * ", + RULE_EXPRESSION, + CalculatorLexer::new, + ) + .expect("pattern compiles"); + + let parsed = parse_top_expression("2 + 8"); + // Addition must not match a multiplication pattern. + assert!(!pattern.match_tree(top_expression(&parsed)).succeeded()); + } + + #[test] + fn trailing_eof_tag_requires_a_rule_that_consumes_it() { + let lexer = CalculatorLexer::new(InputStream::new("")); + let parser = CalculatorParser::new(CommonTokenStream::new(lexer)); + // `start : expression EOF ;` consumes the tag: the pattern matches a + // whole parse. + let pattern = parser + .compile_parse_tree_pattern( + " ", + RULE_START, + CalculatorLexer::new, + ) + .expect("EOF-consuming rule accepts a trailing tag"); + let parsed = parse_top_expression("2 + 8"); + assert!(pattern.match_tree(parsed.tree()).succeeded()); + + // `expression` never consumes EOF, so the tag would be silently + // dropped from the pattern tree; that must be rejected. + assert!( + parser + .compile_parse_tree_pattern( + " + ", + RULE_EXPRESSION, + CalculatorLexer::new, + ) + .is_err(), + "unconsumed trailing tag must not compile" + ); + } +} +"#, + ); +} + #[test] fn visitor_and_typed_walk_dispatch_labeled_left_recursion() { let temp = temporary_directory("typed-tree-walkers");