diff --git a/README.md b/README.md index 1119d709..a4b78890 100644 --- a/README.md +++ b/README.md @@ -94,6 +94,11 @@ Using the same package release for `antlr4-rust-gen` and `antlr-rust-runtime` remains the recommended workflow, but matching the generated-code API is the compile-time requirement. +The bundled generator currently emits revision 3, which exposes parameterized +rule arguments to committed parser-action hooks. The runtime also accepts +revisions 1 and 2 because their older generated recognizers use API surfaces +that remain available. + Generated modules created before this check was introduced cannot be detected retroactively. When first upgrading to a release that includes the check, regenerate every committed lexer and parser once. Thereafter, normal @@ -494,12 +499,13 @@ Generated parsers also expose a parser-side hook escape hatch: `MyParser::with_hooks(tokens, hooks)`, where `hooks` implements `SemanticHooks`. Unknown parser predicates are offered to `SemanticHooks::sempred` before the fallback policy is applied, and unhandled -parser action events are offered to `SemanticHooks::action` after the committed -parse path is selected. Predicate hooks may run speculatively during -prediction, so they must be replay-safe. +parser action events are offered to `SemanticHooks::action` at their grammar +position after the containing path is committed. Predicate hooks may run +speculatively during prediction, so they must be replay-safe; action hooks never +run on speculative or losing paths. -For helper-call predicates written as `helper()`, `this.helper()`, or -`self.helper()`, generated parsers also emit a typed hook adapter +For helper-call predicates and actions written as `helper()`, `this.helper()`, +or `self.helper()`, generated parsers also emit a typed hook adapter (`MyParserHooks` plus `MyParserTypedHooks`) that maps stable manifest coordinates to named Rust methods. A `[[helper]]` pattern can opt into one additional receiver spelling with `receiver = "..."`. For example, an @@ -515,6 +521,10 @@ returns = "bool" lower = "hook" ``` +Use `kind = "parser-action"` and `returns = "unit"` for an action helper. Its +typed method runs exactly once on the committed path, before any later +predicate or nested-rule event. + Lexer callers can use `LexerSemCtx` with `atn::lexer::next_token_with_semantic_hooks` or the compiled-DFA variant to route lexer predicates/actions through the same diff --git a/docs/migration.md b/docs/migration.md index 4a21165c..2be8594e 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -7,6 +7,10 @@ generated-code API revision that is checked against the selected runtime at compile time, so releases that deliberately preserve the source contract can remain compatible without exact SemVer equality. +The current generator emits revision 3 so generated parser-action hooks can +observe parameterized rule arguments. This runtime continues to accept revision +1 and 2 generated modules because their required APIs remain supported. + Generated modules created before the compatibility check was introduced carry no enforceable revision. Regenerate every committed lexer and parser once when first adopting a release with this mechanism. If a later build reports a diff --git a/src/atn/parser.rs b/src/atn/parser.rs index 5fa0c411..d474299d 100644 --- a/src/atn/parser.rs +++ b/src/atn/parser.rs @@ -10,8 +10,9 @@ use crate::dfa::{ use crate::int_stream::IntStream; use crate::prediction::{ AtnConfig, AtnConfigSet, ContextArena, ContextId, EMPTY_CONTEXT, EMPTY_RETURN_STATE, - PredictionContextStats, PredictionFxHasher, PredictionWorkspace, SemanticContext, - all_subsets_conflict, all_subsets_equal, conflicting_alt_subsets, + PredictionContextStats, PredictionFxHasher, PredictionPredicateCall, + PredictionSemanticProvenanceArena, PredictionSemanticProvenanceId, PredictionWorkspace, + SemanticContext, all_subsets_conflict, all_subsets_equal, conflicting_alt_subsets, has_sll_conflict_terminating_prediction, single_viable_alt, }; use crate::token::TOKEN_EOF; @@ -62,6 +63,18 @@ pub struct ParserAtnSimulator<'a> { /// outcomes depend on caller-side evaluation, so any semantic transition /// disables memoization entirely. Computed lazily on first retry. full_context_memo_gate: Option, + /// Semantic configurations that survived the most recent prediction. + /// + /// The simulator defers predicate evaluation to the parser because hooks + /// need live parser state. Keeping the surviving alternative/context pairs + /// lets the committed parser evaluate only simulator-viable paths. + prediction_semantic_candidates: Vec, + /// Whether ATN configs retain rule-call paths for parameterized predicates. + /// + /// This is enabled only by the committed parser when generated rule + /// argument metadata exists, keeping ordinary prediction configs compact. + track_prediction_rule_calls: bool, + semantic_provenance: Option>, } #[derive(Clone, Copy, Debug)] @@ -216,6 +229,20 @@ pub enum ParserAtnPredictionDiagnosticKind { ContextSensitivity, } +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +pub(crate) struct ParserSemanticCandidate { + pub(crate) alt: usize, + pub(crate) context: SemanticContext, + pub(crate) predicate_calls: Vec, +} + +#[derive(Clone, Debug, Eq, Ord, PartialEq, PartialOrd)] +struct CompactParserSemanticCandidate { + alt: usize, + context: SemanticContext, + semantic_provenance: PredictionSemanticProvenanceId, +} + #[derive(Clone, Copy)] struct PredictionCheck { decision: usize, @@ -267,6 +294,7 @@ struct FullContextPrediction { prediction: ParserAtnPrediction, stop_index: usize, resolution: FullContextResolution, + semantic_candidates: Vec, } /// How the full-context loop settled, mirroring the two exits of Java's @@ -294,16 +322,34 @@ fn full_context_prediction( }, stop_index, resolution, + semantic_candidates: semantic_prediction_candidates(configs), + } +} + +fn semantic_prediction_candidates(configs: &AtnConfigSet) -> Vec { + if !configs.has_semantic_context() { + return Vec::new(); } + let mut candidates = configs + .configs() + .iter() + .map(|config| CompactParserSemanticCandidate { + alt: config.alt, + context: config.semantic_context.clone(), + semantic_provenance: config.semantic_provenance_id(), + }) + .collect::>(); + candidates.sort(); + candidates.dedup(); + candidates } #[derive(Clone, Debug, Eq, Hash, PartialEq)] struct ClosureConfigKey { state: usize, alt: usize, - context: ContextId, semantic_context: SemanticContext, - precedence_filter_suppressed: bool, + context_and_provenance: u64, } impl From<&AtnConfig> for ClosureConfigKey { @@ -311,9 +357,9 @@ impl From<&AtnConfig> for ClosureConfigKey { Self { state: config.state, alt: config.alt, - context: config.context, semantic_context: config.semantic_context.clone(), - precedence_filter_suppressed: config.precedence_filter_suppressed, + context_and_provenance: u64::from(config.context.compact()) + | (u64::from(config.semantic_provenance_and_flags()) << 32), } } } @@ -556,6 +602,9 @@ impl<'a> ParserAtnSimulator<'a> { full_context_memo: HashMap::default(), full_context_memo_len: 0, full_context_memo_gate: None, + prediction_semantic_candidates: Vec::new(), + track_prediction_rule_calls: false, + semantic_provenance: None, } } @@ -566,6 +615,7 @@ impl<'a> ParserAtnSimulator<'a> { self.adaptive_closure_work = 0; self.outer_context_cache = None; self.deferred_accept_states.clear(); + self.prediction_semantic_candidates.clear(); self.workspace.reset(); } @@ -575,6 +625,9 @@ impl<'a> ParserAtnSimulator<'a> { /// an overlapping stale simulator cannot republish pre-clear states later. pub fn clear_dfa(&mut self) { self.store = PredictionStore::new(self.atn); + if let Some(semantic_provenance) = self.semantic_provenance.as_mut() { + **semantic_provenance = PredictionSemanticProvenanceArena::default(); + } // The memo keys entries by ContextId into the store's arena the // line above just replaced; stale IDs would alias fresh contexts. self.full_context_memo.clear(); @@ -695,6 +748,9 @@ impl<'a> ParserAtnSimulator<'a> { full_context_memo: HashMap::default(), full_context_memo_len: 0, full_context_memo_gate: None, + prediction_semantic_candidates: Vec::new(), + track_prediction_rule_calls: false, + semantic_provenance: None, } } @@ -702,6 +758,44 @@ impl<'a> ParserAtnSimulator<'a> { &self.store.decision_to_dfa } + pub(crate) fn prediction_semantic_candidates(&self) -> Vec { + self.prediction_semantic_candidates + .iter() + .map(|candidate| ParserSemanticCandidate { + alt: candidate.alt, + context: candidate.context.clone(), + predicate_calls: self.semantic_provenance.as_deref().map_or_else( + Vec::new, + |arena| { + arena + .predicate_calls(candidate.semantic_provenance) + .to_vec() + }, + ), + }) + .collect() + } + + pub(crate) fn set_track_prediction_rule_calls(&mut self, track: bool) { + assert!( + self.shared_cache_key.is_none(), + "shared prediction simulators use a fixed untracked rule-call mode" + ); + if self.track_prediction_rule_calls != track { + assert!( + !self.has_trained_decision, + "prediction rule-call tracking mode cannot change after DFA construction" + ); + } + self.track_prediction_rule_calls = track; + if track { + self.semantic_provenance + .get_or_insert_with(|| Box::new(PredictionSemanticProvenanceArena::default())); + } else { + self.semantic_provenance = None; + } + } + /// Returns adaptive-call and closure-work counters for stable decisions. /// /// A call contributes only when its decision DFA was already non-empty and @@ -937,6 +1031,7 @@ impl<'a> ParserAtnSimulator<'a> { input: &mut T, merge_cache: &mut PredictionWorkspace, ) -> Result { + self.prediction_semantic_candidates.clear(); let decision = request.decision; let learning_revision = self .store @@ -1122,6 +1217,7 @@ impl<'a> ParserAtnSimulator<'a> { .map(|dfa| dfa.configs(state_number).clone()) && let Some(alt) = self.alt_that_finished_decision_entry_rule(&configs) { + self.prediction_semantic_candidates = semantic_prediction_candidates(&configs); return Ok(ParserAtnPrediction { alt, requires_full_context: false, @@ -1155,12 +1251,20 @@ impl<'a> ParserAtnSimulator<'a> { && let Some(prediction) = self.non_greedy_exit_prediction(decision, decision_state, state_number) { + self.record_prediction_semantic_candidates(decision, state_number); return Ok(Some(prediction)); } let Some(info) = self.dfa_prediction_info(decision, state_number) else { return Ok(None); }; let prediction = info.prediction; + let semantic_candidates = self + .store + .decision_to_dfa + .get(decision) + .map(|dfa| semantic_prediction_candidates(dfa.configs(state_number))) + .unwrap_or_default(); + self.prediction_semantic_candidates = semantic_candidates; // SLL-probe stage: the caller only needs to know that this conflict // requires full context; it will re-run with the real outer context. // Returning the SLL prediction here (with requires_full_context set) @@ -1191,7 +1295,7 @@ impl<'a> ParserAtnSimulator<'a> { { #[cfg(feature = "perf-counters")] crate::perf::record_full_context_memo_hit(decision); - return Ok(Some(Self::full_context_retry_prediction( + return Ok(Some(self.full_context_retry_prediction( full_context, info.conflicting_alts, start_index, @@ -1208,7 +1312,7 @@ impl<'a> ParserAtnSimulator<'a> { if memo_allowed { self.record_full_context_memo(memo_key, start_index, input, &full_context); } - return Ok(Some(Self::full_context_retry_prediction( + return Ok(Some(self.full_context_retry_prediction( full_context, info.conflicting_alts, start_index, @@ -1222,12 +1326,20 @@ impl<'a> ParserAtnSimulator<'a> { /// shared by the fresh LL run and the memoized replay so both produce /// byte-identical diagnostics. fn full_context_retry_prediction( + &mut self, full_context: FullContextPrediction, sll_conflicting_alts: Vec, start_index: usize, sll_stop_index: usize, ) -> ParserAtnPrediction { - let (kind, exact, conflicting_alts) = match full_context.resolution { + let FullContextPrediction { + mut prediction, + stop_index, + resolution, + semantic_candidates, + } = full_context; + self.prediction_semantic_candidates = semantic_candidates; + let (kind, exact, conflicting_alts) = match resolution { FullContextResolution::Ambiguous { exact, ref alts } => ( ParserAtnPredictionDiagnosticKind::Ambiguity, exact, @@ -1242,13 +1354,16 @@ impl<'a> ParserAtnSimulator<'a> { sll_conflicting_alts, ), }; - let mut prediction = full_context.prediction; + prediction.has_semantic_context = self + .prediction_semantic_candidates + .iter() + .any(|candidate| candidate.alt == prediction.alt && !candidate.context.is_none()); if conflicting_alts.len() > 1 { prediction.diagnostic = Some(ParserAtnPredictionDiagnostic { kind, start_index, sll_stop_index, - ll_stop_index: full_context.stop_index, + ll_stop_index: stop_index, conflicting_alts, exact, }); @@ -1256,6 +1371,15 @@ impl<'a> ParserAtnSimulator<'a> { prediction } + fn record_prediction_semantic_candidates(&mut self, decision: usize, state_number: DfaStateId) { + self.prediction_semantic_candidates = self + .store + .decision_to_dfa + .get(decision) + .map(|dfa| semantic_prediction_candidates(dfa.configs(state_number))) + .unwrap_or_default(); + } + /// Whether full-context memoization is sound for this ATN. /// /// Predicates make prediction outcomes depend on caller-side evaluation @@ -1994,7 +2118,14 @@ impl<'a> ParserAtnSimulator<'a> { .contexts .parent(config.context, index) .unwrap_or(EMPTY_CONTEXT); - let next = config.moved_to(return_state, parent, &self.store.contexts); + let mut next = config.moved_to(return_state, parent, &self.store.contexts); + if self.track_prediction_rule_calls { + next.exit_prediction_rule( + self.semantic_provenance + .as_deref_mut() + .expect("tracked prediction has a provenance arena"), + ); + } stack.push((next, collect_predicates)); } handled_all_paths @@ -2044,6 +2175,29 @@ impl<'a> ParserAtnSimulator<'a> { }; let mut target = config.moved_to(transition.target(), context, &self.store.contexts); target.semantic_context = semantic_context; + if self.track_prediction_rule_calls { + match transition_kind { + ParserTransitionKind::Rule => { + target.enter_prediction_rule( + self.semantic_provenance + .as_deref_mut() + .expect("tracked prediction has a provenance arena"), + config.state, + transition.arg0() as usize, + ); + } + ParserTransitionKind::Predicate if collect_predicates => { + target.record_prediction_predicate( + self.semantic_provenance + .as_deref_mut() + .expect("tracked prediction has a provenance arena"), + transition.arg0() as usize, + transition.arg1() as usize, + ); + } + _ => {} + } + } Some(target) } @@ -2212,11 +2366,19 @@ fn dfa_state_display(state: ParserDfaStateView<'_>, deferred: bool) -> String { mod tests { use super::*; use crate::atn::AtnStateKind; + use std::mem::size_of; fn finish_atn(builder: ParserAtnBuilder) -> Atn { builder.finish().expect("valid packed parser ATN") } + #[cfg(target_pointer_width = "64")] + #[test] + fn parser_prediction_hot_path_layouts_stay_compact() { + assert!(size_of::() <= 56); + assert!(size_of::() <= 48); + } + #[test] fn union_decision_dfa_preserves_disjoint_coverage() { fn configs( @@ -2494,6 +2656,27 @@ mod tests { assert_eq!(simulator.decision_dfas()[0].states().len(), learned_states); } + #[test] + #[should_panic(expected = "shared prediction simulators use a fixed untracked rule-call mode")] + fn shared_simulator_rejects_rule_call_tracking_mode_changes() { + let atn = Box::leak(Box::new(two_token_decision_atn())); + let mut simulator = ParserAtnSimulator::new_shared(atn); + + simulator.set_track_prediction_rule_calls(true); + } + + #[test] + #[should_panic( + expected = "prediction rule-call tracking mode cannot change after DFA construction" + )] + fn simulator_rejects_rule_call_tracking_mode_changes_after_learning() { + let atn = two_token_decision_atn(); + let mut simulator = ParserAtnSimulator::new(&atn); + assert_eq!(simulator.adaptive_predict(0, [1, 2]), Ok(1)); + + simulator.set_track_prediction_rule_calls(true); + } + #[test] fn shared_simulator_preserves_and_clears_prediction_training_state() { let atn = Box::leak(Box::new(two_token_decision_atn())); diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index ff0ea944..31d45e85 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -1512,11 +1512,16 @@ fn collect_parser_semantics_for_mode( } for action in structural_actions(data)? { let (line, column) = structural_line_column(data, &action.span); + let hook_call = if embedded || !action.authored || action.body.trim().is_empty() { + None + } else { + patterns.hook_helper_call(SemanticsKind::ParserAction, &action.body)? + }; entries.push(SemanticsEntry { kind: SemanticsKind::ParserAction, rule_index: Some(action.rule_index), rule_name: data.rule_names.get(action.rule_index).cloned(), - index: None, + index: Some(action.action_index), atn_state: Some(action.state), line: action.authored.then_some(line), column: action.authored.then_some(column), @@ -1530,7 +1535,7 @@ fn collect_parser_semantics_for_mode( .coordinate_disposition( SemanticsKind::ParserAction, data.rule_names.get(action.rule_index).map(String::as_str), - None, + Some(action.action_index), Some(action.state), ) .unwrap_or_else(|| { @@ -1538,6 +1543,8 @@ fn collect_parser_semantics_for_mode( SemanticsDisposition::Translated } else if !action.authored || action.body.trim().is_empty() { SemanticsDisposition::Synthetic + } else if hook_call.is_some() { + SemanticsDisposition::Hooked } else { policy.unknown_action_disposition() } @@ -1552,6 +1559,11 @@ fn collect_parser_semantics_for_mode( .inline_actions .contains_key(&action.state) .then(|| "PortableBooleanLocal".to_owned()) + .or_else(|| { + hook_call + .as_ref() + .map(|call| format!("Hook({})", rust_function_name(&call.name))) + }) }, }); } @@ -2598,6 +2610,7 @@ struct StructuralRuleCall { target_rule_index: usize, state: usize, arguments: Option, + caller_first_argument: Option, } fn structural_elements<'model>( @@ -2763,6 +2776,12 @@ fn structural_rule_calls(data: &CodegenData<'_>) -> io::Result, }, CallRule { source_state: usize, @@ -4346,7 +4366,6 @@ impl<'a> EmbeddedStepRender<'a> { #[derive(Clone, Copy)] struct PortableLocalStepRender<'a> { declarations: &'a [Vec], - inline_actions: &'a BTreeMap, predicates: &'a BTreeMap<(usize, usize), (String, Option)>, required_generated_rules: &'a BTreeSet, } @@ -4438,6 +4457,7 @@ struct GeneratedParserCompileContext<'a> { inline_action_states: &'a BTreeSet, action_states: &'a BTreeSet, generated_action_states: &'a BTreeSet, + action_indices: &'a BTreeMap, predicate_coordinates: &'a BTreeSet<(usize, usize)>, generated_predicate_coordinates: &'a BTreeSet<(usize, usize)>, } @@ -4445,11 +4465,18 @@ struct GeneratedParserCompileContext<'a> { #[derive(Clone, Debug, Eq, PartialEq)] struct TypedHookMapping { rule_index: usize, - pred_index: usize, + coordinate_index: usize, + kind: ParserTypedHookKind, method_name: String, call: SemanticHelperCall, } +#[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] +enum ParserTypedHookKind { + Predicate, + Action, +} + #[derive(Clone, Copy, Debug, Eq, Ord, PartialEq, PartialOrd)] enum LexerTypedHookKind { Predicate, @@ -4516,6 +4543,7 @@ struct ActionStateSets<'a> { all: &'a BTreeSet, generated: &'a BTreeSet, inline: &'a BTreeSet, + indices: &'a BTreeMap, } #[derive(Clone, Copy)] @@ -4531,6 +4559,7 @@ const fn generated_action_state_sets<'a>( all: context.action_states, generated: context.generated_action_states, inline: context.inline_action_states, + indices: context.action_indices, } } @@ -4564,6 +4593,7 @@ fn parser_generated_rules( inline_action_states: action_states.inline, action_states: action_states.all, generated_action_states: action_states.generated, + action_indices: action_states.indices, predicate_coordinates: predicate_coordinates.all, generated_predicate_coordinates: predicate_coordinates.generated, }; @@ -4629,7 +4659,7 @@ fn generated_atn_preferred_rule_calls( #[cfg(test)] fn generated_adaptive_atn_preferred_rule_calls(rules: &[Option]) -> Vec { - generated_adaptive_atn_preferred_rule_calls_excluding(rules, &BTreeSet::new()) + generated_adaptive_atn_preferred_rule_calls_excluding(rules, &BTreeSet::new(), &BTreeSet::new()) } struct GeneratedAdaptiveAtnRouting { @@ -4689,15 +4719,25 @@ fn generated_atn_preferred_rule_calls_excluding( fn generated_adaptive_atn_preferred_rule_calls_excluding( rules: &[Option], force_generated: &BTreeSet, + effectful_action_states: &BTreeSet, ) -> Vec { - generated_adaptive_atn_routing_excluding(rules, force_generated).candidates + generated_adaptive_atn_routing_excluding(rules, force_generated, effectful_action_states) + .candidates } fn generated_adaptive_atn_routing_excluding( rules: &[Option], force_generated: &BTreeSet, + effectful_action_states: &BTreeSet, ) -> GeneratedAdaptiveAtnRouting { let shapes = generated_rule_shapes(rules); + let action_rules = rules + .iter() + .flatten() + .filter(|rule| generated_steps_have_actions(&rule.steps, effectful_action_states)) + .map(|rule| rule.rule_index) + .collect::>(); + let action_rule_callers = generated_rule_callers_reaching(rules, &action_rules); let seeds = rules .iter() .flatten() @@ -4741,6 +4781,10 @@ fn generated_adaptive_atn_routing_excluding( } } exclude_forced_generated_rules(&mut candidates, force_generated); + // Adaptive retry rewinds and reparses the candidate through the committed + // interpreter. Exclude every generated caller that could already have run + // an action before that retry boundary. + exclude_forced_generated_rules(&mut candidates, &action_rule_callers); let probe_candidate_rules = probe_candidate_rules .into_iter() .map(|candidates_for_probe| { @@ -5021,6 +5065,31 @@ fn generated_steps_shape(steps: &[GeneratedParserStep]) -> GeneratedRuleShape { shape } +fn generated_steps_have_actions( + steps: &[GeneratedParserStep], + effectful_action_states: &BTreeSet, +) -> bool { + steps.iter().any(|step| match step { + GeneratedParserStep::Action { source_state, .. } => { + effectful_action_states.contains(source_state) + } + GeneratedParserStep::Decision { alts, .. } => alts + .iter() + .any(|alt| generated_steps_have_actions(alt, effectful_action_states)), + GeneratedParserStep::StarLoop { body, .. } + | GeneratedParserStep::LeftRecursiveLoop { body, .. } => { + generated_steps_have_actions(body, effectful_action_states) + } + GeneratedParserStep::MatchToken { .. } + | GeneratedParserStep::MatchSet { .. } + | GeneratedParserStep::MatchNotSet { .. } + | GeneratedParserStep::MatchWildcard { .. } + | GeneratedParserStep::Precedence(_) + | GeneratedParserStep::Predicate { .. } + | GeneratedParserStep::CallRule { .. } => false, + }) +} + fn generated_step_shape(step: &GeneratedParserStep) -> GeneratedRuleShape { match step { GeneratedParserStep::Decision { @@ -6209,11 +6278,19 @@ fn compile_generated_parser_transition( follow_state, )), ParserTransitionData::Action { - target, rule_index, .. + target, + rule_index, + action_index, + .. } if action_states.generated.contains(&source_state) => Some(( Some(GeneratedParserStep::Action { source_state, rule_index, + action_index: action_states + .indices + .get(&source_state) + .copied() + .or(action_index), }), target, )), @@ -6295,13 +6372,18 @@ fn generated_adaptive_atn_preferred_rule_count( rules: &[Option], embedded: bool, portable_required_generated_rules: Option<&BTreeSet>, + effectful_action_states: &BTreeSet, ) -> usize { let force_generated_rules = generated_force_generated_rules(rules, embedded, portable_required_generated_rules); - generated_adaptive_atn_preferred_rule_calls_excluding(rules, &force_generated_rules) - .into_iter() - .filter(|preferred| *preferred) - .count() + generated_adaptive_atn_preferred_rule_calls_excluding( + rules, + &force_generated_rules, + effectful_action_states, + ) + .into_iter() + .filter(|preferred| *preferred) + .count() } struct AdaptiveAtnParserRenderSlots { @@ -6366,10 +6448,15 @@ fn render_generated_rule_routing( decision_routing: DecisionRoutingRender<'_>, ) -> (String, usize) { let direct_generated_rule_calls = rules.iter().map(Option::is_some).collect::>(); + let effectful_action_states = inline_action_statements + .iter() + .filter_map(|(state, statement)| (!statement.trim().is_empty()).then_some(*state)) + .collect::>(); let preferred_rule_count = generated_adaptive_atn_preferred_rule_count( rules, embedded.is_some(), portable_locals.map(|portable| portable.required_generated_rules), + &effectful_action_states, ); let dispatch = render_generated_rule_dispatch_with_rule_names( rules, @@ -6405,8 +6492,15 @@ fn render_generated_rule_dispatch_with_rule_names( ); let atn_preferred_rule_calls = generated_atn_preferred_rule_calls_excluding(rules, rule_names, &force_generated_rules); - let adaptive_atn_routing = - generated_adaptive_atn_routing_excluding(rules, &force_generated_rules); + let effectful_action_states = inline_action_statements + .iter() + .filter_map(|(state, statement)| (!statement.trim().is_empty()).then_some(*state)) + .collect::>(); + let adaptive_atn_routing = generated_adaptive_atn_routing_excluding( + rules, + &force_generated_rules, + &effectful_action_states, + ); let adaptive_atn_preferred_rule_slots = indexed_rule_slots(&adaptive_atn_routing.candidates); let adaptive_atn_probe_rule_slots = indexed_probe_slots( &adaptive_atn_routing.probe_candidate_rules, @@ -7332,12 +7426,21 @@ fn render_generated_step( GeneratedParserStep::Action { source_state, rule_index, + action_index, } => { - writeln!( - out, - "{pad}let action = self.base.parser_action_at_current({source_state}, {rule_index}, __rule_start, __consumed_eof);" - ) - .expect("writing to a string cannot fail"); + if let Some(action_index) = action_index { + writeln!( + out, + "{pad}let action = self.base.parser_action_at_current_indexed({source_state}, {rule_index}, {action_index}, __rule_start, __consumed_eof);" + ) + .expect("writing to a string cannot fail"); + } else { + writeln!( + out, + "{pad}let action = self.base.parser_action_at_current({source_state}, {rule_index}, __rule_start, __consumed_eof);" + ) + .expect("writing to a string cannot fail"); + } if let Some(statement) = render_context.inline_action_statements.get(source_state) { if !statement.is_empty() { writeln!(out, "{pad}{statement}").expect("writing to a string cannot fail"); @@ -8014,7 +8117,7 @@ fn semantic_alt_guard_is_unresolved( fn leading_predicates( steps: &[GeneratedParserStep], - portable: Option>, + _portable: Option>, ) -> Vec<(usize, usize)> { let mut predicates = Vec::new(); for step in steps { @@ -8023,15 +8126,11 @@ fn leading_predicates( rule_index, pred_index, } => predicates.push((*rule_index, *pred_index)), - // Portable assignments run only after the alternative is selected, - // so predicates after one are not prediction-visible. - GeneratedParserStep::Action { source_state, .. } - if portable - .is_some_and(|portable| portable.inline_actions.contains_key(source_state)) => - { - break; - } - GeneratedParserStep::Action { .. } | GeneratedParserStep::Precedence(_) => {} + // ANTLR stops collecting prediction-visible predicates at every + // action boundary. The action runs only after the alternative is + // committed, so a later predicate must observe its side effects. + GeneratedParserStep::Action { .. } => break, + GeneratedParserStep::Precedence(_) => {} GeneratedParserStep::MatchToken { .. } | GeneratedParserStep::MatchSet { .. } | GeneratedParserStep::MatchNotSet { .. } @@ -8708,21 +8807,34 @@ fn loop_entry_condition( } } -#[allow(clippy::fn_params_excessive_bools)] -fn render_parser_parse_rule_fallback( +#[derive(Clone, Copy)] +struct ParserFallbackRender<'a> { track_alt_numbers: bool, track_context_alt_numbers: bool, - rule_args: &[(usize, usize, RuleArgTemplate)], + rule_args: &'a [(usize, usize, RuleArgTemplate)], + action_indices: &'a [(usize, usize)], has_action_dispatch: bool, has_predicate_dispatch: bool, - unknown_policy_literal: Option<&str>, -) -> String { + unknown_policy_literal: Option<&'a str>, +} + +fn render_parser_parse_rule_fallback(options: ParserFallbackRender<'_>) -> String { + let ParserFallbackRender { + track_alt_numbers, + track_context_alt_numbers, + rule_args, + action_indices, + has_action_dispatch, + has_predicate_dispatch, + unknown_policy_literal, + } = options; let mut out = String::new(); + let action_indices = render_parser_action_index_array(action_indices); + let rule_args = render_parser_rule_arg_array(rule_args); if has_predicate_dispatch || unknown_policy_literal.is_some() { writeln!( out, - "let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ track_alt_numbers: {track_alt_numbers}, track_context_alt_numbers: {track_context_alt_numbers}, predicates: &[], semantics: Some(parser_semantics()), rule_args: &{}, member_actions: &[], return_actions: &[], unknown_predicate_policy: {} , ..antlr4_runtime::ParserRuntimeOptions::default() }})?;", - render_parser_rule_arg_array(rule_args), + "let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ action_indices: &{action_indices}, track_alt_numbers: {track_alt_numbers}, track_context_alt_numbers: {track_context_alt_numbers}, predicates: &[], semantics: Some(parser_semantics()), rule_args: &{rule_args}, member_actions: &[], return_actions: &[], unknown_predicate_policy: {} , ..antlr4_runtime::ParserRuntimeOptions::default() }})?;", unknown_policy_literal .unwrap_or("antlr4_runtime::UnknownSemanticPolicy::AssumeTrue") ) @@ -8730,13 +8842,13 @@ fn render_parser_parse_rule_fallback( } else if track_alt_numbers || track_context_alt_numbers { writeln!( out, - "let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ track_alt_numbers: {track_alt_numbers}, track_context_alt_numbers: {track_context_alt_numbers}, ..antlr4_runtime::ParserRuntimeOptions::default() }})?;" + "let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ action_indices: &{action_indices}, track_alt_numbers: {track_alt_numbers}, track_context_alt_numbers: {track_context_alt_numbers}, rule_args: &{rule_args}, ..antlr4_runtime::ParserRuntimeOptions::default() }})?;" ) .expect("writing to a string cannot fail"); } else if has_action_dispatch { writeln!( out, - "let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions::default())?;" + "let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions {{ action_indices: &{action_indices}, rule_args: &{rule_args}, ..antlr4_runtime::ParserRuntimeOptions::default() }})?;" ) .expect("writing to a string cannot fail"); } else { @@ -8885,7 +8997,6 @@ impl PortableLocalData { fn step_render(&self) -> Option> { self.has_semantics().then_some(PortableLocalStepRender { declarations: &self.declarations, - inline_actions: &self.inline_actions, predicates: &self.predicates, required_generated_rules: &self.required_generated_rules, }) @@ -8967,7 +9078,7 @@ fn build_structural_portable_local_data( .coordinate_disposition( SemanticsKind::ParserAction, data.rule_names.get(action.rule_index).map(String::as_str), - None, + Some(action.action_index), Some(action.state), ) .is_some() @@ -13658,9 +13769,9 @@ fn collect_noop_action_states( patterns: &SemPatternFile, ) -> io::Result> { let mut noop_action_states = BTreeSet::new(); - let action_state_rules = parser_action_state_rules(data)?; - for state in action_state_rules.keys() { - if parser_action_assume_overridden(patterns, data, &action_state_rules, *state) { + let action_state_coordinates = parser_action_state_coordinates(data)?; + for state in action_state_coordinates.keys() { + if parser_action_assume_overridden(patterns, data, &action_state_coordinates, *state) { noop_action_states.insert(*state); } } @@ -13860,6 +13971,80 @@ fn parser_render_surfaces( } } +struct ParserActionRouting { + inline_statements: BTreeMap, + states: BTreeSet, + generated_states: BTreeSet, + indices: BTreeMap, + committed_indices: Vec<(usize, usize)>, +} + +fn parser_action_routing( + data: &CodegenData<'_>, + embedded: bool, + embedded_data: Option<&EmbeddedParserData>, + portable_local_data: &PortableLocalData, + parameterized_rules: &BTreeSet, + noop_states: &BTreeSet, +) -> io::Result { + let mut inline_statements = embedded_data.map_or_else( + || portable_local_data.inline_actions.clone(), + |embedded| embedded.inline_actions.clone(), + ); + let states = parser_action_states(data)? + .into_iter() + .collect::>(); + let structural_actions = structural_actions(data)?; + let indices = structural_actions + .iter() + .map(|action| (action.state, action.action_index)) + .collect::>(); + let action_rules = structural_actions + .iter() + .map(|action| (action.state, action.rule_index)) + .collect::>(); + let committed_indices = structural_actions + .iter() + .filter(|action| { + !embedded + && action.authored + && !action.body.trim().is_empty() + && !noop_states.contains(&action.state) + }) + .map(|action| (action.state, action.action_index)) + .collect::>(); + let mut generated_states = if embedded { + states.clone() + } else { + noop_states.intersection(&states).copied().collect() + }; + generated_states.extend(portable_local_data.inline_actions.keys().copied()); + if !embedded { + for state in states + .difference(noop_states) + .filter(|state| !portable_local_data.inline_actions.contains_key(state)) + { + let statement = if action_rules + .get(state) + .is_some_and(|rule_index| parameterized_rules.contains(rule_index)) + { + "let _ = self.base.parser_action_hook_with_context_and_local(action, &__ctx, __precedence);" + } else { + "let _ = self.base.parser_action_hook_with_context(action, &__ctx);" + }; + inline_statements.insert(*state, statement.to_owned()); + generated_states.insert(*state); + } + } + Ok(ParserActionRouting { + inline_statements, + states, + generated_states, + indices, + committed_indices, + }) +} + /// Test-facing wrapper over [`render_parser_with_decision_report`] for the /// many render assertions that never look at the manifest rows. #[cfg(test)] @@ -13939,30 +14124,29 @@ fn render_parser_with_decision_report( } else { structural_parser_rule_args(data)? }; - let inline_action_statements = embedded_data.as_ref().map_or_else( - || portable_local_data.inline_actions.clone(), - |embedded| embedded.inline_actions.clone(), - ); + let parameterized_rules = if options.embedded { + BTreeSet::new() + } else { + structural_parameterized_parser_rules(data)? + }; + let ParserActionRouting { + inline_statements: inline_action_statements, + states: action_states, + generated_states: generated_action_states, + indices: action_indices, + committed_indices: committed_action_indices, + } = parser_action_routing( + data, + options.embedded, + embedded_data.as_ref(), + &portable_local_data, + ¶meterized_rules, + &noop_action_states, + )?; let inline_action_states = inline_action_statements .keys() .copied() .collect::>(); - let action_states = parser_action_states(data)? - .into_iter() - .collect::>(); - let mut generated_action_states = if options.embedded { - action_states.clone() - } else { - // Synthetic actions and explicit assume-* overrides are no-op states. - // They should not disable generated parser rules just because they have - // an ATN action transition; real author actions still fall through to - // the interpreted path in non-embedded mode. - noop_action_states - .intersection(&action_states) - .copied() - .collect() - }; - generated_action_states.extend(portable_local_data.inline_actions.keys().copied()); // Under a non-default unknown-coordinate policy every predicate transition // must reach the interpreter, which applies the policy to the complete // structurally bound coordinate inventory. @@ -13993,6 +14177,7 @@ fn render_parser_with_decision_report( all: &action_states, generated: &generated_action_states, inline: &inline_action_states, + indices: &action_indices, }, PredicateCoordinateSets { all: &predicate_coordinates, @@ -14026,14 +14211,15 @@ fn render_parser_with_decision_report( decision_routing, ); let unknown_policy_literal = parser_unknown_policy_literal(options.sem_unknown); - let parse_rule_fallback = render_parser_parse_rule_fallback( + let parse_rule_fallback = render_parser_parse_rule_fallback(ParserFallbackRender { track_alt_numbers, track_context_alt_numbers, - &rule_args, + rule_args: &rule_args, + action_indices: &committed_action_indices, has_action_dispatch, has_predicate_dispatch, unknown_policy_literal, - ); + }); let parser_semantics_function = render_parser_semantics_function(&predicates, data)?; let typed_hook_adapter = render_typed_hook_adapter(&type_name, &parser_typed_hook_mappings(data, patterns)?); @@ -14333,26 +14519,22 @@ where if allow_generated_fallback && __report_error {{ self.base.report_generated_parser_diagnostics(); }} - // A generated predicate that consulted an unimplemented hook - // (returning None under the Error policy) fails the alternative - // and surfaces here as a generic failed-predicate/rule error. - // The documented contract is to fail loud with - // `AntlrError::Unsupported`, so prefer a recorded semantic error - // over the generic one — but only at the top-level entry, mirroring - // the post-parse check below: a nested child keeps its hits so the - // generated parent surfaces them at that boundary instead. if allow_generated_fallback {{ - if let Some(semantic_error) = self.base.take_unknown_semantic_error() {{ - return Err(semantic_error); - }} // A sticky abort (depth cap, listener) wins over an - // error derived from it (e.g. a sync failure after - // recovery absorbed the aborted rule): the caller must - // learn the real cause, and draining un-poisons the - // instance for the next entry-rule call. + // error or semantic miss derived after recovery absorbed + // the aborted rule. Drain any masked semantic miss too, + // so neither condition poisons the next entry. if let Some(abort) = self.base.take_parse_abort() {{ + let _ = self.base.take_unknown_semantic_error(); return Err(abort); }} + // A generated predicate that consulted an unimplemented + // hook fails the alternative and surfaces here as a generic + // failed-predicate/rule error. Prefer the recorded fail-loud + // semantic error when no parser abort occurred. + if let Some(semantic_error) = self.base.take_unknown_semantic_error() {{ + return Err(semantic_error); + }} }} let error = error.into_error(); if allow_generated_fallback && __report_error {{ @@ -14366,26 +14548,19 @@ where }} else {{ self.parse_interpreted_rule_precedence(rule_index, precedence)? }}; - // Surface unknown-predicate coordinates recorded under the Error policy - // at the top-level entry. Generated predicate steps evaluate on the - // committed path and are recovered as rule errors, so a parse that - // consulted an unimplemented hook predicate must fail with - // `AntlrError::Unsupported` instead of returning a recovered `Ok` tree. - if allow_generated_fallback {{ - if let Some(error) = self.base.take_unknown_semantic_error() {{ - return Err(error); - }} - }} if allow_generated_fallback {{ self.base.report_generated_parser_diagnostics(); - if let Some(error) = self.base.take_unknown_semantic_error() {{ - return Err(error); - }} // A sticky abort (depth-cap violation, listener abort) is not a // syntax error: rule-level recovery may have produced a tree - // anyway, but the parse must still fail (and a reused parser - // must start clean). + // and semantic miss anyway, but the abort is the root cause. Drain + // both sticky conditions before returning so parser reuse is clean. if let Some(error) = self.base.take_parse_abort() {{ + let _ = self.base.take_unknown_semantic_error(); + return Err(error); + }} + // Surface unknown predicate/action coordinates recorded under the + // Error policy only after parser aborts have been ruled out. + if let Some(error) = self.base.take_unknown_semantic_error() {{ return Err(error); }} }} @@ -15169,14 +15344,21 @@ fn parser_action_states(data: &CodegenData<'_>) -> io::Result> { Ok(states) } -/// Reads the parser ATN action transitions keyed by source state. -fn parser_action_state_rules(data: &CodegenData<'_>) -> io::Result> { +/// Reads parser ATN action coordinates keyed by source state. +fn parser_action_state_coordinates( + data: &CodegenData<'_>, +) -> io::Result)>> { let atn = data.parser_atn()?; let mut states = BTreeMap::new(); for state in atn.states() { for transition in state.transitions() { - if let ParserTransitionData::Action { rule_index, .. } = transition.data() { - states.insert(state.state_number(), rule_index); + if let ParserTransitionData::Action { + rule_index, + action_index, + .. + } = transition.data() + { + states.insert(state.state_number(), (rule_index, action_index)); } } } @@ -15218,22 +15400,66 @@ fn empty_parser_action_states(data: &CodegenData<'_>) -> io::Result, ) -> io::Result> { - Ok(structural_rule_calls(data)? - .into_iter() - .filter_map(|call| { - let template = parse_rule_arg_template(call.arguments.as_deref()?)?; - Some((call.state, call.target_rule_index, template)) + let mut args = Vec::new(); + for call in structural_rule_calls(data)? { + let Some(value) = call.arguments.as_deref() else { + continue; + }; + let Some(template) = parse_rule_arg_template(value, call.caller_first_argument.as_deref()) + else { + let rule_name = data + .rule_names + .get(call.target_rule_index) + .map_or("", String::as_str); + return Err(io::Error::new( + io::ErrorKind::InvalidData, + format!( + "unsupported parser rule argument expression `{}` for rule `{rule_name}`; use an integer/boolean literal or forward the caller's first declared argument", + value.trim() + ), + )); + }; + args.push((call.state, call.target_rule_index, template)); + } + Ok(args) +} + +fn structural_parameterized_parser_rules(data: &CodegenData<'_>) -> io::Result> { + let semantic = data.semantic.ok_or_else(|| { + io::Error::new( + io::ErrorKind::InvalidData, + "structural grammar model is unavailable", + ) + })?; + Ok(semantic + .recognizer + .rule_numbers + .iter() + .filter_map(|(rule, rule_index)| { + semantic + .bindings + .attributes + .get(rule) + .is_some_and(|attributes| !attributes.arguments.is_empty()) + .then_some(*rule_index) }) .collect()) } -fn parse_rule_arg_template(value: &str) -> Option { +fn parse_rule_arg_template( + value: &str, + caller_first_argument: Option<&str>, +) -> Option { let value = value.trim(); value.parse::().map_or_else( |_| { if matches!(value, "true" | "false") { Some(RuleArgTemplate::Literal(i64::from(value == "true"))) - } else if value == r#""# { + } else if caller_first_argument.is_some_and(|argument| { + value == argument + || value.strip_prefix('$') == Some(argument) + || value == format!(r#""#) + }) { Some(RuleArgTemplate::InheritLocal) } else { None @@ -15412,14 +15638,21 @@ fn render_lexer_predicate_expression(template: &PredicateTemplate) -> String { fn parser_action_assume_overridden( patterns: &SemPatternFile, data: &CodegenData<'_>, - action_state_rules: &BTreeMap, + action_state_coordinates: &BTreeMap)>, state: usize, ) -> bool { - let rule_name = action_state_rules + let (rule_index, action_index) = action_state_coordinates .get(&state) - .and_then(|rule| data.rule_names.get(*rule).map(String::as_str)); + .copied() + .unwrap_or((usize::MAX, None)); + let rule_name = data.rule_names.get(rule_index).map(String::as_str); patterns - .coordinate_override(SemanticsKind::ParserAction, rule_name, None, Some(state)) + .coordinate_override( + SemanticsKind::ParserAction, + rule_name, + action_index, + Some(state), + ) .is_some_and(|override_| { matches!( override_.dispose, @@ -16278,7 +16511,7 @@ fn parser_typed_hook_mappings( ) -> io::Result> { let mut mappings = Vec::new(); for predicate in structural_predicates(data)? { - push_typed_hook_mapping( + push_typed_predicate_hook_mapping( data, patterns, predicate.rule_index, @@ -16287,13 +16520,28 @@ fn parser_typed_hook_mappings( &mut mappings, )?; } - mappings.sort_by_key(|mapping| (mapping.rule_index, mapping.pred_index)); + for action in structural_actions(data)? + .into_iter() + .filter(|action| action.authored && !action.body.trim().is_empty()) + { + if let Some(call) = patterns.hook_helper_call(SemanticsKind::ParserAction, &action.body)? { + mappings.push(TypedHookMapping { + rule_index: action.rule_index, + coordinate_index: action.action_index, + kind: ParserTypedHookKind::Action, + method_name: rust_function_name(&call.name), + call, + }); + } + } + disambiguate_parser_typed_hook_names(&mut mappings); + mappings.sort_by_key(|mapping| (mapping.rule_index, mapping.coordinate_index, mapping.kind)); mappings.dedup(); validate_typed_hook_signatures(&mappings)?; Ok(mappings) } -fn push_typed_hook_mapping( +fn push_typed_predicate_hook_mapping( data: &CodegenData<'_>, patterns: &SemPatternFile, rule_index: usize, @@ -16318,29 +16566,64 @@ fn push_typed_hook_mapping( { mappings.push(TypedHookMapping { rule_index, - pred_index, - method_name: typed_hook_predicate_method_name(&call.name), + coordinate_index: pred_index, + kind: ParserTypedHookKind::Predicate, + method_name: rust_function_name(&call.name), call, }); } Ok(()) } -/// Reserved name of the fixed action-hook method emitted on the typed-hook -/// trait. A predicate-helper method must not normalize to this, or the trait -/// would declare two `custom_action` methods (Rust has no arity overloading). const TYPED_HOOK_ACTION_METHOD: &str = "custom_action"; -/// The typed-hook trait method name for a bare predicate helper, disambiguated -/// so it never collides with the fixed [`TYPED_HOOK_ACTION_METHOD`]. A grammar -/// helper literally named `customAction()` / `custom_action()` normalizes to -/// `custom_action`; suffix it with `_pred` so the generated trait compiles. -fn typed_hook_predicate_method_name(helper: &str) -> String { - let name = rust_function_name(helper); - if name == TYPED_HOOK_ACTION_METHOD { - format!("{name}_pred") - } else { - name +fn disambiguate_parser_typed_hook_names(mappings: &mut [TypedHookMapping]) { + let predicate_names = mappings + .iter() + .filter(|mapping| mapping.kind == ParserTypedHookKind::Predicate) + .map(|mapping| mapping.method_name.clone()) + .collect::>(); + let action_names = mappings + .iter() + .filter(|mapping| mapping.kind == ParserTypedHookKind::Action) + .map(|mapping| mapping.method_name.clone()) + .collect::>(); + let mut allocated = BTreeMap::<(ParserTypedHookKind, String), String>::new(); + let mut used = BTreeSet::from([TYPED_HOOK_ACTION_METHOD.to_owned()]); + for mapping in mappings { + let helper = (mapping.kind, mapping.call.name.clone()); + if let Some(method_name) = allocated.get(&helper) { + mapping.method_name.clone_from(method_name); + continue; + } + if mapping.method_name == TYPED_HOOK_ACTION_METHOD + || (predicate_names.contains(&mapping.method_name) + && action_names.contains(&mapping.method_name)) + { + mapping.method_name.push_str(match mapping.kind { + ParserTypedHookKind::Predicate => "_pred", + ParserTypedHookKind::Action => "_action", + }); + } + let method_name = unique_typed_hook_method_name(&mapping.method_name, &used); + used.insert(method_name.clone()); + allocated.insert(helper, method_name.clone()); + mapping.method_name = method_name; + } +} + +fn unique_typed_hook_method_name(base: &str, used: &BTreeSet) -> String { + if !used.contains(base) { + return base.to_owned(); + } + let stem = base.strip_prefix("r#").unwrap_or(base); + let mut suffix = 2; + loop { + let candidate = format!("{stem}_{suffix}"); + if !used.contains(&candidate) { + return candidate; + } + suffix += 1; } } @@ -16353,7 +16636,7 @@ const fn semantic_literal_kind(literal: &SemanticLiteral) -> SemanticLiteralKind } fn validate_typed_hook_signatures(mappings: &[TypedHookMapping]) -> io::Result<()> { - let mut signatures = BTreeMap::<&str, Vec>::new(); + let mut signatures = BTreeMap::<(&str, ParserTypedHookKind), Vec>::new(); for mapping in mappings { let signature = mapping .call @@ -16361,7 +16644,7 @@ fn validate_typed_hook_signatures(mappings: &[TypedHookMapping]) -> io::Result<( .iter() .map(semantic_literal_kind) .collect::>(); - match signatures.entry(&mapping.method_name) { + match signatures.entry((&mapping.method_name, mapping.kind)) { Entry::Occupied(entry) if entry.get() != &signature => { return Err(io::Error::new( io::ErrorKind::InvalidData, @@ -16415,28 +16698,34 @@ fn render_typed_hook_adapter(type_name: &str, mappings: &[TypedHookMapping]) -> } let trait_name = format!("{type_name}Hooks"); let adapter_name = format!("{type_name}TypedHooks"); - let mut methods = BTreeMap::new(); + let mut methods = BTreeMap::<(String, ParserTypedHookKind), Vec>::new(); for mapping in mappings { methods - .entry(mapping.method_name.clone()) + .entry((mapping.method_name.clone(), mapping.kind)) .or_insert_with(|| mapping.call.arguments.clone()); } let method_decls = methods .iter() - .map(|(method, arguments)| { + .map(|((method, kind), arguments)| { let arguments = render_semantic_method_arguments(arguments); let separator = if arguments.is_empty() { "" } else { ", " }; + let result = if *kind == ParserTypedHookKind::Predicate { + " -> bool" + } else { + "" + }; format!( - " fn {method}(&mut self, ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>{separator}{arguments}) -> bool\n where\n L: TokenSource;" + " fn {method}(&mut self, ctx: &mut antlr4_runtime::ParserSemCtx<'_, L>{separator}{arguments}){result}\n where\n L: TokenSource;" ) }) .collect::>() .join("\n\n"); - let arms = mappings + let predicate_arms = mappings .iter() + .filter(|mapping| mapping.kind == ParserTypedHookKind::Predicate) .map(|mapping| { let rule_index = mapping.rule_index; - let pred_index = mapping.pred_index; + let pred_index = mapping.coordinate_index; let method = &mapping.method_name; let arguments = render_semantic_call_arguments(&mapping.call.arguments); let separator = if arguments.is_empty() { "" } else { ", " }; @@ -16450,6 +16739,28 @@ fn render_typed_hook_adapter(type_name: &str, mappings: &[TypedHookMapping]) -> }) .collect::>() .join("\n"); + let action_arms = mappings + .iter() + .filter(|mapping| mapping.kind == ParserTypedHookKind::Action) + .map(|mapping| { + let rule_index = mapping.rule_index; + let action_index = mapping.coordinate_index; + let method = &mapping.method_name; + let arguments = render_semantic_call_arguments(&mapping.call.arguments); + let separator = if arguments.is_empty() { "" } else { ", " }; + format!( + " ({rule_index}, Some({action_index})) => {{ self.0.{method}(ctx{separator}{arguments}); true }}," + ) + }) + .collect::>() + .join("\n"); + let action_dispatch = if action_arms.is_empty() { + " self.0.custom_action(ctx, action)".to_owned() + } else { + format!( + " match (action.rule_index(), action.action_index()) {{\n{action_arms}\n _ => self.0.custom_action(ctx, action),\n }}" + ) + }; format!( r#"pub trait {trait_name}: Sized {{ {method_decls} @@ -16482,7 +16793,7 @@ where L: TokenSource, {{ match (rule_index, pred_index) {{ -{arms} +{predicate_arms} _ => None, }} }} @@ -16491,7 +16802,7 @@ where where L: TokenSource, {{ - self.0.custom_action(ctx, action) +{action_dispatch} }} }} "# @@ -16719,6 +17030,16 @@ fn render_parser_predicate_array( Ok(format!("[{}]", items.join(", "))) } +/// Renders stable authored parser-action coordinates for committed fallback. +fn render_parser_action_index_array(action_indices: &[(usize, usize)]) -> String { + let items = action_indices + .iter() + .map(|(source_state, action_index)| format!("({source_state}, {action_index})")) + .collect::>() + .join(", "); + format!("[{items}]") +} + /// Renders parser rule-argument metadata for generated calls into the runtime. fn render_parser_rule_arg_array(args: &[(usize, usize, RuleArgTemplate)]) -> String { let items = args @@ -17201,6 +17522,7 @@ mod tests { let decision_by_state = decision_by_state(atn); let action_states = BTreeSet::new(); let generated_action_states = BTreeSet::new(); + let action_indices = BTreeMap::new(); let predicate_coordinates = BTreeSet::new(); let generated_predicate_coordinates = BTreeSet::new(); let context = GeneratedParserCompileContext { @@ -17210,6 +17532,7 @@ mod tests { inline_action_states, action_states: &action_states, generated_action_states: &generated_action_states, + action_indices: &action_indices, predicate_coordinates: &predicate_coordinates, generated_predicate_coordinates: &generated_predicate_coordinates, }; @@ -17681,7 +18004,8 @@ mod tests { "left-recursive routing must remain separate from unconditional cascade routing" ); let preferred = generated_adaptive_atn_preferred_rule_calls(&rules); - let routing = generated_adaptive_atn_routing_excluding(&rules, &BTreeSet::new()); + let routing = + generated_adaptive_atn_routing_excluding(&rules, &BTreeSet::new(), &BTreeSet::new()); assert!( preferred[0], @@ -17708,11 +18032,63 @@ mod tests { let force_generated = generated_rule_callers_reaching(&rules, &BTreeSet::from([1])); assert_eq!( - generated_adaptive_atn_preferred_rule_calls_excluding(&rules, &force_generated), + generated_adaptive_atn_preferred_rule_calls_excluding( + &rules, + &force_generated, + &BTreeSet::new(), + ), vec![false, false, false, false, true] ); } + #[test] + fn adaptive_atn_retries_exclude_effectful_action_rules_and_callers() { + let mut action_seed = left_recursive_rule( + 1, + ATN_PREFERRED_LEFT_RECURSIVE_MIN_DECISION_COST, + ATN_PREFERRED_LEFT_RECURSIVE_MIN_OPERATOR_ALTS, + ); + action_seed.steps.push(GeneratedParserStep::Action { + source_state: 9_001, + rule_index: 1, + action_index: Some(0), + }); + let mut synthetic_action_seed = left_recursive_rule( + 2, + ATN_PREFERRED_LEFT_RECURSIVE_MIN_DECISION_COST, + ATN_PREFERRED_LEFT_RECURSIVE_MIN_OPERATOR_ALTS, + ); + synthetic_action_seed + .steps + .push(GeneratedParserStep::Action { + source_state: 9_002, + rule_index: 2, + action_index: Some(0), + }); + let rules = vec![ + Some(test_rule(0, { + let mut steps = (100..108).map(adaptive_loop).collect::>(); + steps.push(cr(1)); + steps + })), + Some(action_seed), + Some(synthetic_action_seed), + ]; + let effectful_action_states = BTreeSet::from([9_001]); + + let routing = generated_adaptive_atn_routing_excluding( + &rules, + &BTreeSet::new(), + &effectful_action_states, + ); + + assert_eq!(routing.candidates, [false, false, true]); + assert!( + routing.probe_candidate_rules[1].is_empty(), + "an effectful action seed must not request a retry from its generated caller" + ); + } + #[test] fn atn_preferred_rule_calls_propagate_through_expensive_wrappers() { let mut rules = Vec::new(); @@ -17957,6 +18333,7 @@ mod tests { all: &empty_states, generated: &empty_states, inline: &empty_states, + indices: &BTreeMap::new(), }; let predicate_coords = PredicateCoordinateSets { all: &empty_coords, @@ -18020,6 +18397,7 @@ mod tests { all: &BTreeSet::new(), generated: &BTreeSet::new(), inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &BTreeSet::new(), @@ -18040,6 +18418,7 @@ mod tests { all: &BTreeSet::new(), generated: &generated_action_states, inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &BTreeSet::new(), @@ -18050,6 +18429,7 @@ mod tests { Some(GeneratedParserStep::Action { source_state: 4, rule_index: 2, + action_index: Some(0), }), 8 )) @@ -18075,6 +18455,7 @@ mod tests { all: &BTreeSet::new(), generated: &BTreeSet::new(), inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &BTreeSet::new(), @@ -18100,6 +18481,7 @@ mod tests { all: &BTreeSet::new(), generated: &BTreeSet::new(), inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &BTreeSet::new(), @@ -18134,6 +18516,18 @@ mod tests { ); } + #[test] + fn rejects_unsupported_rule_argument_expressions() { + let data = parser_fixture_data("unsupported-rule-argument/T.g4"); + let error = structural_parser_rule_args(&data) + .expect_err("unsupported expressions must not be silently omitted"); + + insta::assert_snapshot!( + error, + @"unsupported parser rule argument expression `1 + 2` for rule `child`; use an integer/boolean literal or forward the caller's first declared argument" + ); + } + #[test] fn compiles_synthetic_noop_action_transitions_as_epsilon() { let action_atn = transition_atn(|_| ParserTransitionSpec::Action { @@ -18152,6 +18546,7 @@ mod tests { all: &BTreeSet::new(), generated: &BTreeSet::new(), inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &BTreeSet::new(), @@ -18182,6 +18577,7 @@ mod tests { all: &action_states, generated: &BTreeSet::new(), inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &BTreeSet::new(), @@ -18211,6 +18607,7 @@ mod tests { all: &BTreeSet::new(), generated: &BTreeSet::new(), inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &BTreeSet::new(), @@ -18243,6 +18640,7 @@ mod tests { all: &BTreeSet::new(), generated: &BTreeSet::new(), inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &predicates, @@ -18378,6 +18776,7 @@ mod tests { all: &BTreeSet::new(), generated: &BTreeSet::new(), inline: &BTreeSet::new(), + indices: &BTreeMap::new(), }, PredicateCoordinateSets { all: &predicates, @@ -18390,11 +18789,24 @@ mod tests { #[test] fn parse_rule_fallback_runs_parser_actions() { - let fallback = render_parser_parse_rule_fallback(false, false, &[], true, false, None); + let rule_args = [(4, 2, RuleArgTemplate::Literal(17))]; + let action_indices = [(5, 0)]; + let fallback = render_parser_parse_rule_fallback(ParserFallbackRender { + track_alt_numbers: false, + track_context_alt_numbers: false, + rule_args: &rule_args, + action_indices: &action_indices, + has_action_dispatch: true, + has_predicate_dispatch: false, + unknown_policy_literal: None, + }); assert!(fallback.contains( "parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence" )); + assert!(fallback.contains( + "rule_args: &[antlr4_runtime::ParserRuleArg { source_state: 4, rule_index: 2, value: 17, inherit_local: false }]" + )); assert!(fallback.contains("for action in actions { self.run_action(action, tree); }")); assert!(fallback.contains("Ok(tree)")); } @@ -19596,14 +20008,17 @@ mod tests { } #[test] - fn non_embedded_parser_action_disables_generated_rule() { + fn non_embedded_parser_action_runs_at_generated_position() { let rendered = render_parser("TParser", &action_parser_data()).expect("parser should render"); assert!( - !rendered.contains("parse_generated_rule_0_dispatch"), - "non-embedded parser action rules must stay on the interpreted path" + rendered.contains("parse_generated_rule_0_dispatch"), + "a hook-routed parser action must remain on the generated path" ); + assert!(rendered.contains("parser_action_at_current_indexed")); + assert!(rendered.contains("parser_action_hook_with_context(action, &__ctx)")); + assert!(rendered.contains("action_indices: &[(")); assert!(rendered.contains("self.base.parser_action_hook(action, tree)")); assert!(!rendered.contains(&format!("{}{}", "Generated", "Action"))); assert!(!rendered.contains(&format!("{}{}", "generated", "_actions"))); @@ -20099,6 +20514,7 @@ mod tests { GeneratedParserStep::Action { source_state: 5, rule_index: 1, + action_index: Some(0), }, GeneratedParserStep::Predicate { rule_index: 1, @@ -20134,7 +20550,6 @@ mod tests { embedded: None, portable_locals: Some(PortableLocalStepRender { declarations: &declarations, - inline_actions: &inline_actions, predicates: &predicates, required_generated_rules: &required_generated_rules, }), @@ -20333,7 +20748,6 @@ mod tests { embedded: None, portable_locals: Some(PortableLocalStepRender { declarations: &declarations, - inline_actions: &BTreeMap::new(), predicates: &predicates, required_generated_rules: &required_generated_rules, }), @@ -21805,8 +22219,8 @@ lower = "cmp(ne, ctx_rule_text(local_type), str(\"var\"))" // Assumed parser actions get explicit no-op arms; hook/error overrides // keep falling through to the parser action hook. let data = predicate_parser_data(); // rule 0 = "s" - let mut action_state_rules = BTreeMap::new(); - action_state_rules.insert(4_usize, 0_usize); // action state 4 belongs to rule `s` + let mut action_state_coordinates = BTreeMap::new(); + action_state_coordinates.insert(4_usize, (0_usize, Some(0_usize))); for dispose in ["assume-true", "assume-false"] { let patterns = parse_sem_patterns(&format!( @@ -21814,23 +22228,44 @@ lower = "cmp(ne, ctx_rule_text(local_type), str(\"var\"))" )) .expect("pattern file parses"); assert!( - parser_action_assume_overridden(&patterns, &data, &action_state_rules, 4), + parser_action_assume_overridden(&patterns, &data, &action_state_coordinates, 4), "dispose {dispose}: an action state in rule `s` is assumed" ); } + let indexed_patterns = parse_sem_patterns( + "version = 1\n[[coordinate]]\nkind = \"action\"\nrule = \"s\"\nindex = 0\ndispose = \"assume-true\"\n", + ) + .expect("indexed pattern file parses"); + assert!( + parser_action_assume_overridden(&indexed_patterns, &data, &action_state_coordinates, 4), + "an index-specific assume override should suppress hook routing" + ); + let other_index_patterns = parse_sem_patterns( + "version = 1\n[[coordinate]]\nkind = \"action\"\nrule = \"s\"\nindex = 1\ndispose = \"assume-true\"\n", + ) + .expect("other-index pattern file parses"); + assert!( + !parser_action_assume_overridden( + &other_index_patterns, + &data, + &action_state_coordinates, + 4 + ), + "an override for another action index must not suppress this hook" + ); let hook_patterns = parse_sem_patterns( "version = 1\n[[coordinate]]\nkind = \"action\"\nrule = \"s\"\ndispose = \"hook\"\n", ) .expect("pattern file parses"); assert!( - !parser_action_assume_overridden(&hook_patterns, &data, &action_state_rules, 4), + !parser_action_assume_overridden(&hook_patterns, &data, &action_state_coordinates, 4), "hook overrides should keep routing through the hook arm" ); assert!( !parser_action_assume_overridden( &SemPatternFile::default(), &data, - &action_state_rules, + &action_state_coordinates, 4 ), "no override -> concrete arm is kept" @@ -22245,27 +22680,94 @@ dispose = "hook" 1, "only the parser-rule helper maps: {mappings:?}" ); - assert_eq!((mappings[0].rule_index, mappings[0].pred_index), (0, 0)); + assert_eq!( + (mappings[0].rule_index, mappings[0].coordinate_index), + (0, 0) + ); assert_eq!(mappings[0].method_name, "is_type_name"); } #[test] fn typed_hook_predicate_method_name_avoids_action_hook_collision() { - // A grammar helper that normalizes to the reserved action-hook method - // name must be disambiguated, or the generated trait would declare two - // `custom_action` methods (Rust has no arity overloading). - assert_eq!( - typed_hook_predicate_method_name("customAction"), - "custom_action_pred" - ); - assert_eq!( - typed_hook_predicate_method_name("custom_action"), - "custom_action_pred" - ); - // An unrelated helper keeps its normalized name. - assert_eq!( - typed_hook_predicate_method_name("isTypeName"), - "is_type_name" + let mut mappings = [ + TypedHookMapping { + rule_index: 0, + coordinate_index: 0, + kind: ParserTypedHookKind::Predicate, + method_name: "custom_action".to_owned(), + call: SemanticHelperCall { + name: "customAction".to_owned(), + arguments: Vec::new(), + negated: false, + }, + }, + TypedHookMapping { + rule_index: 0, + coordinate_index: 1, + kind: ParserTypedHookKind::Predicate, + method_name: "is_type_name".to_owned(), + call: SemanticHelperCall { + name: "isTypeName".to_owned(), + arguments: Vec::new(), + negated: false, + }, + }, + ]; + disambiguate_parser_typed_hook_names(&mut mappings); + assert_eq!(mappings[0].method_name, "custom_action_pred"); + assert_eq!(mappings[1].method_name, "is_type_name"); + } + + #[test] + fn typed_hook_action_method_names_remain_unique_after_suffixing() { + let mut mappings = [ + TypedHookMapping { + rule_index: 0, + coordinate_index: 0, + kind: ParserTypedHookKind::Action, + method_name: "custom_action".to_owned(), + call: SemanticHelperCall { + name: "custom_action".to_owned(), + arguments: Vec::new(), + negated: false, + }, + }, + TypedHookMapping { + rule_index: 0, + coordinate_index: 1, + kind: ParserTypedHookKind::Action, + method_name: "custom_action".to_owned(), + call: SemanticHelperCall { + name: "custom_action".to_owned(), + arguments: Vec::new(), + negated: false, + }, + }, + TypedHookMapping { + rule_index: 0, + coordinate_index: 2, + kind: ParserTypedHookKind::Action, + method_name: "custom_action_action".to_owned(), + call: SemanticHelperCall { + name: "custom_action_action".to_owned(), + arguments: Vec::new(), + negated: false, + }, + }, + ]; + + disambiguate_parser_typed_hook_names(&mut mappings); + + insta::assert_debug_snapshot!( + "typed_hook_action_method_names_remain_unique_after_suffixing", + mappings + .iter() + .map(|mapping| ( + mapping.coordinate_index, + mapping.kind, + mapping.method_name.as_str(), + )) + .collect::>() ); } @@ -22569,6 +23071,31 @@ dispose = "hook" .expect("portable coordinates satisfy strict semantics"); } + #[test] + fn indexed_action_overrides_precede_portable_boolean_lowering() { + let data = portable_bool_parser_data(); + let action = structural_actions(&data) + .expect("portable action inventory should build") + .into_iter() + .next() + .expect("portable fixture has one action"); + + for dispose in ["assume-true", "assume-false", "hook"] { + let patterns = parse_sem_patterns(&format!( + "version = 1\n[[coordinate]]\nkind = \"action\"\nrule = \"s\"\nindex = {}\ndispose = \"{dispose}\"\n", + action.action_index + )) + .expect("indexed action override should parse"); + let portable = build_structural_portable_local_data(&data, &patterns) + .expect("portable local semantics should build"); + + assert!( + portable.inline_actions.is_empty(), + "indexed {dispose} override must suppress portable action lowering" + ); + } + } + #[test] fn semantics_manifest_renders_empty_inventory() { let manifest = render_semantics_manifest( @@ -22678,8 +23205,8 @@ dispose = "hook" .find("if let Some(abort) = self.base.take_parse_abort()") .expect("the Err arm drains a recorded parser abort"); assert!( - diagnostics_at < semantic_at && diagnostics_at < abort_at, - "retained diagnostics must be dispatched before either override can return" + diagnostics_at < abort_at && abort_at < semantic_at, + "retained diagnostics must dispatch first, then parser aborts must precede semantic misses" ); } diff --git a/src/bin/snapshots/antlr4_rust_gen__tests__generated_decision_does_not_hoist_portable_predicate_past_local_action.snap b/src/bin/snapshots/antlr4_rust_gen__tests__generated_decision_does_not_hoist_portable_predicate_past_local_action.snap index 88e8ffdb..b7607555 100644 --- a/src/bin/snapshots/antlr4_rust_gen__tests__generated_decision_does_not_hoist_portable_predicate_past_local_action.snap +++ b/src/bin/snapshots/antlr4_rust_gen__tests__generated_decision_does_not_hoist_portable_predicate_past_local_action.snap @@ -26,7 +26,7 @@ if self.base.report_diagnostic_errors() { } match __prediction.alt { 1 => { - let action = self.base.parser_action_at_current(5, 1, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(5, 1, 0, __rule_start, __consumed_eof); __antlr_local_seen = true; let _ = action; if !(__antlr_local_seen) { diff --git a/src/bin/snapshots/antlr4_rust_gen__tests__generated_module_file_header.snap b/src/bin/snapshots/antlr4_rust_gen__tests__generated_module_file_header.snap index 0bb3a899..4cc8c1aa 100644 --- a/src/bin/snapshots/antlr4_rust_gen__tests__generated_module_file_header.snap +++ b/src/bin/snapshots/antlr4_rust_gen__tests__generated_module_file_header.snap @@ -4,7 +4,7 @@ expression: "generated_module_header.replace(env!(\"CARGO_PKG_VERSION\"),\n\" - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(1, ""); +antlr4_runtime::__antlr4_rust_require_codegen_api!(3, ""); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { diff --git a/src/bin/snapshots/antlr4_rust_gen__tests__typed_hook_action_method_names_remain_unique_after_suffixing.snap b/src/bin/snapshots/antlr4_rust_gen__tests__typed_hook_action_method_names_remain_unique_after_suffixing.snap new file mode 100644 index 00000000..16c5ec8e --- /dev/null +++ b/src/bin/snapshots/antlr4_rust_gen__tests__typed_hook_action_method_names_remain_unique_after_suffixing.snap @@ -0,0 +1,21 @@ +--- +source: src/bin/antlr4-rust-gen.rs +expression: "mappings.iter().map(|mapping|\n(mapping.coordinate_index, mapping.kind,\nmapping.method_name.as_str(),)).collect::>()" +--- +[ + ( + 0, + Action, + "custom_action_action", + ), + ( + 1, + Action, + "custom_action_action", + ), + ( + 2, + Action, + "custom_action_action_2", + ), +] diff --git a/src/bin_support/grammar/generated/antlr_v4_lexer.rs b/src/bin_support/grammar/generated/antlr_v4_lexer.rs index 7b169ab6..062631b7 100644 --- a/src/bin_support/grammar/generated/antlr_v4_lexer.rs +++ b/src/bin_support/grammar/generated/antlr_v4_lexer.rs @@ -1,6 +1,6 @@ // @generated by antlr-rust-runtime v0.25.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(1, "0.25.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(3, "0.25.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { diff --git a/src/bin_support/grammar/generated/antlr_v4_parser.rs b/src/bin_support/grammar/generated/antlr_v4_parser.rs index 106ff167..b4d04a59 100644 --- a/src/bin_support/grammar/generated/antlr_v4_parser.rs +++ b/src/bin_support/grammar/generated/antlr_v4_parser.rs @@ -1,6 +1,6 @@ // @generated by antlr-rust-runtime v0.25.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(1, "0.25.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(3, "0.25.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { @@ -18517,26 +18517,22 @@ where if allow_generated_fallback && __report_error { self.base.report_generated_parser_diagnostics(); } - // A generated predicate that consulted an unimplemented hook - // (returning None under the Error policy) fails the alternative - // and surfaces here as a generic failed-predicate/rule error. - // The documented contract is to fail loud with - // `AntlrError::Unsupported`, so prefer a recorded semantic error - // over the generic one — but only at the top-level entry, mirroring - // the post-parse check below: a nested child keeps its hits so the - // generated parent surfaces them at that boundary instead. if allow_generated_fallback { - if let Some(semantic_error) = self.base.take_unknown_semantic_error() { - return Err(semantic_error); - } // A sticky abort (depth cap, listener) wins over an - // error derived from it (e.g. a sync failure after - // recovery absorbed the aborted rule): the caller must - // learn the real cause, and draining un-poisons the - // instance for the next entry-rule call. + // error or semantic miss derived after recovery absorbed + // the aborted rule. Drain any masked semantic miss too, + // so neither condition poisons the next entry. if let Some(abort) = self.base.take_parse_abort() { + let _ = self.base.take_unknown_semantic_error(); return Err(abort); } + // A generated predicate that consulted an unimplemented + // hook fails the alternative and surfaces here as a generic + // failed-predicate/rule error. Prefer the recorded fail-loud + // semantic error when no parser abort occurred. + if let Some(semantic_error) = self.base.take_unknown_semantic_error() { + return Err(semantic_error); + } } let error = error.into_error(); if allow_generated_fallback && __report_error { @@ -18550,26 +18546,19 @@ where } else { self.parse_interpreted_rule_precedence(rule_index, precedence)? }; - // Surface unknown-predicate coordinates recorded under the Error policy - // at the top-level entry. Generated predicate steps evaluate on the - // committed path and are recovered as rule errors, so a parse that - // consulted an unimplemented hook predicate must fail with - // `AntlrError::Unsupported` instead of returning a recovered `Ok` tree. - if allow_generated_fallback { - if let Some(error) = self.base.take_unknown_semantic_error() { - return Err(error); - } - } if allow_generated_fallback { self.base.report_generated_parser_diagnostics(); - if let Some(error) = self.base.take_unknown_semantic_error() { - return Err(error); - } // A sticky abort (depth-cap violation, listener abort) is not a // syntax error: rule-level recovery may have produced a tree - // anyway, but the parse must still fail (and a reused parser - // must start clean). + // and semantic miss anyway, but the abort is the root cause. Drain + // both sticky conditions before returning so parser reuse is clean. if let Some(error) = self.base.take_parse_abort() { + let _ = self.base.take_unknown_semantic_error(); + return Err(error); + } + // Surface unknown predicate/action coordinates recorded under the + // Error policy only after parser aborts have been ruled out. + if let Some(error) = self.base.take_unknown_semantic_error() { return Err(error); } } @@ -18590,7 +18579,7 @@ where self.base .parse_atn_rule_adaptive_or_fallback(atn(), simulator, rule_index) } else { - let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions { track_alt_numbers: false, track_context_alt_numbers: false, predicates: &[], semantics: Some(parser_semantics()), rule_args: &[], member_actions: &[], return_actions: &[], unknown_predicate_policy: antlr4_runtime::UnknownSemanticPolicy::Error , ..antlr4_runtime::ParserRuntimeOptions::default() })?; + let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions { action_indices: &[], track_alt_numbers: false, track_context_alt_numbers: false, predicates: &[], semantics: Some(parser_semantics()), rule_args: &[], member_actions: &[], return_actions: &[], unknown_predicate_policy: antlr4_runtime::UnknownSemanticPolicy::Error , ..antlr4_runtime::ParserRuntimeOptions::default() })?; let _ = actions; Ok(tree) } diff --git a/src/bin_support/rust_syntax/generated/rust_lexer.rs b/src/bin_support/rust_syntax/generated/rust_lexer.rs index 0dae1282..174ec30c 100644 --- a/src/bin_support/rust_syntax/generated/rust_lexer.rs +++ b/src/bin_support/rust_syntax/generated/rust_lexer.rs @@ -1,6 +1,6 @@ // @generated by antlr-rust-runtime v0.25.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(1, "0.25.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(3, "0.25.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { diff --git a/src/bin_support/rust_syntax/generated/rust_parser.rs b/src/bin_support/rust_syntax/generated/rust_parser.rs index 435334e7..86524c23 100644 --- a/src/bin_support/rust_syntax/generated/rust_parser.rs +++ b/src/bin_support/rust_syntax/generated/rust_parser.rs @@ -1,6 +1,6 @@ // @generated by antlr-rust-runtime v0.25.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(1, "0.25.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(3, "0.25.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { @@ -55118,26 +55118,22 @@ where if allow_generated_fallback && __report_error { self.base.report_generated_parser_diagnostics(); } - // A generated predicate that consulted an unimplemented hook - // (returning None under the Error policy) fails the alternative - // and surfaces here as a generic failed-predicate/rule error. - // The documented contract is to fail loud with - // `AntlrError::Unsupported`, so prefer a recorded semantic error - // over the generic one — but only at the top-level entry, mirroring - // the post-parse check below: a nested child keeps its hits so the - // generated parent surfaces them at that boundary instead. if allow_generated_fallback { - if let Some(semantic_error) = self.base.take_unknown_semantic_error() { - return Err(semantic_error); - } // A sticky abort (depth cap, listener) wins over an - // error derived from it (e.g. a sync failure after - // recovery absorbed the aborted rule): the caller must - // learn the real cause, and draining un-poisons the - // instance for the next entry-rule call. + // error or semantic miss derived after recovery absorbed + // the aborted rule. Drain any masked semantic miss too, + // so neither condition poisons the next entry. if let Some(abort) = self.base.take_parse_abort() { + let _ = self.base.take_unknown_semantic_error(); return Err(abort); } + // A generated predicate that consulted an unimplemented + // hook fails the alternative and surfaces here as a generic + // failed-predicate/rule error. Prefer the recorded fail-loud + // semantic error when no parser abort occurred. + if let Some(semantic_error) = self.base.take_unknown_semantic_error() { + return Err(semantic_error); + } } let error = error.into_error(); if allow_generated_fallback && __report_error { @@ -55151,26 +55147,19 @@ where } else { self.parse_interpreted_rule_precedence(rule_index, precedence)? }; - // Surface unknown-predicate coordinates recorded under the Error policy - // at the top-level entry. Generated predicate steps evaluate on the - // committed path and are recovered as rule errors, so a parse that - // consulted an unimplemented hook predicate must fail with - // `AntlrError::Unsupported` instead of returning a recovered `Ok` tree. - if allow_generated_fallback { - if let Some(error) = self.base.take_unknown_semantic_error() { - return Err(error); - } - } if allow_generated_fallback { self.base.report_generated_parser_diagnostics(); - if let Some(error) = self.base.take_unknown_semantic_error() { - return Err(error); - } // A sticky abort (depth-cap violation, listener abort) is not a // syntax error: rule-level recovery may have produced a tree - // anyway, but the parse must still fail (and a reused parser - // must start clean). + // and semantic miss anyway, but the abort is the root cause. Drain + // both sticky conditions before returning so parser reuse is clean. if let Some(error) = self.base.take_parse_abort() { + let _ = self.base.take_unknown_semantic_error(); + return Err(error); + } + // Surface unknown predicate/action coordinates recorded under the + // Error policy only after parser aborts have been ruled out. + if let Some(error) = self.base.take_unknown_semantic_error() { return Err(error); } } @@ -55191,7 +55180,7 @@ where self.base .parse_atn_rule_adaptive_or_fallback(atn(), simulator, rule_index) } else { - let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions { track_alt_numbers: false, track_context_alt_numbers: false, predicates: &[], semantics: Some(parser_semantics()), rule_args: &[], member_actions: &[], return_actions: &[], unknown_predicate_policy: antlr4_runtime::UnknownSemanticPolicy::Error , ..antlr4_runtime::ParserRuntimeOptions::default() })?; + let (tree, actions) = self.base.parse_atn_rule_with_runtime_options_and_precedence(atn(), rule_index, precedence, antlr4_runtime::ParserRuntimeOptions { action_indices: &[], track_alt_numbers: false, track_context_alt_numbers: false, predicates: &[], semantics: Some(parser_semantics()), rule_args: &[], member_actions: &[], return_actions: &[], unknown_predicate_policy: antlr4_runtime::UnknownSemanticPolicy::Error , ..antlr4_runtime::ParserRuntimeOptions::default() })?; for action in actions { self.run_action(action, tree); } Ok(tree) } @@ -68931,7 +68920,7 @@ where self.base.record_generated_prediction_diagnostic(atn(), 1635, &__prediction); match __prediction.alt { 1 => { - let action = self.base.parser_action_at_current(1623, 75, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(1623, 75, 0, __rule_start, __consumed_eof); let _ = action; let __match = self.base.match_token_recovering(5, 1636, atn())?; __consumed_eof |= __match.consumed_eof(); @@ -69998,7 +69987,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(1705, 84, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(1705, 84, 1, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(1706isize); let __child = self.parse_generated_rule_214_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -70552,7 +70541,7 @@ where self.base.record_generated_prediction_diagnostic(atn(), 1751, &__prediction); match __prediction.alt { 1 => { - let action = self.base.parser_action_at_current(1739, 87, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(1739, 87, 2, __rule_start, __consumed_eof); let _ = action; let __match = self.base.match_token_recovering(5, 1752, atn())?; __consumed_eof |= __match.consumed_eof(); @@ -71724,7 +71713,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(1820, 95, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(1820, 95, 3, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(1821isize); let __child = self.parse_generated_rule_96_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -84697,7 +84686,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(2948, 167, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(2948, 167, 4, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(2949isize); let __child = self.parse_generated_rule_155_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -85366,7 +85355,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3019, 170, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3019, 170, 5, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3020isize); let __child = self.parse_generated_rule_169_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -85514,7 +85503,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3033, 171, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3033, 171, 6, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3034isize); let __child = self.parse_generated_rule_170_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -85678,7 +85667,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3050, 172, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3050, 172, 7, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3051isize); let __child = self.parse_generated_rule_171_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -85826,7 +85815,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3064, 173, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3064, 173, 8, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3065isize); let __child = self.parse_generated_rule_172_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -85980,7 +85969,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3080, 174, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3080, 174, 9, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3081isize); let __child = self.parse_generated_rule_173_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -86098,7 +86087,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3091, 175, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3091, 175, 10, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3092isize); let __child = self.parse_generated_rule_174_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -86216,7 +86205,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3102, 176, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3102, 176, 11, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3103isize); let __child = self.parse_generated_rule_175_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -86448,7 +86437,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3120, 178, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3120, 178, 12, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3121isize); let __child = self.parse_generated_rule_177_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -86566,7 +86555,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3131, 179, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3131, 179, 13, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3132isize); let __child = self.parse_generated_rule_178_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -87141,7 +87130,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3173, 182, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3173, 182, 14, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3174isize); let __child = self.parse_generated_rule_156_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -87509,7 +87498,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3212, 184, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3212, 184, 15, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3213isize); let __child = self.parse_generated_rule_183_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -87657,7 +87646,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3226, 185, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3226, 185, 16, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3227isize); let __child = self.parse_generated_rule_184_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -87821,7 +87810,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3243, 186, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3243, 186, 17, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3244isize); let __child = self.parse_generated_rule_185_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -87969,7 +87958,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3257, 187, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3257, 187, 18, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3258isize); let __child = self.parse_generated_rule_186_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -88123,7 +88112,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3273, 188, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3273, 188, 19, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3274isize); let __child = self.parse_generated_rule_187_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -88241,7 +88230,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3284, 189, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3284, 189, 20, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3285isize); let __child = self.parse_generated_rule_188_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -88359,7 +88348,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3295, 190, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3295, 190, 21, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3296isize); let __child = self.parse_generated_rule_189_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -88787,7 +88776,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3327, 192, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3327, 192, 22, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3328isize); let __child = self.parse_generated_rule_191_dispatch(0, false).map_err(GeneratedRuleError::into_error); @@ -88905,7 +88894,7 @@ where let mut __consumed_eof = false; let mut __sync_error: Option = None; let __result = (|| -> Result<(), antlr4_runtime::AntlrError> { - let action = self.base.parser_action_at_current(3338, 193, __rule_start, __consumed_eof); + let action = self.base.parser_action_at_current_indexed(3338, 193, 23, __rule_start, __consumed_eof); let _ = action; let __invoking_marker = self.base.push_invoking_state(3339isize); let __child = self.parse_generated_rule_192_dispatch(0, false).map_err(GeneratedRuleError::into_error); diff --git a/src/bin_support/rust_syntax/generated/semantics.json b/src/bin_support/rust_syntax/generated/semantics.json index a0b73c2f..28f1002e 100644 --- a/src/bin_support/rust_syntax/generated/semantics.json +++ b/src/bin_support/rust_syntax/generated/semantics.json @@ -13,30 +13,30 @@ "kind": "parser", "name": "RustParser", "coordinates": [ - {"kind": "parser-action", "rule": "path_parent", "rule_index": 75, "index": null, "atn_state": 1623, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "lifetime_bound", "rule_index": 84, "index": null, "atn_state": 1705, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "ty_path_parent", "rule_index": 87, "index": null, "atn_state": 1739, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "bound", "rule_index": 95, "index": null, "atn_state": 1820, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "post_expr", "rule_index": 167, "index": null, "atn_state": 2948, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "cast_expr", "rule_index": 170, "index": null, "atn_state": 3019, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "mul_expr", "rule_index": 171, "index": null, "atn_state": 3033, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "add_expr", "rule_index": 172, "index": null, "atn_state": 3050, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "shift_expr", "rule_index": 173, "index": null, "atn_state": 3064, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "bit_and_expr", "rule_index": 174, "index": null, "atn_state": 3080, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "bit_xor_expr", "rule_index": 175, "index": null, "atn_state": 3091, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "bit_or_expr", "rule_index": 176, "index": null, "atn_state": 3102, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "and_expr", "rule_index": 178, "index": null, "atn_state": 3120, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "or_expr", "rule_index": 179, "index": null, "atn_state": 3131, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "post_expr_no_struct", "rule_index": 182, "index": null, "atn_state": 3173, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "cast_expr_no_struct", "rule_index": 184, "index": null, "atn_state": 3212, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "mul_expr_no_struct", "rule_index": 185, "index": null, "atn_state": 3226, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "add_expr_no_struct", "rule_index": 186, "index": null, "atn_state": 3243, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "shift_expr_no_struct", "rule_index": 187, "index": null, "atn_state": 3257, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "bit_and_expr_no_struct", "rule_index": 188, "index": null, "atn_state": 3273, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "bit_xor_expr_no_struct", "rule_index": 189, "index": null, "atn_state": 3284, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "bit_or_expr_no_struct", "rule_index": 190, "index": null, "atn_state": 3295, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "and_expr_no_struct", "rule_index": 192, "index": null, "atn_state": 3327, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, - {"kind": "parser-action", "rule": "or_expr_no_struct", "rule_index": 193, "index": null, "atn_state": 3338, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null} + {"kind": "parser-action", "rule": "path_parent", "rule_index": 75, "index": 0, "atn_state": 1623, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "lifetime_bound", "rule_index": 84, "index": 1, "atn_state": 1705, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "ty_path_parent", "rule_index": 87, "index": 2, "atn_state": 1739, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "bound", "rule_index": 95, "index": 3, "atn_state": 1820, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "post_expr", "rule_index": 167, "index": 4, "atn_state": 2948, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "cast_expr", "rule_index": 170, "index": 5, "atn_state": 3019, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "mul_expr", "rule_index": 171, "index": 6, "atn_state": 3033, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "add_expr", "rule_index": 172, "index": 7, "atn_state": 3050, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "shift_expr", "rule_index": 173, "index": 8, "atn_state": 3064, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "bit_and_expr", "rule_index": 174, "index": 9, "atn_state": 3080, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "bit_xor_expr", "rule_index": 175, "index": 10, "atn_state": 3091, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "bit_or_expr", "rule_index": 176, "index": 11, "atn_state": 3102, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "and_expr", "rule_index": 178, "index": 12, "atn_state": 3120, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "or_expr", "rule_index": 179, "index": 13, "atn_state": 3131, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "post_expr_no_struct", "rule_index": 182, "index": 14, "atn_state": 3173, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "cast_expr_no_struct", "rule_index": 184, "index": 15, "atn_state": 3212, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "mul_expr_no_struct", "rule_index": 185, "index": 16, "atn_state": 3226, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "add_expr_no_struct", "rule_index": 186, "index": 17, "atn_state": 3243, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "shift_expr_no_struct", "rule_index": 187, "index": 18, "atn_state": 3257, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "bit_and_expr_no_struct", "rule_index": 188, "index": 19, "atn_state": 3273, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "bit_xor_expr_no_struct", "rule_index": 189, "index": 20, "atn_state": 3284, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "bit_or_expr_no_struct", "rule_index": 190, "index": 21, "atn_state": 3295, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "and_expr_no_struct", "rule_index": 192, "index": 22, "atn_state": 3327, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "or_expr_no_struct", "rule_index": 193, "index": 23, "atn_state": 3338, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null} ] } ] diff --git a/src/lib.rs b/src/lib.rs index 68789c63..1ced1bd4 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -4,20 +4,22 @@ extern crate self as antlr4_runtime; /// Current generated-source/runtime contract revision emitted by the bundled generator. #[doc(hidden)] -pub const __ANTLR4_RUST_CODEGEN_API: u32 = 1; +pub const __ANTLR4_RUST_CODEGEN_API: u32 = 3; /// Verifies that generated source is compatible with the selected runtime. #[doc(hidden)] #[macro_export] macro_rules! __antlr4_rust_require_codegen_api { (1, $generator_version:literal) => {}; + (2, $generator_version:literal) => {}; + (3, $generator_version:literal) => {}; ($requested:literal, $generator_version:literal) => { compile_error!(concat!( "antlr4-rust generated-code API mismatch: antlr4-rust-gen v", $generator_version, " emitted generated-code API revision ", stringify!($requested), - ", but the selected antlr-rust-runtime supports revision 1; regenerate this \ + ", but the selected antlr-rust-runtime supports revisions 1, 2, and 3; regenerate this \ recognizer with a compatible antlr4-rust-gen or select a compatible \ antlr-rust-runtime dependency" )); diff --git a/src/parser.rs b/src/parser.rs index f961f9a8..d10af2a6 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -75,6 +75,7 @@ type FxHashSet = HashSet; use crate::atn::AtnStateKind; use crate::atn::parser::{ ParserAtnPrediction, ParserAtnPredictionDiagnosticKind, ParserAtnSimulator, + ParserAtnSimulatorError, ParserSemanticCandidate, }; use crate::atn::parser_atn::{ ParserAtn as Atn, ParserAtnState as AtnState, ParserIntervalSet, ParserTransition, @@ -86,6 +87,7 @@ use crate::char_stream::CharStream; use crate::errors::{AntlrError, SyntaxErrorEvent}; use crate::int_stream::IntStream; use crate::lexer::{LexerCustomAction, LexerLifecycleCtx, LexerSemCtx}; +use crate::prediction::SemanticContext; use crate::recognizer::{Recognizer, RecognizerData}; use crate::semir::{self, AStmt, ArithOp, CmpOp, ExprId, HookId, MemberEnv, PExpr, SemIr, StmtId}; use crate::token::{ @@ -405,6 +407,7 @@ const FAST_RECOGNIZER_DEFERRED_FILL_AT: usize = 64; pub struct ParserAction { source_state: usize, rule_index: usize, + action_index: Option, start_index: usize, stop_index: Option, rule_init: bool, @@ -422,6 +425,26 @@ impl ParserAction { Self { source_state, rule_index, + action_index: None, + start_index, + stop_index, + rule_init: false, + expected_state: None, + } + } + + /// Creates an indexed action event for a recognized parser path. + pub const fn new_indexed( + source_state: usize, + rule_index: usize, + action_index: usize, + start_index: usize, + stop_index: Option, + ) -> Self { + Self { + source_state, + rule_index, + action_index: Some(action_index), start_index, stop_index, rule_init: false, @@ -438,6 +461,7 @@ impl ParserAction { Self { source_state: usize::MAX, rule_index, + action_index: None, start_index, stop_index: None, rule_init: true, @@ -455,6 +479,11 @@ impl ParserAction { self.rule_index } + /// Stable source-order action index in the grammar. + pub const fn action_index(&self) -> Option { + self.action_index + } + /// Token-stream index where the active rule began. pub const fn start_index(&self) -> usize { self.start_index @@ -533,9 +562,8 @@ where self.rule_name.as_deref() } - /// Predicate/action index inside the owning rule. Parser actions keyed only - /// by ATN source state report `usize::MAX` here; use [`Self::action`] for - /// the stable action event. + /// Predicate/action index inside the owning rule. Legacy parser actions + /// without source-index metadata report `usize::MAX`. #[must_use] pub const fn coordinate_index(&self) -> usize { self.coordinate_index @@ -1190,8 +1218,15 @@ pub struct ParserSemantics { /// Optional generated-runtime metadata for metadata-driven parser execution. #[derive(Clone, Copy, Debug, Default)] pub struct ParserRuntimeOptions<'a> { - /// Rule indexes whose `@init` actions should be replayed. + /// Rule indexes whose `@init` actions should run at rule entry or be + /// returned for legacy replay when no semantic hook handles them. pub init_action_rules: &'a [usize], + /// Stable parser-action indexes keyed by authored ATN source state. + /// + /// A non-empty table selects committed interpreted execution: mapped + /// actions run at their grammar position instead of being replayed after + /// the complete rule has been recognized. + pub action_indices: &'a [(usize, usize)], /// Whether generated parse-tree contexts should retain alternative numbers. pub track_alt_numbers: bool, /// Whether generated typed contexts should retain private dispatch alternatives. @@ -4201,6 +4236,7 @@ fn atn_has_predicate_transitions(atn: &Atn) -> bool { /// same runtime context. fn can_use_fast_predicate_recognizer(atn: &Atn, options: &ParserRuntimeOptions<'_>) -> bool { options.init_action_rules.is_empty() + && options.action_indices.is_empty() && !options.track_alt_numbers && options .predicates @@ -4271,6 +4307,7 @@ struct EpsilonActionStep { source_state: usize, target: usize, action_rule_index: Option, + action_index: Option, left_recursive_boundary: Option, decision: Option, decision_start_index: Option, @@ -4816,6 +4853,32 @@ where steps: usize, } +struct CommittedAtnParser<'atn, 'sim, 'options, S, H = NoSemanticHooks> +where + S: TokenSource, + H: SemanticHooks, +{ + parser: &'sim mut BaseParser, + atn: &'atn Atn, + simulator: ParserAtnSimulator<'atn>, + options: ParserRuntimeOptions<'options>, + decision_by_state: Vec>, + action_index_by_state: FxHashMap, + deferred_actions: Vec, +} + +struct CommittedRuleOutcome { + tree: ParseTree, + consumed_eof: bool, +} + +struct CommittedDecisionContext<'a> { + precedence: i32, + local_int_arg: Option<(usize, i64)>, + context: &'a mut ParserRuleContext, + entered_loops: &'a mut BTreeSet, +} + /// Outcome of a generated token / set / not-set match that may recover. /// /// Generated parsers append `children` to the current rule context. `consumed_eof` @@ -7129,14 +7192,89 @@ where ParserAction::new(source_state, rule_index, start_index, stop_index) } + /// Builds an indexed generated parser-action event at the current input position. + pub fn parser_action_at_current_indexed( + &mut self, + source_state: usize, + rule_index: usize, + action_index: usize, + start_index: usize, + consumed_eof: bool, + ) -> ParserAction { + let stop_index = self.rule_stop_token_index(self.input.index(), consumed_eof); + ParserAction::new_indexed( + source_state, + rule_index, + action_index, + start_index, + stop_index, + ) + } + /// Offers a committed parser action event to the user semantic hook. /// /// Generated parsers call this for action source states that were present /// in the ATN but not translated into a built-in Rust action template. pub fn parser_action_hook(&mut self, action: ParserAction, tree: ParseTree) -> bool { + self.parser_action_hook_inner(action, None, Some(tree), None, true) + } + + /// Offers an action to semantic hooks at its committed grammar position. + /// + /// The current rule context contains children completed before the action; + /// the full rule tree is not available until the rule returns. + pub fn parser_action_hook_with_context( + &mut self, + action: ParserAction, + context: &ParserRuleContext, + ) -> bool { + self.parser_action_hook_inner(action, Some(context), None, None, true) + } + + /// Offers an action with the current generated rule's integer argument. + /// + /// Generated parameterized rules use the same integer carrier as generated + /// predicate evaluation. The context exposes it through + /// [`ParserSemCtx::local_int_arg`]. + pub fn parser_action_hook_with_context_and_local( + &mut self, + action: ParserAction, + context: &ParserRuleContext, + local_int_arg: i32, + ) -> bool { + self.parser_action_hook_inner( + action, + Some(context), + None, + Some((action.rule_index(), i64::from(local_int_arg))), + true, + ) + } + + /// Offers a rule-init action at rule entry while preserving legacy replay. + /// + /// A declined init is returned to the generated caller, so it is not an + /// unhandled action yet and must not trip the fail-loud policy here. + fn parser_rule_init_hook_with_context( + &mut self, + action: ParserAction, + context: &ParserRuleContext, + local_int_arg: Option<(usize, i64)>, + ) -> bool { + debug_assert!(action.is_rule_init()); + self.parser_action_hook_inner(action, Some(context), None, local_int_arg, false) + } + + fn parser_action_hook_inner( + &mut self, + action: ParserAction, + context: Option<&ParserRuleContext>, + tree: Option, + local_int_arg: Option<(usize, i64)>, + record_unhandled: bool, + ) -> bool { let rule_index = action.rule_index(); let rule_name = self.rule_names().get(rule_index).cloned(); - let context = None; let input = &mut self.input; let semantic_hooks = &mut self.semantic_hooks; let member_values = &self.int_members; @@ -7144,11 +7282,11 @@ where input, tree_storage: &self.tree, rule_index, - coordinate_index: usize::MAX, + coordinate_index: action.action_index().unwrap_or(usize::MAX), rule_name, context, - tree: Some(tree), - local_int_arg: None, + tree, + local_int_arg, member_values, action: Some(action), }; @@ -7158,7 +7296,10 @@ where // committed action is silently dropped — record it so the parse entry // can fail loud under the fail-loud boundary, mirroring unknown // predicates. `assume-*` policies opt out of the fail-loud recording. - if !handled && matches!(self.unknown_predicate_policy, UnknownSemanticPolicy::Error) { + if record_unhandled + && !handled + && matches!(self.unknown_predicate_policy, UnknownSemanticPolicy::Error) + { let coordinate = (rule_index, action.source_state()); if !self.unhandled_action_hits.contains(&coordinate) { self.unhandled_action_hits.push(coordinate); @@ -7807,6 +7948,70 @@ where self.parse_atn_rule_with_runtime_options_and_precedence(atn, rule_index, 0, options) } + fn parse_atn_rule_committed_with_runtime_options( + &mut self, + atn: &Atn, + rule_index: usize, + precedence: i32, + options: ParserRuntimeOptions<'_>, + ) -> Result<(ParseTree, Vec), AntlrError> { + let top_level_entry = self.is_top_level_entry(); + self.unknown_predicate_policy = options.unknown_predicate_policy; + let prior_unknown_predicate_hits = std::mem::take(&mut self.unknown_predicate_hits); + let prior_unhandled_action_hits = std::mem::take(&mut self.unhandled_action_hits); + self.clear_prediction_diagnostics(); + self.reset_per_parse_caches(); + self.reset_recognition_arena(); + + let mut decision_by_state = vec![None; atn.states().len()]; + for (decision, state_number) in atn.decision_to_state().iter().enumerate() { + if let Some(slot) = decision_by_state.get_mut(state_number) { + *slot = Some(decision); + } + } + let mut action_index_by_state = FxHashMap::default(); + for &(state, index) in options.action_indices { + action_index_by_state.entry(state).or_insert(index); + } + let mut simulator = ParserAtnSimulator::new(atn); + simulator.set_track_prediction_rule_calls(!options.rule_args.is_empty()); + let (result, deferred_actions) = { + let mut committed = CommittedAtnParser { + parser: self, + atn, + simulator, + options, + decision_by_state, + action_index_by_state, + deferred_actions: Vec::new(), + }; + let result = committed.parse_rule(rule_index, precedence, None, None); + (result, committed.deferred_actions) + }; + + if top_level_entry { + self.report_generated_parser_diagnostics(); + } + let semantic_error = self.unknown_semantic_error(); + self.restore_prior_unknown_predicate_hits(prior_unknown_predicate_hits); + self.restore_prior_unhandled_action_hits(prior_unhandled_action_hits); + if top_level_entry && let Some(error) = self.take_parse_abort() { + self.reset_unknown_semantic_hits(); + return Err(error); + } + if let Some(error) = semantic_error { + if top_level_entry { + self.reset_unknown_semantic_hits(); + } + return Err(error); + } + let result = result.map(|outcome| (outcome.tree, deferred_actions)); + if top_level_entry && let Err(error) = &result { + self.report_unrecovered_parser_error(error); + } + result + } + /// Parses a generated rule with action replay, parser predicate support, /// and an initial left-recursive precedence threshold. pub fn parse_atn_rule_with_runtime_options_and_precedence( @@ -7816,6 +8021,11 @@ where precedence: i32, options: ParserRuntimeOptions<'_>, ) -> Result<(ParseTree, Vec), AntlrError> { + if !options.action_indices.is_empty() { + return self.parse_atn_rule_committed_with_runtime_options( + atn, rule_index, precedence, options, + ); + } let report_unrecovered_error = self.is_top_level_entry(); let ParserRuntimeOptions { init_action_rules, @@ -7827,6 +8037,7 @@ where member_actions, return_actions, unknown_predicate_policy, + .. } = options; let capture_alt_numbers = track_alt_numbers || track_context_alt_numbers; if init_action_rules.is_empty() @@ -10288,9 +10499,13 @@ where let transition_data = transition.data(); match &transition_data { Transition::Epsilon { target } | Transition::Action { target, .. } => { - let action_rule_index = match &transition_data { - Transition::Action { rule_index, .. } => Some(*rule_index), - _ => None, + let (action_rule_index, action_index) = match &transition_data { + Transition::Action { + rule_index, + action_index, + .. + } => (Some(*rule_index), *action_index), + _ => (None, None), }; outcomes.extend(self.recognize_epsilon_or_action_step( atn, @@ -10299,6 +10514,7 @@ where source_state: state_number, target: *target, action_rule_index, + action_index, left_recursive_boundary: left_recursive_boundary(atn, state, *target), decision, decision_start_index: next_decision_start_index, @@ -10759,11 +10975,25 @@ where expected, } = scratch; let action = step.action_rule_index.map(|rule_index| { - ParserAction::new( - step.source_state, - rule_index, - request.rule_start_index, - self.rule_stop_token_index(request.index, request.consumed_eof), + let stop_index = self.rule_stop_token_index(request.index, request.consumed_eof); + step.action_index.map_or_else( + || { + ParserAction::new( + step.source_state, + rule_index, + request.rule_start_index, + stop_index, + ) + }, + |action_index| { + ParserAction::new_indexed( + step.source_state, + rule_index, + action_index, + request.rule_start_index, + stop_index, + ) + }, ) }); let next_member_values = if action.is_some() { @@ -11499,6 +11729,21 @@ where self.unknown_predicate_hits = merged; } + /// Re-inserts unhandled action coordinates recorded before a nested + /// committed parse so only that child parse's misses affect its result. + fn restore_prior_unhandled_action_hits(&mut self, prior: Vec<(usize, usize)>) { + if prior.is_empty() { + return; + } + let mut merged = prior; + for coordinate in std::mem::take(&mut self.unhandled_action_hits) { + if !merged.contains(&coordinate) { + merged.push(coordinate); + } + } + self.unhandled_action_hits = merged; + } + /// Applies the active [`UnknownSemanticPolicy`] to a predicate coordinate /// that has no entry in the generated predicate table. /// @@ -12305,633 +12550,1433 @@ where } } -/// Detects the loop edge where ANTLR would call `pushNewRecursionContext` for a -/// transformed left-recursive rule. -fn left_recursive_boundary(atn: &Atn, state: AtnState<'_>, target: usize) -> Option { - if !state.precedence_rule_decision() { - return None; - } - let target_state = atn.state(target)?; - if target_state.kind() == AtnStateKind::LoopEnd { - return None; - } - state.rule_index() -} +impl CommittedAtnParser<'_, '_, '_, S, H> +where + S: TokenSource, + H: SemanticHooks, +{ + fn parse_rule( + &mut self, + rule_index: usize, + precedence: i32, + inherited_local_int_arg: Option<(usize, i64)>, + init_expected_state: Option, + ) -> Result { + let start_state = self + .atn + .rule_to_start_state() + .get(rule_index) + .ok_or_else(|| { + AntlrError::Unsupported(format!("rule {rule_index} has no start state")) + })?; + let stop_state = self + .atn + .rule_to_stop_state() + .get(rule_index) + .filter(|state| *state != usize::MAX) + .ok_or_else(|| { + AntlrError::Unsupported(format!("rule {rule_index} has no stop state")) + })?; + let left_recursive = self + .atn + .state(start_state) + .is_some_and(AtnState::left_recursive_rule); + if let Some(error) = self.parser.rule_depth_cap_violation() { + return Err(error); + } + if let Some(error) = self.parser.parse_listener_enter_rule(rule_index) { + return Err(error); + } + let mut context = if left_recursive { + self.parser.enter_recursion_rule( + invoking_state_number(start_state), + rule_index, + precedence, + ) + } else { + self.parser + .enter_rule(invoking_state_number(start_state), rule_index) + }; + let rule_start_index = self.parser.current_visible_index(); + let local_int_arg = + usize::try_from(context.invoking_state()) + .ok() + .and_then(|source_state| { + rule_local_int_arg( + self.options.rule_args, + source_state, + rule_index, + inherited_local_int_arg, + ) + }); + if self.options.init_action_rules.contains(&rule_index) { + let action = ParserAction::new_rule_init( + rule_index, + rule_start_index, + init_expected_state.or(Some(start_state)), + ); + if !self + .parser + .parser_rule_init_hook_with_context(action, &context, local_int_arg) + { + self.deferred_actions.push(action); + } + } + let mut consumed_eof = false; + let result = self.walk_rule( + rule_index, + start_state, + stop_state, + precedence, + rule_start_index, + local_int_arg, + left_recursive, + &mut context, + &mut consumed_eof, + ); -/// Selects the first outer alternative observed for a rule path. -/// -/// ANTLR's alt-numbered tree contexts store the rule alternative chosen at the -/// outer decision. The metadata recognizer only needs this when a generated -/// grammar opts into that target template; otherwise the value remains `0` and -/// parse-tree rendering is unchanged. -fn next_alt_number( - state: AtnState<'_>, - transition_count: usize, - transition_index: usize, - current_alt_number: usize, - track_alt_numbers: bool, -) -> usize { - if !track_alt_numbers || current_alt_number != 0 || transition_count <= 1 { - return current_alt_number; - } - if matches!( - state.kind(), - AtnStateKind::Basic - | AtnStateKind::BlockStart - | AtnStateKind::PlusBlockStart - | AtnStateKind::StarBlockStart - | AtnStateKind::StarLoopEntry - ) && !state.precedence_rule_decision() - { - return transition_index + 1; + let result = match result { + Ok(()) => Ok(if left_recursive { + self.parser.finish_recursion_rule(context, consumed_eof) + } else { + self.parser.finish_rule(context, consumed_eof) + }), + Err(error) if self.parser.bail_on_error() => { + if left_recursive { + self.parser.unroll_recursion_context(); + } else { + self.parser.exit_rule(); + } + Err(error) + } + Err(error) => { + self.parser + .recover_generated_rule(&mut context, self.atn, error); + Ok(if left_recursive { + self.parser.finish_recursion_rule(context, consumed_eof) + } else { + self.parser.finish_rule(context, consumed_eof) + }) + } + }; + self.parser.parse_listener_exit_rule(rule_index); + result.map(|tree| CommittedRuleOutcome { tree, consumed_eof }) } - current_alt_number -} -/// Converts an ATN state number into the signed invoking-state slot used by -/// ANTLR parse-tree contexts, saturating only for impossible platform widths. -fn invoking_state_number(state_number: usize) -> isize { - isize::try_from(state_number).unwrap_or(isize::MAX) -} + #[allow(clippy::too_many_arguments)] + fn walk_rule( + &mut self, + rule_index: usize, + mut state_number: usize, + stop_state: usize, + precedence: i32, + rule_start_index: usize, + local_int_arg: Option<(usize, i64)>, + left_recursive: bool, + context: &mut ParserRuleContext, + consumed_eof: &mut bool, + ) -> Result<(), AntlrError> { + let mut entered_loops = BTreeSet::new(); + let mut visited_coordinates = FxHashSet::default(); + let mut guarded_input_index = self.parser.input.index(); + while state_number != stop_state { + let input_index = self.parser.input.index(); + if input_index != guarded_input_index { + visited_coordinates.clear(); + guarded_input_index = input_index; + } + if !visited_coordinates.insert((state_number, input_index)) { + return Err(AntlrError::Unsupported(format!( + "committed parser encountered a non-consuming ATN cycle at state \ + {state_number}" + ))); + } + let state = self.atn.state(state_number).ok_or_else(|| { + AntlrError::Unsupported(format!("missing parser ATN state {state_number}")) + })?; + if state.is_rule_stop() { + return Err(AntlrError::Unsupported(format!( + "rule {rule_index} reached unexpected stop state {state_number}" + ))); + } + let transition_index = { + let mut decision_context = CommittedDecisionContext { + precedence, + local_int_arg, + context, + entered_loops: &mut entered_loops, + }; + self.transition_index(state, &mut decision_context)? + }; + let transition = state.transitions().get(transition_index).ok_or_else(|| { + AntlrError::Unsupported(format!( + "missing transition {transition_index} from parser ATN state {state_number}" + )) + })?; -const fn packed_i32(value: u32) -> i32 { - i32::from_le_bytes(value.to_le_bytes()) -} - -fn direct_precedence(precedence: i32) -> usize { - usize::try_from(precedence.max(0)).unwrap_or_default() -} - -fn token_input_display(token: &impl Token) -> String { - format!("'{}'", token.text().unwrap_or("")) -} + let next_alt = next_alt_number( + state, + state.transitions().len(), + transition_index, + context.alt_number(), + self.options.track_alt_numbers, + ); + if self.options.track_alt_numbers && context.alt_number() == 0 && next_alt != 0 { + context.set_alt_number(next_alt); + } + let next_context_alt = next_alt_number( + state, + state.transitions().len(), + transition_index, + context.context_alt_number(), + self.options.track_context_alt_numbers, + ); + if self.options.track_context_alt_numbers + && context.context_alt_number() == 0 + && next_context_alt != 0 + { + context.set_context_alt_number(next_context_alt); + } -fn display_input_text(text: &str) -> String { - let mut out = String::new(); - for ch in text.chars() { - match ch { - '\n' => out.push_str("\\n"), - '\r' => out.push_str("\\r"), - '\t' => out.push_str("\\t"), - other => out.push(other), + if left_recursive + && left_recursive_boundary(self.atn, state, transition.target()).is_some() + { + if let Some(error) = self.parser.rule_depth_cap_violation() { + return Err(error); + } + self.parser.parse_listener_exit_rule(rule_index); + self.parser.push_new_recursion_context_with_previous( + invoking_state_number( + self.atn + .rule_to_start_state() + .get(rule_index) + .unwrap_or(state_number), + ), + rule_index, + context, + ); + if let Some(error) = self.parser.parse_listener_enter_rule(rule_index) { + return Err(error); + } + } + state_number = self.apply_transition( + state_number, + transition, + precedence, + rule_start_index, + local_int_arg, + context, + consumed_eof, + )?; } + Ok(()) } - out -} -fn diagnostic_for_token(token: Option, message: String) -> ParserDiagnostic { - let (line, column, offending) = token.map_or((0, 0, None), |token| { - (token.line(), token.column(), Some(token.token_id())) - }); - ParserDiagnostic { - line, - column, - message, - offending, - } -} + fn transition_index( + &mut self, + state: AtnState<'_>, + decision_context: &mut CommittedDecisionContext<'_>, + ) -> Result { + let transition_count = state.transitions().len(); + if transition_count == 1 { + return Ok(0); + } + let Some(decision) = self + .decision_by_state + .get(state.state_number()) + .copied() + .flatten() + else { + return Err(AntlrError::Unsupported(format!( + "parser ATN state {} has {transition_count} transitions but is not a decision", + state.state_number() + ))); + }; -fn expected_symbols_display(symbols: &BTreeSet, vocabulary: &Vocabulary) -> String { - expected_symbols_display_iter(symbols.iter().copied(), vocabulary) -} + let decision_start = self.parser.input.index(); + let overridden_transition = if self.parser.semantic_hooks.observes_parser_decisions() { + self.parser + .semantic_hooks + .parser_decision_override(decision, decision_start, transition_count) + .and_then(|alternative| alternative.checked_sub(1)) + .filter(|alternative| *alternative < transition_count) + } else { + None + }; + if let Some(selected) = overridden_transition { + self.update_loop_selection(state, selected, decision_context); + return Ok(selected); + } + + if !state.precedence_rule_decision() { + let loop_back = match state.kind() { + AtnStateKind::PlusLoopBack | AtnStateKind::StarLoopBack => true, + AtnStateKind::StarLoopEntry => decision_context + .entered_loops + .contains(&state.state_number()), + _ => false, + }; + let children = self.parser.sync_decision( + self.atn, + state.state_number(), + !decision_context.context.has_matched_child(), + loop_back, + )?; + for child in children { + self.parser.add_parse_child(decision_context.context, child); + } + } -fn expected_symbols_display_iter( - symbols: impl IntoIterator, - vocabulary: &Vocabulary, -) -> String { - let items = symbols - .into_iter() - .map(|symbol| expected_symbol_display(symbol, vocabulary)) - .collect::>(); - if let [single] = items.as_slice() { - return single.clone(); - } - format!("{{{}}}", items.join(", ")) -} + let prediction_precedence = if state.precedence_rule_decision() { + usize::try_from(decision_context.precedence.max(0)).unwrap_or_default() + } else { + 0 + }; + let prediction_context = { + let return_states = self + .parser + .prediction_context_return_states(self.atn) + .collect::>(); + self.simulator + .intern_prediction_context(self.parser.rule_context_version(), return_states) + }; + self.simulator.set_exact_ambig_detection( + self.parser.prediction_mode() == PredictionMode::LlExactAmbigDetection, + ); + let prediction_mode = self.parser.prediction_mode(); + let prediction = match self.simulator.adaptive_predict_stream_info_sll_probe( + decision, + prediction_precedence, + &mut self.parser.input, + ) { + Ok(prediction) + if prediction.requires_full_context && prediction_mode != PredictionMode::Sll => + { + self.simulator.adaptive_predict_stream_info_with_context( + decision, + prediction_precedence, + &mut self.parser.input, + prediction_context, + ) + } + prediction => prediction, + }; + let mut prediction = match prediction { + Ok(prediction) => prediction, + Err(ParserAtnSimulatorError::NoViableAlt { index, .. }) + if state.precedence_rule_decision() => + { + let enter_alt = state.transitions().iter().position(|transition| { + self.atn + .state(transition.target()) + .is_some_and(|target| target.kind() != AtnStateKind::LoopEnd) + }); + let exit_alt = state.transitions().iter().position(|transition| { + self.atn + .state(transition.target()) + .is_some_and(|target| target.kind() == AtnStateKind::LoopEnd) + }); + let selected = if self.parser.left_recursive_loop_enter_matches( + self.atn, + state.state_number(), + decision_context.precedence, + ) { + enter_alt + } else { + exit_alt + }; + let Some(selected) = selected else { + return Err(self + .parser + .no_viable_alternative_error_at(decision_start, index)); + }; + ParserAtnPrediction { + alt: selected + 1, + requires_full_context: true, + has_semantic_context: true, + diagnostic: None, + } + } + Err(ParserAtnSimulatorError::NoViableAlt { index, .. }) => { + return Err(self + .parser + .no_viable_alternative_error_at(decision_start, index)); + } + Err(ParserAtnSimulatorError::PredictionRequiresMoreLookahead) => { + return Err(self.parser.no_viable_alternative_error(decision_start)); + } + Err(error) => { + return Err(AntlrError::Unsupported(format!( + "committed parser prediction failed at decision {decision}: {error:?}" + ))); + } + }; + let mut selected = prediction + .alt + .checked_sub(1) + .filter(|index| *index < transition_count) + .ok_or_else(|| self.parser.no_viable_alternative_error(decision_start))?; + + let semantic_candidates = self.simulator.prediction_semantic_candidates(); + if !semantic_candidates.is_empty() { + let predicted_alt = prediction.alt; + let mut semantic_results = BTreeMap::new(); + let selected_alt = selected + 1; + let selected_matches = self.semantic_alternative_matches( + selected_alt, + decision_context, + &semantic_candidates, + ); + semantic_results.insert(selected_alt, selected_matches); + if !selected_matches { + let alternatives = semantic_candidates + .iter() + .map(|candidate| candidate.alt) + .filter(|alternative| *alternative != 0 && *alternative <= transition_count) + .collect::>(); + selected = alternatives + .into_iter() + .find(|alternative| { + let matches = self.semantic_alternative_matches( + *alternative, + decision_context, + &semantic_candidates, + ); + semantic_results.insert(*alternative, matches); + matches + }) + .and_then(|alternative| alternative.checked_sub(1)) + .ok_or_else(|| self.parser.no_viable_alternative_error(decision_start))?; + } + if self.parser.report_diagnostic_errors + && let Some(diagnostic) = prediction.diagnostic.as_ref() + { + for alternative in diagnostic.conflicting_alts.clone() { + if semantic_results.contains_key(&alternative) + || !semantic_candidates + .iter() + .any(|candidate| candidate.alt == alternative) + { + continue; + } + let matches = self.semantic_alternative_matches( + alternative, + decision_context, + &semantic_candidates, + ); + semantic_results.insert(alternative, matches); + } + } + Self::filter_prediction_diagnostic( + &mut prediction, + predicted_alt, + selected + 1, + &semantic_results, + ); + } + self.parser.record_generated_prediction_diagnostic( + self.atn, + state.state_number(), + &prediction, + ); -fn expected_symbol_display(symbol: i32, vocabulary: &Vocabulary) -> String { - if symbol == TOKEN_EOF { - return "".to_owned(); + self.update_loop_selection(state, selected, decision_context); + Ok(selected) } - vocabulary.display_name(symbol) -} -fn caller_follow_token_info_for_stream( - input: &mut CommonTokenStream, - index: usize, -) -> (i32, bool, bool) { - // Generated callers own statement separators; leave them available when - // an interpreted child rule can either stop before or consume one. - if index >= FAST_RECOGNIZER_DEFERRED_FILL_AT && !input.is_filled() { - input.fill(); + fn semantic_alternative_matches( + &mut self, + alternative: usize, + decision_context: &CommittedDecisionContext<'_>, + candidates: &[ParserSemanticCandidate], + ) -> bool { + candidates + .iter() + .filter(|candidate| candidate.alt == alternative) + .any(|candidate| { + self.semantic_context_matches(&candidate.context, decision_context, candidate) + }) } - let token_type = input.token_type_at_index(index); - let visible_channel = input.channel(); - let token = input.get(index); - let is_boundary = token - .as_ref() - .and_then(Token::text) - .is_some_and(is_caller_follow_boundary_text); - let is_boundary_gap = token.as_ref().is_some_and(|token| { - token.channel() != visible_channel - || is_caller_follow_boundary_gap_text(token.text_or_empty()) - }); - (token_type, is_boundary, is_boundary_gap) -} -fn is_caller_follow_boundary_text(text: &str) -> bool { - text.chars().any(|ch| ch == ';' || ch == '\n') - && text.chars().all(|ch| ch.is_whitespace() || ch == ';') -} + fn filter_prediction_diagnostic( + prediction: &mut ParserAtnPrediction, + predicted_alt: usize, + selected_alt: usize, + semantic_results: &BTreeMap, + ) { + prediction.alt = selected_alt; + if selected_alt != predicted_alt { + prediction.diagnostic = None; + return; + } + if let Some(diagnostic) = prediction.diagnostic.as_mut() { + diagnostic + .conflicting_alts + .retain(|alternative| semantic_results.get(alternative).copied().unwrap_or(true)); + if diagnostic.conflicting_alts.len() < 2 { + prediction.diagnostic = None; + } + } + } -fn is_caller_follow_boundary_gap_text(text: &str) -> bool { - text.chars().all(|ch| ch.is_whitespace() || ch == ';') -} + fn semantic_context_matches( + &mut self, + semantic_context: &SemanticContext, + decision_context: &CommittedDecisionContext<'_>, + candidate: &ParserSemanticCandidate, + ) -> bool { + match semantic_context { + SemanticContext::None => true, + SemanticContext::Predicate { + rule_index, + pred_index, + .. + } => { + let mut matched_provenance = false; + for predicate_call in candidate + .predicate_calls + .iter() + .filter(|call| call.rule_index == *rule_index && call.pred_index == *pred_index) + { + matched_provenance = true; + let mut local_int_arg = decision_context.local_int_arg; + for rule_call in &predicate_call.rule_calls { + local_int_arg = rule_local_int_arg( + self.options.rule_args, + rule_call.source_state, + rule_call.rule_index, + local_int_arg, + ); + } + if !self.semantic_predicate_matches( + *rule_index, + *pred_index, + decision_context, + local_int_arg, + ) { + return false; + } + } + if matched_provenance { + true + } else { + self.semantic_predicate_matches( + *rule_index, + *pred_index, + decision_context, + decision_context.local_int_arg, + ) + } + } + SemanticContext::Precedence { precedence } => { + *precedence >= decision_context.precedence + } + SemanticContext::And(children) => { + for child in children { + if !self.semantic_context_matches(child, decision_context, candidate) { + return false; + } + } + true + } + SemanticContext::Or(children) => { + for child in children { + if self.semantic_context_matches(child, decision_context, candidate) { + return true; + } + } + false + } + } + } -/// Returns whether `state` belongs to an ANTLR-transformed left-recursive rule. -/// Inline insertion in those precedence loops can synthesize a missing operand -/// before an operator and then block the legitimate loop-exit path. -fn state_is_left_recursive_rule(atn: &Atn, state: AtnState<'_>) -> bool { - let Some(rule_index) = state.rule_index() else { - return false; - }; - atn.rule_to_start_state() - .get(rule_index) - .and_then(|state_number| atn.state(state_number)) - .is_some_and(AtnState::left_recursive_rule) -} + fn semantic_predicate_matches( + &mut self, + rule_index: usize, + pred_index: usize, + decision_context: &CommittedDecisionContext<'_>, + local_int_arg: Option<(usize, i64)>, + ) -> bool { + let member_values = self.parser.int_members.clone(); + self.parser.parser_predicate_matches(PredicateEval { + index: self.parser.input.index(), + rule_index, + pred_index, + predicates: self.options.predicates, + semantics: self.options.semantics, + context: Some(&*decision_context.context), + local_int_arg, + member_values: &member_values, + }) + } -/// Picks the better of two `parse_atn_rule` passes (with and without the -/// FIRST-set prefilter). A clean outcome (no diagnostics) always wins over a -/// recovered one; among recovered outcomes the second pass is preferred -/// because the no-prefilter walk reaches ANTLR-style recovery inside child -/// rules. If both passes failed, the second pass's expected-token snapshot -/// is returned so the caller renders the same diagnostic ANTLR would. -fn select_better_top_outcome( - first: Result<(FastRecognizeOutcome, ExpectedTokens, usize), ExpectedTokens>, - second: Result<(FastRecognizeOutcome, ExpectedTokens, usize), ExpectedTokens>, - arena: &RecognitionArena, -) -> Result<(FastRecognizeOutcome, ExpectedTokens, usize), ExpectedTokens> { - match (first, second) { - (Ok(first), Ok(second)) => { - if arena.diagnostics(first.0.diagnostics).next().is_none() { - Ok(first) + fn update_loop_selection( + &self, + state: AtnState<'_>, + selected: usize, + decision_context: &mut CommittedDecisionContext<'_>, + ) { + if state.kind() == AtnStateKind::StarLoopEntry { + let enters = self + .atn + .state( + state + .transitions() + .get(selected) + .expect("selected transition is in bounds") + .target(), + ) + .is_some_and(|target| target.kind() != AtnStateKind::LoopEnd); + if enters { + decision_context.entered_loops.insert(state.state_number()); } else { - Ok(second) + decision_context.entered_loops.remove(&state.state_number()); } } - (Ok(first), Err(_)) => Ok(first), - (Err(_), Ok(second)) => Ok(second), - (Err(_), Err(second_expected)) => Err(second_expected), } -} -/// Chooses the outermost parse result that consumed the most input. -/// -/// The recognizer intentionally keeps shorter endpoints available while walking -/// nested rule transitions so callers can satisfy following tokens such as -/// `expr 'and' expr`. Only the public rule entry commits to one endpoint. -fn select_best_fast_outcome( - outcomes: impl Iterator, - prediction_mode: PredictionMode, - caller_follow: Option<&TokenBitSet>, - mut token_info_at: impl FnMut(usize) -> (i32, bool, bool), - arena: &RecognitionArena, -) -> Option { - let mut best = None; - let mut best_caller_follow = None; - for outcome in outcomes { - if matches!( - prediction_mode, - PredictionMode::Ll | PredictionMode::LlExactAmbigDetection - ) && outcome.diagnostics.is_empty() - && let Some(follow) = caller_follow - { - let (token_type, is_boundary, _) = token_info_at(outcome.index); - if is_boundary && follow.contains(token_type) { - let replace = - best_caller_follow - .as_ref() - .is_none_or(|existing: &FastRecognizeOutcome| { - (outcome.index, outcome.consumed_eof) - < (existing.index, existing.consumed_eof) - }); - if replace { - best_caller_follow = Some(outcome); + #[allow(clippy::too_many_arguments)] + fn apply_transition( + &mut self, + source_state: usize, + transition: ParserTransition<'_>, + precedence: i32, + rule_start_index: usize, + local_int_arg: Option<(usize, i64)>, + context: &mut ParserRuleContext, + consumed_eof: &mut bool, + ) -> Result { + self.parser.set_state(invoking_state_number(source_state)); + match transition.data() { + Transition::Epsilon { target } => Ok(target), + Transition::Atom { target, label } => { + let matched = self + .parser + .match_token_recovering(label, target, self.atn)?; + *consumed_eof |= matched.consumed_eof(); + for child in matched.into_child_iter() { + self.parser.add_parse_child(context, child); + } + Ok(target) + } + Transition::Range { + target, + start, + stop, + } => { + let matched = + self.parser + .match_set_recovering(&[(start, stop)], target, self.atn)?; + *consumed_eof |= matched.consumed_eof(); + for child in matched.into_child_iter() { + self.parser.add_parse_child(context, child); + } + Ok(target) + } + Transition::Set { target, set } => { + let matched = self + .parser + .match_token_set_recovering(set, target, self.atn)?; + *consumed_eof |= matched.consumed_eof(); + for child in matched.into_child_iter() { + self.parser.add_parse_child(context, child); + } + Ok(target) + } + Transition::NotSet { target, set } => { + let matched = self.parser.match_not_token_set_recovering( + set, + 1, + self.atn.max_token_type(), + target, + self.atn, + )?; + *consumed_eof |= matched.consumed_eof(); + for child in matched.into_child_iter() { + self.parser.add_parse_child(context, child); + } + Ok(target) + } + Transition::Wildcard { target } => { + let matched = self.parser.match_not_set_recovering( + &[], + 1, + self.atn.max_token_type(), + target, + self.atn, + )?; + *consumed_eof |= matched.consumed_eof(); + for child in matched.into_child_iter() { + self.parser.add_parse_child(context, child); + } + Ok(target) + } + Transition::Rule { + rule_index, + follow_state, + precedence: rule_precedence, + .. + } => { + let marker = self + .parser + .push_invoking_state(invoking_state_number(source_state)); + let child = if self.parser.generated_rule_stack_check_due() { + grow_generated_rule_stack(|| { + self.parse_rule( + rule_index, + rule_precedence, + local_int_arg, + Some(follow_state), + ) + }) + } else { + self.parse_rule( + rule_index, + rule_precedence, + local_int_arg, + Some(follow_state), + ) + }; + self.parser.discard_invoking_state(marker); + let child = child?; + *consumed_eof |= child.consumed_eof; + self.parser.add_parse_child(context, child.tree); + Ok(follow_state) + } + Transition::Predicate { + target, + rule_index, + pred_index, + .. + } => { + let member_values = self.parser.int_members.clone(); + if self.parser.parser_predicate_matches(PredicateEval { + index: self.parser.input.index(), + rule_index, + pred_index, + predicates: self.options.predicates, + semantics: self.options.semantics, + context: Some(context), + local_int_arg, + member_values: &member_values, + }) { + return Ok(target); + } + if let Some(message) = self + .options + .semantics + .and_then(|semantics| { + self.parser.parser_semantic_ir_predicate_failure_message( + rule_index, pred_index, semantics, + ) + }) + .or_else(|| { + self.parser.parser_predicate_failure_message( + rule_index, + pred_index, + self.options.predicates, + ) + }) + { + return Err(self + .parser + .failed_predicate_option_error(rule_index, message)); + } + Err(self.parser.failed_predicate_error("semantic predicate")) + } + Transition::Action { + target, rule_index, .. + } => { + self.apply_translated_actions(source_state, rule_index, context); + if let Some(action_index) = self.action_index(source_state) { + let action = self.parser.parser_action_at_current_indexed( + source_state, + rule_index, + action_index, + rule_start_index, + *consumed_eof, + ); + let _ = self.parser.parser_action_hook_inner( + action, + Some(context), + None, + local_int_arg, + true, + ); + } + Ok(target) + } + Transition::Precedence { + target, + precedence: transition_precedence, + } => { + if transition_precedence >= precedence { + Ok(target) + } else { + Err(self + .parser + .failed_predicate_error(format!("precpred(_ctx, {transition_precedence})"))) } } } - let Some(existing) = best else { - best = Some(outcome); - continue; - }; - let outcome_position = (outcome.index, outcome.consumed_eof); - let best_position = (existing.index, existing.consumed_eof); - let better = match prediction_mode { - PredictionMode::Ll | PredictionMode::LlExactAmbigDetection => outcome_is_better( - outcome_position, - outcome.diagnostics, - best_position, - existing.diagnostics, - arena, - ), - PredictionMode::Sll => outcome.index > existing.index, - }; - best = Some(if better { outcome } else { existing }); } - let should_use_caller_follow = - best_caller_follow - .as_ref() - .zip(best.as_ref()) - .is_some_and(|(candidate, selected)| { - if !selected.diagnostics.is_empty() { - return true; - } - candidate.index < selected.index - && (candidate.index..selected.index).all(|index| token_info_at(index).2) - }); - if should_use_caller_follow { - best_caller_follow - } else { - best + + fn apply_translated_actions( + &mut self, + source_state: usize, + rule_index: usize, + context: &mut ParserRuleContext, + ) { + apply_member_actions( + source_state, + self.options.member_actions, + self.options.semantics, + &mut self.parser.int_members, + ); + let return_values = return_values_after_action( + source_state, + rule_index, + self.options.return_actions, + self.options.semantics, + &BTreeMap::new(), + ); + for (name, value) in return_values { + context.set_int_return(name, value); + } + } + + fn action_index(&self, source_state: usize) -> Option { + self.action_index_by_state.get(&source_state).copied() } } -fn select_best_outcome( - outcomes: impl Iterator, - prediction_mode: PredictionMode, - arena: &RecognitionArena, -) -> Option { - let outcomes = outcomes.collect::>(); - let prefer_first_tie = outcomes - .iter() - .any(|outcome| arena.sequence_needs_stable_tie(outcome.nodes)); - outcomes.into_iter().reduce(|best, outcome| { - let outcome_position = (outcome.index, outcome.consumed_eof); - let best_position = (best.index, best.consumed_eof); - let better = match prediction_mode { - PredictionMode::Ll | PredictionMode::LlExactAmbigDetection => { - outcome_is_better( - outcome_position, - outcome.diagnostics, - best_position, - best.diagnostics, - arena, - ) || (outcome_position == best_position - && arena.diagnostics_len(outcome.diagnostics) - == arena.diagnostics_len(best.diagnostics) - && arena.diagnostics_recovery_rank(outcome.diagnostics) - == arena.diagnostics_recovery_rank(best.diagnostics) - && (outcome.decisions < best.decisions - || (!prefer_first_tie - && outcome.decisions == best.decisions - && outcome.actions > best.actions))) - } - PredictionMode::Sll => { - outcome_position > best_position - || (outcome_position == best_position - && !prefer_first_tie - && (outcome.decisions < best.decisions - || (outcome.decisions == best.decisions - && outcome_is_better( - outcome_position, - outcome.diagnostics, - best_position, - best.diagnostics, - arena, - )))) - } - }; - if better { - return outcome; - } - best - }) +/// Detects the loop edge where ANTLR would call `pushNewRecursionContext` for a +/// transformed left-recursive rule. +fn left_recursive_boundary(atn: &Atn, state: AtnState<'_>, target: usize) -> Option { + if !state.precedence_rule_decision() { + return None; + } + let target_state = atn.state(target)?; + if target_state.kind() == AtnStateKind::LoopEnd { + return None; + } + state.rule_index() } -/// Records the serialized transition order at parser decision states. +/// Selects the first outer alternative observed for a rule path. /// -/// When two clean paths consume the same input, ANTLR's adaptive prediction -/// chooses by alternative order. Keeping this compact trace lets the metadata -/// recognizer distinguish greedy and non-greedy optional blocks without a full -/// prediction simulator. -fn transition_decision( - atn: &Atn, +/// ANTLR's alt-numbered tree contexts store the rule alternative chosen at the +/// outer decision. The metadata recognizer only needs this when a generated +/// grammar opts into that target template; otherwise the value remains `0` and +/// parse-tree rendering is unchanged. +fn next_alt_number( state: AtnState<'_>, transition_count: usize, transition_index: usize, - predicates: &[(usize, usize, ParserPredicate)], -) -> Option { - if transition_count <= 1 || decision_reaches_unsupported_predicate(atn, state, predicates) { - return None; + current_alt_number: usize, + track_alt_numbers: bool, +) -> usize { + if !track_alt_numbers || current_alt_number != 0 || transition_count <= 1 { + return current_alt_number; } - Some(transition_index) + if matches!( + state.kind(), + AtnStateKind::Basic + | AtnStateKind::BlockStart + | AtnStateKind::PlusBlockStart + | AtnStateKind::StarBlockStart + | AtnStateKind::StarLoopEntry + ) && !state.precedence_rule_decision() + { + return transition_index + 1; + } + current_alt_number } -/// Reports whether a state should reset the active no-viable decision start. -/// -/// Loop entry/back states are continuations of the surrounding adaptive -/// prediction; resetting at those states would turn LL-star failures back into -/// ordinary mismatches. -fn starts_prediction_decision(state: AtnState<'_>, transition_count: usize) -> bool { - transition_count > 1 - && !matches!( - state.kind(), - AtnStateKind::PlusLoopBack | AtnStateKind::StarLoopBack | AtnStateKind::StarLoopEntry - ) +/// Converts an ATN state number into the signed invoking-state slot used by +/// ANTLR parse-tree contexts, saturating only for impossible platform widths. +fn invoking_state_number(state_number: usize) -> isize { + isize::try_from(state_number).unwrap_or(isize::MAX) } -/// Marks a farthest expected-token set as no-viable when multiple alternatives -/// failed after the active decision had already consumed input. -fn record_no_viable_if_ambiguous( - expected: &mut ExpectedTokens, - decision_start_index: Option, - index: usize, -) { - if expected.index == Some(index) && expected.symbols.len() > 1 { - if let Some(decision_start) = no_viable_decision_start(decision_start_index, index) { - expected.record_no_viable(decision_start, index); - } - } +const fn packed_i32(value: u32) -> i32 { + i32::from_le_bytes(value.to_le_bytes()) } -/// Records a no-viable decision caused by a failed semantic predicate before -/// any consuming transition can contribute an expected-token set. -const fn record_predicate_no_viable( - expected: &mut ExpectedTokens, - decision_start_index: Option, - index: usize, -) { - if let Some(decision_start) = decision_start_index { - expected.record_no_viable(decision_start, index); - } +fn direct_precedence(precedence: i32) -> usize { + usize::try_from(precedence.max(0)).unwrap_or_default() } -/// Returns the active decision start only when the error is past that start. -const fn no_viable_decision_start( - decision_start_index: Option, - index: usize, -) -> Option { - match decision_start_index { - Some(start) if index > start => Some(start), - _ => None, - } +fn token_input_display(token: &impl Token) -> String { + format!("'{}'", token.text().unwrap_or("")) } -/// Restores expected-token bookkeeping when a child rule found a clean -/// consuming path; failures in longer child alternatives should not pollute the -/// caller's final expectation set. -fn restore_expected( - children: &[RecognizeOutcome], - child_start_index: usize, - expected: &mut ExpectedTokens, - snapshot: ExpectedTokens, - preserve_child_expected: bool, -) { - if preserve_child_expected { - return; +fn display_input_text(text: &str) -> String { + let mut out = String::new(); + for ch in text.chars() { + match ch { + '\n' => out.push_str("\\n"), + '\r' => out.push_str("\\r"), + '\t' => out.push_str("\\t"), + other => out.push(other), + } } - if children - .iter() - .any(|child| child.diagnostics.is_empty() && child.index > child_start_index) - { - *expected = snapshot; + out +} + +fn diagnostic_for_token(token: Option, message: String) -> ParserDiagnostic { + let (line, column, offending) = token.map_or((0, 0, None), |token| { + (token.line(), token.column(), Some(token.token_id())) + }); + ParserDiagnostic { + line, + column, + message, + offending, } } -/// Reports whether a decision can reach a predicate the generator did not -/// translate. Static alternative order is unsafe for those context predicates. -fn decision_reaches_unsupported_predicate( - atn: &Atn, - state: AtnState<'_>, - predicates: &[(usize, usize, ParserPredicate)], -) -> bool { - state.transitions().iter().any(|transition| { - transition_reaches_unsupported_predicate(atn, transition, predicates, &mut BTreeSet::new()) - }) +fn expected_symbols_display(symbols: &BTreeSet, vocabulary: &Vocabulary) -> String { + expected_symbols_display_iter(symbols.iter().copied(), vocabulary) } -/// Walks epsilon-like edges from one transition to find unsupported predicates. -fn transition_reaches_unsupported_predicate( - atn: &Atn, - transition: ParserTransition<'_>, - predicates: &[(usize, usize, ParserPredicate)], - visited: &mut BTreeSet, -) -> bool { - match &transition.data() { - Transition::Predicate { - rule_index, - pred_index, - .. - } => !predicates - .iter() - .any(|(rule, pred, _)| rule == rule_index && pred == pred_index), - Transition::Epsilon { target } - | Transition::Action { target, .. } - | Transition::Rule { target, .. } => { - state_reaches_unsupported_predicate(atn, *target, predicates, visited) - } - Transition::Precedence { .. } - | Transition::Atom { .. } - | Transition::Range { .. } - | Transition::Set { .. } - | Transition::NotSet { .. } - | Transition::Wildcard { .. } => false, +fn expected_symbols_display_iter( + symbols: impl IntoIterator, + vocabulary: &Vocabulary, +) -> String { + let items = symbols + .into_iter() + .map(|symbol| expected_symbol_display(symbol, vocabulary)) + .collect::>(); + if let [single] = items.as_slice() { + return single.clone(); } + format!("{{{}}}", items.join(", ")) } -/// Finds an unsupported predicate reachable before a consuming transition. -fn state_reaches_unsupported_predicate( - atn: &Atn, - state_number: usize, - predicates: &[(usize, usize, ParserPredicate)], - visited: &mut BTreeSet, -) -> bool { - if !visited.insert(state_number) { - return false; +fn expected_symbol_display(symbol: i32, vocabulary: &Vocabulary) -> String { + if symbol == TOKEN_EOF { + return "".to_owned(); } - let Some(state) = atn.state(state_number) else { - return false; - }; - state.transitions().iter().any(|transition| { - transition_reaches_unsupported_predicate(atn, transition, predicates, visited) - }) + vocabulary.display_name(symbol) } -/// Adds a decision step to the front of an already-recognized suffix path. -fn prepend_decision(outcome: &mut RecognizeOutcome, decision: Option) { - if let Some(decision) = decision { - outcome.decisions.insert(0, decision); +fn caller_follow_token_info_for_stream( + input: &mut CommonTokenStream, + index: usize, +) -> (i32, bool, bool) { + // Generated callers own statement separators; leave them available when + // an interpreted child rule can either stop before or consume one. + if index >= FAST_RECOGNIZER_DEFERRED_FILL_AT && !input.is_filled() { + input.fill(); } + let token_type = input.token_type_at_index(index); + let visible_channel = input.channel(); + let token = input.get(index); + let is_boundary = token + .as_ref() + .and_then(Token::text) + .is_some_and(is_caller_follow_boundary_text); + let is_boundary_gap = token.as_ref().is_some_and(|token| { + token.channel() != visible_channel + || is_caller_follow_boundary_gap_text(token.text_or_empty()) + }); + (token_type, is_boundary, is_boundary_gap) } -fn outcome_is_better( - outcome_position: (usize, bool), - outcome_diagnostics: DiagnosticSeqId, - best_position: (usize, bool), - best_diagnostics: DiagnosticSeqId, - arena: &RecognitionArena, -) -> bool { - let outcome_len = arena.diagnostics_len(outcome_diagnostics); - let best_len = arena.diagnostics_len(best_diagnostics); - outcome_position > best_position - || (outcome_position == best_position - && (outcome_len < best_len - || (outcome_len == best_len - && arena.diagnostics_recovery_rank(outcome_diagnostics) - < arena.diagnostics_recovery_rank(best_diagnostics)))) +fn is_caller_follow_boundary_text(text: &str) -> bool { + text.chars().any(|ch| ch == ';' || ch == '\n') + && text.chars().all(|ch| ch.is_whitespace() || ch == ';') } -fn discard_recovered_fast_outcomes_if_clean_path_exists(outcomes: &mut Vec) { - if outcomes - .iter() - .any(|outcome| outcome.diagnostics.is_empty()) - { - outcomes.retain(|outcome| outcome.diagnostics.is_empty()); - } +fn is_caller_follow_boundary_gap_text(text: &str) -> bool { + text.chars().all(|ch| ch.is_whitespace() || ch == ';') } -fn discard_recovered_outcomes_if_clean_path_exists( - outcomes: &mut Vec, - arena: &RecognitionArena, -) { - if outcomes - .iter() - .any(|outcome| outcome_has_rule_failure_diagnostic(outcome, arena)) - { - return; - } - if outcomes - .iter() - .any(|outcome| outcome.diagnostics.is_empty()) - { - outcomes.retain(|outcome| outcome.diagnostics.is_empty()); - } +/// Returns whether `state` belongs to an ANTLR-transformed left-recursive rule. +/// Inline insertion in those precedence loops can synthesize a missing operand +/// before an operator and then block the legitimate loop-exit path. +fn state_is_left_recursive_rule(atn: &Atn, state: AtnState<'_>) -> bool { + let Some(rule_index) = state.rule_index() else { + return false; + }; + atn.rule_to_start_state() + .get(rule_index) + .and_then(|state_number| atn.state(state_number)) + .is_some_and(AtnState::left_recursive_rule) } -/// Reports whether a recovered outcome came from an explicit predicate -/// fail-option and therefore should compete with shorter clean loop exits. -fn outcome_has_rule_failure_diagnostic( - outcome: &RecognizeOutcome, +/// Picks the better of two `parse_atn_rule` passes (with and without the +/// FIRST-set prefilter). A clean outcome (no diagnostics) always wins over a +/// recovered one; among recovered outcomes the second pass is preferred +/// because the no-prefilter walk reaches ANTLR-style recovery inside child +/// rules. If both passes failed, the second pass's expected-token snapshot +/// is returned so the caller renders the same diagnostic ANTLR would. +fn select_better_top_outcome( + first: Result<(FastRecognizeOutcome, ExpectedTokens, usize), ExpectedTokens>, + second: Result<(FastRecognizeOutcome, ExpectedTokens, usize), ExpectedTokens>, arena: &RecognitionArena, -) -> bool { - arena - .diagnostics(outcome.diagnostics) - .any(|diagnostic| diagnostic.message.starts_with("rule ")) +) -> Result<(FastRecognizeOutcome, ExpectedTokens, usize), ExpectedTokens> { + match (first, second) { + (Ok(first), Ok(second)) => { + if arena.diagnostics(first.0.diagnostics).next().is_none() { + Ok(first) + } else { + Ok(second) + } + } + (Ok(first), Err(_)) => Ok(first), + (Err(_), Ok(second)) => Ok(second), + (Err(_), Err(second_expected)) => Err(second_expected), + } } -/// Removes equivalent endpoints before memoizing a state result while -/// preserving ATN transition-discovery order. -/// -/// Outcomes are compared on observable recognition state — the input index, -/// EOF consumption, and diagnostics — without descending into the parse-tree -/// fragment carried by `nodes`. Two paths reaching the same point with -/// different node trees would otherwise prevent memoization from collapsing -/// equivalent suffixes and explode the speculative-path cache. +/// Chooses the outermost parse result that consumed the most input. /// -/// The first occurrence per recognition key wins, which matches ANTLR's -/// greedy alternative selection: serialized ATNs put greedy `*`/`+` loop-back -/// transitions before loop-exit, so the first-discovered outcome carries the -/// greedy parse-tree fragment. -fn dedupe_fast_outcomes(outcomes: &mut Vec, arena: &RecognitionArena) { - if outcomes.len() < 2 { - return; - } - let mut seen = FxHashSet::with_capacity_and_hasher(outcomes.len(), FxBuildHasher::default()); - outcomes.retain(|outcome| { - seen.insert(( - outcome.index, - outcome.consumed_eof, - arena.diagnostics_len(outcome.diagnostics), - arena.diagnostics_recovery_rank(outcome.diagnostics), - )) - }); +/// The recognizer intentionally keeps shorter endpoints available while walking +/// nested rule transitions so callers can satisfy following tokens such as +/// `expr 'and' expr`. Only the public rule entry commits to one endpoint. +fn select_best_fast_outcome( + outcomes: impl Iterator, + prediction_mode: PredictionMode, + caller_follow: Option<&TokenBitSet>, + mut token_info_at: impl FnMut(usize) -> (i32, bool, bool), + arena: &RecognitionArena, +) -> Option { + let mut best = None; + let mut best_caller_follow = None; + for outcome in outcomes { + if matches!( + prediction_mode, + PredictionMode::Ll | PredictionMode::LlExactAmbigDetection + ) && outcome.diagnostics.is_empty() + && let Some(follow) = caller_follow + { + let (token_type, is_boundary, _) = token_info_at(outcome.index); + if is_boundary && follow.contains(token_type) { + let replace = + best_caller_follow + .as_ref() + .is_none_or(|existing: &FastRecognizeOutcome| { + (outcome.index, outcome.consumed_eof) + < (existing.index, existing.consumed_eof) + }); + if replace { + best_caller_follow = Some(outcome); + } + } + } + let Some(existing) = best else { + best = Some(outcome); + continue; + }; + let outcome_position = (outcome.index, outcome.consumed_eof); + let best_position = (existing.index, existing.consumed_eof); + let better = match prediction_mode { + PredictionMode::Ll | PredictionMode::LlExactAmbigDetection => outcome_is_better( + outcome_position, + outcome.diagnostics, + best_position, + existing.diagnostics, + arena, + ), + PredictionMode::Sll => outcome.index > existing.index, + }; + best = Some(if better { outcome } else { existing }); + } + let should_use_caller_follow = + best_caller_follow + .as_ref() + .zip(best.as_ref()) + .is_some_and(|(candidate, selected)| { + if !selected.diagnostics.is_empty() { + return true; + } + candidate.index < selected.index + && (candidate.index..selected.index).all(|index| token_info_at(index).2) + }); + if should_use_caller_follow { + best_caller_follow + } else { + best + } } -const FAST_OUTCOME_INLINE_KEYS: usize = 8; -const FAST_OUTCOME_BITS_PER_WORD: usize = 64; -const MAX_FAST_OUTCOME_DENSE_BYTES: usize = 64 * 1024; -const MAX_RETAINED_FAST_OUTCOME_SPARSE_KEYS: usize = 65_536; +fn select_best_outcome( + outcomes: impl Iterator, + prediction_mode: PredictionMode, + arena: &RecognitionArena, +) -> Option { + let outcomes = outcomes.collect::>(); + let prefer_first_tie = outcomes + .iter() + .any(|outcome| arena.sequence_needs_stable_tie(outcome.nodes)); + outcomes.into_iter().reduce(|best, outcome| { + let outcome_position = (outcome.index, outcome.consumed_eof); + let best_position = (best.index, best.consumed_eof); + let better = match prediction_mode { + PredictionMode::Ll | PredictionMode::LlExactAmbigDetection => { + outcome_is_better( + outcome_position, + outcome.diagnostics, + best_position, + best.diagnostics, + arena, + ) || (outcome_position == best_position + && arena.diagnostics_len(outcome.diagnostics) + == arena.diagnostics_len(best.diagnostics) + && arena.diagnostics_recovery_rank(outcome.diagnostics) + == arena.diagnostics_recovery_rank(best.diagnostics) + && (outcome.decisions < best.decisions + || (!prefer_first_tie + && outcome.decisions == best.decisions + && outcome.actions > best.actions))) + } + PredictionMode::Sll => { + outcome_position > best_position + || (outcome_position == best_position + && !prefer_first_tie + && (outcome.decisions < best.decisions + || (outcome.decisions == best.decisions + && outcome_is_better( + outcome_position, + outcome.diagnostics, + best_position, + best.diagnostics, + arena, + )))) + } + }; + if better { + return outcome; + } + best + }) +} -#[derive(Clone, Copy, Debug, Eq, PartialEq)] -enum FastOutcomeDedupStrategy { - Inline, - Dense, - Sparse, +/// Records the serialized transition order at parser decision states. +/// +/// When two clean paths consume the same input, ANTLR's adaptive prediction +/// chooses by alternative order. Keeping this compact trace lets the metadata +/// recognizer distinguish greedy and non-greedy optional blocks without a full +/// prediction simulator. +fn transition_decision( + atn: &Atn, + state: AtnState<'_>, + transition_count: usize, + transition_index: usize, + predicates: &[(usize, usize, ParserPredicate)], +) -> Option { + if transition_count <= 1 || decision_reaches_unsupported_predicate(atn, state, predicates) { + return None; + } + Some(transition_index) } -impl FastOutcomeDedupScratch { - fn prepare_dense(&mut self, word_count: usize) { - while let Some(word_index) = self.touched_dense_words.pop() { - self.dense_words[usize::try_from(word_index).expect("u32 fits in usize")] = 0; - } - if self.dense_words.len() < word_count { - self.dense_words.resize(word_count, 0); +/// Reports whether a state should reset the active no-viable decision start. +/// +/// Loop entry/back states are continuations of the surrounding adaptive +/// prediction; resetting at those states would turn LL-star failures back into +/// ordinary mismatches. +fn starts_prediction_decision(state: AtnState<'_>, transition_count: usize) -> bool { + transition_count > 1 + && !matches!( + state.kind(), + AtnStateKind::PlusLoopBack | AtnStateKind::StarLoopBack | AtnStateKind::StarLoopEntry + ) +} + +/// Marks a farthest expected-token set as no-viable when multiple alternatives +/// failed after the active decision had already consumed input. +fn record_no_viable_if_ambiguous( + expected: &mut ExpectedTokens, + decision_start_index: Option, + index: usize, +) { + if expected.index == Some(index) && expected.symbols.len() > 1 { + if let Some(decision_start) = no_viable_decision_start(decision_start_index, index) { + expected.record_no_viable(decision_start, index); } } } -fn clean_fast_outcome_dense_layout(outcomes: &[FastRecognizeOutcome]) -> Option<(usize, usize)> { - let first_index = outcomes.first()?.index; - let (min_index, max_index) = outcomes[1..].iter().fold( - (first_index, first_index), - |(min_index, max_index), outcome| { - (min_index.min(outcome.index), max_index.max(outcome.index)) - }, - ); - let index_span = max_index.checked_sub(min_index)?.checked_add(1)?; - let bit_count = index_span.checked_mul(2)?; - let word_count = - bit_count.checked_add(FAST_OUTCOME_BITS_PER_WORD - 1)? / FAST_OUTCOME_BITS_PER_WORD; - let dense_bytes = word_count.checked_mul(size_of::())?; - let sparse_key_bytes = outcomes.len().checked_mul(size_of::<(usize, bool)>())?; - (dense_bytes <= MAX_FAST_OUTCOME_DENSE_BYTES && dense_bytes <= sparse_key_bytes) - .then_some((min_index, word_count)) +/// Records a no-viable decision caused by a failed semantic predicate before +/// any consuming transition can contribute an expected-token set. +const fn record_predicate_no_viable( + expected: &mut ExpectedTokens, + decision_start_index: Option, + index: usize, +) { + if let Some(decision_start) = decision_start_index { + expected.record_no_viable(decision_start, index); + } } -#[cfg(feature = "perf-counters")] -fn record_clean_fast_outcome_dedup( - strategy: FastOutcomeDedupStrategy, - input_len: usize, - output_len: usize, - dense_words: usize, -) { - let counter = match strategy { - FastOutcomeDedupStrategy::Inline => &perf_counters::OUTCOME_DEDUPE_INLINE, - FastOutcomeDedupStrategy::Dense => &perf_counters::OUTCOME_DEDUPE_DENSE, - FastOutcomeDedupStrategy::Sparse => &perf_counters::OUTCOME_DEDUPE_SPARSE, - }; - perf_counters::inc( - &perf_counters::OUTCOME_DEDUPE_INPUTS, - u64::try_from(input_len).unwrap_or(u64::MAX), - ); - perf_counters::inc( - &perf_counters::OUTCOME_DEDUPE_REMOVED, - u64::try_from(input_len - output_len).unwrap_or(u64::MAX), - ); - perf_counters::inc(counter, 1); - perf_counters::inc( - &perf_counters::OUTCOME_DEDUPE_DENSE_WORDS, - u64::try_from(dense_words).unwrap_or(u64::MAX), - ); +/// Returns the active decision start only when the error is past that start. +const fn no_viable_decision_start( + decision_start_index: Option, + index: usize, +) -> Option { + match decision_start_index { + Some(start) if index > start => Some(start), + _ => None, + } } -/// Removes duplicate clean endpoints while preserving transition-discovery -/// order. Tiny lists stay on the stack; larger compact ranges use a direct -/// bitmap, and only wide sparse ranges pay for hashing. -fn dedupe_clean_fast_outcomes( - outcomes: &mut Vec, - scratch: &mut FastOutcomeDedupScratch, -) -> FastOutcomeDedupStrategy { - #[cfg(feature = "perf-counters")] - let input_len = outcomes.len(); - if outcomes.len() <= FAST_OUTCOME_INLINE_KEYS { - let mut inline_keys = [(0, false); FAST_OUTCOME_INLINE_KEYS]; - let mut inline_len = 0_usize; - outcomes.retain(|outcome| { - let key = (outcome.index, outcome.consumed_eof); - if inline_keys[..inline_len].contains(&key) { - return false; +/// Restores expected-token bookkeeping when a child rule found a clean +/// consuming path; failures in longer child alternatives should not pollute the +/// caller's final expectation set. +fn restore_expected( + children: &[RecognizeOutcome], + child_start_index: usize, + expected: &mut ExpectedTokens, + snapshot: ExpectedTokens, + preserve_child_expected: bool, +) { + if preserve_child_expected { + return; + } + if children + .iter() + .any(|child| child.diagnostics.is_empty() && child.index > child_start_index) + { + *expected = snapshot; + } +} + +/// Reports whether a decision can reach a predicate the generator did not +/// translate. Static alternative order is unsafe for those context predicates. +fn decision_reaches_unsupported_predicate( + atn: &Atn, + state: AtnState<'_>, + predicates: &[(usize, usize, ParserPredicate)], +) -> bool { + state.transitions().iter().any(|transition| { + transition_reaches_unsupported_predicate(atn, transition, predicates, &mut BTreeSet::new()) + }) +} + +/// Walks epsilon-like edges from one transition to find unsupported predicates. +fn transition_reaches_unsupported_predicate( + atn: &Atn, + transition: ParserTransition<'_>, + predicates: &[(usize, usize, ParserPredicate)], + visited: &mut BTreeSet, +) -> bool { + match &transition.data() { + Transition::Predicate { + rule_index, + pred_index, + .. + } => !predicates + .iter() + .any(|(rule, pred, _)| rule == rule_index && pred == pred_index), + Transition::Epsilon { target } + | Transition::Action { target, .. } + | Transition::Rule { target, .. } => { + state_reaches_unsupported_predicate(atn, *target, predicates, visited) + } + Transition::Precedence { .. } + | Transition::Atom { .. } + | Transition::Range { .. } + | Transition::Set { .. } + | Transition::NotSet { .. } + | Transition::Wildcard { .. } => false, + } +} + +/// Finds an unsupported predicate reachable before a consuming transition. +fn state_reaches_unsupported_predicate( + atn: &Atn, + state_number: usize, + predicates: &[(usize, usize, ParserPredicate)], + visited: &mut BTreeSet, +) -> bool { + if !visited.insert(state_number) { + return false; + } + let Some(state) = atn.state(state_number) else { + return false; + }; + state.transitions().iter().any(|transition| { + transition_reaches_unsupported_predicate(atn, transition, predicates, visited) + }) +} + +/// Adds a decision step to the front of an already-recognized suffix path. +fn prepend_decision(outcome: &mut RecognizeOutcome, decision: Option) { + if let Some(decision) = decision { + outcome.decisions.insert(0, decision); + } +} + +fn outcome_is_better( + outcome_position: (usize, bool), + outcome_diagnostics: DiagnosticSeqId, + best_position: (usize, bool), + best_diagnostics: DiagnosticSeqId, + arena: &RecognitionArena, +) -> bool { + let outcome_len = arena.diagnostics_len(outcome_diagnostics); + let best_len = arena.diagnostics_len(best_diagnostics); + outcome_position > best_position + || (outcome_position == best_position + && (outcome_len < best_len + || (outcome_len == best_len + && arena.diagnostics_recovery_rank(outcome_diagnostics) + < arena.diagnostics_recovery_rank(best_diagnostics)))) +} + +fn discard_recovered_fast_outcomes_if_clean_path_exists(outcomes: &mut Vec) { + if outcomes + .iter() + .any(|outcome| outcome.diagnostics.is_empty()) + { + outcomes.retain(|outcome| outcome.diagnostics.is_empty()); + } +} + +fn discard_recovered_outcomes_if_clean_path_exists( + outcomes: &mut Vec, + arena: &RecognitionArena, +) { + if outcomes + .iter() + .any(|outcome| outcome_has_rule_failure_diagnostic(outcome, arena)) + { + return; + } + if outcomes + .iter() + .any(|outcome| outcome.diagnostics.is_empty()) + { + outcomes.retain(|outcome| outcome.diagnostics.is_empty()); + } +} + +/// Reports whether a recovered outcome came from an explicit predicate +/// fail-option and therefore should compete with shorter clean loop exits. +fn outcome_has_rule_failure_diagnostic( + outcome: &RecognizeOutcome, + arena: &RecognitionArena, +) -> bool { + arena + .diagnostics(outcome.diagnostics) + .any(|diagnostic| diagnostic.message.starts_with("rule ")) +} + +/// Removes equivalent endpoints before memoizing a state result while +/// preserving ATN transition-discovery order. +/// +/// Outcomes are compared on observable recognition state — the input index, +/// EOF consumption, and diagnostics — without descending into the parse-tree +/// fragment carried by `nodes`. Two paths reaching the same point with +/// different node trees would otherwise prevent memoization from collapsing +/// equivalent suffixes and explode the speculative-path cache. +/// +/// The first occurrence per recognition key wins, which matches ANTLR's +/// greedy alternative selection: serialized ATNs put greedy `*`/`+` loop-back +/// transitions before loop-exit, so the first-discovered outcome carries the +/// greedy parse-tree fragment. +fn dedupe_fast_outcomes(outcomes: &mut Vec, arena: &RecognitionArena) { + if outcomes.len() < 2 { + return; + } + let mut seen = FxHashSet::with_capacity_and_hasher(outcomes.len(), FxBuildHasher::default()); + outcomes.retain(|outcome| { + seen.insert(( + outcome.index, + outcome.consumed_eof, + arena.diagnostics_len(outcome.diagnostics), + arena.diagnostics_recovery_rank(outcome.diagnostics), + )) + }); +} + +const FAST_OUTCOME_INLINE_KEYS: usize = 8; +const FAST_OUTCOME_BITS_PER_WORD: usize = 64; +const MAX_FAST_OUTCOME_DENSE_BYTES: usize = 64 * 1024; +const MAX_RETAINED_FAST_OUTCOME_SPARSE_KEYS: usize = 65_536; + +#[derive(Clone, Copy, Debug, Eq, PartialEq)] +enum FastOutcomeDedupStrategy { + Inline, + Dense, + Sparse, +} + +impl FastOutcomeDedupScratch { + fn prepare_dense(&mut self, word_count: usize) { + while let Some(word_index) = self.touched_dense_words.pop() { + self.dense_words[usize::try_from(word_index).expect("u32 fits in usize")] = 0; + } + if self.dense_words.len() < word_count { + self.dense_words.resize(word_count, 0); + } + } +} + +fn clean_fast_outcome_dense_layout(outcomes: &[FastRecognizeOutcome]) -> Option<(usize, usize)> { + let first_index = outcomes.first()?.index; + let (min_index, max_index) = outcomes[1..].iter().fold( + (first_index, first_index), + |(min_index, max_index), outcome| { + (min_index.min(outcome.index), max_index.max(outcome.index)) + }, + ); + let index_span = max_index.checked_sub(min_index)?.checked_add(1)?; + let bit_count = index_span.checked_mul(2)?; + let word_count = + bit_count.checked_add(FAST_OUTCOME_BITS_PER_WORD - 1)? / FAST_OUTCOME_BITS_PER_WORD; + let dense_bytes = word_count.checked_mul(size_of::())?; + let sparse_key_bytes = outcomes.len().checked_mul(size_of::<(usize, bool)>())?; + (dense_bytes <= MAX_FAST_OUTCOME_DENSE_BYTES && dense_bytes <= sparse_key_bytes) + .then_some((min_index, word_count)) +} + +#[cfg(feature = "perf-counters")] +fn record_clean_fast_outcome_dedup( + strategy: FastOutcomeDedupStrategy, + input_len: usize, + output_len: usize, + dense_words: usize, +) { + let counter = match strategy { + FastOutcomeDedupStrategy::Inline => &perf_counters::OUTCOME_DEDUPE_INLINE, + FastOutcomeDedupStrategy::Dense => &perf_counters::OUTCOME_DEDUPE_DENSE, + FastOutcomeDedupStrategy::Sparse => &perf_counters::OUTCOME_DEDUPE_SPARSE, + }; + perf_counters::inc( + &perf_counters::OUTCOME_DEDUPE_INPUTS, + u64::try_from(input_len).unwrap_or(u64::MAX), + ); + perf_counters::inc( + &perf_counters::OUTCOME_DEDUPE_REMOVED, + u64::try_from(input_len - output_len).unwrap_or(u64::MAX), + ); + perf_counters::inc(counter, 1); + perf_counters::inc( + &perf_counters::OUTCOME_DEDUPE_DENSE_WORDS, + u64::try_from(dense_words).unwrap_or(u64::MAX), + ); +} + +/// Removes duplicate clean endpoints while preserving transition-discovery +/// order. Tiny lists stay on the stack; larger compact ranges use a direct +/// bitmap, and only wide sparse ranges pay for hashing. +fn dedupe_clean_fast_outcomes( + outcomes: &mut Vec, + scratch: &mut FastOutcomeDedupScratch, +) -> FastOutcomeDedupStrategy { + #[cfg(feature = "perf-counters")] + let input_len = outcomes.len(); + if outcomes.len() <= FAST_OUTCOME_INLINE_KEYS { + let mut inline_keys = [(0, false); FAST_OUTCOME_INLINE_KEYS]; + let mut inline_len = 0_usize; + outcomes.retain(|outcome| { + let key = (outcome.index, outcome.consumed_eof); + if inline_keys[..inline_len].contains(&key) { + return false; } inline_keys[inline_len] = key; inline_len += 1; @@ -13512,175 +14557,999 @@ mod tests { ); } } - atn.set_rule_to_start_state(starts.clone()) + atn.set_rule_to_start_state(starts.clone()) + .expect("rule start states"); + atn.set_rule_to_stop_state(stops.clone()) + .expect("rule stop states"); + for rule_index in 0..depth - 1 { + let follow_state = if consuming_follows { + follows[rule_index] + } else { + stops[rule_index] + }; + atn.add_transition( + starts[rule_index], + ParserTransitionSpec::Rule { + target: starts[rule_index + 1], + rule_index: rule_index + 1, + follow_state, + precedence: 0, + }, + ) + .expect("nested rule transition"); + if branching { + atn.add_transition( + starts[rule_index], + ParserTransitionSpec::Atom { + target: stops[rule_index], + label: 2, + }, + ) + .expect("dead branch transition"); + } + if consuming_follows { + atn.add_transition( + follow_state, + ParserTransitionSpec::Atom { + target: stops[rule_index], + label: 1, + }, + ) + .expect("consuming follow transition"); + } + } + let token_set = atn.add_interval_set([(1, 1)]).expect("token set"); + atn.add_transition( + starts[depth - 1], + ParserTransitionSpec::Set { + target: stops[depth - 1], + set: token_set, + }, + ) + .expect("terminal set transition"); + if branching { + atn.add_transition( + starts[depth - 1], + ParserTransitionSpec::Atom { + target: stops[depth - 1], + label: 2, + }, + ) + .expect("dead leaf branch transition"); + } + finish_atn(atn) + } + + fn ordinary_star_loop_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(2); + for (state_number, kind, rule_index) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::StarLoopEntry, 0), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::StarLoopBack, 0), + (4, AtnStateKind::LoopEnd, 0), + (5, AtnStateKind::Basic, 0), + (6, AtnStateKind::RuleStop, 0), + (7, AtnStateKind::RuleStart, 1), + (8, AtnStateKind::Basic, 1), + (9, AtnStateKind::RuleStop, 1), + ] { + assert_eq!( + atn.add_state(kind, Some(rule_index)) + .expect("state") + .index(), + state_number + ); + } + atn.set_rule_to_start_state(vec![0, 7]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![6, 9]) + .expect("rule stop states"); + atn.add_decision_state(1).expect("decision state"); + atn.set_loop_back_state(4, 3).expect("loop back state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 4 }) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Rule { + target: 7, + rule_index: 1, + follow_state: 3, + precedence: 0, + }, + ) + .expect("transition"); + atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); + atn.add_transition( + 5, + ParserTransitionSpec::Atom { + target: 6, + label: TOKEN_EOF, + }, + ) + .expect("transition"); + atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 8 }) + .expect("transition"); + atn.add_transition( + 8, + ParserTransitionSpec::Atom { + target: 9, + label: 1, + }, + ) + .expect("transition"); + finish_atn(atn) + } + + /// ATN for `s : (X | X X)* EOF`. + fn ambiguous_ordinary_star_loop_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for (state_number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::StarLoopEntry), + (2, AtnStateKind::StarBlockStart), + (3, AtnStateKind::Basic), + (4, AtnStateKind::BlockEnd), + (5, AtnStateKind::StarLoopBack), + (6, AtnStateKind::LoopEnd), + (7, AtnStateKind::Basic), + (8, AtnStateKind::RuleStop), + ] { + assert_eq!( + atn.add_state(kind, Some(0)).expect("state").index(), + state_number + ); + } + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![8]) + .expect("rule stop states"); + atn.set_end_state(2, 4).expect("block end state"); + atn.set_loop_back_state(6, 5).expect("loop back state"); + atn.add_decision_state(1).expect("decision state"); + atn.add_decision_state(2).expect("decision state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Atom { + target: 4, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Atom { + target: 3, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition( + 3, + ParserTransitionSpec::Atom { + target: 4, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) + .expect("transition"); + atn.add_transition( + 7, + ParserTransitionSpec::Atom { + target: 8, + label: TOKEN_EOF, + }, + ) + .expect("transition"); + finish_atn(atn) + } + + fn ordinary_plus_loop_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(2); + for (state_number, kind, rule_index) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::Basic, 0), + (2, AtnStateKind::PlusLoopBack, 0), + (3, AtnStateKind::LoopEnd, 0), + (4, AtnStateKind::Basic, 0), + (5, AtnStateKind::RuleStop, 0), + (6, AtnStateKind::RuleStart, 1), + (7, AtnStateKind::Basic, 1), + (8, AtnStateKind::RuleStop, 1), + ] { + assert_eq!( + atn.add_state(kind, Some(rule_index)) + .expect("state") + .index(), + state_number + ); + } + atn.set_rule_to_start_state(vec![0, 6]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![5, 8]) + .expect("rule stop states"); + atn.add_decision_state(2).expect("decision state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition( + 1, + ParserTransitionSpec::Rule { + target: 6, + rule_index: 1, + follow_state: 2, + precedence: 0, + }, + ) + .expect("transition"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("transition"); + atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 }) + .expect("transition"); + atn.add_transition( + 4, + ParserTransitionSpec::Atom { + target: 5, + label: TOKEN_EOF, + }, + ) + .expect("transition"); + atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) + .expect("transition"); + atn.add_transition( + 7, + ParserTransitionSpec::Atom { + target: 8, + label: 1, + }, + ) + .expect("transition"); + finish_atn(atn) + } + + fn repeated_x_tokens(count: usize) -> Vec { + let mut tokens = (0..count) + .map(|_| TestToken::new(1).with_text("x")) + .collect::>(); + tokens.push(TestToken::eof("parser-test", count, 1, count)); + tokens + } + + fn left_recursive_loop_with_caller_follow_atn(caller_symbol: i32) -> Atn { + let mut atn = ParserAtnBuilder::new(2); + assert_eq!( + atn.add_state(AtnStateKind::RuleStart, Some(0)) + .expect("state") + .index(), + 0 + ); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(0)) + .expect("state") + .index(), + 1 + ); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(0)) + .expect("state") + .index(), + 2 + ); + assert_eq!( + atn.add_state(AtnStateKind::RuleStart, Some(1)) + .expect("state") + .index(), + 3 + ); + atn.set_left_recursive_rule(3) + .expect("left-recursive rule start"); + assert_eq!( + atn.add_state(AtnStateKind::StarLoopEntry, Some(1)) + .expect("state") + .index(), + 4 + ); + atn.set_precedence_rule_decision(4) + .expect("precedence decision"); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(1)) + .expect("state") + .index(), + 5 + ); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(1)) + .expect("state") + .index(), + 6 + ); + assert_eq!( + atn.add_state(AtnStateKind::LoopEnd, Some(1)) + .expect("state") + .index(), + 7 + ); + assert_eq!( + atn.add_state(AtnStateKind::RuleStop, Some(1)) + .expect("state") + .index(), + 8 + ); + assert_eq!( + atn.add_state(AtnStateKind::RuleStop, Some(0)) + .expect("state") + .index(), + 9 + ); + atn.set_rule_to_start_state(vec![0, 3]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![9, 8]) + .expect("rule stop states"); + atn.add_transition( + 1, + ParserTransitionSpec::Rule { + target: 3, + rule_index: 1, + follow_state: 2, + precedence: 0, + }, + ) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Atom { + target: 9, + label: caller_symbol, + }, + ) + .expect("transition"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 7 }) + .expect("transition"); + atn.add_transition( + 5, + ParserTransitionSpec::Precedence { + target: 6, + precedence: 1, + }, + ) + .expect("transition"); + atn.add_transition( + 6, + ParserTransitionSpec::Atom { + target: 4, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 8 }) + .expect("transition"); + finish_atn(atn) + } + + fn labeled_left_recursive_operator_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(4); + for (state, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::BlockStart), + (2, AtnStateKind::StarLoopEntry), + (3, AtnStateKind::StarBlockStart), + (4, AtnStateKind::Basic), + (5, AtnStateKind::Basic), + (6, AtnStateKind::Basic), + (7, AtnStateKind::StarLoopBack), + (8, AtnStateKind::LoopEnd), + (9, AtnStateKind::RuleStop), + ] { + assert_eq!(atn.add_state(kind, Some(0)).expect("state").index(), state); + } + atn.set_left_recursive_rule(0) + .expect("left-recursive rule start"); + atn.set_precedence_rule_decision(2) + .expect("precedence decision"); + atn.set_loop_back_state(8, 7).expect("loop-back state"); + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![9]) + .expect("rule stop states"); + for state in [1, 2, 3] { + atn.add_decision_state(state).expect("decision state"); + } + for (source, target) in [(0, 1), (2, 3), (2, 8), (7, 2), (8, 9)] { + atn.add_transition(source, ParserTransitionSpec::Epsilon { target }) + .expect("epsilon transition"); + } + for (source, target, label) in [(1, 2, 1), (1, 2, 2), (4, 6, 4), (5, 6, 3), (6, 7, 1)] { + atn.add_transition(source, ParserTransitionSpec::Atom { target, label }) + .expect("token transition"); + } + for (target, precedence) in [(4, 2), (5, 1)] { + atn.add_transition(3, ParserTransitionSpec::Precedence { target, precedence }) + .expect("operator precedence"); + } + finish_atn(atn) + } + + fn parser_inside_left_recursive_callee(symbol: i32) -> BaseParser { + let mut parser = mini_parser(vec![ + TestToken::new(symbol).with_text("lookahead"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.rule_context_stack = vec![ + RuleContextFrame { + rule_index: 0, + invoking_state: -1, + }, + RuleContextFrame { + rule_index: 1, + invoking_state: 1, + }, + ]; + parser + } + + fn left_recursive_loop_with_shared_gt_prefix_atn() -> Atn { + // StarLoopEntry with two operator alts that share leading token 1 (`>`): + // prec 2: token 1, token 1 (shift `>>`) + // prec 1: token 1 (relational `>`) + let mut atn = ParserAtnBuilder::new(1); + for (state, kind, rule) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::StarLoopEntry, 0), + (2, AtnStateKind::Basic, 0), // ops hub + (3, AtnStateKind::Basic, 0), // shift prec + (4, AtnStateKind::Basic, 0), // shift first > + (5, AtnStateKind::Basic, 0), // shift second > + (6, AtnStateKind::Basic, 0), // rel prec + (7, AtnStateKind::Basic, 0), // rel > + (8, AtnStateKind::LoopEnd, 0), + (9, AtnStateKind::RuleStop, 0), + ] { + assert_eq!( + atn.add_state(kind, Some(rule)).expect("state").index(), + state + ); + if state == 0 { + atn.set_left_recursive_rule(state) + .expect("left-recursive rule start"); + } else if state == 1 { + atn.set_precedence_rule_decision(state) + .expect("precedence decision"); + } + } + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![9]) + .expect("rule stop states"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("ops"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 8 }) + .expect("exit"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("to shift"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("to rel"); + atn.add_transition( + 3, + ParserTransitionSpec::Precedence { + target: 4, + precedence: 2, + }, + ) + .expect("shift prec"); + atn.add_transition( + 4, + ParserTransitionSpec::Atom { + target: 5, + label: 1, + }, + ) + .expect("shift first >"); + atn.add_transition( + 5, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("shift second >"); + atn.add_transition( + 6, + ParserTransitionSpec::Precedence { + target: 7, + precedence: 1, + }, + ) + .expect("rel prec"); + atn.add_transition( + 7, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("rel >"); + atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) + .expect("loop end"); + finish_atn(atn) + } + + fn left_recursive_loop_with_rule_wrapped_gt_prefix_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(2); + for (state, kind, rule) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::StarLoopEntry, 0), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::Basic, 0), + (4, AtnStateKind::Basic, 0), + (5, AtnStateKind::Basic, 0), + (6, AtnStateKind::Basic, 0), + (7, AtnStateKind::Basic, 0), + (8, AtnStateKind::LoopEnd, 0), + (9, AtnStateKind::RuleStop, 0), + (10, AtnStateKind::RuleStart, 1), + (11, AtnStateKind::Basic, 1), + (12, AtnStateKind::RuleStop, 1), + ] { + assert_eq!( + atn.add_state(kind, Some(rule)).expect("state").index(), + state + ); + if state == 0 { + atn.set_left_recursive_rule(state) + .expect("left-recursive rule start"); + } else if state == 1 { + atn.set_precedence_rule_decision(state) + .expect("precedence decision"); + } + } + atn.set_rule_to_start_state(vec![0, 10]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![9, 12]) + .expect("rule stop states"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("ops"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 8 }) + .expect("exit"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("to shift"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("to relational"); + atn.add_transition( + 3, + ParserTransitionSpec::Precedence { + target: 4, + precedence: 2, + }, + ) + .expect("shift precedence"); + atn.add_transition( + 4, + ParserTransitionSpec::Rule { + target: 10, + rule_index: 1, + follow_state: 5, + precedence: 0, + }, + ) + .expect("first shift token helper"); + atn.add_transition( + 5, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("second shift token"); + atn.add_transition( + 6, + ParserTransitionSpec::Precedence { + target: 7, + precedence: 1, + }, + ) + .expect("relational precedence"); + atn.add_transition( + 7, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("relational token"); + atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) + .expect("loop end"); + atn.add_transition(10, ParserTransitionSpec::Epsilon { target: 11 }) + .expect("helper entry"); + atn.add_transition( + 11, + ParserTransitionSpec::Atom { + target: 12, + label: 1, + }, + ) + .expect("first shift token"); + finish_atn(atn) + } + + fn left_recursive_loop_with_predicate_and_multi_token_prefix_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for (state, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::StarLoopEntry), + (2, AtnStateKind::Basic), + (3, AtnStateKind::Basic), + (4, AtnStateKind::Basic), + (5, AtnStateKind::Basic), + (6, AtnStateKind::Basic), + (7, AtnStateKind::Basic), + (8, AtnStateKind::Basic), + (9, AtnStateKind::LoopEnd), + (10, AtnStateKind::RuleStop), + ] { + assert_eq!(atn.add_state(kind, Some(0)).expect("state").index(), state); + if state == 0 { + atn.set_left_recursive_rule(state) + .expect("left-recursive rule start"); + } else if state == 1 { + atn.set_precedence_rule_decision(state) + .expect("precedence decision"); + } + } + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![10]) + .expect("rule stop states"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("ops"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 9 }) + .expect("exit"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("to multi-token operator"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("to predicate operator"); + atn.add_transition( + 3, + ParserTransitionSpec::Precedence { + target: 4, + precedence: 2, + }, + ) + .expect("multi-token precedence"); + atn.add_transition( + 4, + ParserTransitionSpec::Atom { + target: 5, + label: 1, + }, + ) + .expect("multi-token first"); + atn.add_transition( + 5, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("multi-token second"); + atn.add_transition( + 6, + ParserTransitionSpec::Precedence { + target: 7, + precedence: 2, + }, + ) + .expect("predicate precedence"); + atn.add_transition( + 7, + ParserTransitionSpec::Predicate { + target: 8, + rule_index: 0, + pred_index: 0, + context_dependent: false, + }, + ) + .expect("operator predicate"); + atn.add_transition( + 8, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("predicate single token"); + atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 10 }) + .expect("loop end"); + finish_atn(atn) + } + + fn left_recursive_loop_with_nullable_operator_prefix_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(2); + for (state, kind, rule) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::StarLoopEntry, 0), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::Basic, 0), + (4, AtnStateKind::Basic, 0), + (5, AtnStateKind::LoopEnd, 0), + (6, AtnStateKind::RuleStop, 0), + (7, AtnStateKind::RuleStart, 1), + (8, AtnStateKind::RuleStop, 1), + (9, AtnStateKind::Basic, 1), + ] { + assert_eq!( + atn.add_state(kind, Some(rule)).expect("state").index(), + state + ); + if state == 0 { + atn.set_left_recursive_rule(state) + .expect("left-recursive rule start"); + } else if state == 1 { + atn.set_precedence_rule_decision(state) + .expect("precedence decision"); + } + } + atn.set_rule_to_start_state(vec![0, 7]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![6, 8]) + .expect("rule stop states"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Precedence { + target: 3, + precedence: 3, + }, + ) + .expect("transition"); + atn.add_transition( + 3, + ParserTransitionSpec::Rule { + target: 7, + rule_index: 1, + follow_state: 4, + precedence: 0, + }, + ) + .expect("transition"); + atn.add_transition( + 4, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("transition"); + atn.add_transition( + 7, + ParserTransitionSpec::Precedence { + target: 9, + precedence: 1, + }, + ) + .expect("transition"); + atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 8 }) + .expect("transition"); + finish_atn(atn) + } + + fn left_recursive_loop_with_predicate_guarded_operator_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(2); + for (state, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::StarLoopEntry), + (2, AtnStateKind::Basic), + (3, AtnStateKind::Basic), + (4, AtnStateKind::Basic), + (5, AtnStateKind::LoopEnd), + (6, AtnStateKind::RuleStop), + ] { + assert_eq!(atn.add_state(kind, Some(0)).expect("state").index(), state); + if state == 0 { + atn.set_left_recursive_rule(state) + .expect("left-recursive rule start"); + } else if state == 1 { + atn.set_precedence_rule_decision(state) + .expect("precedence decision"); + } + } + atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); - atn.set_rule_to_stop_state(stops.clone()) + atn.set_rule_to_stop_state(vec![6]) .expect("rule stop states"); - for rule_index in 0..depth - 1 { - let follow_state = if consuming_follows { - follows[rule_index] - } else { - stops[rule_index] - }; - atn.add_transition( - starts[rule_index], - ParserTransitionSpec::Rule { - target: starts[rule_index + 1], - rule_index: rule_index + 1, - follow_state, - precedence: 0, - }, - ) - .expect("nested rule transition"); - if branching { - atn.add_transition( - starts[rule_index], - ParserTransitionSpec::Atom { - target: stops[rule_index], - label: 2, - }, - ) - .expect("dead branch transition"); - } - if consuming_follows { - atn.add_transition( - follow_state, - ParserTransitionSpec::Atom { - target: stops[rule_index], - label: 1, - }, - ) - .expect("consuming follow transition"); - } - } - let token_set = atn.add_interval_set([(1, 1)]).expect("token set"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); atn.add_transition( - starts[depth - 1], - ParserTransitionSpec::Set { - target: stops[depth - 1], - set: token_set, + 2, + ParserTransitionSpec::Precedence { + target: 3, + precedence: 1, }, ) - .expect("terminal set transition"); - if branching { - atn.add_transition( - starts[depth - 1], - ParserTransitionSpec::Atom { - target: stops[depth - 1], - label: 2, - }, - ) - .expect("dead leaf branch transition"); - } + .expect("transition"); + atn.add_transition( + 3, + ParserTransitionSpec::Predicate { + target: 4, + rule_index: 0, + pred_index: 0, + context_dependent: false, + }, + ) + .expect("transition"); + atn.add_transition( + 4, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("transition"); finish_atn(atn) } - fn ordinary_star_loop_atn() -> Atn { + fn left_recursive_loop_with_nullable_follow_call_atn(caller_symbol: i32) -> Atn { let mut atn = ParserAtnBuilder::new(2); - for (state_number, kind, rule_index) in [ + for (state, kind, rule) in [ (0, AtnStateKind::RuleStart, 0), - (1, AtnStateKind::StarLoopEntry, 0), + (1, AtnStateKind::Basic, 0), (2, AtnStateKind::Basic, 0), - (3, AtnStateKind::StarLoopBack, 0), - (4, AtnStateKind::LoopEnd, 0), - (5, AtnStateKind::Basic, 0), - (6, AtnStateKind::RuleStop, 0), - (7, AtnStateKind::RuleStart, 1), + (3, AtnStateKind::Basic, 0), + (4, AtnStateKind::RuleStop, 0), + (5, AtnStateKind::RuleStart, 1), + (6, AtnStateKind::StarLoopEntry, 1), + (7, AtnStateKind::Basic, 1), (8, AtnStateKind::Basic, 1), - (9, AtnStateKind::RuleStop, 1), + (9, AtnStateKind::LoopEnd, 1), + (10, AtnStateKind::RuleStop, 1), + (11, AtnStateKind::RuleStart, 2), + (12, AtnStateKind::RuleStop, 2), ] { assert_eq!( - atn.add_state(kind, Some(rule_index)) - .expect("state") - .index(), - state_number + atn.add_state(kind, Some(rule)).expect("state").index(), + state ); + if state == 5 { + atn.set_left_recursive_rule(state) + .expect("left-recursive rule start"); + } else if state == 6 { + atn.set_precedence_rule_decision(state) + .expect("precedence decision"); + } } - atn.set_rule_to_start_state(vec![0, 7]) + atn.set_rule_to_start_state(vec![0, 5, 11]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![6, 9]) + atn.set_rule_to_stop_state(vec![4, 10, 12]) .expect("rule stop states"); - atn.add_decision_state(1).expect("decision state"); - atn.set_loop_back_state(4, 3).expect("loop back state"); atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 4 }) - .expect("transition"); atn.add_transition( - 2, + 1, ParserTransitionSpec::Rule { - target: 7, + target: 5, rule_index: 1, + follow_state: 2, + precedence: 0, + }, + ) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Rule { + target: 11, + rule_index: 2, follow_state: 3, precedence: 0, }, ) .expect("transition"); - atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) - .expect("transition"); atn.add_transition( - 5, + 3, ParserTransitionSpec::Atom { - target: 6, - label: TOKEN_EOF, + target: 4, + label: caller_symbol, }, ) .expect("transition"); - atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 8 }) + atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) + .expect("transition"); + atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 9 }) .expect("transition"); + atn.add_transition( + 7, + ParserTransitionSpec::Precedence { + target: 8, + precedence: 1, + }, + ) + .expect("transition"); atn.add_transition( 8, ParserTransitionSpec::Atom { - target: 9, + target: 6, label: 1, }, ) .expect("transition"); + atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 10 }) + .expect("transition"); + atn.add_transition(11, ParserTransitionSpec::Epsilon { target: 12 }) + .expect("transition"); finish_atn(atn) } - /// ATN for `s : (X | X X)* EOF`. - fn ambiguous_ordinary_star_loop_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(1); - for (state_number, kind) in [ - (0, AtnStateKind::RuleStart), - (1, AtnStateKind::StarLoopEntry), - (2, AtnStateKind::StarBlockStart), - (3, AtnStateKind::Basic), - (4, AtnStateKind::BlockEnd), - (5, AtnStateKind::StarLoopBack), - (6, AtnStateKind::LoopEnd), - (7, AtnStateKind::Basic), - (8, AtnStateKind::RuleStop), + fn left_recursive_loop_with_nullable_parent_return_atn(caller_symbol: i32) -> Atn { + let mut atn = ParserAtnBuilder::new(2); + for (state, kind, rule) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::Basic, 0), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::RuleStop, 0), + (4, AtnStateKind::RuleStart, 1), + (5, AtnStateKind::Basic, 1), + (6, AtnStateKind::Basic, 1), + (7, AtnStateKind::RuleStop, 1), + (8, AtnStateKind::RuleStart, 2), + (9, AtnStateKind::StarLoopEntry, 2), + (10, AtnStateKind::Basic, 2), + (11, AtnStateKind::Basic, 2), + (12, AtnStateKind::LoopEnd, 2), + (13, AtnStateKind::RuleStop, 2), ] { assert_eq!( - atn.add_state(kind, Some(0)).expect("state").index(), - state_number + atn.add_state(kind, Some(rule)).expect("state").index(), + state ); + if state == 8 { + atn.set_left_recursive_rule(state) + .expect("left-recursive rule start"); + } else if state == 9 { + atn.set_precedence_rule_decision(state) + .expect("precedence decision"); + } } - atn.set_rule_to_start_state(vec![0]) + atn.set_rule_to_start_state(vec![0, 4, 8]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![8]) + atn.set_rule_to_stop_state(vec![3, 7, 13]) .expect("rule stop states"); - atn.set_end_state(2, 4).expect("block end state"); - atn.set_loop_back_state(6, 5).expect("loop back state"); - atn.add_decision_state(1).expect("decision state"); - atn.add_decision_state(2).expect("decision state"); atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 6 }) - .expect("transition"); atn.add_transition( - 2, - ParserTransitionSpec::Atom { + 1, + ParserTransitionSpec::Rule { target: 4, - label: 1, + rule_index: 1, + follow_state: 2, + precedence: 0, }, ) .expect("transition"); @@ -13688,88 +15557,113 @@ mod tests { 2, ParserTransitionSpec::Atom { target: 3, - label: 1, + label: caller_symbol, }, ) .expect("transition"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); atn.add_transition( - 3, - ParserTransitionSpec::Atom { - target: 4, - label: 1, + 5, + ParserTransitionSpec::Rule { + target: 8, + rule_index: 2, + follow_state: 6, + precedence: 0, }, ) .expect("transition"); - atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) + atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) .expect("transition"); - atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 1 }) + atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 10 }) .expect("transition"); - atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) + atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 12 }) .expect("transition"); atn.add_transition( - 7, + 10, + ParserTransitionSpec::Precedence { + target: 11, + precedence: 1, + }, + ) + .expect("transition"); + atn.add_transition( + 11, ParserTransitionSpec::Atom { - target: 8, - label: TOKEN_EOF, + target: 9, + label: 1, }, ) .expect("transition"); + atn.add_transition(12, ParserTransitionSpec::Epsilon { target: 13 }) + .expect("transition"); finish_atn(atn) } - fn ordinary_plus_loop_atn() -> Atn { + fn left_recursive_loop_with_recursive_operand_return_atn(caller_symbol: i32) -> Atn { let mut atn = ParserAtnBuilder::new(2); - for (state_number, kind, rule_index) in [ + for (state, kind, rule) in [ (0, AtnStateKind::RuleStart, 0), (1, AtnStateKind::Basic, 0), - (2, AtnStateKind::PlusLoopBack, 0), - (3, AtnStateKind::LoopEnd, 0), - (4, AtnStateKind::Basic, 0), - (5, AtnStateKind::RuleStop, 0), - (6, AtnStateKind::RuleStart, 1), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::RuleStop, 0), + (4, AtnStateKind::RuleStart, 1), + (5, AtnStateKind::StarLoopEntry, 1), + (6, AtnStateKind::Basic, 1), (7, AtnStateKind::Basic, 1), - (8, AtnStateKind::RuleStop, 1), + (8, AtnStateKind::Basic, 1), + (9, AtnStateKind::Basic, 1), + (10, AtnStateKind::LoopEnd, 1), + (11, AtnStateKind::RuleStop, 1), ] { assert_eq!( - atn.add_state(kind, Some(rule_index)) - .expect("state") - .index(), - state_number + atn.add_state(kind, Some(rule)).expect("state").index(), + state ); + if state == 4 { + atn.set_left_recursive_rule(state) + .expect("left-recursive rule start"); + } else if state == 5 { + atn.set_precedence_rule_decision(state) + .expect("precedence decision"); + } } - atn.set_rule_to_start_state(vec![0, 6]) + atn.set_rule_to_start_state(vec![0, 4]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![5, 8]) + atn.set_rule_to_stop_state(vec![3, 11]) .expect("rule stop states"); - atn.add_decision_state(2).expect("decision state"); atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) .expect("transition"); atn.add_transition( 1, ParserTransitionSpec::Rule { - target: 6, + target: 4, rule_index: 1, follow_state: 2, precedence: 0, }, ) .expect("transition"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) - .expect("transition"); - atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 }) - .expect("transition"); atn.add_transition( - 4, + 2, ParserTransitionSpec::Atom { - target: 5, - label: TOKEN_EOF, + target: 3, + label: caller_symbol, }, ) .expect("transition"); - atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("transition"); + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 10 }) .expect("transition"); + atn.add_transition( + 6, + ParserTransitionSpec::Precedence { + target: 7, + precedence: 1, + }, + ) + .expect("transition"); atn.add_transition( 7, ParserTransitionSpec::Atom { @@ -13778,176 +15672,242 @@ mod tests { }, ) .expect("transition"); + atn.add_transition( + 8, + ParserTransitionSpec::Rule { + target: 4, + rule_index: 1, + follow_state: 9, + precedence: 2, + }, + ) + .expect("transition"); + atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); + atn.add_transition(10, ParserTransitionSpec::Epsilon { target: 11 }) + .expect("transition"); finish_atn(atn) } - fn repeated_x_tokens(count: usize) -> Vec { - let mut tokens = (0..count) - .map(|_| TestToken::new(1).with_text("x")) - .collect::>(); - tokens.push(TestToken::eof("parser-test", count, 1, count)); - tokens + #[test] + fn left_recursive_loop_defers_overlapping_caller_lookahead() { + let overlapping_atn = left_recursive_loop_with_caller_follow_atn(1); + let unambiguous_atn = left_recursive_loop_with_caller_follow_atn(2); + + let mut overlapping = parser_inside_left_recursive_callee(1); + assert_eq!( + overlapping.left_recursive_loop_enter_prediction(&overlapping_atn, 4, 0), + None + ); + + let mut unambiguous_enter = parser_inside_left_recursive_callee(1); + assert_eq!( + unambiguous_enter.left_recursive_loop_enter_prediction(&unambiguous_atn, 4, 0), + Some(true) + ); + + let mut unambiguous_exit = parser_inside_left_recursive_callee(2); + assert_eq!( + unambiguous_exit.left_recursive_loop_enter_prediction(&unambiguous_atn, 4, 0), + Some(false) + ); + + assert_eq!( + overlapping.left_recursive_loop_enter_prediction(&unambiguous_atn, 4, 0), + Some(true), + "overlap results must not leak across ATNs" + ); } - fn left_recursive_loop_with_caller_follow_atn(caller_symbol: i32) -> Atn { - let mut atn = ParserAtnBuilder::new(2); + #[test] + fn left_recursive_loop_enters_after_nullable_operator_prefix() { + let atn = left_recursive_loop_with_nullable_operator_prefix_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("operator"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: -1, + }]; + assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(0)) - .expect("state") - .index(), - 0 + parser.left_recursive_loop_enter_prediction(&atn, 1, 0), + Some(true) ); assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 1 + parser.left_recursive_loop_enter_prediction(&atn, 1, 0), + Some(true), + "cached operator lookahead must preserve the nullable prefix return path" ); assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 2 + parser.left_recursive_loop_enter_prediction(&atn, 1, 2), + Some(true), + "the nullable child must use its rule-call precedence, not the caller precedence" ); + } + + #[test] + fn left_recursive_loop_defers_multi_token_prefix_that_shadows_lower_single_token() { + // Models Java `>` (relational, prec 1, one token) vs `>>` (shift, prec 2, + // two tokens). At prec 2 only shift is viable; one-token lookahead on `>` + // must defer so StarLoopEntry adaptive predict can exit when the second + // `>` is absent (as in `a < b > c`). + let atn = left_recursive_loop_with_shared_gt_prefix_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text(">"), + TestToken::new(2).with_text("id"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: -1, + }]; + assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(1)) - .expect("state") - .index(), - 3 + parser.left_recursive_loop_enter_prediction(&atn, 1, 0), + Some(true), + "at low precedence relational `>` is a single-token operator" ); - atn.set_left_recursive_rule(3) - .expect("left-recursive rule start"); assert_eq!( - atn.add_state(AtnStateKind::StarLoopEntry, Some(1)) - .expect("state") - .index(), - 4 + parser.left_recursive_loop_enter_prediction(&atn, 1, 1), + Some(true), + "relational remains single-token at its own precedence" ); - atn.set_precedence_rule_decision(4) - .expect("precedence decision"); assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(1)) - .expect("state") - .index(), - 5 + parser.left_recursive_loop_enter_prediction(&atn, 1, 2), + None, + "at shift precedence, bare `>` must not force enter" + ); + } + + #[test] + fn left_recursive_loop_preserves_rule_wrapped_operator_continuation() { + let atn = left_recursive_loop_with_rule_wrapped_gt_prefix_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text(">"), + TestToken::new(2).with_text("id"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: -1, + }]; + + assert_eq!( + parser.left_recursive_loop_enter_prediction(&atn, 1, 0), + Some(true), + "the direct relational alternative remains a one-token operator" + ); + assert_eq!( + parser.left_recursive_loop_enter_prediction(&atn, 1, 2), + None, + "a token matched in the helper rule must return to the second shift token" + ); + } + + #[test] + fn left_recursive_loop_preserves_predicate_and_multi_token_reachability() { + let atn = left_recursive_loop_with_predicate_and_multi_token_prefix_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text(">"), + TestToken::new(2).with_text("id"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: -1, + }]; + + assert_eq!( + parser.left_recursive_loop_enter_prediction(&atn, 1, 2), + None, + "a predicate-gated single-token path must not be hidden by a multi-token path" + ); + } + + #[test] + fn left_recursive_loop_defers_predicate_guarded_operator() { + let atn = left_recursive_loop_with_predicate_guarded_operator_atn(); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("operator"), + TestToken::eof("parser-test", 1, 1, 1), + ], + RejectingPredicateHooks::default(), ); + parser.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: -1, + }]; + assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(1)) - .expect("state") - .index(), - 6 + parser.left_recursive_loop_enter_prediction(&atn, 1, 0), + None, + "a false predicate must be evaluated before entering the operator alternative" ); assert_eq!( - atn.add_state(AtnStateKind::LoopEnd, Some(1)) - .expect("state") - .index(), - 7 + parser.left_recursive_loop_enter_prediction(&atn, 1, 0), + None, + "cached predicate-dependent lookahead must keep deferring" ); + } + + #[test] + fn left_recursive_loop_defers_through_nullable_caller_rule_call() { + let atn = left_recursive_loop_with_nullable_follow_call_atn(1); + let mut parser = parser_inside_left_recursive_callee(1); + assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(1)) - .expect("state") - .index(), - 8 + parser.left_recursive_loop_enter_prediction(&atn, 6, 0), + None ); assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(0)) - .expect("state") - .index(), - 9 + parser.left_recursive_loop_enter_prediction(&atn, 6, 0), + None, + "the cached overlap must preserve the nullable child return path" ); - atn.set_rule_to_start_state(vec![0, 3]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![9, 8]) - .expect("rule stop states"); - atn.add_transition( - 1, - ParserTransitionSpec::Rule { - target: 3, - rule_index: 1, - follow_state: 2, - precedence: 0, - }, - ) - .expect("transition"); - atn.add_transition( - 2, - ParserTransitionSpec::Atom { - target: 9, - label: caller_symbol, + } + + #[test] + fn left_recursive_loop_defers_through_nullable_parent_return() { + let atn = left_recursive_loop_with_nullable_parent_return_atn(1); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("lookahead"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.rule_context_stack = vec![ + RuleContextFrame { + rule_index: 0, + invoking_state: -1, }, - ) - .expect("transition"); - atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) - .expect("transition"); - atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 7 }) - .expect("transition"); - atn.add_transition( - 5, - ParserTransitionSpec::Precedence { - target: 6, - precedence: 1, + RuleContextFrame { + rule_index: 1, + invoking_state: 1, }, - ) - .expect("transition"); - atn.add_transition( - 6, - ParserTransitionSpec::Atom { - target: 4, - label: 1, + RuleContextFrame { + rule_index: 2, + invoking_state: 5, }, - ) - .expect("transition"); - atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 8 }) - .expect("transition"); - finish_atn(atn) - } + ]; - fn labeled_left_recursive_operator_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(4); - for (state, kind) in [ - (0, AtnStateKind::RuleStart), - (1, AtnStateKind::BlockStart), - (2, AtnStateKind::StarLoopEntry), - (3, AtnStateKind::StarBlockStart), - (4, AtnStateKind::Basic), - (5, AtnStateKind::Basic), - (6, AtnStateKind::Basic), - (7, AtnStateKind::StarLoopBack), - (8, AtnStateKind::LoopEnd), - (9, AtnStateKind::RuleStop), - ] { - assert_eq!(atn.add_state(kind, Some(0)).expect("state").index(), state); - } - atn.set_left_recursive_rule(0) - .expect("left-recursive rule start"); - atn.set_precedence_rule_decision(2) - .expect("precedence decision"); - atn.set_loop_back_state(8, 7).expect("loop-back state"); - atn.set_rule_to_start_state(vec![0]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![9]) - .expect("rule stop states"); - for state in [1, 2, 3] { - atn.add_decision_state(state).expect("decision state"); - } - for (source, target) in [(0, 1), (2, 3), (2, 8), (7, 2), (8, 9)] { - atn.add_transition(source, ParserTransitionSpec::Epsilon { target }) - .expect("epsilon transition"); - } - for (source, target, label) in [(1, 2, 1), (1, 2, 2), (4, 6, 4), (5, 6, 3), (6, 7, 1)] { - atn.add_transition(source, ParserTransitionSpec::Atom { target, label }) - .expect("token transition"); - } - for (target, precedence) in [(4, 2), (5, 1)] { - atn.add_transition(3, ParserTransitionSpec::Precedence { target, precedence }) - .expect("operator precedence"); - } - finish_atn(atn) + assert_eq!( + parser.left_recursive_loop_enter_prediction(&atn, 9, 0), + None, + "a nullable caller must unwind to its parent's consuming follow path" + ); + assert_eq!( + parser.left_recursive_loop_enter_prediction(&atn, 9, 0), + None, + "the caller-overlap cache must not retain a false negative" + ); } - fn parser_inside_left_recursive_callee(symbol: i32) -> BaseParser { + #[test] + fn left_recursive_loop_defers_after_recursive_operand_returns_to_loop() { + let atn = left_recursive_loop_with_recursive_operand_return_atn(1); let mut parser = mini_parser(vec![ - TestToken::new(symbol).with_text("lookahead"), + TestToken::new(1).with_text("lookahead"), TestToken::eof("parser-test", 1, 1, 1), ]); parser.rule_context_stack = vec![ @@ -13959,605 +15919,722 @@ mod tests { rule_index: 1, invoking_state: 1, }, + RuleContextFrame { + rule_index: 1, + invoking_state: 8, + }, ]; - parser + + assert_eq!( + parser.left_recursive_loop_enter_prediction(&atn, 5, 0), + None, + "a recursive operand return must preserve its parent caller context" + ); + assert_eq!( + parser.left_recursive_loop_enter_prediction(&atn, 5, 0), + None, + "the caller-overlap cache must preserve the loop-boundary return" + ); } - fn left_recursive_loop_with_shared_gt_prefix_atn() -> Atn { - // StarLoopEntry with two operator alts that share leading token 1 (`>`): - // prec 2: token 1, token 1 (shift `>>`) - // prec 1: token 1 (relational `>`) + fn token_then_eof_atn() -> Atn { + AtnDeserializer::new(&SerializedAtn::from_i32(&[ + 4, 1, 2, // version, parser, max token type + 3, // states + 2, 0, // rule start + 1, 0, // basic + 7, 0, // rule stop + 0, // non-greedy states + 0, // precedence states + 1, // rules + 0, // rule 0 start + 0, // modes + 0, // sets + 2, // transitions + 0, 1, 5, 1, 0, 0, // match token 1 + 1, 2, 5, -1, 0, 0, // match EOF + 0, // decisions + ])) + .deserialize_parser() + .expect("artificial parser ATN should deserialize") + } + + fn epsilon_cycle_atn() -> Atn { let mut atn = ParserAtnBuilder::new(1); - for (state, kind, rule) in [ - (0, AtnStateKind::RuleStart, 0), - (1, AtnStateKind::StarLoopEntry, 0), - (2, AtnStateKind::Basic, 0), // ops hub - (3, AtnStateKind::Basic, 0), // shift prec - (4, AtnStateKind::Basic, 0), // shift first > - (5, AtnStateKind::Basic, 0), // shift second > - (6, AtnStateKind::Basic, 0), // rel prec - (7, AtnStateKind::Basic, 0), // rel > - (8, AtnStateKind::LoopEnd, 0), - (9, AtnStateKind::RuleStop, 0), + for (state_number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::Basic), + (2, AtnStateKind::RuleStop), ] { assert_eq!( - atn.add_state(kind, Some(rule)).expect("state").index(), - state + atn.add_state(kind, Some(0)).expect("state").index(), + state_number + ); + } + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![2]) + .expect("rule stop states"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("self-cycle transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("exit transition"); + finish_atn(atn) + } + + fn committed_non_consuming_cycle_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for (state_number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::Basic), + (2, AtnStateKind::RuleStop), + ] { + assert_eq!( + atn.add_state(kind, Some(0)).expect("state").index(), + state_number ); - if state == 0 { - atn.set_left_recursive_rule(state) - .expect("left-recursive rule start"); - } else if state == 1 { - atn.set_precedence_rule_decision(state) - .expect("precedence decision"); - } } atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![9]) + atn.set_rule_to_stop_state(vec![2]) .expect("rule stop states"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("ops"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 8 }) - .expect("exit"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) - .expect("to shift"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 6 }) - .expect("to rel"); - atn.add_transition( - 3, - ParserTransitionSpec::Precedence { - target: 4, - precedence: 2, - }, - ) - .expect("shift prec"); - atn.add_transition( - 4, - ParserTransitionSpec::Atom { - target: 5, - label: 1, - }, - ) - .expect("shift first >"); - atn.add_transition( - 5, - ParserTransitionSpec::Atom { - target: 1, - label: 1, - }, - ) - .expect("shift second >"); - atn.add_transition( - 6, - ParserTransitionSpec::Precedence { - target: 7, - precedence: 1, - }, - ) - .expect("rel prec"); - atn.add_transition( - 7, - ParserTransitionSpec::Atom { - target: 1, - label: 1, - }, - ) - .expect("rel >"); - atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) - .expect("loop end"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("cycle entry"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("self-cycle transition"); finish_atn(atn) } - fn left_recursive_loop_with_rule_wrapped_gt_prefix_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(2); - for (state, kind, rule) in [ - (0, AtnStateKind::RuleStart, 0), - (1, AtnStateKind::StarLoopEntry, 0), - (2, AtnStateKind::Basic, 0), - (3, AtnStateKind::Basic, 0), - (4, AtnStateKind::Basic, 0), - (5, AtnStateKind::Basic, 0), - (6, AtnStateKind::Basic, 0), - (7, AtnStateKind::Basic, 0), - (8, AtnStateKind::LoopEnd, 0), - (9, AtnStateKind::RuleStop, 0), - (10, AtnStateKind::RuleStart, 1), - (11, AtnStateKind::Basic, 1), - (12, AtnStateKind::RuleStop, 1), + fn eof_then_action_atn() -> Atn { + AtnDeserializer::new(&SerializedAtn::from_i32(&[ + 4, 1, 1, // version, parser, max token type + 3, // states + 2, 0, // rule start + 1, 0, // basic + 7, 0, // rule stop + 0, // non-greedy states + 0, // precedence states + 1, // rules + 0, // rule 0 start + 0, // modes + 0, // sets + 2, // transitions + 0, 1, 5, -1, 0, 0, // match EOF + 1, 2, 6, 0, 0, 0, // parser action + 0, // decisions + ])) + .deserialize_parser() + .expect("artificial parser ATN should deserialize") + } + + fn noop_action_then_token_then_eof_atn() -> Atn { + AtnDeserializer::new(&SerializedAtn::from_i32(&[ + 4, 1, 2, // version, parser, max token type + 4, // states + 2, 0, // rule start + 1, 0, // basic + 1, 0, // basic + 7, 0, // rule stop + 0, // non-greedy states + 0, // precedence states + 1, // rules + 0, // rule 0 start + 0, // modes + 0, // sets + 3, // transitions + 0, 1, 6, 0, -1, 0, // no-op parser action + 1, 2, 5, 1, 0, 0, // match token 1 + 2, 3, 5, -1, 0, 0, // match EOF + 0, // decisions + ])) + .deserialize_parser() + .expect("artificial no-op action ATN should deserialize") + } + + fn committed_action_then_predicate_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for (state_number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::Basic), + (2, AtnStateKind::Basic), + (3, AtnStateKind::Basic), + (4, AtnStateKind::RuleStop), ] { assert_eq!( - atn.add_state(kind, Some(rule)).expect("state").index(), - state + atn.add_state(kind, Some(0)).expect("state").index(), + state_number ); - if state == 0 { - atn.set_left_recursive_rule(state) - .expect("left-recursive rule start"); - } else if state == 1 { - atn.set_precedence_rule_decision(state) - .expect("precedence decision"); - } } - atn.set_rule_to_start_state(vec![0, 10]) + atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![9, 12]) + atn.set_rule_to_stop_state(vec![4]) .expect("rule stop states"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("ops"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 8 }) - .expect("exit"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) - .expect("to shift"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 6 }) - .expect("to relational"); - atn.add_transition( - 3, - ParserTransitionSpec::Precedence { - target: 4, - precedence: 2, - }, - ) - .expect("shift precedence"); - atn.add_transition( - 4, - ParserTransitionSpec::Rule { - target: 10, - rule_index: 1, - follow_state: 5, - precedence: 0, - }, - ) - .expect("first shift token helper"); atn.add_transition( - 5, - ParserTransitionSpec::Atom { + 0, + ParserTransitionSpec::Action { target: 1, - label: 1, + rule_index: 0, + action_index: None, + context_dependent: false, }, ) - .expect("second shift token"); + .expect("action transition"); atn.add_transition( - 6, - ParserTransitionSpec::Precedence { - target: 7, - precedence: 1, + 1, + ParserTransitionSpec::Predicate { + target: 2, + rule_index: 0, + pred_index: 0, + context_dependent: false, }, ) - .expect("relational precedence"); + .expect("predicate transition"); atn.add_transition( - 7, + 2, ParserTransitionSpec::Atom { - target: 1, + target: 3, label: 1, }, ) - .expect("relational token"); - atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) - .expect("loop end"); - atn.add_transition(10, ParserTransitionSpec::Epsilon { target: 11 }) - .expect("helper entry"); + .expect("token transition"); atn.add_transition( - 11, + 3, ParserTransitionSpec::Atom { - target: 12, - label: 1, + target: 4, + label: TOKEN_EOF, }, ) - .expect("first shift token"); + .expect("EOF transition"); finish_atn(atn) } - fn left_recursive_loop_with_predicate_and_multi_token_prefix_atn() -> Atn { + /// ATN for `parent : child[42] {Parent();}; child[int value] : {Child();} EOF;`. + fn parameterized_child_action_eof_atn() -> Atn { let mut atn = ParserAtnBuilder::new(1); - for (state, kind) in [ - (0, AtnStateKind::RuleStart), - (1, AtnStateKind::StarLoopEntry), - (2, AtnStateKind::Basic), - (3, AtnStateKind::Basic), - (4, AtnStateKind::Basic), - (5, AtnStateKind::Basic), - (6, AtnStateKind::Basic), - (7, AtnStateKind::Basic), - (8, AtnStateKind::Basic), - (9, AtnStateKind::LoopEnd), - (10, AtnStateKind::RuleStop), + for (state_number, kind, rule_index) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::Basic, 0), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::RuleStop, 0), + (4, AtnStateKind::RuleStart, 1), + (5, AtnStateKind::Basic, 1), + (6, AtnStateKind::RuleStop, 1), ] { - assert_eq!(atn.add_state(kind, Some(0)).expect("state").index(), state); - if state == 0 { - atn.set_left_recursive_rule(state) - .expect("left-recursive rule start"); - } else if state == 1 { - atn.set_precedence_rule_decision(state) - .expect("precedence decision"); - } + assert_eq!( + atn.add_state(kind, Some(rule_index)) + .expect("state") + .index(), + state_number + ); } - atn.set_rule_to_start_state(vec![0]) + atn.set_rule_to_start_state(vec![0, 4]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![10]) + atn.set_rule_to_stop_state(vec![3, 6]) .expect("rule stop states"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("ops"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 9 }) - .expect("exit"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) - .expect("to multi-token operator"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 6 }) - .expect("to predicate operator"); atn.add_transition( - 3, - ParserTransitionSpec::Precedence { + 0, + ParserTransitionSpec::Rule { target: 4, - precedence: 2, + rule_index: 1, + follow_state: 1, + precedence: 0, }, ) - .expect("multi-token precedence"); + .expect("parameterized child call"); + atn.add_transition( + 1, + ParserTransitionSpec::Action { + target: 2, + rule_index: 0, + action_index: None, + context_dependent: false, + }, + ) + .expect("parent action"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("parent stop"); atn.add_transition( 4, - ParserTransitionSpec::Atom { + ParserTransitionSpec::Action { target: 5, - label: 1, + rule_index: 1, + action_index: None, + context_dependent: false, }, ) - .expect("multi-token first"); + .expect("child action"); atn.add_transition( 5, ParserTransitionSpec::Atom { - target: 1, - label: 1, + target: 6, + label: TOKEN_EOF, }, ) - .expect("multi-token second"); + .expect("child EOF"); + finish_atn(atn) + } + + fn action_then_nested_rule_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for (state_number, kind, rule_index) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::Basic, 0), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::RuleStop, 0), + (4, AtnStateKind::RuleStart, 1), + (5, AtnStateKind::RuleStop, 1), + ] { + assert_eq!( + atn.add_state(kind, Some(rule_index)) + .expect("state") + .index(), + state_number + ); + } + atn.set_rule_to_start_state(vec![0, 4]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![3, 5]) + .expect("rule stop states"); atn.add_transition( - 6, - ParserTransitionSpec::Precedence { - target: 7, - precedence: 2, + 0, + ParserTransitionSpec::Action { + target: 1, + rule_index: 0, + action_index: None, + context_dependent: false, }, ) - .expect("predicate precedence"); + .expect("parent action"); atn.add_transition( - 7, - ParserTransitionSpec::Predicate { - target: 8, - rule_index: 0, - pred_index: 0, - context_dependent: false, + 1, + ParserTransitionSpec::Rule { + target: 4, + rule_index: 1, + follow_state: 2, + precedence: 0, }, ) - .expect("operator predicate"); + .expect("nested rule call"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("parent stop"); atn.add_transition( - 8, + 4, ParserTransitionSpec::Atom { - target: 1, - label: 1, + target: 5, + label: TOKEN_EOF, }, ) - .expect("predicate single token"); - atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 10 }) - .expect("loop end"); + .expect("child EOF"); finish_atn(atn) } - fn left_recursive_loop_with_nullable_operator_prefix_atn() -> Atn { + fn losing_alternative_action_atn() -> Atn { let mut atn = ParserAtnBuilder::new(2); - for (state, kind, rule) in [ - (0, AtnStateKind::RuleStart, 0), - (1, AtnStateKind::StarLoopEntry, 0), - (2, AtnStateKind::Basic, 0), - (3, AtnStateKind::Basic, 0), - (4, AtnStateKind::Basic, 0), - (5, AtnStateKind::LoopEnd, 0), - (6, AtnStateKind::RuleStop, 0), - (7, AtnStateKind::RuleStart, 1), - (8, AtnStateKind::RuleStop, 1), - (9, AtnStateKind::Basic, 1), + for (state_number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::BlockStart), + (2, AtnStateKind::Basic), + (3, AtnStateKind::Basic), + (4, AtnStateKind::BlockEnd), + (5, AtnStateKind::RuleStop), ] { assert_eq!( - atn.add_state(kind, Some(rule)).expect("state").index(), - state + atn.add_state(kind, Some(0)).expect("state").index(), + state_number ); - if state == 0 { - atn.set_left_recursive_rule(state) - .expect("left-recursive rule start"); - } else if state == 1 { - atn.set_precedence_rule_decision(state) - .expect("precedence decision"); - } } - atn.set_rule_to_start_state(vec![0, 7]) + atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![6, 8]) + atn.set_rule_to_stop_state(vec![5]) .expect("rule stop states"); + atn.set_end_state(1, 4).expect("block end state"); + atn.add_decision_state(1).expect("decision state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("entry transition"); atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 5 }) - .expect("transition"); + .expect("first alternative"); atn.add_transition( - 2, - ParserTransitionSpec::Precedence { - target: 3, - precedence: 3, + 1, + ParserTransitionSpec::Atom { + target: 4, + label: 2, }, ) - .expect("transition"); + .expect("second alternative"); atn.add_transition( - 3, - ParserTransitionSpec::Rule { - target: 7, - rule_index: 1, - follow_state: 4, - precedence: 0, + 2, + ParserTransitionSpec::Action { + target: 3, + rule_index: 0, + action_index: None, + context_dependent: false, }, ) - .expect("transition"); + .expect("losing action"); atn.add_transition( - 4, + 3, ParserTransitionSpec::Atom { - target: 1, + target: 4, label: 1, }, ) - .expect("transition"); - atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) - .expect("transition"); + .expect("first alternative token"); atn.add_transition( - 7, - ParserTransitionSpec::Precedence { - target: 9, - precedence: 1, + 4, + ParserTransitionSpec::Atom { + target: 5, + label: TOKEN_EOF, }, ) - .expect("transition"); - atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 8 }) - .expect("transition"); + .expect("EOF transition"); finish_atn(atn) } - fn left_recursive_loop_with_predicate_guarded_operator_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(2); - for (state, kind) in [ + fn committed_action_star_loop_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for (state_number, kind) in [ (0, AtnStateKind::RuleStart), (1, AtnStateKind::StarLoopEntry), (2, AtnStateKind::Basic), (3, AtnStateKind::Basic), - (4, AtnStateKind::Basic), + (4, AtnStateKind::StarLoopBack), (5, AtnStateKind::LoopEnd), (6, AtnStateKind::RuleStop), ] { - assert_eq!(atn.add_state(kind, Some(0)).expect("state").index(), state); - if state == 0 { - atn.set_left_recursive_rule(state) - .expect("left-recursive rule start"); - } else if state == 1 { - atn.set_precedence_rule_decision(state) - .expect("precedence decision"); - } + assert_eq!( + atn.add_state(kind, Some(0)).expect("state").index(), + state_number + ); } atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); atn.set_rule_to_stop_state(vec![6]) .expect("rule stop states"); + atn.add_decision_state(1).expect("decision state"); + atn.set_loop_back_state(5, 4).expect("loop back state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("entry transition"); atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("transition"); + .expect("loop body"); atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 5 }) - .expect("transition"); + .expect("loop exit"); atn.add_transition( 2, - ParserTransitionSpec::Precedence { + ParserTransitionSpec::Action { target: 3, - precedence: 1, + rule_index: 0, + action_index: None, + context_dependent: false, }, ) - .expect("transition"); + .expect("loop action"); atn.add_transition( 3, - ParserTransitionSpec::Predicate { + ParserTransitionSpec::Atom { target: 4, - rule_index: 0, - pred_index: 0, - context_dependent: false, + label: 1, }, ) - .expect("transition"); + .expect("loop token"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("loop back"); atn.add_transition( - 4, + 5, ParserTransitionSpec::Atom { - target: 1, - label: 1, + target: 6, + label: TOKEN_EOF, }, ) - .expect("transition"); - atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) - .expect("transition"); + .expect("EOF transition"); finish_atn(atn) } - fn left_recursive_loop_with_nullable_follow_call_atn(caller_symbol: i32) -> Atn { - let mut atn = ParserAtnBuilder::new(2); - for (state, kind, rule) in [ - (0, AtnStateKind::RuleStart, 0), - (1, AtnStateKind::Basic, 0), - (2, AtnStateKind::Basic, 0), - (3, AtnStateKind::Basic, 0), - (4, AtnStateKind::RuleStop, 0), - (5, AtnStateKind::RuleStart, 1), - (6, AtnStateKind::StarLoopEntry, 1), - (7, AtnStateKind::Basic, 1), - (8, AtnStateKind::Basic, 1), - (9, AtnStateKind::LoopEnd, 1), - (10, AtnStateKind::RuleStop, 1), - (11, AtnStateKind::RuleStart, 2), - (12, AtnStateKind::RuleStop, 2), + fn committed_action_left_recursive_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(4); + for (state, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::BlockStart), + (2, AtnStateKind::StarLoopEntry), + (3, AtnStateKind::StarBlockStart), + (4, AtnStateKind::Basic), + (5, AtnStateKind::Basic), + (6, AtnStateKind::Basic), + (7, AtnStateKind::StarLoopBack), + (8, AtnStateKind::LoopEnd), + (9, AtnStateKind::RuleStop), + (10, AtnStateKind::Basic), ] { - assert_eq!( - atn.add_state(kind, Some(rule)).expect("state").index(), - state - ); - if state == 5 { - atn.set_left_recursive_rule(state) - .expect("left-recursive rule start"); - } else if state == 6 { - atn.set_precedence_rule_decision(state) - .expect("precedence decision"); - } + assert_eq!(atn.add_state(kind, Some(0)).expect("state").index(), state); + } + atn.set_left_recursive_rule(0) + .expect("left-recursive rule start"); + atn.set_precedence_rule_decision(2) + .expect("precedence decision"); + atn.set_loop_back_state(8, 7).expect("loop-back state"); + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![9]) + .expect("rule stop states"); + for state in [1, 2, 3] { + atn.add_decision_state(state).expect("decision state"); + } + for (source, target) in [(0, 1), (2, 3), (2, 8), (7, 2), (8, 9)] { + atn.add_transition(source, ParserTransitionSpec::Epsilon { target }) + .expect("epsilon transition"); + } + for (source, target, label) in [(1, 2, 1), (1, 2, 2), (4, 6, 4), (5, 6, 3)] { + atn.add_transition(source, ParserTransitionSpec::Atom { target, label }) + .expect("token transition"); + } + for (target, precedence) in [(4, 2), (5, 1)] { + atn.add_transition(3, ParserTransitionSpec::Precedence { target, precedence }) + .expect("operator precedence"); } - atn.set_rule_to_start_state(vec![0, 5, 11]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![4, 10, 12]) - .expect("rule stop states"); - atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - atn.add_transition( - 1, - ParserTransitionSpec::Rule { - target: 5, - rule_index: 1, - follow_state: 2, - precedence: 0, - }, - ) - .expect("transition"); - atn.add_transition( - 2, - ParserTransitionSpec::Rule { - target: 11, - rule_index: 2, - follow_state: 3, - precedence: 0, - }, - ) - .expect("transition"); - atn.add_transition( - 3, - ParserTransitionSpec::Atom { - target: 4, - label: caller_symbol, - }, - ) - .expect("transition"); - atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) - .expect("transition"); - atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 9 }) - .expect("transition"); atn.add_transition( - 7, - ParserTransitionSpec::Precedence { - target: 8, - precedence: 1, + 6, + ParserTransitionSpec::Action { + target: 10, + rule_index: 0, + action_index: None, + context_dependent: false, }, ) - .expect("transition"); + .expect("operator action"); atn.add_transition( - 8, + 10, ParserTransitionSpec::Atom { - target: 6, + target: 7, label: 1, }, ) - .expect("transition"); - atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 10 }) - .expect("transition"); - atn.add_transition(11, ParserTransitionSpec::Epsilon { target: 12 }) - .expect("transition"); + .expect("right operand"); finish_atn(atn) } - fn left_recursive_loop_with_nullable_parent_return_atn(caller_symbol: i32) -> Atn { + fn two_alt_decision_atn() -> Atn { let mut atn = ParserAtnBuilder::new(2); - for (state, kind, rule) in [ - (0, AtnStateKind::RuleStart, 0), - (1, AtnStateKind::Basic, 0), - (2, AtnStateKind::Basic, 0), - (3, AtnStateKind::RuleStop, 0), - (4, AtnStateKind::RuleStart, 1), - (5, AtnStateKind::Basic, 1), - (6, AtnStateKind::Basic, 1), - (7, AtnStateKind::RuleStop, 1), - (8, AtnStateKind::RuleStart, 2), - (9, AtnStateKind::StarLoopEntry, 2), - (10, AtnStateKind::Basic, 2), - (11, AtnStateKind::Basic, 2), - (12, AtnStateKind::LoopEnd, 2), - (13, AtnStateKind::RuleStop, 2), - ] { - assert_eq!( - atn.add_state(kind, Some(rule)).expect("state").index(), - state - ); - if state == 8 { - atn.set_left_recursive_rule(state) - .expect("left-recursive rule start"); - } else if state == 9 { - atn.set_precedence_rule_decision(state) - .expect("precedence decision"); - } - } - atn.set_rule_to_start_state(vec![0, 4, 8]) + assert_eq!( + atn.add_state(AtnStateKind::RuleStart, Some(0)) + .expect("state") + .index(), + 0 + ); + assert_eq!( + atn.add_state(AtnStateKind::BlockStart, Some(0)) + .expect("state") + .index(), + 1 + ); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(0)) + .expect("state") + .index(), + 2 + ); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(0)) + .expect("state") + .index(), + 3 + ); + assert_eq!( + atn.add_state(AtnStateKind::BlockEnd, Some(0)) + .expect("state") + .index(), + 4 + ); + assert_eq!( + atn.add_state(AtnStateKind::RuleStop, Some(0)) + .expect("state") + .index(), + 5 + ); + atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![3, 7, 13]) + atn.set_rule_to_stop_state(vec![5]) .expect("rule stop states"); + atn.add_decision_state(1).expect("decision state"); atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) .expect("transition"); atn.add_transition( 1, - ParserTransitionSpec::Rule { - target: 4, - rule_index: 1, - follow_state: 2, - precedence: 0, + ParserTransitionSpec::Atom { + target: 2, + label: 1, }, ) .expect("transition"); atn.add_transition( - 2, + 1, ParserTransitionSpec::Atom { target: 3, - label: caller_symbol, + label: 2, }, ) .expect("transition"); + atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 4 }) + .expect("transition"); + atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 }) + .expect("transition"); atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) .expect("transition"); + finish_atn(atn) + } + + /// ATN for `start : (A)? B EOF ;` (A=1, B=2, C=3, max token type 3). + /// State 1 is the nullable optional-block decision; its sync set is {A, B}. + fn optional_then_b_eof_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(3); + assert_eq!( + atn.add_state(AtnStateKind::RuleStart, Some(0)) + .expect("state") + .index(), + 0 + ); + assert_eq!( + atn.add_state(AtnStateKind::BlockStart, Some(0)) + .expect("state") + .index(), + 1 + ); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(0)) + .expect("state") + .index(), + 2 + ); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(0)) + .expect("state") + .index(), + 3 + ); + assert_eq!( + atn.add_state(AtnStateKind::Basic, Some(0)) + .expect("state") + .index(), + 4 + ); + assert_eq!( + atn.add_state(AtnStateKind::RuleStop, Some(0)) + .expect("state") + .index(), + 5 + ); + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![5]) + .expect("rule stop states"); + atn.add_decision_state(1).expect("decision state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + // Optional block: match A then fall through, or skip straight to state 3. atn.add_transition( - 5, - ParserTransitionSpec::Rule { - target: 8, - rule_index: 2, - follow_state: 6, - precedence: 0, + 1, + ParserTransitionSpec::Atom { + target: 3, + label: 1, }, ) .expect("transition"); - atn.add_transition(6, ParserTransitionSpec::Epsilon { target: 7 }) - .expect("transition"); - atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 10 }) - .expect("transition"); - atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 12 }) + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 }) .expect("transition"); + // Match B, then EOF. atn.add_transition( - 10, - ParserTransitionSpec::Precedence { - target: 11, - precedence: 1, + 3, + ParserTransitionSpec::Atom { + target: 4, + label: 2, }, ) .expect("transition"); atn.add_transition( - 11, + 4, ParserTransitionSpec::Atom { - target: 9, - label: 1, + target: 5, + label: TOKEN_EOF, }, ) .expect("transition"); - atn.add_transition(12, ParserTransitionSpec::Epsilon { target: 13 }) - .expect("transition"); finish_atn(atn) } - fn left_recursive_loop_with_recursive_operand_return_atn(caller_symbol: i32) -> Atn { + #[test] + fn sync_decision_deletes_only_a_single_token() { + // ANTLR sync recovery deletes exactly one token, only when LA(2) is + // expected. `(A)? B EOF` at the optional-block decision: + // - `C B` -> single-token deletion: one error node for the extra `C`. + // - `C C B` -> LA(2) is `C` (not expected), so NO deletion; sync returns + // without consuming and records the expected set for the + // subsequent mismatch (the parser must not over-consume both + // `C`s and accept the input). + let atn = optional_then_b_eof_atn(); + + let mut single = mini_parser(vec![ + TestToken::new(3).with_text("c"), + TestToken::new(2).with_text("b"), + TestToken::eof("parser-test", 1, 2, 2), + ]); + single.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }]; + let children = single + .sync_decision(&atn, 1, true, false) + .expect("single extraneous token recovers"); + assert_eq!(children.len(), 1); + assert_eq!(single.node(children[0]).kind(), NodeKind::Error); + assert_eq!(single.number_of_syntax_errors(), 1); + // Exactly one token consumed (the cursor now sits on `b`). + assert_eq!(single.la(1), 2); + + let mut double = mini_parser(vec![ + TestToken::new(3).with_text("c"), + TestToken::new(3).with_text("c"), + TestToken::new(2).with_text("b"), + TestToken::eof("parser-test", 1, 3, 3), + ]); + double.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }]; + let result = double.sync_decision(&atn, 1, true, false); + // No single-token deletion fires (LA(2) is `c`, not expected): sync must NOT + // consume either `c`. It reports the mismatch at the first `c` (so the parser + // does not over-consume both and accept the input). Nothing is consumed, so + // the cursor still sits on the first `c` for rule-level recovery. + let error = result.expect_err("two extraneous tokens must not be deleted by sync"); + match error { + AntlrError::ParserError { message, .. } => { + assert!(message.starts_with("mismatched input"), "got: {message}"); + } + other => panic!("expected a mismatched-input ParserError, got {other:?}"), + } + assert_eq!(double.la(1), 3); + } + + /// The real serialized ATN that `antlr4-rust-gen` emits for + /// `grammar T; s : A* EOF; A:'a'; C:'c';` — a `*` loop whose follow set after + /// the loop is `EOF`. The loop decision is state 5. + fn star_loop_then_eof_atn() -> Atn { + AtnDeserializer::new(&SerializedAtn::from_i32(&[ + 4, 1, 3, 11, 2, 0, 7, 0, 1, 0, 5, 0, 4, 8, 0, 10, 0, 12, 0, 7, 9, 0, 1, 0, 1, 0, 1, 0, + 0, 0, 1, 0, 0, 0, 10, 0, 5, 1, 0, 0, 0, 2, 4, 5, 1, 0, 0, 3, 2, 1, 0, 0, 0, 4, 7, 1, 0, + 0, 0, 5, 3, 1, 0, 0, 0, 5, 6, 1, 0, 0, 0, 6, 8, 1, 0, 0, 0, 7, 5, 1, 0, 0, 0, 8, 9, 5, + 0, 0, 1, 9, 1, 1, 0, 0, 0, 1, 5, + ])) + .deserialize_parser() + .expect("star-loop-then-EOF ATN should deserialize") + } + + /// ATN for `entry : nested EOF; nested : A*;`. + /// + /// State 5 is nullable within `nested`; its caller follow is EOF. + fn nested_star_rule_atn() -> Atn { let mut atn = ParserAtnBuilder::new(2); - for (state, kind, rule) in [ + for (state_number, kind, rule_index) in [ (0, AtnStateKind::RuleStart, 0), (1, AtnStateKind::Basic, 0), (2, AtnStateKind::Basic, 0), @@ -14565,28 +16642,23 @@ mod tests { (4, AtnStateKind::RuleStart, 1), (5, AtnStateKind::StarLoopEntry, 1), (6, AtnStateKind::Basic, 1), - (7, AtnStateKind::Basic, 1), - (8, AtnStateKind::Basic, 1), - (9, AtnStateKind::Basic, 1), - (10, AtnStateKind::LoopEnd, 1), - (11, AtnStateKind::RuleStop, 1), + (7, AtnStateKind::StarLoopBack, 1), + (8, AtnStateKind::LoopEnd, 1), + (9, AtnStateKind::RuleStop, 1), ] { assert_eq!( - atn.add_state(kind, Some(rule)).expect("state").index(), - state + atn.add_state(kind, Some(rule_index)) + .expect("state") + .index(), + state_number ); - if state == 4 { - atn.set_left_recursive_rule(state) - .expect("left-recursive rule start"); - } else if state == 5 { - atn.set_precedence_rule_decision(state) - .expect("precedence decision"); - } } atn.set_rule_to_start_state(vec![0, 4]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![3, 11]) + atn.set_rule_to_stop_state(vec![3, 9]) .expect("rule stop states"); + atn.add_decision_state(5).expect("decision state"); + atn.set_loop_back_state(8, 7).expect("loop back state"); atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) .expect("transition"); atn.add_transition( @@ -14603,389 +16675,292 @@ mod tests { 2, ParserTransitionSpec::Atom { target: 3, - label: caller_symbol, + label: TOKEN_EOF, }, ) .expect("transition"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) .expect("transition"); - atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 10 }) + atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 8 }) .expect("transition"); atn.add_transition( 6, - ParserTransitionSpec::Precedence { - target: 7, - precedence: 1, - }, - ) - .expect("transition"); - atn.add_transition( - 7, ParserTransitionSpec::Atom { - target: 8, + target: 7, label: 1, }, ) .expect("transition"); - atn.add_transition( - 8, - ParserTransitionSpec::Rule { - target: 4, - rule_index: 1, - follow_state: 9, - precedence: 2, - }, - ) - .expect("transition"); - atn.add_transition(9, ParserTransitionSpec::Epsilon { target: 5 }) + atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 5 }) .expect("transition"); - atn.add_transition(10, ParserTransitionSpec::Epsilon { target: 11 }) + atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) .expect("transition"); finish_atn(atn) } - #[test] - fn left_recursive_loop_defers_overlapping_caller_lookahead() { - let overlapping_atn = left_recursive_loop_with_caller_follow_atn(1); - let unambiguous_atn = left_recursive_loop_with_caller_follow_atn(2); - - let mut overlapping = parser_inside_left_recursive_callee(1); + /// ATN for `s : a+ Y ; a : X ;`. + /// + /// At EOF, recovery can synthesize an empty failed `a` child. The enclosing + /// `+` loop must not treat that zero-width child as a successful iteration + /// and then re-enter the loop at the same token index. + fn plus_loop_with_recovering_body_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(2); assert_eq!( - overlapping.left_recursive_loop_enter_prediction(&overlapping_atn, 4, 0), - None + atn.add_state(AtnStateKind::RuleStart, Some(0)) + .expect("state") + .index(), + 0 ); - - let mut unambiguous_enter = parser_inside_left_recursive_callee(1); assert_eq!( - unambiguous_enter.left_recursive_loop_enter_prediction(&unambiguous_atn, 4, 0), - Some(true) + atn.add_state(AtnStateKind::PlusBlockStart, Some(0)) + .expect("state") + .index(), + 1 ); - - let mut unambiguous_exit = parser_inside_left_recursive_callee(2); assert_eq!( - unambiguous_exit.left_recursive_loop_enter_prediction(&unambiguous_atn, 4, 0), - Some(false) + atn.add_state(AtnStateKind::Basic, Some(0)) + .expect("state") + .index(), + 2 ); - assert_eq!( - overlapping.left_recursive_loop_enter_prediction(&unambiguous_atn, 4, 0), - Some(true), - "overlap results must not leak across ATNs" + atn.add_state(AtnStateKind::BlockEnd, Some(0)) + .expect("state") + .index(), + 3 ); - } - - #[test] - fn left_recursive_loop_enters_after_nullable_operator_prefix() { - let atn = left_recursive_loop_with_nullable_operator_prefix_atn(); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("operator"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - parser.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: -1, - }]; - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 0), - Some(true) + atn.add_state(AtnStateKind::PlusLoopBack, Some(0)) + .expect("state") + .index(), + 4 ); assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 0), - Some(true), - "cached operator lookahead must preserve the nullable prefix return path" + atn.add_state(AtnStateKind::LoopEnd, Some(0)) + .expect("state") + .index(), + 5 ); assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 2), - Some(true), - "the nullable child must use its rule-call precedence, not the caller precedence" + atn.add_state(AtnStateKind::RuleStop, Some(0)) + .expect("state") + .index(), + 6 ); - } - - #[test] - fn left_recursive_loop_defers_multi_token_prefix_that_shadows_lower_single_token() { - // Models Java `>` (relational, prec 1, one token) vs `>>` (shift, prec 2, - // two tokens). At prec 2 only shift is viable; one-token lookahead on `>` - // must defer so StarLoopEntry adaptive predict can exit when the second - // `>` is absent (as in `a < b > c`). - let atn = left_recursive_loop_with_shared_gt_prefix_atn(); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text(">"), - TestToken::new(2).with_text("id"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - parser.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: -1, - }]; - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 0), - Some(true), - "at low precedence relational `>` is a single-token operator" + atn.add_state(AtnStateKind::RuleStart, Some(1)) + .expect("state") + .index(), + 7 ); assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 1), - Some(true), - "relational remains single-token at its own precedence" + atn.add_state(AtnStateKind::Basic, Some(1)) + .expect("state") + .index(), + 8 ); assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 2), - None, - "at shift precedence, bare `>` must not force enter" + atn.add_state(AtnStateKind::RuleStop, Some(1)) + .expect("state") + .index(), + 9 ); + atn.set_rule_to_start_state(vec![0, 7]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![6, 9]) + .expect("rule stop states"); + atn.set_end_state(1, 3).expect("block end state"); + atn.set_loop_back_state(5, 4).expect("loop back state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Rule { + target: 7, + rule_index: 1, + follow_state: 3, + precedence: 0, + }, + ) + .expect("transition"); + atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 }) + .expect("transition"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("transition"); + atn.add_transition( + 5, + ParserTransitionSpec::Atom { + target: 6, + label: 2, + }, + ) + .expect("transition"); + atn.add_transition( + 7, + ParserTransitionSpec::Atom { + target: 8, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) + .expect("transition"); + finish_atn(atn) } #[test] - fn left_recursive_loop_preserves_rule_wrapped_operator_continuation() { - let atn = left_recursive_loop_with_rule_wrapped_gt_prefix_atn(); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text(">"), - TestToken::new(2).with_text("id"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - parser.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: -1, - }]; + fn runtime_options_default_exits_recovering_empty_plus_iteration() { + let atn = plus_loop_with_recovering_body_atn(); + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 0), - Some(true), - "the direct relational alternative remains a one-token operator" - ); - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 2), - None, - "a token matched in the helper rule must return to the second shift token" - ); + let error = parser + .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) + .expect_err("EOF recovery should report a bounded mismatch"); + + let AntlrError::ParserError { message, .. } = error else { + panic!("expected ParserError, got {error:?}"); + }; + insta::assert_snapshot!(message, @"mismatched input '' expecting {'x', 2}"); + assert_eq!(parser.number_of_syntax_errors(), 1); + assert_eq!(parser.input.index(), 0, "EOF remains unconsumed"); } #[test] - fn left_recursive_loop_preserves_predicate_and_multi_token_reachability() { - let atn = left_recursive_loop_with_predicate_and_multi_token_prefix_atn(); + fn sync_decision_deletes_token_before_eof_at_loop_back() { + // `s : A* EOF` on `c`: the loop decision (state 5) can recover onto EOF. + // At the loop ENTRY (loop_back = false) a single unexpected token before + // EOF is deleted as an error node (then the generated EOF match consumes + // the real EOF) — matching ANTLR's `(s c )` + "extraneous input". + // EOF must be a valid scan-stop for this to fire. + let atn = star_loop_then_eof_atn(); let mut parser = mini_parser(vec![ - TestToken::new(1).with_text(">"), - TestToken::new(2).with_text("id"), + TestToken::new(2).with_text("c"), TestToken::eof("parser-test", 1, 1, 1), ]); parser.rule_context_stack = vec![RuleContextFrame { rule_index: 0, - invoking_state: -1, + invoking_state: 0, }]; - + let children = parser + .sync_decision(&atn, 5, true, false) + .expect("single token before EOF recovers"); + assert_eq!(children.len(), 1); + assert_eq!(parser.node(children[0]).kind(), NodeKind::Error); + assert_eq!(parser.number_of_syntax_errors(), 1); assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 2), - None, - "a predicate-gated single-token path must not be hidden by a multi-token path" + parser.la(1), + TOKEN_EOF, + "EOF is left for the rule's EOF match" ); } #[test] - fn left_recursive_loop_defers_predicate_guarded_operator() { - let atn = left_recursive_loop_with_predicate_guarded_operator_atn(); - let mut parser = mini_parser_with_hooks( - vec![ - TestToken::new(1).with_text("operator"), - TestToken::eof("parser-test", 1, 1, 1), - ], - RejectingPredicateHooks::default(), - ); + fn sync_decision_does_not_delete_two_tokens_before_eof_at_loop_entry() { + // `s : A* EOF` on `c c`: at the loop ENTRY (loop_back = false) ANTLR does + // single-token deletion, which fails because LA(2) = `c` is not expected — + // so it reports `mismatched input` and consumes nothing (ANTLR: `(s c c)` + // with no EOF). The scan must NOT multi-token-consume both `c`s here. + let atn = star_loop_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(2).with_text("c"), + TestToken::new(2).with_text("c"), + TestToken::eof("parser-test", 1, 2, 2), + ]); parser.rule_context_stack = vec![RuleContextFrame { rule_index: 0, - invoking_state: -1, + invoking_state: 0, }]; - - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 0), - None, - "a false predicate must be evaluated before entering the operator alternative" - ); - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 1, 0), - None, - "cached predicate-dependent lookahead must keep deferring" - ); - } - - #[test] - fn left_recursive_loop_defers_through_nullable_caller_rule_call() { - let atn = left_recursive_loop_with_nullable_follow_call_atn(1); - let mut parser = parser_inside_left_recursive_callee(1); - - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 6, 0), - None - ); + let error = parser + .sync_decision(&atn, 5, true, false) + .expect_err("two tokens at the loop entry must not be deleted"); + match error { + AntlrError::ParserError { message, .. } => { + assert!(message.starts_with("mismatched input"), "got: {message}"); + } + other => panic!("expected mismatched-input ParserError, got {other:?}"), + } assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 6, 0), - None, - "the cached overlap must preserve the nullable child return path" + parser.la(1), + 2, + "nothing consumed; cursor still on first `c`" ); } #[test] - fn left_recursive_loop_defers_through_nullable_parent_return() { - let atn = left_recursive_loop_with_nullable_parent_return_atn(1); + fn sync_decision_consumes_until_eof_at_loop_back() { + // Same `s : A* EOF` decision, but at a loop-BACK (loop_back = true, i.e. + // after ≥1 `A` matched). ANTLR uses multi-token `consumeUntil(recoverSet)` + // there, so two unexpected tokens before EOF are BOTH deleted and the rule + // recovers (matching `(s a c c )` for input `a c c`). Here we feed the + // post-`a` state directly: `c c ` with loop_back = true. + let atn = star_loop_then_eof_atn(); let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("lookahead"), - TestToken::eof("parser-test", 1, 1, 1), + TestToken::new(2).with_text("c"), + TestToken::new(2).with_text("c"), + TestToken::eof("parser-test", 1, 2, 2), ]); - parser.rule_context_stack = vec![ - RuleContextFrame { - rule_index: 0, - invoking_state: -1, - }, - RuleContextFrame { - rule_index: 1, - invoking_state: 1, - }, - RuleContextFrame { - rule_index: 2, - invoking_state: 5, - }, - ]; - - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 9, 0), - None, - "a nullable caller must unwind to its parent's consuming follow path" - ); - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 9, 0), - None, - "the caller-overlap cache must not retain a false negative" + parser.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }]; + let children = parser + .sync_decision(&atn, 5, false, true) + .expect("loop-back multi-token deletion recovers onto EOF"); + assert_eq!(children.len(), 2, "both `c`s deleted as error nodes"); + assert!( + children + .iter() + .all(|child| parser.node(*child).kind() == NodeKind::Error) ); + assert_eq!(parser.number_of_syntax_errors(), 1); + assert_eq!(parser.la(1), TOKEN_EOF, "EOF left for the rule's EOF match"); } #[test] - fn left_recursive_loop_defers_after_recursive_operand_returns_to_loop() { - let atn = left_recursive_loop_with_recursive_operand_return_atn(1); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("lookahead"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - parser.rule_context_stack = vec![ - RuleContextFrame { - rule_index: 0, - invoking_state: -1, - }, - RuleContextFrame { - rule_index: 1, - invoking_state: 1, - }, - RuleContextFrame { - rule_index: 1, - invoking_state: 8, - }, - ]; - - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 5, 0), - None, - "a recursive operand return must preserve its parent caller context" - ); - assert_eq!( - parser.left_recursive_loop_enter_prediction(&atn, 5, 0), - None, - "the caller-overlap cache must preserve the loop-boundary return" - ); - } + fn sync_decision_returns_before_recovery_for_nullable_exit() { + let atn = nested_star_rule_atn(); + for (current_context_empty, loop_back) in [(true, false), (false, true)] { + let mut parser = mini_parser(vec![ + TestToken::new(2).with_text("c"), + TestToken::new(1).with_text("a"), + TestToken::eof("parser-test", 1, 2, 2), + ]); + parser.rule_context_stack = vec![ + RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }, + RuleContextFrame { + rule_index: 1, + invoking_state: 1, + }, + ]; - fn token_then_eof_atn() -> Atn { - AtnDeserializer::new(&SerializedAtn::from_i32(&[ - 4, 1, 2, // version, parser, max token type - 3, // states - 2, 0, // rule start - 1, 0, // basic - 7, 0, // rule stop - 0, // non-greedy states - 0, // precedence states - 1, // rules - 0, // rule 0 start - 0, // modes - 0, // sets - 2, // transitions - 0, 1, 5, 1, 0, 0, // match token 1 - 1, 2, 5, -1, 0, 0, // match EOF - 0, // decisions - ])) - .deserialize_parser() - .expect("artificial parser ATN should deserialize") - } + let children = parser + .sync_decision(&atn, 5, current_context_empty, loop_back) + .expect("nullable synchronization is a no-op"); - fn epsilon_cycle_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(1); - for (state_number, kind) in [ - (0, AtnStateKind::RuleStart), - (1, AtnStateKind::Basic), - (2, AtnStateKind::RuleStop), - ] { + assert!(children.is_empty()); + assert_eq!(parser.la(1), 2, "the caller must receive the current token"); + assert_eq!(parser.number_of_syntax_errors(), 0); assert_eq!( - atn.add_state(kind, Some(0)).expect("state").index(), - state_number + parser + .generated_sync_expected + .as_ref() + .expect("nullable sync preserves expected symbols") + .to_btree_set(), + BTreeSet::from([TOKEN_EOF, 1]) ); } - atn.set_rule_to_start_state(vec![0]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![2]) - .expect("rule stop states"); - atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("self-cycle transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("exit transition"); - finish_atn(atn) - } - - fn eof_then_action_atn() -> Atn { - AtnDeserializer::new(&SerializedAtn::from_i32(&[ - 4, 1, 1, // version, parser, max token type - 3, // states - 2, 0, // rule start - 1, 0, // basic - 7, 0, // rule stop - 0, // non-greedy states - 0, // precedence states - 1, // rules - 0, // rule 0 start - 0, // modes - 0, // sets - 2, // transitions - 0, 1, 5, -1, 0, 0, // match EOF - 1, 2, 6, 0, 0, 0, // parser action - 0, // decisions - ])) - .deserialize_parser() - .expect("artificial parser ATN should deserialize") - } - - fn noop_action_then_token_then_eof_atn() -> Atn { - AtnDeserializer::new(&SerializedAtn::from_i32(&[ - 4, 1, 2, // version, parser, max token type - 4, // states - 2, 0, // rule start - 1, 0, // basic - 1, 0, // basic - 7, 0, // rule stop - 0, // non-greedy states - 0, // precedence states - 1, // rules - 0, // rule 0 start - 0, // modes - 0, // sets - 3, // transitions - 0, 1, 6, 0, -1, 0, // no-op parser action - 1, 2, 5, 1, 0, 0, // match token 1 - 2, 3, 5, -1, 0, 0, // match EOF - 0, // decisions - ])) - .deserialize_parser() - .expect("artificial no-op action ATN should deserialize") } - fn two_alt_decision_atn() -> Atn { + fn predicate_after_token_atn() -> Atn { let mut atn = ParserAtnBuilder::new(2); assert_eq!( atn.add_state(AtnStateKind::RuleStart, Some(0)) @@ -14994,7 +16969,7 @@ mod tests { 0 ); assert_eq!( - atn.add_state(AtnStateKind::BlockStart, Some(0)) + atn.add_state(AtnStateKind::Basic, Some(0)) .expect("state") .index(), 1 @@ -15011,213 +16986,351 @@ mod tests { .index(), 3 ); - assert_eq!( - atn.add_state(AtnStateKind::BlockEnd, Some(0)) - .expect("state") - .index(), - 4 - ); assert_eq!( atn.add_state(AtnStateKind::RuleStop, Some(0)) .expect("state") .index(), - 5 + 4 ); atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![5]) + atn.set_rule_to_stop_state(vec![4]) + .expect("rule stop states"); + atn.add_transition( + 0, + ParserTransitionSpec::Atom { + target: 1, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition( + 1, + ParserTransitionSpec::Predicate { + target: 2, + rule_index: 0, + pred_index: 0, + context_dependent: false, + }, + ) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Atom { + target: 3, + label: 2, + }, + ) + .expect("transition"); + atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 }) + .expect("transition"); + finish_atn(atn) + } + + fn predicate_gated_same_lookahead_atn(pred_indexes: [usize; 2]) -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for (state_number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::BlockStart), + (2, AtnStateKind::Basic), + (3, AtnStateKind::Basic), + (4, AtnStateKind::Basic), + (5, AtnStateKind::Basic), + (6, AtnStateKind::BlockEnd), + (7, AtnStateKind::RuleStop), + ] { + assert_eq!( + atn.add_state(kind, Some(0)).expect("state").index(), + state_number + ); + } + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![7]) + .expect("rule stop states"); + atn.add_decision_state(1).expect("decision state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Predicate { + target: 4, + rule_index: 0, + pred_index: pred_indexes[0], + context_dependent: false, + }, + ) + .expect("transition"); + atn.add_transition( + 3, + ParserTransitionSpec::Predicate { + target: 5, + rule_index: 0, + pred_index: pred_indexes[1], + context_dependent: false, + }, + ) + .expect("transition"); + atn.add_transition( + 4, + ParserTransitionSpec::Atom { + target: 6, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition( + 5, + ParserTransitionSpec::Atom { + target: 6, + label: 1, + }, + ) + .expect("transition"); + atn.add_transition( + 6, + ParserTransitionSpec::Atom { + target: 7, + label: TOKEN_EOF, + }, + ) + .expect("transition"); + finish_atn(atn) + } + + /// ATN for `s : A B | {false}? A C | {true}? A C;`. + fn semantic_fallback_viability_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(3); + for (state_number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::BlockStart), + (2, AtnStateKind::Basic), + (3, AtnStateKind::Basic), + (4, AtnStateKind::Basic), + (5, AtnStateKind::Basic), + (6, AtnStateKind::Basic), + (7, AtnStateKind::Basic), + (8, AtnStateKind::Basic), + (9, AtnStateKind::BlockEnd), + (10, AtnStateKind::RuleStop), + ] { + assert_eq!( + atn.add_state(kind, Some(0)).expect("state").index(), + state_number + ); + } + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![10]) + .expect("rule stop states"); + atn.set_end_state(1, 9).expect("block end state"); + atn.add_decision_state(1).expect("decision state"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("entry transition"); + atn.add_transition( + 1, + ParserTransitionSpec::Atom { + target: 2, + label: 1, + }, + ) + .expect("first alternative"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("second alternative"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 6 }) + .expect("third alternative"); + atn.add_transition( + 2, + ParserTransitionSpec::Atom { + target: 9, + label: 2, + }, + ) + .expect("first alternative suffix"); + for (source, target, pred_index) in [(3, 4, 0), (6, 7, 1)] { + atn.add_transition( + source, + ParserTransitionSpec::Predicate { + target, + rule_index: 0, + pred_index, + context_dependent: false, + }, + ) + .expect("predicate transition"); + } + for (source, target, label) in [(4, 5, 1), (5, 9, 3), (7, 8, 1), (8, 9, 3)] { + atn.add_transition(source, ParserTransitionSpec::Atom { target, label }) + .expect("predicate alternative token"); + } + atn.add_transition( + 9, + ParserTransitionSpec::Atom { + target: 10, + label: TOKEN_EOF, + }, + ) + .expect("EOF transition"); + finish_atn(atn) + } + + /// ATN for `s : gated | A; gated : {false}? A;`. + fn rule_call_predicate_decision_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for (state_number, kind, rule_index) in [ + (0, AtnStateKind::RuleStart, 0), + (1, AtnStateKind::BlockStart, 0), + (2, AtnStateKind::Basic, 0), + (3, AtnStateKind::Basic, 0), + (4, AtnStateKind::BlockEnd, 0), + (5, AtnStateKind::RuleStop, 0), + (6, AtnStateKind::RuleStart, 1), + (7, AtnStateKind::Basic, 1), + (8, AtnStateKind::RuleStop, 1), + ] { + assert_eq!( + atn.add_state(kind, Some(rule_index)) + .expect("state") + .index(), + state_number + ); + } + atn.set_rule_to_start_state(vec![0, 6]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![5, 8]) .expect("rule stop states"); + atn.set_end_state(1, 4).expect("block end state"); atn.add_decision_state(1).expect("decision state"); atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); + .expect("entry transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("gated alternative entry"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 }) + .expect("direct alternative entry"); atn.add_transition( - 1, + 2, + ParserTransitionSpec::Rule { + target: 6, + rule_index: 1, + follow_state: 4, + precedence: 0, + }, + ) + .expect("gated alternative"); + atn.add_transition( + 3, ParserTransitionSpec::Atom { - target: 2, + target: 4, label: 1, }, ) - .expect("transition"); + .expect("direct alternative"); atn.add_transition( - 1, + 4, ParserTransitionSpec::Atom { - target: 3, - label: 2, + target: 5, + label: TOKEN_EOF, }, ) - .expect("transition"); - atn.add_transition(2, ParserTransitionSpec::Epsilon { target: 4 }) - .expect("transition"); - atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 }) - .expect("transition"); - atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) - .expect("transition"); + .expect("EOF transition"); + atn.add_transition( + 6, + ParserTransitionSpec::Predicate { + target: 7, + rule_index: 1, + pred_index: 0, + context_dependent: false, + }, + ) + .expect("callee predicate"); + atn.add_transition( + 7, + ParserTransitionSpec::Atom { + target: 8, + label: 1, + }, + ) + .expect("callee token"); finish_atn(atn) } - /// ATN for `start : (A)? B EOF ;` (A=1, B=2, C=3, max token type 3). - /// State 1 is the nullable optional-block decision; its sync set is {A, B}. - fn optional_then_b_eof_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(3); - assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(0)) - .expect("state") - .index(), - 0 - ); - assert_eq!( - atn.add_state(AtnStateKind::BlockStart, Some(0)) - .expect("state") - .index(), - 1 - ); - assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 2 - ); - assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 3 - ); - assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 4 - ); - assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(0)) - .expect("state") - .index(), - 5 - ); + /// ATN for `s : ({true}? A)* EOF;`. + fn predicate_gated_star_loop_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(2); + for (state_number, kind) in [ + (0, AtnStateKind::RuleStart), + (1, AtnStateKind::StarLoopEntry), + (2, AtnStateKind::Basic), + (3, AtnStateKind::Basic), + (4, AtnStateKind::StarLoopBack), + (5, AtnStateKind::LoopEnd), + (6, AtnStateKind::RuleStop), + ] { + assert_eq!( + atn.add_state(kind, Some(0)).expect("state").index(), + state_number + ); + } atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![5]) + atn.set_rule_to_stop_state(vec![6]) .expect("rule stop states"); atn.add_decision_state(1).expect("decision state"); + atn.set_loop_back_state(5, 4).expect("loop back state"); atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - // Optional block: match A then fall through, or skip straight to state 3. + .expect("entry transition"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + .expect("loop enter"); + atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 5 }) + .expect("loop exit"); atn.add_transition( - 1, - ParserTransitionSpec::Atom { + 2, + ParserTransitionSpec::Predicate { target: 3, - label: 1, + rule_index: 0, + pred_index: 0, + context_dependent: false, }, ) - .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 }) - .expect("transition"); - // Match B, then EOF. + .expect("loop predicate"); atn.add_transition( 3, ParserTransitionSpec::Atom { target: 4, - label: 2, + label: 1, }, ) - .expect("transition"); + .expect("loop token"); + atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("loop back"); atn.add_transition( - 4, + 5, ParserTransitionSpec::Atom { - target: 5, + target: 6, label: TOKEN_EOF, }, ) - .expect("transition"); + .expect("EOF transition"); finish_atn(atn) } - #[test] - fn sync_decision_deletes_only_a_single_token() { - // ANTLR sync recovery deletes exactly one token, only when LA(2) is - // expected. `(A)? B EOF` at the optional-block decision: - // - `C B` -> single-token deletion: one error node for the extra `C`. - // - `C C B` -> LA(2) is `C` (not expected), so NO deletion; sync returns - // without consuming and records the expected set for the - // subsequent mismatch (the parser must not over-consume both - // `C`s and accept the input). - let atn = optional_then_b_eof_atn(); - - let mut single = mini_parser(vec![ - TestToken::new(3).with_text("c"), - TestToken::new(2).with_text("b"), - TestToken::eof("parser-test", 1, 2, 2), - ]); - single.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }]; - let children = single - .sync_decision(&atn, 1, true, false) - .expect("single extraneous token recovers"); - assert_eq!(children.len(), 1); - assert_eq!(single.node(children[0]).kind(), NodeKind::Error); - assert_eq!(single.number_of_syntax_errors(), 1); - // Exactly one token consumed (the cursor now sits on `b`). - assert_eq!(single.la(1), 2); - - let mut double = mini_parser(vec![ - TestToken::new(3).with_text("c"), - TestToken::new(3).with_text("c"), - TestToken::new(2).with_text("b"), - TestToken::eof("parser-test", 1, 3, 3), - ]); - double.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }]; - let result = double.sync_decision(&atn, 1, true, false); - // No single-token deletion fires (LA(2) is `c`, not expected): sync must NOT - // consume either `c`. It reports the mismatch at the first `c` (so the parser - // does not over-consume both and accept the input). Nothing is consumed, so - // the cursor still sits on the first `c` for rule-level recovery. - let error = result.expect_err("two extraneous tokens must not be deleted by sync"); - match error { - AntlrError::ParserError { message, .. } => { - assert!(message.starts_with("mismatched input"), "got: {message}"); - } - other => panic!("expected a mismatched-input ParserError, got {other:?}"), - } - assert_eq!(double.la(1), 3); - } - - /// The real serialized ATN that `antlr4-rust-gen` emits for - /// `grammar T; s : A* EOF; A:'a'; C:'c';` — a `*` loop whose follow set after - /// the loop is `EOF`. The loop decision is state 5. - fn star_loop_then_eof_atn() -> Atn { - AtnDeserializer::new(&SerializedAtn::from_i32(&[ - 4, 1, 3, 11, 2, 0, 7, 0, 1, 0, 5, 0, 4, 8, 0, 10, 0, 12, 0, 7, 9, 0, 1, 0, 1, 0, 1, 0, - 0, 0, 1, 0, 0, 0, 10, 0, 5, 1, 0, 0, 0, 2, 4, 5, 1, 0, 0, 3, 2, 1, 0, 0, 0, 4, 7, 1, 0, - 0, 0, 5, 3, 1, 0, 0, 0, 5, 6, 1, 0, 0, 0, 6, 8, 1, 0, 0, 0, 7, 5, 1, 0, 0, 0, 8, 9, 5, - 0, 0, 1, 9, 1, 1, 0, 0, 0, 1, 5, - ])) - .deserialize_parser() - .expect("star-loop-then-EOF ATN should deserialize") - } - - /// ATN for `entry : nested EOF; nested : A*;`. - /// - /// State 5 is nullable within `nested`; its caller follow is EOF. - fn nested_star_rule_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(2); - for (state_number, kind, rule_index) in [ - (0, AtnStateKind::RuleStart, 0), - (1, AtnStateKind::Basic, 0), - (2, AtnStateKind::Basic, 0), - (3, AtnStateKind::RuleStop, 0), - (4, AtnStateKind::RuleStart, 1), - (5, AtnStateKind::StarLoopEntry, 1), - (6, AtnStateKind::Basic, 1), - (7, AtnStateKind::StarLoopBack, 1), - (8, AtnStateKind::LoopEnd, 1), - (9, AtnStateKind::RuleStop, 1), - ] { + fn nested_nullable_context_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + for state_number in 0..=20 { + let kind = match state_number { + 0 | 10 | 16 => AtnStateKind::RuleStart, + 9 | 15 | 20 => AtnStateKind::RuleStop, + _ => AtnStateKind::Basic, + }; + let rule_index = match state_number { + 0..=9 => 0, + 10..=15 => 1, + _ => 2, + }; assert_eq!( atn.add_state(kind, Some(rule_index)) .expect("state") @@ -15225,59 +17338,46 @@ mod tests { state_number ); } - atn.set_rule_to_start_state(vec![0, 4]) + atn.set_rule_to_start_state(vec![0, 10, 16]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![3, 9]) + atn.set_rule_to_stop_state(vec![9, 15, 20]) .expect("rule stop states"); - atn.add_decision_state(5).expect("decision state"); - atn.set_loop_back_state(8, 7).expect("loop back state"); - atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); atn.add_transition( 1, ParserTransitionSpec::Rule { - target: 4, + target: 10, rule_index: 1, - follow_state: 2, + follow_state: 8, precedence: 0, }, ) .expect("transition"); atn.add_transition( - 2, + 8, ParserTransitionSpec::Atom { - target: 3, - label: TOKEN_EOF, + target: 9, + label: 1, }, ) .expect("transition"); - atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) - .expect("transition"); - atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 6 }) - .expect("transition"); - atn.add_transition(5, ParserTransitionSpec::Epsilon { target: 8 }) + atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) .expect("transition"); atn.add_transition( - 6, - ParserTransitionSpec::Atom { - target: 7, - label: 1, + 2, + ParserTransitionSpec::Rule { + target: 16, + rule_index: 2, + follow_state: 14, + precedence: 0, }, ) .expect("transition"); - atn.add_transition(7, ParserTransitionSpec::Epsilon { target: 5 }) - .expect("transition"); - atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) + atn.add_transition(14, ParserTransitionSpec::Epsilon { target: 15 }) .expect("transition"); finish_atn(atn) } - /// ATN for `s : a+ Y ; a : X ;`. - /// - /// At EOF, recovery can synthesize an empty failed `a` child. The enclosing - /// `+` loop must not treat that zero-width child as a successful iteration - /// and then re-enter the loop at the same token index. - fn plus_loop_with_recovering_body_atn() -> Atn { + fn generated_match_recovery_atn() -> Atn { let mut atn = ParserAtnBuilder::new(2); assert_eq!( atn.add_state(AtnStateKind::RuleStart, Some(0)) @@ -15286,7 +17386,7 @@ mod tests { 0 ); assert_eq!( - atn.add_state(AtnStateKind::PlusBlockStart, Some(0)) + atn.add_state(AtnStateKind::Basic, Some(0)) .expect("state") .index(), 1 @@ -15298,1484 +17398,1895 @@ mod tests { 2 ); assert_eq!( - atn.add_state(AtnStateKind::BlockEnd, Some(0)) + atn.add_state(AtnStateKind::RuleStop, Some(0)) .expect("state") .index(), 3 ); assert_eq!( - atn.add_state(AtnStateKind::PlusLoopBack, Some(0)) + atn.add_state(AtnStateKind::RuleStart, Some(1)) .expect("state") .index(), 4 ); assert_eq!( - atn.add_state(AtnStateKind::LoopEnd, Some(0)) + atn.add_state(AtnStateKind::RuleStop, Some(1)) .expect("state") .index(), 5 ); + atn.set_rule_to_start_state(vec![0, 4]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![3, 5]) + .expect("rule stop states"); + atn.add_transition( + 1, + ParserTransitionSpec::Rule { + target: 4, + rule_index: 1, + follow_state: 2, + precedence: 0, + }, + ) + .expect("transition"); + atn.add_transition( + 2, + ParserTransitionSpec::Atom { + target: 3, + label: TOKEN_EOF, + }, + ) + .expect("transition"); + finish_atn(atn) + } + + fn complement_set_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); + assert_eq!( + atn.add_state(AtnStateKind::RuleStart, Some(0)) + .expect("state") + .index(), + 0 + ); assert_eq!( atn.add_state(AtnStateKind::RuleStop, Some(0)) .expect("state") .index(), - 6 + 1 ); + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![1]) + .expect("rule stop states"); + let excluded = atn.add_interval_set([(1, 1)]).expect("excluded set"); + atn.add_transition( + 0, + ParserTransitionSpec::NotSet { + target: 1, + set: excluded, + }, + ) + .expect("transition"); + finish_atn(atn) + } + + /// ATN for `start : . EOF ;`: a wildcard whose follow state explicitly matches + /// EOF. State 0 (`RuleStart`) -wildcard-> 2 -EOF-> 1 (`RuleStop`). + fn wildcard_then_eof_atn() -> Atn { + let mut atn = ParserAtnBuilder::new(1); assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(1)) + atn.add_state(AtnStateKind::RuleStart, Some(0)) .expect("state") .index(), - 7 + 0 ); assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(1)) + atn.add_state(AtnStateKind::RuleStop, Some(0)) .expect("state") .index(), - 8 + 1 ); assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(1)) + atn.add_state(AtnStateKind::Basic, Some(0)) .expect("state") .index(), - 9 + 2 ); - atn.set_rule_to_start_state(vec![0, 7]) + atn.set_rule_to_start_state(vec![0]) .expect("rule start states"); - atn.set_rule_to_stop_state(vec![6, 9]) + atn.set_rule_to_stop_state(vec![1]) .expect("rule stop states"); - atn.set_end_state(1, 3).expect("block end state"); - atn.set_loop_back_state(5, 4).expect("loop back state"); - atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) + atn.add_transition(0, ParserTransitionSpec::Wildcard { target: 2 }) .expect("transition"); atn.add_transition( 2, - ParserTransitionSpec::Rule { - target: 7, - rule_index: 1, - follow_state: 3, - precedence: 0, - }, - ) - .expect("transition"); - atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 }) - .expect("transition"); - atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - atn.add_transition(4, ParserTransitionSpec::Epsilon { target: 5 }) - .expect("transition"); - atn.add_transition( - 5, - ParserTransitionSpec::Atom { - target: 6, - label: 2, - }, - ) - .expect("transition"); - atn.add_transition( - 7, ParserTransitionSpec::Atom { - target: 8, - label: 1, + target: 1, + label: TOKEN_EOF, }, ) .expect("transition"); - atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) - .expect("transition"); finish_atn(atn) } #[test] - fn runtime_options_default_exits_recovering_empty_plus_iteration() { - let atn = plus_loop_with_recovering_body_atn(); + fn parser_matches_token_and_reports_mismatch() { + let source = Source { + tokens: vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ], + index: 0, + }; + let data = RecognizerData::new( + "Mini.g4", + Vocabulary::new([None, Some("'x'")], [None, Some("X")], [None::<&str>, None]), + ); + let mut parser = BaseParser::new(CommonTokenStream::new(source), data); + let matched = parser.match_token(1).expect("token 1 should match"); + assert_eq!(parser.node(matched).text(), "x"); + assert!(parser.match_token(1).is_err()); + } + + #[test] + fn parser_matches_token_sets() { + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + + let matched = parser + .match_set(&[(1, 1), (3, 4)]) + .expect("token set should match"); + assert_eq!(parser.node(matched).text(), "x"); + assert!(parser.match_not_set(&[(1, 1)], 1, 4).is_err()); + } + + #[test] + fn generated_rule_api_tracks_state_and_precedence() { + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + + let context = parser.enter_rule(7, 2); + assert_eq!(context.rule_index(), 2); + assert_eq!(parser.state(), 7); + assert_eq!( + parser.rule_context_stack, + vec![RuleContextFrame { + rule_index: 2, + invoking_state: 7 + }] + ); + + let recursive = parser.enter_recursion_rule(11, 3, 4); + assert_eq!(recursive.rule_index(), 3); + assert!(parser.precpred(4)); + assert!(parser.precpred(5)); + assert!(!parser.precpred(3)); + + let next = parser.push_new_recursion_context(13, 3); + assert_eq!(next.invoking_state(), 13); + parser.unroll_recursion_context(); + assert_eq!(parser.precedence_stack, vec![0]); + assert_eq!( + parser.rule_context_stack, + vec![RuleContextFrame { + rule_index: 2, + invoking_state: 7 + }] + ); + + parser.exit_rule(); + assert!(parser.rule_context_stack.is_empty()); + } + + #[test] + fn reset_rewinds_input_and_clears_parser_owned_parse_state() { + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + let matched = parser.match_token(1).expect("token should match"); + assert_eq!(parser.node(matched).text(), "x"); + parser.record_generated_syntax_error(); + parser.set_int_member(7, 11); + parser.set_build_parse_trees(false); + parser.set_report_diagnostic_errors(true); + parser.set_prediction_mode(PredictionMode::Sll); + parser.set_bail_on_error(true); + let _context = parser.enter_recursion_rule(9, 0, 4); + parser.pending_invoking_states.push(5); + parser.unknown_predicate_hits.push((0, 1)); + parser.unhandled_action_hits.push((0, 2)); + + parser.reset(); + + assert_eq!(parser.input.index(), 0); + assert_eq!(parser.la(1), 1); + assert_eq!(parser.state(), -1); + assert_eq!(parser.number_of_syntax_errors(), 0); + assert_eq!(parser.parse_tree_storage().node_count(), 0); + assert!(parser.rule_context_stack.is_empty()); + assert!(parser.pending_invoking_states.is_empty()); + assert_eq!(parser.precedence_stack, [0]); + assert!(parser.unknown_predicate_hits.is_empty()); + assert!(parser.unhandled_action_hits.is_empty()); + assert_eq!(parser.int_member(7), Some(11)); + assert!(!parser.build_parse_trees()); + assert!(parser.report_diagnostic_errors()); + assert_eq!(parser.prediction_mode(), PredictionMode::Sll); + assert!(parser.bail_on_error()); + } + + #[test] + fn set_token_stream_replaces_input_and_resets_parser() { + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("old"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.consume(); + parser.record_generated_syntax_error(); + let replacement = CommonTokenStream::new(Source { + tokens: vec![ + TestToken::new(2).with_text("new"), + TestToken::eof("parser-test", 1, 1, 1), + ], + index: 0, + }); + + parser.set_token_stream(replacement); + + assert_eq!(parser.input.index(), 0); + assert_eq!(parser.la(1), 2); + assert_eq!(parser.input.text_all(), "new"); + assert_eq!(parser.number_of_syntax_errors(), 0); + } + + #[test] + fn active_invocation_states_exclude_the_root_frame() { let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - let error = parser - .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) - .expect_err("EOF recovery should report a bounded mismatch"); + let _root = parser.enter_rule(0, 0); + assert!(parser.active_invocation_states().is_empty()); - let AntlrError::ParserError { message, .. } = error else { - panic!("expected ParserError, got {error:?}"); - }; - insta::assert_snapshot!(message, @"mismatched input '' expecting {'x', 2}"); - assert_eq!(parser.number_of_syntax_errors(), 1); - assert_eq!(parser.input.index(), 0, "EOF remains unconsumed"); + let marker = parser.push_invoking_state(6); + let _child = parser.enter_rule(2, 1); + parser.discard_invoking_state(marker); + assert_eq!(parser.active_invocation_states(), [6]); + + let marker = parser.push_invoking_state(13); + let _grandchild = parser.enter_rule(4, 2); + parser.discard_invoking_state(marker); + assert_eq!(parser.active_invocation_states(), [13, 6]); + + parser.exit_rule(); + parser.exit_rule(); + parser.exit_rule(); } #[test] - fn sync_decision_deletes_token_before_eof_at_loop_back() { - // `s : A* EOF` on `c`: the loop decision (state 5) can recover onto EOF. - // At the loop ENTRY (loop_back = false) a single unexpected token before - // EOF is deleted as an error node (then the generated EOF match consumes - // the real EOF) — matching ANTLR's `(s c )` + "extraneous input". - // EOF must be a valid scan-stop for this to fire. - let atn = star_loop_then_eof_atn(); + fn parser_predicates_support_token_adjacency() { let mut parser = mini_parser(vec![ - TestToken::new(2).with_text("c"), - TestToken::eof("parser-test", 1, 1, 1), + TestToken::new(1).with_text("=").with_span(0, 0), + TestToken::new(1).with_text(">").with_span(1, 1), + TestToken::eof("parser-test", 2, 1, 2), ]); - parser.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }]; - let children = parser - .sync_decision(&atn, 5, true, false) - .expect("single token before EOF recovers"); - assert_eq!(children.len(), 1); - assert_eq!(parser.node(children[0]).kind(), NodeKind::Error); - assert_eq!(parser.number_of_syntax_errors(), 1); - assert_eq!( - parser.la(1), - TOKEN_EOF, - "EOF is left for the rule's EOF match" - ); + parser.consume(); + parser.consume(); + + let predicates = [(0, 0, ParserPredicate::TokenPairAdjacent)]; + + assert!(parser.parser_semantic_predicate_matches(&predicates, 0, 0)); + + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("=").with_span(0, 0), + TestToken::new(1) + .with_text(" ") + .with_channel(HIDDEN_CHANNEL) + .with_span(1, 1), + TestToken::new(1).with_text(">").with_span(2, 2), + TestToken::eof("parser-test", 3, 1, 3), + ]); + parser.consume(); + parser.consume(); + + assert!(!parser.parser_semantic_predicate_matches(&predicates, 0, 0)); } #[test] - fn sync_decision_does_not_delete_two_tokens_before_eof_at_loop_entry() { - // `s : A* EOF` on `c c`: at the loop ENTRY (loop_back = false) ANTLR does - // single-token deletion, which fails because LA(2) = `c` is not expected — - // so it reports `mismatched input` and consumes nothing (ANTLR: `(s c c)` - // with no EOF). The scan must NOT multi-token-consume both `c`s here. - let atn = star_loop_then_eof_atn(); + fn parser_predicates_support_context_child_text_checks() { let mut parser = mini_parser(vec![ - TestToken::new(2).with_text("c"), - TestToken::new(2).with_text("c"), - TestToken::eof("parser-test", 1, 2, 2), + TestToken::new(1).with_text("var"), + TestToken::eof("parser-test", 1, 1, 1), ]); - parser.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }]; - let error = parser - .sync_decision(&atn, 5, true, false) - .expect_err("two tokens at the loop entry must not be deleted"); - match error { - AntlrError::ParserError { message, .. } => { - assert!(message.starts_with("mismatched input"), "got: {message}"); - } - other => panic!("expected mismatched-input ParserError, got {other:?}"), - } + let mut context = ParserRuleContext::new(1, 0); + let mut child_context = ParserRuleContext::new(2, 0); + let terminal = parser.terminal_tree(TokenId::try_from(0).expect("test token ID")); + parser.tree.add_child(&mut child_context, terminal); + let child = parser.rule_node(child_context); + parser.tree.add_child(&mut context, child); + let predicates = [( + 1, + 0, + ParserPredicate::ContextChildRuleTextNotEquals { + rule_index: 2, + text: "var", + }, + )]; + + assert!( + !parser.parser_semantic_predicate_matches_with_context_and_local( + &predicates, + 1, + 0, + &context, + 0, + ) + ); + } + + #[test] + fn context_expected_symbols_walks_nullable_parent_contexts() { + let atn = nested_nullable_context_atn(); + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + parser.rule_context_stack = vec![ + RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }, + RuleContextFrame { + rule_index: 1, + invoking_state: 1, + }, + RuleContextFrame { + rule_index: 2, + invoking_state: 2, + }, + ]; + + let expected = parser.context_expected_symbols(&atn); + + assert!(expected.contains(&1)); + assert!(expected.contains(&TOKEN_EOF)); + } + + #[test] + fn prediction_context_return_states_track_rule_stack_changes() { + let atn = nested_nullable_context_atn(); + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + parser.rule_context_stack = vec![ + RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }, + RuleContextFrame { + rule_index: 1, + invoking_state: 1, + }, + RuleContextFrame { + rule_index: 2, + invoking_state: 2, + }, + ]; + + let initial_version = parser.rule_context_version(); + let first: Vec<_> = parser.prediction_context_return_states(&atn).collect(); + let second: Vec<_> = parser.prediction_context_return_states(&atn).collect(); + assert_eq!(first, second); + assert_eq!(parser.rule_context_version(), initial_version); + + parser.exit_rule(); + let after_pop: Vec<_> = parser.prediction_context_return_states(&atn).collect(); + assert_ne!(first, after_pop); + assert_ne!(parser.rule_context_version(), initial_version); + } + + #[test] + fn generated_match_token_recovers_missing_token_from_context_follow() { + let atn = generated_match_recovery_atn(); + let data = RecognizerData::new( + "Mini.g4", + Vocabulary::new( + [None, Some("'X'"), Some("'Y'")], + [None, Some("X"), Some("Y")], + [None::<&str>, None, None], + ), + ); + let mut parser = BaseParser::new( + CommonTokenStream::new(Source { + tokens: vec![TestToken::eof("parser-test", 3, 1, 3)], + index: 0, + }), + data, + ); + parser.rule_context_stack = vec![ + RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }, + RuleContextFrame { + rule_index: 1, + invoking_state: 1, + }, + ]; + assert_eq!(parser.number_of_syntax_errors(), 0); + + let node = parser + .match_token_recovering(2, 5, &atn) + .expect("generated match should insert missing token"); + + assert_eq!(node.children().len(), 1); + assert_eq!(parser.node(node.children()[0]).text(), ""); + assert_eq!( + node.clone() + .into_child_iter() + .map(|child| parser.node(child).text()) + .collect::>(), + [""] + ); + // Single-token insertion synthesizes a missing token and consumes nothing, + // so no EOF terminal is consumed even though lookahead is EOF. + assert!(!node.consumed_eof()); + assert_eq!(parser.la(1), TOKEN_EOF); + assert_eq!(parser.number_of_syntax_errors(), 1); assert_eq!( - parser.la(1), - 2, - "nothing consumed; cursor still on first `c`" + parser.generated_parser_diagnostics, + [ParserDiagnostic { + line: 1, + column: 3, + message: "missing 'Y' at ''".to_owned(), + offending: parser.input.lt_id(1), + }] ); } #[test] - fn sync_decision_consumes_until_eof_at_loop_back() { - // Same `s : A* EOF` decision, but at a loop-BACK (loop_back = true, i.e. - // after ≥1 `A` matched). ANTLR uses multi-token `consumeUntil(recoverSet)` - // there, so two unexpected tokens before EOF are BOTH deleted and the rule - // recovers (matching `(s a c c )` for input `a c c`). Here we feed the - // post-`a` state directly: `c c ` with loop_back = true. - let atn = star_loop_then_eof_atn(); - let mut parser = mini_parser(vec![ - TestToken::new(2).with_text("c"), - TestToken::new(2).with_text("c"), - TestToken::eof("parser-test", 1, 2, 2), - ]); - parser.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }]; - let children = parser - .sync_decision(&atn, 5, false, true) - .expect("loop-back multi-token deletion recovers onto EOF"); - assert_eq!(children.len(), 2, "both `c`s deleted as error nodes"); - assert!( - children - .iter() - .all(|child| parser.node(*child).kind() == NodeKind::Error) + fn generated_match_token_counts_single_token_deletion_recovery() { + let atn = generated_match_recovery_atn(); + let data = RecognizerData::new( + "Mini.g4", + Vocabulary::new( + [None, Some("'X'"), Some("'Y'"), Some("'Z'")], + [None, Some("X"), Some("Y"), Some("Z")], + [None::<&str>, None, None, None], + ), + ); + let mut parser = BaseParser::new( + CommonTokenStream::new(Source { + tokens: vec![ + TestToken::new(3).with_text("z"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 3, 1, 3), + ], + index: 0, + }), + data, ); - assert_eq!(parser.number_of_syntax_errors(), 1); - assert_eq!(parser.la(1), TOKEN_EOF, "EOF left for the rule's EOF match"); - } - - #[test] - fn sync_decision_returns_before_recovery_for_nullable_exit() { - let atn = nested_star_rule_atn(); - for (current_context_empty, loop_back) in [(true, false), (false, true)] { - let mut parser = mini_parser(vec![ - TestToken::new(2).with_text("c"), - TestToken::new(1).with_text("a"), - TestToken::eof("parser-test", 1, 2, 2), - ]); - parser.rule_context_stack = vec![ - RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }, - RuleContextFrame { - rule_index: 1, - invoking_state: 1, - }, - ]; - - let children = parser - .sync_decision(&atn, 5, current_context_empty, loop_back) - .expect("nullable synchronization is a no-op"); - assert!(children.is_empty()); - assert_eq!(parser.la(1), 2, "the caller must receive the current token"); - assert_eq!(parser.number_of_syntax_errors(), 0); - assert_eq!( - parser - .generated_sync_expected - .as_ref() - .expect("nullable sync preserves expected symbols") - .to_btree_set(), - BTreeSet::from([TOKEN_EOF, 1]) - ); - } - } + let node = parser + .match_token_recovering(2, 5, &atn) + .expect("generated match should delete the extraneous token"); - fn predicate_after_token_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(2); - assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(0)) - .expect("state") - .index(), - 0 - ); + assert_eq!(node.children().len(), 2); + assert_eq!(parser.node(node.children()[0]).kind(), NodeKind::Error); + assert_eq!(parser.node(node.children()[0]).text(), "z"); + assert_eq!(parser.node(node.children()[1]).text(), "y"); assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 1 + node.into_child_iter() + .map(|child| parser.node(child).text()) + .collect::>(), + ["z", "y"] ); - assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 2 + assert_eq!(parser.number_of_syntax_errors(), 1); + } + + #[test] + fn generated_match_token_iterates_single_success_without_a_children_vec() { + let atn = generated_match_recovery_atn(); + let data = RecognizerData::new( + "Mini.g4", + Vocabulary::new( + [None, Some("'X'"), Some("'Y'")], + [None, Some("X"), Some("Y")], + [None::<&str>, None, None], + ), ); - assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 3 + let mut parser = BaseParser::new( + CommonTokenStream::new(Source { + tokens: vec![ + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 1, 1, 1), + ], + index: 0, + }), + data, ); + + let node = parser + .match_token_recovering(2, 5, &atn) + .expect("generated match should consume the expected token"); + assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(0)) - .expect("state") - .index(), - 4 + node.into_child_iter() + .map(|child| parser.node(child).text()) + .collect::>(), + ["y"] ); - atn.set_rule_to_start_state(vec![0]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![4]) - .expect("rule stop states"); - atn.add_transition( - 0, - ParserTransitionSpec::Atom { - target: 1, - label: 1, - }, - ) - .expect("transition"); - atn.add_transition( - 1, - ParserTransitionSpec::Predicate { - target: 2, - rule_index: 0, - pred_index: 0, - context_dependent: false, - }, - ) - .expect("transition"); - atn.add_transition( - 2, - ParserTransitionSpec::Atom { - target: 3, - label: 2, - }, - ) - .expect("transition"); - atn.add_transition(3, ParserTransitionSpec::Epsilon { target: 4 }) - .expect("transition"); - finish_atn(atn) + assert_eq!(parser.number_of_syntax_errors(), 0); } - fn predicate_gated_same_lookahead_atn(pred_indexes: [usize; 2]) -> Atn { - let mut atn = ParserAtnBuilder::new(1); - for (state_number, kind) in [ - (0, AtnStateKind::RuleStart), - (1, AtnStateKind::BlockStart), - (2, AtnStateKind::Basic), - (3, AtnStateKind::Basic), - (4, AtnStateKind::Basic), - (5, AtnStateKind::Basic), - (6, AtnStateKind::BlockEnd), - (7, AtnStateKind::RuleStop), - ] { - assert_eq!( - atn.add_state(kind, Some(0)).expect("state").index(), - state_number - ); - } - atn.set_rule_to_start_state(vec![0]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![7]) - .expect("rule stop states"); - atn.add_decision_state(1).expect("decision state"); - atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 2 }) - .expect("transition"); - atn.add_transition(1, ParserTransitionSpec::Epsilon { target: 3 }) - .expect("transition"); - atn.add_transition( - 2, - ParserTransitionSpec::Predicate { - target: 4, - rule_index: 0, - pred_index: pred_indexes[0], - context_dependent: false, - }, - ) - .expect("transition"); - atn.add_transition( - 3, - ParserTransitionSpec::Predicate { - target: 5, + #[test] + fn generated_diagnostic_restore_rolls_back_syntax_error_count() { + let atn = generated_match_recovery_atn(); + let data = RecognizerData::new( + "Mini.g4", + Vocabulary::new( + [None, Some("'X'"), Some("'Y'")], + [None, Some("X"), Some("Y")], + [None::<&str>, None, None], + ), + ); + let mut parser = BaseParser::new( + CommonTokenStream::new(Source { + tokens: vec![TestToken::eof("parser-test", 3, 1, 3)], + index: 0, + }), + data, + ); + parser.rule_context_stack = vec![ + RuleContextFrame { rule_index: 0, - pred_index: pred_indexes[1], - context_dependent: false, - }, - ) - .expect("transition"); - atn.add_transition( - 4, - ParserTransitionSpec::Atom { - target: 6, - label: 1, - }, - ) - .expect("transition"); - atn.add_transition( - 5, - ParserTransitionSpec::Atom { - target: 6, - label: 1, + invoking_state: 0, }, - ) - .expect("transition"); - atn.add_transition( - 6, - ParserTransitionSpec::Atom { - target: 7, - label: TOKEN_EOF, + RuleContextFrame { + rule_index: 1, + invoking_state: 1, }, - ) - .expect("transition"); - finish_atn(atn) + ]; + let marker = parser.generated_diagnostics_checkpoint(); + + let _ = parser + .match_token_recovering(2, 5, &atn) + .expect("generated match should insert missing token"); + assert_eq!(parser.number_of_syntax_errors(), 1); + + parser.restore_generated_diagnostics(marker); + + assert_eq!(parser.number_of_syntax_errors(), 0); + assert!(parser.generated_parser_diagnostics.is_empty()); } - fn nested_nullable_context_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(1); - for state_number in 0..=20 { - let kind = match state_number { - 0 | 10 | 16 => AtnStateKind::RuleStart, - 9 | 15 | 20 => AtnStateKind::RuleStop, - _ => AtnStateKind::Basic, - }; - let rule_index = match state_number { - 0..=9 => 0, - 10..=15 => 1, - _ => 2, - }; - assert_eq!( - atn.add_state(kind, Some(rule_index)) - .expect("state") - .index(), - state_number - ); - } - atn.set_rule_to_start_state(vec![0, 10, 16]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![9, 15, 20]) - .expect("rule stop states"); - atn.add_transition( - 1, - ParserTransitionSpec::Rule { - target: 10, - rule_index: 1, - follow_state: 8, - precedence: 0, - }, + #[test] + fn generated_prediction_diagnostics_use_adaptive_context() { + let atn = two_alt_decision_atn(); + let data = RecognizerData::new( + "Mini.g4", + Vocabulary::new( + [None, Some("'x'"), Some("'y'")], + [None, Some("X"), Some("Y")], + [None::<&str>, None, None], + ), ) - .expect("transition"); - atn.add_transition( - 8, - ParserTransitionSpec::Atom { - target: 9, - label: 1, + .with_rule_names(["s"]); + let mut parser = BaseParser::new( + CommonTokenStream::new(Source { + tokens: vec![ + TestToken::new(1) + .with_text("x") + .with_position(1, 0) + .with_span(0, 0), + TestToken::new(2) + .with_text("y") + .with_position(1, 2) + .with_span(1, 1), + TestToken::eof("parser-test", 2, 1, 3), + ], + index: 0, + }), + data, + ); + parser.set_report_diagnostic_errors(true); + + parser.record_generated_prediction_diagnostic( + &atn, + 1, + &ParserAtnPrediction { + alt: 1, + requires_full_context: true, + has_semantic_context: false, + diagnostic: Some(ParserAtnPredictionDiagnostic { + kind: ParserAtnPredictionDiagnosticKind::ContextSensitivity, + start_index: 0, + sll_stop_index: 1, + ll_stop_index: 0, + conflicting_alts: vec![1, 2], + exact: false, + }), }, - ) - .expect("transition"); - atn.add_transition(8, ParserTransitionSpec::Epsilon { target: 9 }) - .expect("transition"); - atn.add_transition( - 2, - ParserTransitionSpec::Rule { - target: 16, - rule_index: 2, - follow_state: 14, - precedence: 0, + ); + // Ambiguities from the default LL prediction mode are non-exact, so — + // matching Java's exactOnly DiagnosticErrorListener — only the + // attempting-full-context line is reported. Exact-ambiguity mode + // reports the ambiguity itself. + parser.record_generated_prediction_diagnostic( + &atn, + 1, + &ParserAtnPrediction { + alt: 1, + requires_full_context: true, + has_semantic_context: false, + diagnostic: Some(ParserAtnPredictionDiagnostic { + kind: ParserAtnPredictionDiagnosticKind::Ambiguity, + start_index: 0, + sll_stop_index: 1, + ll_stop_index: 1, + conflicting_alts: vec![1, 2], + exact: false, + }), }, - ) - .expect("transition"); - atn.add_transition(14, ParserTransitionSpec::Epsilon { target: 15 }) - .expect("transition"); - finish_atn(atn) + ); + + // The full-context/context-sensitivity diagnostic trace (order + decision + input windows) + // is one snapshot rather than three ParserDiagnostic literals. + insta::assert_debug_snapshot!( + "generated_prediction_diagnostics_use_adaptive_context", + parser.generated_parser_diagnostics + ); } - fn generated_match_recovery_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(2); + #[test] + fn generated_match_not_set_recovers_empty_complement_at_eof() { + let atn = complement_set_atn(); + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + parser.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }]; + + let node = parser + .match_not_token_set_recovering( + atn.token_set(0).expect("excluded token set"), + 1, + 1, + 1, + &atn, + ) + .expect("empty complement should recover at EOF"); + + assert_eq!(node.children().len(), 1); + // Recovery synthesizes a missing token without consuming EOF, so the + // enclosing rule must not record EOF as its stop token. + assert!(!node.consumed_eof()); + assert_eq!(parser.la(1), TOKEN_EOF); assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(0)) - .expect("state") - .index(), - 0 + parser.generated_parser_diagnostics, + [ParserDiagnostic { + line: 1, + column: 1, + message: "missing {} at ''".to_owned(), + offending: parser.input.lt_id(1), + }] ); - assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 1 + } + + #[test] + fn wildcard_recovers_via_insertion_when_follow_expects_eof_at_eof() { + // `start : . EOF ;` on empty input. The wildcard is modeled as an + // empty-complement not-set; at EOF the follow state (the explicit EOF + // match) expects EOF, so even in the start rule recovery must perform + // single-token insertion (``) rather than aborting — matching + // ANTLR's `(start )` / "missing ... at ''". + let atn = wildcard_then_eof_atn(); + let data = RecognizerData::new( + "Mini.g4", + Vocabulary::new([None, Some("'x'")], [None, Some("X")], [None::<&str>, None]), ); - assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 2 + let mut parser = BaseParser::new( + CommonTokenStream::new(Source { + tokens: vec![TestToken::eof("parser-test", 1, 1, 1)], + index: 0, + }), + data, + ); + parser.rule_context_stack = vec![RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }]; + + let node = parser + .match_not_set_recovering(&[], 1, atn.max_token_type(), 2, &atn) + .expect("wildcard at EOF should recover by insertion when follow expects EOF"); + + // A single `` error node is inserted; EOF is not consumed. + assert_eq!(node.children().len(), 1); + assert!(!node.consumed_eof()); + assert!( + parser + .node(node.children()[0]) + .text() + .starts_with("'".to_owned(), + offending: parser.input.lt_id(1), + }] ); - assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(1)) - .expect("state") - .index(), - 4 + } + + #[test] + fn generated_rule_recovery_consumes_to_parent_follow() { + let atn = generated_match_recovery_atn(); + let data = RecognizerData::new( + "Mini.g4", + Vocabulary::new( + [None, Some("'X'"), Some("'Y'"), Some("'Z'")], + [None, Some("X"), Some("Y"), Some("Z")], + [None::<&str>, None, None, None], + ), ); - assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(1)) - .expect("state") - .index(), - 5 + let mut parser = BaseParser::new( + CommonTokenStream::new(Source { + tokens: vec![ + TestToken::new(3).with_text("z"), + TestToken::eof("parser-test", 1, 1, 1), + ], + index: 0, + }), + data, ); - atn.set_rule_to_start_state(vec![0, 4]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![3, 5]) - .expect("rule stop states"); - atn.add_transition( - 1, - ParserTransitionSpec::Rule { - target: 4, - rule_index: 1, - follow_state: 2, - precedence: 0, - }, - ) - .expect("transition"); - atn.add_transition( - 2, - ParserTransitionSpec::Atom { - target: 3, - label: TOKEN_EOF, + let _parent = parser.enter_rule(0, 0); + let marker = parser.push_invoking_state(1); + let mut child = parser.enter_rule(4, 1); + parser.discard_invoking_state(marker); + + // The anchor recorded where the error was built must survive into the + // dispatched diagnostic even though recovery consumes past it below. + let offending = parser.input.lt_id(1); + assert!(offending.is_some(), "the 'z' token should be buffered"); + parser.recover_generated_rule( + &mut child, + &atn, + AntlrError::ParserError { + line: 1, + column: 0, + message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), + offending, }, - ) - .expect("transition"); - finish_atn(atn) - } + ); + let tree = parser.finish_rule(child, false); - fn complement_set_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(1); + assert_eq!(parser.la(1), TOKEN_EOF); assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(0)) - .expect("state") - .index(), - 0 + parser.node(tree).to_string_tree_with_names(&["s", "a"]), + "(a z)" ); + assert_eq!(parser.number_of_syntax_errors(), 1); assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(0)) - .expect("state") - .index(), - 1 + parser.generated_parser_diagnostics, + [ParserDiagnostic { + line: 1, + column: 0, + message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), + offending, + }] ); - atn.set_rule_to_start_state(vec![0]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![1]) - .expect("rule stop states"); - let excluded = atn.add_interval_set([(1, 1)]).expect("excluded set"); - atn.add_transition( - 0, - ParserTransitionSpec::NotSet { - target: 1, - set: excluded, - }, - ) - .expect("transition"); - finish_atn(atn) + parser.exit_rule(); } - /// ATN for `start : . EOF ;`: a wildcard whose follow state explicitly matches - /// EOF. State 0 (`RuleStart`) -wildcard-> 2 -EOF-> 1 (`RuleStop`). - fn wildcard_then_eof_atn() -> Atn { - let mut atn = ParserAtnBuilder::new(1); - assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(0)) - .expect("state") - .index(), - 0 + #[test] + fn generated_rule_recovery_forces_progress_after_repeated_error_state() { + let atn = nested_nullable_context_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.rule_context_stack = vec![ + RuleContextFrame { + rule_index: 0, + invoking_state: 0, + }, + RuleContextFrame { + rule_index: 1, + invoking_state: 1, + }, + RuleContextFrame { + rule_index: 2, + invoking_state: 2, + }, + ]; + parser.set_state(20); + let mut context = ParserRuleContext::new(2, 2); + + parser.recover_generated_rule( + &mut context, + &atn, + AntlrError::NoViableAlternative { + input: "'x'".to_owned(), + }, ); - assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(0)) - .expect("state") - .index(), - 1 + assert_eq!(parser.input.index(), 0); + + parser.set_state(21); + parser.recover_generated_rule( + &mut context, + &atn, + AntlrError::NoViableAlternative { + input: "'x'".to_owned(), + }, ); + assert_eq!(parser.input.index(), 0); assert_eq!( - atn.add_state(AtnStateKind::Basic, Some(0)) - .expect("state") - .index(), - 2 + parser.generated_recovery_error_states, + BTreeSet::from([20, 21]) ); - atn.set_rule_to_start_state(vec![0]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![1]) - .expect("rule stop states"); - atn.add_transition(0, ParserTransitionSpec::Wildcard { target: 2 }) - .expect("transition"); - atn.add_transition( - 2, - ParserTransitionSpec::Atom { - target: 1, - label: TOKEN_EOF, + + parser.set_state(20); + parser.recover_generated_rule( + &mut context, + &atn, + AntlrError::NoViableAlternative { + input: "'x'".to_owned(), }, - ) - .expect("transition"); - finish_atn(atn) + ); + + assert_eq!(parser.input.index(), 1); + assert_eq!(parser.la(1), TOKEN_EOF); + assert!(context.has_matched_child()); + assert_eq!(parser.generated_recovery_error_states, BTreeSet::from([20])); + + parser.match_eof().expect("EOF should match"); + assert_eq!(parser.generated_recovery_error_index, None); + assert!(parser.generated_recovery_error_states.is_empty()); + } + + #[test] + fn greedy_ll1_alt_handles_nullable_loop_exit() { + let mut body_symbols = TokenBitSet::default(); + body_symbols.insert(1); + let entry = DecisionLookahead { + transitions: vec![ + TransitionLookSet { + symbols: body_symbols, + nullable: false, + }, + TransitionLookSet { + symbols: TokenBitSet::default(), + nullable: true, + }, + ], + }; + + assert_eq!(ll1_unique_alt(&entry, 2), None); + assert_eq!(ll1_greedy_alt(&entry, 2, false), Some(1)); + assert_eq!(ll1_greedy_alt(&entry, 1, false), None); + assert_eq!(ll1_greedy_alt(&entry, 1, true), None); + } + + #[test] + fn ordinary_repetition_builds_tree_in_input_order() { + for atn in [ordinary_star_loop_atn(), ordinary_plus_loop_atn()] { + let mut parser = mini_parser(repeated_x_tokens(3)); + let tree = parser + .parse_atn_rule(&atn, 0) + .expect("ordinary repetition should parse"); + + let root = parser + .node(tree) + .as_rule() + .expect("entry result should be a rule"); + let body_rules = root.child_rules(1).collect::>(); + assert_eq!(root.text(), "xxx"); + assert_eq!(body_rules.len(), 3); + assert_eq!( + body_rules + .iter() + .map(|rule| rule.start_id().expect("body start").index()) + .collect::>(), + [0, 1, 2] + ); + assert_eq!( + body_rules + .iter() + .map(|rule| rule.stop_id().expect("body stop").index()) + .collect::>(), + [0, 1, 2] + ); + assert_eq!(parser.number_of_syntax_errors(), 0); + } } #[test] - fn parser_matches_token_and_reports_mismatch() { - let source = Source { - tokens: vec![ - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), - ], - index: 0, - }; - let data = RecognizerData::new( - "Mini.g4", - Vocabulary::new([None, Some("'x'")], [None, Some("X")], [None::<&str>, None]), - ); - let mut parser = BaseParser::new(CommonTokenStream::new(source), data); - let matched = parser.match_token(1).expect("token 1 should match"); - assert_eq!(parser.node(matched).text(), "x"); - assert!(parser.match_token(1).is_err()); + fn deeply_nested_deferred_rules_materialize_on_small_stack() { + const DEPTH: usize = 20_000; + + std::thread::Builder::new() + .name("deferred-rule-materialization".to_owned()) + .stack_size(256 * 1024) + .spawn(|| { + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 0, 1, 0)]); + let mut root = FastDeferredNodeId::EMPTY; + for depth in 0..DEPTH { + root = parser + .recognition_arena + .deferred_rule_node(FastDeferredRule { + rule_index: u32::try_from(depth).expect("depth fits in u32"), + invoking_state: i32::try_from(depth).expect("depth fits in i32"), + start_index: 0, + stop_index: None, + deferred_children: root, + children: NodeSeqId::EMPTY, + }); + } + + let (mut children, alt_number) = + parser.materialize_fast_deferred_nodes(root, NodeSeqId::EMPTY); + assert_eq!(alt_number, 0); + for expected_rule in (0..DEPTH).rev() { + let mut nodes = parser.recognition_arena.iter(children); + let node = nodes.next().expect("nested rule node"); + assert!(nodes.next().is_none(), "each rule has one child"); + let ArenaRecognizedNode::Rule { + rule_index, + children: nested, + .. + } = parser.recognition_arena.node(node) + else { + panic!("expected nested rule"); + }; + assert_eq!(rule_index as usize, expected_rule); + children = nested; + } + assert!(children.is_empty()); + }) + .expect("small-stack thread should start") + .join() + .expect("deferred rules should materialize without recursion"); } #[test] - fn parser_matches_token_sets() { + fn deferred_alternatives_preserve_left_recursive_contexts() { let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), + TestToken::new(1).with_text("1"), + TestToken::new(2).with_text("+"), + TestToken::new(1).with_text("2"), + TestToken::eof("parser-test", 3, 1, 3), ]); + let base = parser.arena_token_node(0, false); + let operator = parser.arena_token_node(1, false); + let right = parser.arena_token_node(2, false); - let matched = parser - .match_set(&[(1, 1), (3, 4)]) - .expect("token set should match"); - assert_eq!(parser.node(matched).text(), "x"); - assert!(parser.match_not_set(&[(1, 1)], 1, 4).is_err()); + let base = parser.recognition_arena.prepend(NodeSeqId::EMPTY, base); + let base = parser.recognition_arena.deferred_fragment(base); + let operator = parser.recognition_arena.prepend(NodeSeqId::EMPTY, operator); + let operator = parser.recognition_arena.deferred_fragment(operator); + let right = parser.recognition_arena.prepend(NodeSeqId::EMPTY, right); + let right = parser.recognition_arena.deferred_fragment(right); + let base_alt = parser.recognition_arena.deferred_alternative(1); + let boundary = parser.recognition_arena.deferred_left_recursive_boundary(0); + let operator_alt = parser.recognition_arena.deferred_alternative(6); + + let mut deferred = FastDeferredNodeId::EMPTY; + for fragment in [base_alt, base, boundary, operator_alt, operator, right] { + deferred = parser + .recognition_arena + .concat_deferred_nodes(deferred, fragment); + } + let (nodes, root_alt_number) = + parser.materialize_fast_deferred_nodes(deferred, NodeSeqId::EMPTY); + let nodes = parser + .recognition_arena + .fold_left_recursive_boundaries(nodes); + + let mut root = ParserRuleContext::new(0, -1); + root.set_context_alt_number(root_alt_number); + let mut cursor = nodes; + while let Some(link) = parser.recognition_arena.link(cursor) { + let child = parser + .arena_recognized_node_tree(link.head, false, true) + .expect("materialized child should become a public tree"); + parser.tree.add_child(&mut root, child); + cursor = link.tail; + } + let tree = parser.rule_node(root); + let contexts = parser + .node(tree) + .descendants() + .filter_map(Node::as_rule) + .map(|rule| { + ( + rule.rule_index(), + rule.alt_number(), + rule.context_alt_number(), + rule.text(), + ) + }) + .collect::>(); + + insta::assert_debug_snapshot!( + "deferred_alternatives_preserve_left_recursive_contexts", + contexts + ); } #[test] - fn generated_rule_api_tracks_state_and_precedence() { - let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + fn fast_recognizer_preserves_labeled_left_recursive_operator_context() { + let atn = labeled_left_recursive_operator_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("a"), + TestToken::new(3).with_text("+"), + TestToken::new(1).with_text("b"), + TestToken::eof("parser-test", 3, 1, 3), + ]); - let context = parser.enter_rule(7, 2); - assert_eq!(context.rule_index(), 2); - assert_eq!(parser.state(), 7); - assert_eq!( - parser.rule_context_stack, - vec![RuleContextFrame { - rule_index: 2, - invoking_state: 7 - }] + let (tree, _) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + track_context_alt_numbers: true, + ..ParserRuntimeOptions::default() + }, + ) + .expect("labeled left-recursive addition should parse"); + let contexts = parser + .node(tree) + .descendants() + .filter_map(Node::as_rule) + .map(|rule| { + let operator = rule + .children() + .next() + .and_then(Node::as_rule) + .is_some_and(|child| child.rule_index() == rule.rule_index()); + (operator, rule.context_alt_number(), rule.text()) + }) + .collect::>(); + + insta::assert_debug_snapshot!( + "fast_recognizer_preserves_labeled_left_recursive_operator_context", + contexts ); + assert!(!parser.recognition_arena.deferred_nodes.is_empty()); + assert_eq!(parser.number_of_syntax_errors(), 0); + } - let recursive = parser.enter_recursion_rule(11, 3, 4); - assert_eq!(recursive.rule_index(), 3); - assert!(parser.precpred(4)); - assert!(parser.precpred(5)); - assert!(!parser.precpred(3)); + #[test] + fn deeply_nested_rule_calls_grow_the_stack() { + const DEPTH: usize = 4_096; + const STACK_SIZE: usize = 256 * 1024; + let atn = nested_rule_chain_atn(DEPTH); + std::thread::Builder::new() + .name("nested-adaptive-set-rules".to_owned()) + .stack_size(STACK_SIZE) + .spawn(move || { + let mut parser = mini_parser(vec![TestToken::new(1).with_text("x")]); + parser.set_build_parse_trees(false); + // This test isolates recognizer depth from the separately + // cached FIRST-set metadata walk. + parser.fast_first_set_prefilter = false; + parser + .parse_atn_rule(&atn, 0) + .expect("nested rule chain should grow the native stack"); + assert_eq!(parser.input.index(), 1); + }) + .expect("small-stack thread should start") + .join() + .expect("nested rule chain should not overflow its stack"); + } - let next = parser.push_new_recursion_context(13, 3); - assert_eq!(next.invoking_state(), 13); - parser.unroll_recursion_context(); - assert_eq!(parser.precedence_stack, vec![0]); - assert_eq!( - parser.rule_context_stack, - vec![RuleContextFrame { - rule_index: 2, - invoking_state: 7 - }] - ); + #[test] + fn deeply_nested_branching_rules_grow_the_stack() { + const DEPTH: usize = 4_096; + const STACK_SIZE: usize = 256 * 1024; + let atn = nested_rule_graph_atn(DEPTH, true, false); + std::thread::Builder::new() + .name("nested-branching-rules".to_owned()) + .stack_size(STACK_SIZE) + .spawn(move || { + let mut parser = mini_parser(vec![TestToken::new(1).with_text("x")]); + parser.set_build_parse_trees(false); + parser + .parse_atn_rule(&atn, 0) + .expect("branching rule chain should grow the native stack"); + assert_eq!(parser.input.index(), 1); + }) + .expect("small-stack thread should start") + .join() + .expect("branching rule chain should not overflow its stack"); + } - parser.exit_rule(); - assert!(parser.rule_context_stack.is_empty()); + #[test] + fn deeply_nested_rule_follows_grow_the_stack() { + const DEPTH: usize = 4_096; + const STACK_SIZE: usize = 256 * 1024; + let atn = nested_rule_graph_atn(DEPTH, false, true); + std::thread::Builder::new() + .name("nested-rule-follows".to_owned()) + .stack_size(STACK_SIZE) + .spawn(move || { + let mut parser = mini_parser(repeated_x_tokens(DEPTH)); + parser.set_build_parse_trees(false); + parser.fast_first_set_prefilter = false; + parser + .parse_atn_rule(&atn, 0) + .expect("rule follow chain should grow the native stack"); + assert_eq!(parser.input.index(), DEPTH); + }) + .expect("small-stack thread should start") + .join() + .expect("nested rule follow chain should not overflow its stack"); } #[test] - fn reset_rewinds_input_and_clears_parser_owned_parse_state() { - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - let matched = parser.match_token(1).expect("token should match"); - assert_eq!(parser.node(matched).text(), "x"); - parser.record_generated_syntax_error(); - parser.set_int_member(7, 11); - parser.set_build_parse_trees(false); - parser.set_report_diagnostic_errors(true); - parser.set_prediction_mode(PredictionMode::Sll); - parser.set_bail_on_error(true); - let _context = parser.enter_recursion_rule(9, 0, 4); - parser.pending_invoking_states.push(5); - parser.unknown_predicate_hits.push((0, 1)); - parser.unhandled_action_hits.push((0, 2)); - - parser.reset(); - - assert_eq!(parser.input.index(), 0); - assert_eq!(parser.la(1), 1); - assert_eq!(parser.state(), -1); - assert_eq!(parser.number_of_syntax_errors(), 0); - assert_eq!(parser.parse_tree_storage().node_count(), 0); - assert!(parser.rule_context_stack.is_empty()); - assert!(parser.pending_invoking_states.is_empty()); - assert_eq!(parser.precedence_stack, [0]); - assert!(parser.unknown_predicate_hits.is_empty()); - assert!(parser.unhandled_action_hits.is_empty()); - assert_eq!(parser.int_member(7), Some(11)); - assert!(!parser.build_parse_trees()); - assert!(parser.report_diagnostic_errors()); - assert_eq!(parser.prediction_mode(), PredictionMode::Sll); - assert!(parser.bail_on_error()); + fn deeply_nested_recovery_grows_the_stack() { + const DEPTH: usize = 4_096; + const STACK_SIZE: usize = 256 * 1024; + let atn = nested_rule_chain_atn(DEPTH); + std::thread::Builder::new() + .name("nested-rule-recovery".to_owned()) + .stack_size(STACK_SIZE) + .spawn(move || { + let mut parser = mini_parser(vec![ + TestToken::new(2).with_text("z"), + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 2, 1, 2), + ]); + parser.set_build_parse_trees(false); + parser.fast_first_set_prefilter = false; + parser + .parse_atn_rule(&atn, 0) + .expect("nested recovery should grow the native stack"); + assert_eq!(parser.input.index(), 2); + assert_eq!(parser.number_of_syntax_errors(), 1); + }) + .expect("small-stack thread should start") + .join() + .expect("nested rule recovery should not overflow its stack"); } #[test] - fn set_token_stream_replaces_input_and_resets_parser() { - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("old"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - parser.consume(); - parser.record_generated_syntax_error(); - let replacement = CommonTokenStream::new(Source { - tokens: vec![ - TestToken::new(2).with_text("new"), - TestToken::eof("parser-test", 1, 1, 1), - ], - index: 0, - }); + fn ambiguous_ordinary_repetition_merges_equivalent_coordinates() { + const REPETITIONS: usize = 64; - parser.set_token_stream(replacement); + let atn = ambiguous_ordinary_star_loop_atn(); + let mut parser = mini_parser(repeated_x_tokens(REPETITIONS)); + let tree = parser + .parse_atn_rule(&atn, 0) + .expect("ambiguous ordinary repetition should parse"); - assert_eq!(parser.input.index(), 0); - assert_eq!(parser.la(1), 2); - assert_eq!(parser.input.text_all(), "new"); + let root = parser + .node(tree) + .as_rule() + .expect("entry result should be a rule"); + assert_eq!(root.text(), format!("{}", "x".repeat(REPETITIONS))); + assert_eq!(parser.input.index(), REPETITIONS); + assert!( + parser.recognition_arena.deferred_nodes.len() <= REPETITIONS * 8, + "equivalent segmentations should keep deferred storage linear" + ); assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn active_invocation_states_exclude_the_root_frame() { - let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - - let _root = parser.enter_rule(0, 0); - assert!(parser.active_invocation_states().is_empty()); - - let marker = parser.push_invoking_state(6); - let _child = parser.enter_rule(2, 1); - parser.discard_invoking_state(marker); - assert_eq!(parser.active_invocation_states(), [6]); + fn long_ordinary_repetition_does_not_consume_native_stack() { + const REPETITIONS: usize = 20_000; - let marker = parser.push_invoking_state(13); - let _grandchild = parser.enter_rule(4, 2); - parser.discard_invoking_state(marker); - assert_eq!(parser.active_invocation_states(), [13, 6]); + for atn in [ordinary_star_loop_atn(), ordinary_plus_loop_atn()] { + let mut parser = mini_parser(repeated_x_tokens(REPETITIONS)); + parser.set_build_parse_trees(false); + parser + .parse_atn_rule(&atn, 0) + .expect("long ordinary repetition should parse"); - parser.exit_rule(); - parser.exit_rule(); - parser.exit_rule(); + assert_eq!(parser.input.index(), REPETITIONS); + assert_eq!(parser.number_of_syntax_errors(), 0); + } } #[test] - fn parser_predicates_support_token_adjacency() { - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("=").with_span(0, 0), - TestToken::new(1).with_text(">").with_span(1, 1), - TestToken::eof("parser-test", 2, 1, 2), - ]); - parser.consume(); - parser.consume(); - - let predicates = [(0, 0, ParserPredicate::TokenPairAdjacent)]; + fn long_rule_repetition_materializes_tree_with_linear_arena_growth() { + const REPETITIONS: usize = 2_000; + let expected_text = format!("{}", "x".repeat(REPETITIONS)); - assert!(parser.parser_semantic_predicate_matches(&predicates, 0, 0)); + for atn in [ordinary_star_loop_atn(), ordinary_plus_loop_atn()] { + let mut parser = mini_parser(repeated_x_tokens(REPETITIONS)); + let tree = parser + .parse_atn_rule(&atn, 0) + .expect("long rule repetition should parse"); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("=").with_span(0, 0), - TestToken::new(1) - .with_text(" ") - .with_channel(HIDDEN_CHANNEL) - .with_span(1, 1), - TestToken::new(1).with_text(">").with_span(2, 2), - TestToken::eof("parser-test", 3, 1, 3), - ]); - parser.consume(); - parser.consume(); + let root = parser + .node(tree) + .as_rule() + .expect("entry result should be a rule"); + assert_eq!(root.text(), expected_text); + assert_eq!(root.child_rules(1).count(), REPETITIONS); + let first_body = root.child_rules(1).next().expect("first body rule"); + let last_body = root.child_rules(1).next_back().expect("last body rule"); + assert_eq!(first_body.start_id().expect("first body start").index(), 0); + assert_eq!( + last_body.stop_id().expect("last body stop").index(), + REPETITIONS - 1 + ); - assert!(!parser.parser_semantic_predicate_matches(&predicates, 0, 0)); + let stats = parser.recognition_arena_stats(); + assert_eq!( + (stats.total_nodes, stats.live_nodes, stats.dead_nodes), + (REPETITIONS, REPETITIONS, 0) + ); + assert_eq!( + (stats.total_links, stats.live_links, stats.dead_links), + (REPETITIONS, REPETITIONS, 0) + ); + assert_eq!(parser.recognition_arena.deferred_rules.len(), REPETITIONS); + assert_eq!( + parser.recognition_arena.deferred_nodes.len(), + REPETITIONS * 2 - 1 + ); + assert_eq!(parser.number_of_syntax_errors(), 0); + } } #[test] - fn parser_predicates_support_context_child_text_checks() { - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("var"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - let mut context = ParserRuleContext::new(1, 0); - let mut child_context = ParserRuleContext::new(2, 0); - let terminal = parser.terminal_tree(TokenId::try_from(0).expect("test token ID")); - parser.tree.add_child(&mut child_context, terminal); - let child = parser.rule_node(child_context); - parser.tree.add_child(&mut context, child); - let predicates = [( - 1, - 0, - ParserPredicate::ContextChildRuleTextNotEquals { - rule_index: 2, - text: "var", - }, - )]; + fn clean_memo_probe_selects_sparse_promote_and_reprobe_modes() { + let key = |state_number| FastRecognizeKey { + state_number, + stop_state: 10, + index: state_number, + rule_start_index: 0, + decision_start_index: None, + precedence: 0, + recovery_symbols_id: 0, + recovery_state: None, + }; - assert!( - !parser.parser_semantic_predicate_matches_with_context_and_local( - &predicates, - 1, - 0, - &context, - 0, - ) + let mut sparse = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + for state_number in 0..(CLEAN_MEMO_PROBE_LIMIT - 1) { + assert!(sparse.clean_memo_enabled_for_key(&key(state_number))); + } + assert!(!sparse.clean_memo_enabled_for_key(&key(CLEAN_MEMO_PROBE_LIMIT))); + assert_eq!(sparse.clean_memo_mode, CleanMemoMode::Sparse); + + let mut promote = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + let repeated = key(1); + for _ in 0..=CLEAN_MEMO_REPEAT_LIMIT { + assert!(promote.clean_memo_enabled_for_key(&repeated)); + } + assert_eq!(promote.clean_memo_mode, CleanMemoMode::Promote); + + for _ in 1..CLEAN_MEMO_REPROBE_INTERVAL { + assert!(!sparse.clean_memo_enabled_for_key(&repeated)); + } + assert!(sparse.clean_memo_enabled_for_key(&repeated)); + assert_eq!(sparse.clean_memo_mode, CleanMemoMode::Probe); + for _ in 0..CLEAN_MEMO_REPEAT_LIMIT { + assert!(sparse.clean_memo_enabled_for_key(&repeated)); + } + assert_eq!(sparse.clean_memo_mode, CleanMemoMode::Promote); + } + + #[test] + fn fast_recognize_memo_capacity_scales_from_small_floor_to_bounded_maximum() { + assert_eq!( + fast_recognize_memo_capacity(0), + FAST_RECOGNIZE_MIN_MEMO_CAPACITY + ); + assert_eq!( + fast_recognize_memo_capacity(FAST_RECOGNIZE_MIN_MEMO_CAPACITY / 8), + FAST_RECOGNIZE_MIN_MEMO_CAPACITY + ); + assert_eq!(fast_recognize_memo_capacity(1_000), 8_000); + assert_eq!( + fast_recognize_memo_capacity(usize::MAX), + FAST_RECOGNIZE_MAX_MEMO_CAPACITY ); } #[test] - fn context_expected_symbols_walks_nullable_parent_contexts() { - let atn = nested_nullable_context_atn(); - let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - parser.rule_context_stack = vec![ - RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }, - RuleContextFrame { - rule_index: 1, - invoking_state: 1, - }, - RuleContextFrame { - rule_index: 2, - invoking_state: 2, + fn fast_recognize_scratch_reuses_small_tables_and_releases_oversized_memo() { + let mut scratch = FastRecognizeTopScratch::default(); + scratch.prepare(FAST_RECOGNIZE_MIN_MEMO_CAPACITY); + let retained_capacity = scratch.memo.capacity(); + assert!(retained_capacity >= FAST_RECOGNIZE_MIN_MEMO_CAPACITY); + assert!(retained_capacity <= FAST_RECOGNIZE_MAX_RETAINED_MEMO_CAPACITY); + + let larger_capacity = retained_capacity + 1; + scratch.prepare(larger_capacity); + let grown_capacity = scratch.memo.capacity(); + assert!(grown_capacity >= larger_capacity); + assert!(grown_capacity <= FAST_RECOGNIZE_MAX_RETAINED_MEMO_CAPACITY); + + scratch.memo.insert( + FastRecognizeKey { + state_number: 0, + stop_state: 0, + index: 0, + rule_start_index: 0, + decision_start_index: None, + precedence: 0, + recovery_symbols_id: 0, + recovery_state: None, }, - ]; + Rc::from([FastRecognizeOutcome { + index: 0, + consumed_eof: false, + diagnostics: DiagnosticSeqId::EMPTY, + deferred_nodes: FastDeferredNodeId::EMPTY, + nodes: NodeSeqId::EMPTY, + }]), + ); + scratch.release_oversized_memo(); + assert!(scratch.memo.is_empty()); + assert_eq!(scratch.memo.capacity(), grown_capacity); - let expected = parser.context_expected_symbols(&atn); + scratch + .memo + .reserve(FAST_RECOGNIZE_MAX_RETAINED_MEMO_CAPACITY * 2); + assert!(scratch.memo.capacity() > FAST_RECOGNIZE_MAX_RETAINED_MEMO_CAPACITY); - assert!(expected.contains(&1)); - assert!(expected.contains(&TOKEN_EOF)); + scratch.release_oversized_memo(); + assert!(scratch.memo.is_empty()); + assert_eq!(scratch.memo.capacity(), 0); } #[test] - fn prediction_context_return_states_track_rule_stack_changes() { - let atn = nested_nullable_context_atn(); - let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - parser.rule_context_stack = vec![ - RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }, - RuleContextFrame { - rule_index: 1, - invoking_state: 1, + fn clean_empty_multi_alt_outcomes_are_memoized() { + let mut atn = ParserAtnBuilder::new(2); + assert_eq!( + atn.add_state(AtnStateKind::RuleStart, Some(0)) + .expect("state") + .index(), + 0 + ); + assert_eq!( + atn.add_state(AtnStateKind::BlockStart, Some(0)) + .expect("state") + .index(), + 1 + ); + assert_eq!( + atn.add_state(AtnStateKind::RuleStop, Some(0)) + .expect("state") + .index(), + 2 + ); + atn.set_rule_to_start_state(vec![0]) + .expect("rule start states"); + atn.set_rule_to_stop_state(vec![2]) + .expect("rule stop states"); + atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) + .expect("transition"); + atn.add_transition( + 1, + ParserTransitionSpec::Atom { + target: 2, + label: 1, }, - RuleContextFrame { - rule_index: 2, - invoking_state: 2, + ) + .expect("transition"); + atn.add_transition( + 1, + ParserTransitionSpec::Atom { + target: 2, + label: 2, }, - ]; + ) + .expect("transition"); + let atn = finish_atn(atn); - let initial_version = parser.rule_context_version(); - let first: Vec<_> = parser.prediction_context_return_states(&atn).collect(); - let second: Vec<_> = parser.prediction_context_return_states(&atn).collect(); - assert_eq!(first, second); - assert_eq!(parser.rule_context_version(), initial_version); + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 0, 1, 0)]); + parser.fast_recovery_enabled = false; + let mut visiting = FxHashSet::default(); + let mut memo = FxHashMap::default(); + let mut expected = ExpectedTokens::default(); + let outcomes = parser.recognize_state_fast( + &atn, + FastRecognizeRequest { + state_number: 1, + stop_state: 2, + index: 0, + rule_start_index: 0, + decision_start_index: None, + precedence: 0, + depth: 0, + recovery_symbols: parser.empty_recovery_symbols(), + recovery_state: None, + }, + FastRecognizeScratch { + predicate_context: None, + visiting: &mut visiting, + memo: &mut memo, + expected: &mut expected, + native_depth: 0, + }, + ); - parser.exit_rule(); - let after_pop: Vec<_> = parser.prediction_context_return_states(&atn).collect(); - assert_ne!(first, after_pop); - assert_ne!(parser.rule_context_version(), initial_version); - } + assert!(outcomes.is_empty()); + assert_eq!(memo.len(), 1); + assert!(memo.values().next().expect("memo entry").is_empty()); - #[test] - fn generated_match_token_recovers_missing_token_from_context_follow() { - let atn = generated_match_recovery_atn(); - let data = RecognizerData::new( - "Mini.g4", - Vocabulary::new( - [None, Some("'X'"), Some("'Y'")], - [None, Some("X"), Some("Y")], - [None::<&str>, None, None], - ), - ); - let mut parser = BaseParser::new( - CommonTokenStream::new(Source { - tokens: vec![TestToken::eof("parser-test", 3, 1, 3)], + parser.clean_memo_mode = CleanMemoMode::Sparse; + visiting.clear(); + memo.clear(); + expected = ExpectedTokens::default(); + let sparse_outcomes = parser.recognize_state_fast( + &atn, + FastRecognizeRequest { + state_number: 1, + stop_state: 2, index: 0, - }), - data, - ); - parser.rule_context_stack = vec![ - RuleContextFrame { - rule_index: 0, - invoking_state: 0, + rule_start_index: 0, + decision_start_index: None, + precedence: 0, + depth: 0, + recovery_symbols: parser.empty_recovery_symbols(), + recovery_state: None, }, - RuleContextFrame { - rule_index: 1, - invoking_state: 1, + FastRecognizeScratch { + predicate_context: None, + visiting: &mut visiting, + memo: &mut memo, + expected: &mut expected, + native_depth: 0, }, - ]; - assert_eq!(parser.number_of_syntax_errors(), 0); + ); - let node = parser - .match_token_recovering(2, 5, &atn) - .expect("generated match should insert missing token"); + assert!(sparse_outcomes.is_empty()); + assert!(memo.is_empty()); + } - assert_eq!(node.children().len(), 1); - assert_eq!(parser.node(node.children()[0]).text(), ""); - assert_eq!( - node.clone() - .into_child_iter() - .map(|child| parser.node(child).text()) - .collect::>(), - [""] - ); - // Single-token insertion synthesizes a missing token and consumes nothing, - // so no EOF terminal is consumed even though lookahead is EOF. - assert!(!node.consumed_eof()); - assert_eq!(parser.la(1), TOKEN_EOF); - assert_eq!(parser.number_of_syntax_errors(), 1); - assert_eq!( - parser.generated_parser_diagnostics, - [ParserDiagnostic { - line: 1, - column: 3, - message: "missing 'Y' at ''".to_owned(), - offending: parser.input.lt_id(1), - }] - ); + #[test] + fn wildcard_matches_non_eof_only() { + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + let matched = parser.match_wildcard().expect("wildcard"); + assert_eq!(parser.node(matched).text(), "x"); + assert!(parser.match_wildcard().is_err()); } #[test] - fn generated_match_token_counts_single_token_deletion_recovery() { - let atn = generated_match_recovery_atn(); - let data = RecognizerData::new( - "Mini.g4", - Vocabulary::new( - [None, Some("'X'"), Some("'Y'"), Some("'Z'")], - [None, Some("X"), Some("Y"), Some("Z")], - [None::<&str>, None, None, None], - ), - ); - let mut parser = BaseParser::new( - CommonTokenStream::new(Source { - tokens: vec![ - TestToken::new(3).with_text("z"), - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 3, 1, 3), - ], - index: 0, - }), - data, - ); + fn add_parse_child_records_match_even_without_tree_building() { + // `sync_decision`'s "is the current context empty" flag must reflect real + // matches, not parse-tree children: when `build_parse_trees(false)`, + // `children` stays empty but `has_matched_child` must still flip so nested + // recovery does not wrongly suppress single-token deletion. + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); + let token = TestToken::new(1).with_text("x"); - let node = parser - .match_token_recovering(2, 5, &atn) - .expect("generated match should delete the extraneous token"); + parser.set_build_parse_trees(false); + let mut ctx = ParserRuleContext::new(0, 0); + assert!(!ctx.has_matched_child()); + let child = parser.terminal_tree(token.id); + parser.add_parse_child(&mut ctx, child); + // Tree building is off, so no child is stored... + assert_eq!(ctx.child_count(), 0); + assert_eq!(parser.parse_tree_storage().node_count(), 0); + // ...but the match is recorded, so the context is no longer "empty". + assert!(ctx.has_matched_child()); - assert_eq!(node.children().len(), 2); - assert_eq!(parser.node(node.children()[0]).kind(), NodeKind::Error); - assert_eq!(parser.node(node.children()[0]).text(), "z"); - assert_eq!(parser.node(node.children()[1]).text(), "y"); - assert_eq!( - node.into_child_iter() - .map(|child| parser.node(child).text()) - .collect::>(), - ["z", "y"] - ); - assert_eq!(parser.number_of_syntax_errors(), 1); + // With tree building on, the child is stored and the match is recorded. + parser.set_build_parse_trees(true); + let mut ctx = ParserRuleContext::new(0, 0); + let child = parser.terminal_tree(token.id); + parser.add_parse_child(&mut ctx, child); + assert_eq!(ctx.child_count(), 1); + assert!(ctx.has_matched_child()); } #[test] - fn generated_match_token_iterates_single_success_without_a_children_vec() { - let atn = generated_match_recovery_atn(); - let data = RecognizerData::new( - "Mini.g4", - Vocabulary::new( - [None, Some("'X'"), Some("'Y'")], - [None, Some("X"), Some("Y")], - [None::<&str>, None, None], - ), - ); - let mut parser = BaseParser::new( - CommonTokenStream::new(Source { - tokens: vec![ - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 1, 1, 1), - ], - index: 0, - }), - data, - ); + fn disabled_tree_building_does_not_grow_flat_storage() { + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::new(1).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), + ]); + parser.set_build_parse_trees(false); + let mut context = ParserRuleContext::new(0, -1); - let node = parser - .match_token_recovering(2, 5, &atn) - .expect("generated match should consume the expected token"); + for _ in 0..2 { + let child = parser.match_token(1).expect("token should match"); + parser.add_parse_child(&mut context, child); + } + let current = parser.input.lt_id(1).expect("EOF token"); + let error = parser.error_tree(current); + parser.add_parse_child(&mut context, error); + let root = parser.rule_node(context); assert_eq!( - node.into_child_iter() - .map(|child| parser.node(child).text()) - .collect::>(), - ["y"] + parser.parse_tree_storage().stats(), + ParseTreeStats::default() + ); + assert!( + parser + .parse_tree_storage() + .node(parser.token_store(), root) + .is_none(), + "the no-tree sentinel must not resolve to stored data" ); - assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn generated_diagnostic_restore_rolls_back_syntax_error_count() { - let atn = generated_match_recovery_atn(); - let data = RecognizerData::new( - "Mini.g4", - Vocabulary::new( - [None, Some("'X'"), Some("'Y'")], - [None, Some("X"), Some("Y")], - [None::<&str>, None, None], - ), - ); - let mut parser = BaseParser::new( - CommonTokenStream::new(Source { - tokens: vec![TestToken::eof("parser-test", 3, 1, 3)], - index: 0, - }), - data, - ); - parser.rule_context_stack = vec![ - RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }, - RuleContextFrame { - rule_index: 1, - invoking_state: 1, - }, - ]; - let marker = parser.generated_diagnostics_checkpoint(); - - let _ = parser - .match_token_recovering(2, 5, &atn) - .expect("generated match should insert missing token"); - assert_eq!(parser.number_of_syntax_errors(), 1); + fn disabled_tree_building_skips_recognition_rule_node_storage() { + let atn = ordinary_star_loop_atn(); + let mut parser = mini_parser(repeated_x_tokens(3)); + parser.set_build_parse_trees(false); - parser.restore_generated_diagnostics(marker); + parser + .parse_atn_rule(&atn, 0) + .expect("ordinary repetition should parse without a tree"); - assert_eq!(parser.number_of_syntax_errors(), 0); - assert!(parser.generated_parser_diagnostics.is_empty()); + assert_eq!(parser.input.index(), 3); + assert!(parser.recognition_arena.nodes.is_empty()); + assert!(parser.recognition_arena.seq_links.is_empty()); + assert!(parser.recognition_arena.deferred_nodes.is_empty()); + assert!(parser.recognition_arena.deferred_rules.is_empty()); + assert!(!parser.fast_token_nodes_enabled); + assert!(parser.fast_recognize_scratch.memo.is_empty()); } #[test] - fn generated_prediction_diagnostics_use_adaptive_context() { - let atn = two_alt_decision_atn(); - let data = RecognizerData::new( - "Mini.g4", - Vocabulary::new( - [None, Some("'x'"), Some("'y'")], - [None, Some("X"), Some("Y")], - [None::<&str>, None, None], - ), - ) - .with_rule_names(["s"]); - let mut parser = BaseParser::new( - CommonTokenStream::new(Source { - tokens: vec![ - TestToken::new(1) - .with_text("x") - .with_position(1, 0) - .with_span(0, 0), - TestToken::new(2) - .with_text("y") - .with_position(1, 2) - .with_span(1, 1), - TestToken::eof("parser-test", 2, 1, 3), - ], - index: 0, - }), - data, - ); - parser.set_report_diagnostic_errors(true); - - parser.record_generated_prediction_diagnostic( - &atn, - 1, - &ParserAtnPrediction { - alt: 1, - requires_full_context: true, - has_semantic_context: false, - diagnostic: Some(ParserAtnPredictionDiagnostic { - kind: ParserAtnPredictionDiagnosticKind::ContextSensitivity, - start_index: 0, - sll_stop_index: 1, - ll_stop_index: 0, - conflicting_alts: vec![1, 2], - exact: false, - }), - }, - ); - // Ambiguities from the default LL prediction mode are non-exact, so — - // matching Java's exactOnly DiagnosticErrorListener — only the - // attempting-full-context line is reported. Exact-ambiguity mode - // reports the ambiguity itself. - parser.record_generated_prediction_diagnostic( - &atn, - 1, - &ParserAtnPrediction { - alt: 1, - requires_full_context: true, - has_semantic_context: false, - diagnostic: Some(ParserAtnPredictionDiagnostic { - kind: ParserAtnPredictionDiagnosticKind::Ambiguity, - start_index: 0, - sll_stop_index: 1, - ll_stop_index: 1, - conflicting_alts: vec![1, 2], - exact: false, - }), - }, - ); + fn parser_interprets_simple_atn_rule() { + let atn = token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); - // The full-context/context-sensitivity diagnostic trace (order + decision + input windows) - // is one snapshot rather than three ParserDiagnostic literals. - insta::assert_debug_snapshot!( - "generated_prediction_diagnostics_use_adaptive_context", - parser.generated_parser_diagnostics + let tree = parser + .parse_atn_rule(&atn, 0) + .expect("artificial parser rule should parse"); + assert_eq!(parser.node(tree).text(), "x"); + assert_eq!(parser.number_of_syntax_errors(), 0); + assert_eq!( + parser + .node(tree) + .first_rule_stop(0) + .expect("rule should stop at EOF") + .token_type(), + TOKEN_EOF + ); + + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + let (tree, actions) = parser + .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) + .expect("runtime-option parser rule should parse"); + assert!(actions.is_empty()); + assert_eq!( + parser + .node(tree) + .first_rule_stop(0) + .expect("rule should stop at EOF") + .token_type(), + TOKEN_EOF ); } #[test] - fn generated_match_not_set_recovers_empty_complement_at_eof() { - let atn = complement_set_atn(); - let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - parser.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }]; + fn runtime_options_default_ignores_noop_action_transitions() { + let atn = noop_action_then_token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); - let node = parser - .match_not_token_set_recovering( - atn.token_set(0).expect("excluded token set"), - 1, - 1, - 1, - &atn, - ) - .expect("empty complement should recover at EOF"); + let (tree, actions) = parser + .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) + .expect("no-op parser action should not force action replay"); - assert_eq!(node.children().len(), 1); - // Recovery synthesizes a missing token without consuming EOF, so the - // enclosing rule must not record EOF as its stop token. - assert!(!node.consumed_eof()); - assert_eq!(parser.la(1), TOKEN_EOF); - assert_eq!( - parser.generated_parser_diagnostics, - [ParserDiagnostic { - line: 1, - column: 1, - message: "missing {} at ''".to_owned(), - offending: parser.input.lt_id(1), - }] + assert_eq!(parser.node(tree).text(), "x"); + assert!( + actions.is_empty(), + "action_index=None transitions are ANTLR metadata, not replay actions" ); + assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn wildcard_recovers_via_insertion_when_follow_expects_eof_at_eof() { - // `start : . EOF ;` on empty input. The wildcard is modeled as an - // empty-complement not-set; at EOF the follow state (the explicit EOF - // match) expects EOF, so even in the start rule recovery must perform - // single-token insertion (``) rather than aborting — matching - // ANTLR's `(start )` / "missing ... at ''". - let atn = wildcard_then_eof_atn(); - let data = RecognizerData::new( - "Mini.g4", - Vocabulary::new([None, Some("'x'")], [None, Some("X")], [None::<&str>, None]), - ); - let mut parser = BaseParser::new( - CommonTokenStream::new(Source { - tokens: vec![TestToken::eof("parser-test", 1, 1, 1)], - index: 0, - }), - data, - ); - parser.rule_context_stack = vec![RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }]; + fn parser_exposes_buffered_token_stream_after_parse() { + let atn = token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); - let node = parser - .match_not_set_recovering(&[], 1, atn.max_token_type(), 2, &atn) - .expect("wildcard at EOF should recover by insertion when follow expects EOF"); + let tree = parser + .parse_atn_rule(&atn, 0) + .expect("artificial parser rule should parse"); + assert_eq!(parser.node(tree).text(), "x"); - // A single `` error node is inserted; EOF is not consumed. - assert_eq!(node.children().len(), 1); - assert!(!node.consumed_eof()); - assert!( - parser - .node(node.children()[0]) - .text() - .starts_with(">(); + assert_eq!(buffered.len(), 2); + assert_eq!(buffered[0].text(), Some("x")); + assert_eq!(buffered[0].token_id().index(), 0); + assert_eq!(buffered[1].token_type(), TOKEN_EOF); + assert_eq!(stream.token_source().index, source_index_after_parse); + drop(buffered); + + let stream = parser.into_token_stream(); + assert_eq!(stream.token_source().index, source_index_after_parse); + assert_eq!( + stream.tokens().next().expect("first token").text(), + Some("x") ); - assert_eq!(parser.la(1), TOKEN_EOF); assert_eq!( - parser.generated_parser_diagnostics, - [ParserDiagnostic { - line: 1, - column: 1, - message: "missing 'x' at ''".to_owned(), - offending: parser.input.lt_id(1), - }] + stream.tokens().nth(1).expect("EOF token").token_type(), + TOKEN_EOF ); } #[test] - fn generated_rule_recovery_consumes_to_parent_follow() { - let atn = generated_match_recovery_atn(); - let data = RecognizerData::new( - "Mini.g4", - Vocabulary::new( - [None, Some("'X'"), Some("'Y'"), Some("'Z'")], - [None, Some("X"), Some("Y"), Some("Z")], - [None::<&str>, None, None, None], - ), - ); - let mut parser = BaseParser::new( - CommonTokenStream::new(Source { - tokens: vec![ - TestToken::new(3).with_text("z"), - TestToken::eof("parser-test", 1, 1, 1), - ], - index: 0, - }), - data, - ); - let _parent = parser.enter_rule(0, 0); - let marker = parser.push_invoking_state(1); - let mut child = parser.enter_rule(4, 1); - parser.discard_invoking_state(marker); + fn parsed_file_exposes_all_buffered_tokens() { + let atn = token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(99) + .with_text(" comment") + .with_channel(HIDDEN_CHANNEL), + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 9, 1, 9), + ]); - // The anchor recorded where the error was built must survive into the - // dispatched diagnostic even though recovery consumes past it below. - let offending = parser.input.lt_id(1); - assert!(offending.is_some(), "the 'z' token should be buffered"); - parser.recover_generated_rule( - &mut child, - &atn, - AntlrError::ParserError { - line: 1, - column: 0, - message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), - offending, - }, + let tree = parser + .parse_atn_rule(&atn, 0) + .expect("artificial parser rule should parse"); + let parsed = parser.into_parsed_file(tree); + + // Snapshot the full buffered stream — hidden-channel comment, default-channel token, EOF — + // as (type, channel, text) triples; contents make the count self-evident. + insta::assert_debug_snapshot!( + "parsed_file_exposes_all_buffered_tokens", + parsed + .tokens() + .iter() + .map(|token| (token.token_type(), token.channel(), token.text())) + .collect::>() ); - let tree = parser.finish_rule(child, false); + assert_eq!(parsed.tokens().into_iter().count(), 3); + } - assert_eq!(parser.la(1), TOKEN_EOF); + #[test] + fn parser_syntax_error_count_tracks_interpreted_recovery() { + let atn = token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), + ]); + + let tree = parser + .parse_atn_rule(&atn, 0) + .expect("invalid token should recover into an error node"); + + assert_eq!(parser.number_of_syntax_errors(), 1); assert_eq!( - parser.node(tree).to_string_tree_with_names(&["s", "a"]), - "(a z)" + parser + .node(tree) + .first_error_token() + .expect("recovery should embed an error token") + .text(), + Some("y") ); + } + + #[test] + fn failed_interpreted_parse_notifies_error_listener() { + let atn = token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(2) + .with_text("y") + .with_span(0, 0) + .with_byte_span(0, 1) + .with_position(3, 5), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.remove_error_listeners(); + let diagnostics = Arc::new(Mutex::new(Vec::new())); + parser.add_error_listener(RecordingErrorListener { + diagnostics: Arc::clone(&diagnostics), + }); + + let error = parser + .parse_atn_rule(&atn, 0) + .expect_err("start-rule mismatch should remain a parser error"); + assert_eq!(parser.number_of_syntax_errors(), 1); - assert_eq!( - parser.generated_parser_diagnostics, - [ParserDiagnostic { - line: 1, - column: 0, - message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), - offending, - }] + assert!(matches!(&error, AntlrError::ParserError { .. })); + insta::assert_debug_snapshot!( + "failed_interpreted_parse_notifies_error_listener", + *diagnostics.lock().expect("recorded diagnostics lock") ); - parser.exit_rule(); } #[test] - fn generated_rule_recovery_forces_progress_after_repeated_error_state() { - let atn = nested_nullable_context_atn(); + fn adaptive_direct_rule_uses_simulator_decision() { + let atn = two_alt_decision_atn(); + let mut simulator = ParserAtnSimulator::new(&atn); + let mut parser = mini_parser(vec![ + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + + let tree = parser + .parse_atn_rule_adaptive_or_fallback(&atn, &mut simulator, 0) + .expect("direct adaptive rule should parse"); + + assert_eq!(parser.node(tree).text(), "y"); + assert_eq!(parser.input.index(), 1); + } + + #[test] + fn adaptive_direct_rule_restores_input_on_fallback() { + let atn = predicate_after_token_atn(); + let mut simulator = ParserAtnSimulator::new(&atn); let mut parser = mini_parser(vec![ TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), ]); - parser.rule_context_stack = vec![ - RuleContextFrame { - rule_index: 0, - invoking_state: 0, - }, - RuleContextFrame { - rule_index: 1, - invoking_state: 1, - }, - RuleContextFrame { - rule_index: 2, - invoking_state: 2, - }, - ]; - parser.set_state(20); - let mut context = ParserRuleContext::new(2, 2); - parser.recover_generated_rule( - &mut context, - &atn, - AntlrError::NoViableAlternative { - input: "'x'".to_owned(), - }, - ); - assert_eq!(parser.input.index(), 0); + let tree = parser + .parse_atn_rule_adaptive_or_fallback(&atn, &mut simulator, 0) + .expect("fallback recognizer should parse"); - parser.set_state(21); - parser.recover_generated_rule( - &mut context, - &atn, - AntlrError::NoViableAlternative { - input: "'x'".to_owned(), - }, - ); - assert_eq!(parser.input.index(), 0); - assert_eq!( - parser.generated_recovery_error_states, - BTreeSet::from([20, 21]) - ); + assert_eq!(parser.node(tree).text(), "xy"); + assert_eq!(parser.input.index(), 2); + let stats = parser.parse_tree_storage().stats(); + assert_eq!(stats.nodes, parser.node(tree).descendants().count()); + assert_eq!(stats.edges, stats.nodes.saturating_sub(1)); + assert_eq!(stats.scratch_links, 0); + } - parser.set_state(20); - parser.recover_generated_rule( - &mut context, - &atn, - AntlrError::NoViableAlternative { - input: "'x'".to_owned(), - }, - ); + #[test] + fn unknown_predicate_policy_defaults_to_assume_true() { + let atn = predicate_after_token_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), + ]); - assert_eq!(parser.input.index(), 1); - assert_eq!(parser.la(1), TOKEN_EOF); - assert!(context.has_matched_child()); - assert_eq!(parser.generated_recovery_error_states, BTreeSet::from([20])); + let (tree, _) = parser + .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) + .expect("unknown predicate should pass under the default policy"); - parser.match_eof().expect("EOF should match"); - assert_eq!(parser.generated_recovery_error_index, None); - assert!(parser.generated_recovery_error_states.is_empty()); + assert_eq!(parser.node(tree).text(), "xy"); + assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn greedy_ll1_alt_handles_nullable_loop_exit() { - let mut body_symbols = TokenBitSet::default(); - body_symbols.insert(1); - let entry = DecisionLookahead { - transitions: vec![ - TransitionLookSet { - symbols: body_symbols, - nullable: false, - }, - TransitionLookSet { - symbols: TokenBitSet::default(), - nullable: true, + fn private_context_alt_tracking_keeps_fast_predicate_recognition() { + let atn = predicate_gated_same_lookahead_atn([0, 1]); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + + let (tree, _) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + predicates: &[ + (0, 0, ParserPredicate::False), + (0, 1, ParserPredicate::True), + ], + track_context_alt_numbers: true, + ..ParserRuntimeOptions::default() }, - ], - }; + ) + .expect("the second predicate-gated alternative should match"); - assert_eq!(ll1_unique_alt(&entry, 2), None); - assert_eq!(ll1_greedy_alt(&entry, 2, false), Some(1)); - assert_eq!(ll1_greedy_alt(&entry, 1, false), None); - assert_eq!(ll1_greedy_alt(&entry, 1, true), None); + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + insta::assert_debug_snapshot!( + "private_context_alt_tracking_keeps_fast_predicate_recognition", + (root.alt_number(), root.context_alt_number(), root.text()) + ); + assert_eq!(parser.number_of_syntax_errors(), 0); + assert_eq!(parser.fast_predicate_cache.get(&(0, 0, 0)), Some(&false)); + assert_eq!(parser.fast_predicate_cache.get(&(0, 0, 1)), Some(&true)); } #[test] - fn ordinary_repetition_builds_tree_in_input_order() { - for atn in [ordinary_star_loop_atn(), ordinary_plus_loop_atn()] { - let mut parser = mini_parser(repeated_x_tokens(3)); - let tree = parser - .parse_atn_rule(&atn, 0) - .expect("ordinary repetition should parse"); + fn nested_interpreted_parse_preserves_prior_unknown_predicate_hits() { + // A generated parent may record an unknown-predicate coordinate, then + // descend into an interpreted child. The child's interpreter entry must + // not wipe the parent's recorded hit before the top-level surfaces it. + let atn = token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); - let root = parser - .node(tree) - .as_rule() - .expect("entry result should be a rule"); - let body_rules = root.child_rules(1).collect::>(); - assert_eq!(root.text(), "xxx"); - assert_eq!(body_rules.len(), 3); - assert_eq!( - body_rules - .iter() - .map(|rule| rule.start_id().expect("body start").index()) - .collect::>(), - [0, 1, 2] - ); - assert_eq!( - body_rules - .iter() - .map(|rule| rule.stop_id().expect("body stop").index()) - .collect::>(), - [0, 1, 2] - ); - assert_eq!(parser.number_of_syntax_errors(), 0); - } + // Simulate the parent having recorded a fail-loud coordinate. + parser.unknown_predicate_hits.push((7, 3)); + + // Run an interpreted child parse that records no coordinate of its own. + parser + .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) + .expect("child rule parses"); + + // The parent's coordinate must still be present for the top-level entry. + let error = parser + .take_unknown_semantic_error() + .expect("parent's recorded coordinate must survive the nested interpreted parse"); + let AntlrError::Unsupported(message) = error else { + panic!("expected AntlrError::Unsupported, got {error:?}"); + }; + assert!(message.contains("pred_index=3"), "message: {message}"); } #[test] - fn deeply_nested_deferred_rules_materialize_on_small_stack() { - const DEPTH: usize = 20_000; + fn nested_committed_parse_preserves_prior_unhandled_action_hits() { + let atn = token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.unhandled_action_hits.push((7, 42)); - std::thread::Builder::new() - .name("deferred-rule-materialization".to_owned()) - .stack_size(256 * 1024) - .spawn(|| { - let mut parser = mini_parser(vec![TestToken::eof("parser-test", 0, 1, 0)]); - let mut root = FastDeferredNodeId::EMPTY; - for depth in 0..DEPTH { - root = parser - .recognition_arena - .deferred_rule_node(FastDeferredRule { - rule_index: u32::try_from(depth).expect("depth fits in u32"), - invoking_state: i32::try_from(depth).expect("depth fits in i32"), - start_index: 0, - stop_index: None, - deferred_children: root, - children: NodeSeqId::EMPTY, - }); - } + parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("a child with no action miss must not observe its parent's miss"); - let (mut children, alt_number) = - parser.materialize_fast_deferred_nodes(root, NodeSeqId::EMPTY); - assert_eq!(alt_number, 0); - for expected_rule in (0..DEPTH).rev() { - let mut nodes = parser.recognition_arena.iter(children); - let node = nodes.next().expect("nested rule node"); - assert!(nodes.next().is_none(), "each rule has one child"); - let ArenaRecognizedNode::Rule { - rule_index, - children: nested, - .. - } = parser.recognition_arena.node(node) - else { - panic!("expected nested rule"); - }; - assert_eq!(rule_index as usize, expected_rule); - children = nested; - } - assert!(children.is_empty()); - }) - .expect("small-stack thread should start") - .join() - .expect("deferred rules should materialize without recursion"); + let error = parser + .take_unknown_semantic_error() + .expect("the parent's action miss must survive the nested committed parse"); + let AntlrError::Unsupported(message) = error else { + panic!("expected AntlrError::Unsupported, got {error:?}"); + }; + assert!( + message.contains("rule_index=7") && message.contains("state=42"), + "message: {message}" + ); } #[test] - fn deferred_alternatives_preserve_left_recursive_contexts() { + fn unknown_predicate_policy_assume_false_kills_the_guarded_path() { + let atn = predicate_after_token_atn(); let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("1"), - TestToken::new(2).with_text("+"), - TestToken::new(1).with_text("2"), - TestToken::eof("parser-test", 3, 1, 3), + TestToken::new(1).with_text("x"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), ]); - let base = parser.arena_token_node(0, false); - let operator = parser.arena_token_node(1, false); - let right = parser.arena_token_node(2, false); - - let base = parser.recognition_arena.prepend(NodeSeqId::EMPTY, base); - let base = parser.recognition_arena.deferred_fragment(base); - let operator = parser.recognition_arena.prepend(NodeSeqId::EMPTY, operator); - let operator = parser.recognition_arena.deferred_fragment(operator); - let right = parser.recognition_arena.prepend(NodeSeqId::EMPTY, right); - let right = parser.recognition_arena.deferred_fragment(right); - let base_alt = parser.recognition_arena.deferred_alternative(1); - let boundary = parser.recognition_arena.deferred_left_recursive_boundary(0); - let operator_alt = parser.recognition_arena.deferred_alternative(6); - - let mut deferred = FastDeferredNodeId::EMPTY; - for fragment in [base_alt, base, boundary, operator_alt, operator, right] { - deferred = parser - .recognition_arena - .concat_deferred_nodes(deferred, fragment); - } - let (nodes, root_alt_number) = - parser.materialize_fast_deferred_nodes(deferred, NodeSeqId::EMPTY); - let nodes = parser - .recognition_arena - .fold_left_recursive_boundaries(nodes); - let mut root = ParserRuleContext::new(0, -1); - root.set_context_alt_number(root_alt_number); - let mut cursor = nodes; - while let Some(link) = parser.recognition_arena.link(cursor) { - let child = parser - .arena_recognized_node_tree(link.head, false, true) - .expect("materialized child should become a public tree"); - parser.tree.add_child(&mut root, child); - cursor = link.tail; - } - let tree = parser.rule_node(root); - let contexts = parser - .node(tree) - .descendants() - .filter_map(Node::as_rule) - .map(|rule| { - ( - rule.rule_index(), - rule.alt_number(), - rule.context_alt_number(), - rule.text(), - ) - }) - .collect::>(); + let result = parser.parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + unknown_predicate_policy: UnknownSemanticPolicy::AssumeFalse, + ..ParserRuntimeOptions::default() + }, + ); - insta::assert_debug_snapshot!( - "deferred_alternatives_preserve_left_recursive_contexts", - contexts + assert!( + result.is_err(), + "the only path is predicate-guarded, so assume-false must fail the parse" ); } #[test] - fn fast_recognizer_preserves_labeled_left_recursive_operator_context() { - let atn = labeled_left_recursive_operator_atn(); + fn predicate_failure_message_keeps_semantic_recovery_path() { + let atn = predicate_after_token_atn(); let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("a"), - TestToken::new(3).with_text("+"), - TestToken::new(1).with_text("b"), - TestToken::eof("parser-test", 3, 1, 3), + TestToken::new(1).with_text("x"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), ]); let (tree, _) = parser @@ -16783,1075 +19294,1322 @@ mod tests { &atn, 0, ParserRuntimeOptions { - track_context_alt_numbers: true, + predicates: &[( + 0, + 0, + ParserPredicate::FalseWithMessage { + message: "predicate rejected input", + }, + )], ..ParserRuntimeOptions::default() }, ) - .expect("labeled left-recursive addition should parse"); - let contexts = parser - .node(tree) - .descendants() - .filter_map(Node::as_rule) - .map(|rule| { - let operator = rule - .children() - .next() - .and_then(Node::as_rule) - .is_some_and(|child| child.rule_index() == rule.rule_index()); - (operator, rule.context_alt_number(), rule.text()) - }) - .collect::>(); + .expect("failure-message predicates recover through the semantic interpreter"); - insta::assert_debug_snapshot!( - "fast_recognizer_preserves_labeled_left_recursive_operator_context", - contexts + assert_eq!(parser.node(tree).text(), "xy"); + assert_eq!(parser.number_of_syntax_errors(), 1); + assert!( + parser.fast_predicate_cache.is_empty(), + "failure-message predicates need the semantic interpreter's recovery outcome" ); - assert!(!parser.recognition_arena.deferred_nodes.is_empty()); - assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn deeply_nested_rule_calls_grow_the_stack() { - const DEPTH: usize = 4_096; - const STACK_SIZE: usize = 256 * 1024; - let atn = nested_rule_chain_atn(DEPTH); - std::thread::Builder::new() - .name("nested-adaptive-set-rules".to_owned()) - .stack_size(STACK_SIZE) - .spawn(move || { - let mut parser = mini_parser(vec![TestToken::new(1).with_text("x")]); - parser.set_build_parse_trees(false); - // This test isolates recognizer depth from the separately - // cached FIRST-set metadata walk. - parser.fast_first_set_prefilter = false; - parser - .parse_atn_rule(&atn, 0) - .expect("nested rule chain should grow the native stack"); - assert_eq!(parser.input.index(), 1); - }) - .expect("small-stack thread should start") - .join() - .expect("nested rule chain should not overflow its stack"); + fn unknown_predicate_policy_error_names_the_coordinate() { + let atn = predicate_after_token_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), + ]); + + let error = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + unknown_predicate_policy: UnknownSemanticPolicy::Error, + ..ParserRuntimeOptions::default() + }, + ) + .expect_err("evaluating an unknown predicate under Error policy must fail"); + + let AntlrError::Unsupported(message) = error else { + panic!("expected AntlrError::Unsupported, got {error:?}"); + }; + assert!( + message.contains("unsupported semantic predicate"), + "message should name the failure class: {message}" + ); + assert!( + message.contains("pred_index=0"), + "message should carry the coordinate: {message}" + ); } #[test] - fn deeply_nested_branching_rules_grow_the_stack() { - const DEPTH: usize = 4_096; - const STACK_SIZE: usize = 256 * 1024; - let atn = nested_rule_graph_atn(DEPTH, true, false); - std::thread::Builder::new() - .name("nested-branching-rules".to_owned()) - .stack_size(STACK_SIZE) - .spawn(move || { - let mut parser = mini_parser(vec![TestToken::new(1).with_text("x")]); - parser.set_build_parse_trees(false); - parser - .parse_atn_rule(&atn, 0) - .expect("branching rule chain should grow the native stack"); - assert_eq!(parser.input.index(), 1); - }) - .expect("small-stack thread should start") - .join() - .expect("branching rule chain should not overflow its stack"); + fn fail_loud_hits_do_not_leak_into_a_reused_interpreter_parse() { + // A parser reused after a fail-loud parse must not carry the old + // coordinates into a later parse. The fail-loud return keeps the hits + // (so a generated parent can surface a recovered child's coordinate), + // and the next parse's entry stashes/replaces them, so a subsequent + // clean parse surfaces no stale error. + let atn = predicate_after_token_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), + ]); + + parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + unknown_predicate_policy: UnknownSemanticPolicy::Error, + ..ParserRuntimeOptions::default() + }, + ) + .expect_err("first parse fails loud under the Error policy"); + + // The failed parse kept its coordinate on the parser (so a generated + // parent could surface a recovered child). A top-level reuse resets the + // hits — generated parsers call `reset_unknown_semantic_hits` at their + // public entry; direct interpreter-API callers do the same. + parser.reset_unknown_semantic_hits(); + assert!( + parser.take_unknown_semantic_error().is_none(), + "reset must drop stale unknown-predicate coordinates before a reused parse" + ); + } + + #[derive(Debug, Default)] + struct RecordingHooks { + predicates: Vec<(usize, usize, usize, Option)>, + actions: Vec<(usize, String, Option)>, + action_trees: Vec>, + } + + impl SemanticHooks for RecordingHooks { + fn sempred( + &mut self, + ctx: &mut ParserSemCtx<'_, S>, + rule_index: usize, + pred_index: usize, + ) -> Option + where + S: TokenSource, + { + self.predicates.push(( + ctx.input_index(), + rule_index, + pred_index, + ctx.token_text(1) + .and_then(|token| token.text().map(str::to_owned)), + )); + Some(true) + } + + fn action(&mut self, ctx: &mut ParserSemCtx<'_, S>, action: ParserAction) -> bool + where + S: TokenSource, + { + self.actions.push(( + action.source_state(), + ctx.action_text(), + ctx.rule_name().map(str::to_owned), + )); + self.action_trees.push(ctx.tree().map(Node::text)); + true + } } - #[test] - fn deeply_nested_rule_follows_grow_the_stack() { - const DEPTH: usize = 4_096; - const STACK_SIZE: usize = 256 * 1024; - let atn = nested_rule_graph_atn(DEPTH, false, true); - std::thread::Builder::new() - .name("nested-rule-follows".to_owned()) - .stack_size(STACK_SIZE) - .spawn(move || { - let mut parser = mini_parser(repeated_x_tokens(DEPTH)); - parser.set_build_parse_trees(false); - parser.fast_first_set_prefilter = false; - parser - .parse_atn_rule(&atn, 0) - .expect("rule follow chain should grow the native stack"); - assert_eq!(parser.input.index(), DEPTH); - }) - .expect("small-stack thread should start") - .join() - .expect("nested rule follow chain should not overflow its stack"); + #[derive(Debug, Default)] + struct StatefulActionHooks { + entered: bool, + events: Vec, } - #[test] - fn deeply_nested_recovery_grows_the_stack() { - const DEPTH: usize = 4_096; - const STACK_SIZE: usize = 256 * 1024; - let atn = nested_rule_chain_atn(DEPTH); - std::thread::Builder::new() - .name("nested-rule-recovery".to_owned()) - .stack_size(STACK_SIZE) - .spawn(move || { - let mut parser = mini_parser(vec![ - TestToken::new(2).with_text("z"), - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 2, 1, 2), - ]); - parser.set_build_parse_trees(false); - parser.fast_first_set_prefilter = false; - parser - .parse_atn_rule(&atn, 0) - .expect("nested recovery should grow the native stack"); - assert_eq!(parser.input.index(), 2); - assert_eq!(parser.number_of_syntax_errors(), 1); - }) - .expect("small-stack thread should start") - .join() - .expect("nested rule recovery should not overflow its stack"); + impl SemanticHooks for StatefulActionHooks { + fn sempred( + &mut self, + _ctx: &mut ParserSemCtx<'_, S>, + _rule_index: usize, + _pred_index: usize, + ) -> Option + where + S: TokenSource, + { + self.events.push(format!("predicate:{}", self.entered)); + Some(self.entered) + } + + fn action(&mut self, _ctx: &mut ParserSemCtx<'_, S>, action: ParserAction) -> bool + where + S: TokenSource, + { + self.events.push(format!( + "action:{}", + action + .action_index() + .map_or_else(|| "legacy".to_owned(), |index| index.to_string()) + )); + self.entered = true; + true + } } - #[test] - fn ambiguous_ordinary_repetition_merges_equivalent_coordinates() { - const REPETITIONS: usize = 64; + #[derive(Debug, Default)] + struct InitOrderingHooks { + initialized: bool, + events: Vec, + } - let atn = ambiguous_ordinary_star_loop_atn(); - let mut parser = mini_parser(repeated_x_tokens(REPETITIONS)); - let tree = parser - .parse_atn_rule(&atn, 0) - .expect("ambiguous ordinary repetition should parse"); + impl SemanticHooks for InitOrderingHooks { + fn sempred( + &mut self, + _ctx: &mut ParserSemCtx<'_, S>, + _rule_index: usize, + _pred_index: usize, + ) -> Option + where + S: TokenSource, + { + self.events.push(format!("predicate:{}", self.initialized)); + Some(self.initialized) + } - let root = parser - .node(tree) - .as_rule() - .expect("entry result should be a rule"); - assert_eq!(root.text(), format!("{}", "x".repeat(REPETITIONS))); - assert_eq!(parser.input.index(), REPETITIONS); - assert!( - parser.recognition_arena.deferred_nodes.len() <= REPETITIONS * 8, - "equivalent segmentations should keep deferred storage linear" - ); - assert_eq!(parser.number_of_syntax_errors(), 0); + fn action(&mut self, _ctx: &mut ParserSemCtx<'_, S>, action: ParserAction) -> bool + where + S: TokenSource, + { + if action.is_rule_init() { + self.initialized = true; + self.events.push("init".to_owned()); + } else { + self.events.push(format!( + "action:{}:initialized={}", + action + .action_index() + .map_or_else(|| "legacy".to_owned(), |index| index.to_string()), + self.initialized + )); + } + true + } } - #[test] - fn long_ordinary_repetition_does_not_consume_native_stack() { - const REPETITIONS: usize = 20_000; - - for atn in [ordinary_star_loop_atn(), ordinary_plus_loop_atn()] { - let mut parser = mini_parser(repeated_x_tokens(REPETITIONS)); - parser.set_build_parse_trees(false); - parser - .parse_atn_rule(&atn, 0) - .expect("long ordinary repetition should parse"); + #[derive(Debug, Default)] + struct ActionContextHooks { + actions: Vec<(usize, Option, Option)>, + } - assert_eq!(parser.input.index(), REPETITIONS); - assert_eq!(parser.number_of_syntax_errors(), 0); + impl SemanticHooks for ActionContextHooks { + fn action(&mut self, ctx: &mut ParserSemCtx<'_, S>, action: ParserAction) -> bool + where + S: TokenSource, + { + self.actions.push(( + action.action_index().unwrap_or(usize::MAX), + ctx.local_int_arg(), + action.stop_index(), + )); + true } } - #[test] - fn long_rule_repetition_materializes_tree_with_linear_arena_growth() { - const REPETITIONS: usize = 2_000; - let expected_text = format!("{}", "x".repeat(REPETITIONS)); + #[derive(Debug, Default)] + struct DecliningActionHooks { + actions: Vec, + } - for atn in [ordinary_star_loop_atn(), ordinary_plus_loop_atn()] { - let mut parser = mini_parser(repeated_x_tokens(REPETITIONS)); - let tree = parser - .parse_atn_rule(&atn, 0) - .expect("long rule repetition should parse"); + impl SemanticHooks for DecliningActionHooks { + fn action(&mut self, _ctx: &mut ParserSemCtx<'_, S>, action: ParserAction) -> bool + where + S: TokenSource, + { + self.actions.push(action.source_state()); + false + } + } - let root = parser - .node(tree) - .as_rule() - .expect("entry result should be a rule"); - assert_eq!(root.text(), expected_text); - assert_eq!(root.child_rules(1).count(), REPETITIONS); - let first_body = root.child_rules(1).next().expect("first body rule"); - let last_body = root.child_rules(1).next_back().expect("last body rule"); - assert_eq!(first_body.start_id().expect("first body start").index(), 0); - assert_eq!( - last_body.stop_id().expect("last body stop").index(), - REPETITIONS - 1 - ); + #[derive(Debug, Default)] + struct ForcedSecondAlternativeHooks { + decisions: Vec<(usize, usize, usize)>, + } - let stats = parser.recognition_arena_stats(); - assert_eq!( - (stats.total_nodes, stats.live_nodes, stats.dead_nodes), - (REPETITIONS, REPETITIONS, 0) - ); - assert_eq!( - (stats.total_links, stats.live_links, stats.dead_links), - (REPETITIONS, REPETITIONS, 0) - ); - assert_eq!(parser.recognition_arena.deferred_rules.len(), REPETITIONS); - assert_eq!( - parser.recognition_arena.deferred_nodes.len(), - REPETITIONS * 2 - 1 - ); - assert_eq!(parser.number_of_syntax_errors(), 0); + impl SemanticHooks for ForcedSecondAlternativeHooks { + fn observes_parser_decisions(&self) -> bool { + true + } + + fn parser_decision_override( + &mut self, + decision: usize, + input_index: usize, + alternative_count: usize, + ) -> Option { + self.decisions + .push((decision, input_index, alternative_count)); + Some(2) } } - #[test] - fn clean_memo_probe_selects_sparse_promote_and_reprobe_modes() { - let key = |state_number| FastRecognizeKey { - state_number, - stop_state: 10, - index: state_number, - rule_start_index: 0, - decision_start_index: None, - precedence: 0, - recovery_symbols_id: 0, - recovery_state: None, - }; + struct RecordingParseListener { + events: Arc>>, + } - let mut sparse = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - for state_number in 0..(CLEAN_MEMO_PROBE_LIMIT - 1) { - assert!(sparse.clean_memo_enabled_for_key(&key(state_number))); + impl ParseListener for RecordingParseListener { + fn enter_every_rule(&mut self, event: &EnterRuleEvent<'_>) -> Result<(), AntlrError> { + self.events + .lock() + .expect("parse-listener event lock") + .push(format!("enter:{}", event.rule_index)); + Ok(()) } - assert!(!sparse.clean_memo_enabled_for_key(&key(CLEAN_MEMO_PROBE_LIMIT))); - assert_eq!(sparse.clean_memo_mode, CleanMemoMode::Sparse); - let mut promote = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - let repeated = key(1); - for _ in 0..=CLEAN_MEMO_REPEAT_LIMIT { - assert!(promote.clean_memo_enabled_for_key(&repeated)); + fn exit_every_rule(&mut self, rule_index: usize) { + self.events + .lock() + .expect("parse-listener event lock") + .push(format!("exit:{rule_index}")); } - assert_eq!(promote.clean_memo_mode, CleanMemoMode::Promote); + } - for _ in 1..CLEAN_MEMO_REPROBE_INTERVAL { - assert!(!sparse.clean_memo_enabled_for_key(&repeated)); - } - assert!(sparse.clean_memo_enabled_for_key(&repeated)); - assert_eq!(sparse.clean_memo_mode, CleanMemoMode::Probe); - for _ in 0..CLEAN_MEMO_REPEAT_LIMIT { - assert!(sparse.clean_memo_enabled_for_key(&repeated)); + #[derive(Debug, Default)] + struct RejectingPredicateHooks { + predicates: Vec<(usize, usize, usize, Option)>, + } + + impl SemanticHooks for RejectingPredicateHooks { + fn sempred( + &mut self, + ctx: &mut ParserSemCtx<'_, S>, + rule_index: usize, + pred_index: usize, + ) -> Option + where + S: TokenSource, + { + self.predicates.push(( + ctx.input_index(), + rule_index, + pred_index, + ctx.token_text(1) + .and_then(|token| token.text().map(str::to_owned)), + )); + Some(false) } - assert_eq!(sparse.clean_memo_mode, CleanMemoMode::Promote); } #[test] - fn fast_recognize_memo_capacity_scales_from_small_floor_to_bounded_maximum() { - assert_eq!( - fast_recognize_memo_capacity(0), - FAST_RECOGNIZE_MIN_MEMO_CAPACITY - ); - assert_eq!( - fast_recognize_memo_capacity(FAST_RECOGNIZE_MIN_MEMO_CAPACITY / 8), - FAST_RECOGNIZE_MIN_MEMO_CAPACITY + fn fast_predicate_cache_replays_hook_once_per_coordinate_and_input() { + let atn = predicate_gated_same_lookahead_atn([0, 0]); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ], + RecordingHooks::default(), ); - assert_eq!(fast_recognize_memo_capacity(1_000), 8_000); - assert_eq!( - fast_recognize_memo_capacity(usize::MAX), - FAST_RECOGNIZE_MAX_MEMO_CAPACITY + + let (tree, _) = parser + .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) + .expect("both alternatives share one replay-safe predicate result"); + + assert_eq!(parser.node(tree).text(), "x"); + assert_eq!( + parser.semantic_hooks.predicates, + vec![(0, 0, 0, Some("x".to_owned()))] ); + assert_eq!(parser.fast_predicate_cache.get(&(0, 0, 0)), Some(&true)); } #[test] - fn fast_recognize_scratch_reuses_small_tables_and_releases_oversized_memo() { - let mut scratch = FastRecognizeTopScratch::default(); - scratch.prepare(FAST_RECOGNIZE_MIN_MEMO_CAPACITY); - let retained_capacity = scratch.memo.capacity(); - assert!(retained_capacity >= FAST_RECOGNIZE_MIN_MEMO_CAPACITY); - assert!(retained_capacity <= FAST_RECOGNIZE_MAX_RETAINED_MEMO_CAPACITY); - - let larger_capacity = retained_capacity + 1; - scratch.prepare(larger_capacity); - let grown_capacity = scratch.memo.capacity(); - assert!(grown_capacity >= larger_capacity); - assert!(grown_capacity <= FAST_RECOGNIZE_MAX_RETAINED_MEMO_CAPACITY); - - scratch.memo.insert( - FastRecognizeKey { - state_number: 0, - stop_state: 0, - index: 0, - rule_start_index: 0, - decision_start_index: None, - precedence: 0, - recovery_symbols_id: 0, - recovery_state: None, - }, - Rc::from([FastRecognizeOutcome { - index: 0, - consumed_eof: false, - diagnostics: DiagnosticSeqId::EMPTY, - deferred_nodes: FastDeferredNodeId::EMPTY, - nodes: NodeSeqId::EMPTY, - }]), + fn semantic_hook_handles_unknown_predicate_before_error_policy() { + let atn = predicate_after_token_atn(); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("x"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), + ], + RecordingHooks::default(), ); - scratch.release_oversized_memo(); - assert!(scratch.memo.is_empty()); - assert_eq!(scratch.memo.capacity(), grown_capacity); - scratch - .memo - .reserve(FAST_RECOGNIZE_MAX_RETAINED_MEMO_CAPACITY * 2); - assert!(scratch.memo.capacity() > FAST_RECOGNIZE_MAX_RETAINED_MEMO_CAPACITY); + let (tree, _) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + unknown_predicate_policy: UnknownSemanticPolicy::Error, + ..ParserRuntimeOptions::default() + }, + ) + .expect("hook supplies the missing predicate result"); - scratch.release_oversized_memo(); - assert!(scratch.memo.is_empty()); - assert_eq!(scratch.memo.capacity(), 0); + assert_eq!(parser.node(tree).text(), "xy"); + assert_eq!( + parser.semantic_hooks.predicates, + vec![(1, 0, 0, Some("y".to_owned()))] + ); + assert_eq!(parser.fast_predicate_cache.get(&(1, 0, 0)), Some(&true)); } #[test] - fn clean_empty_multi_alt_outcomes_are_memoized() { - let mut atn = ParserAtnBuilder::new(2); - assert_eq!( - atn.add_state(AtnStateKind::RuleStart, Some(0)) - .expect("state") - .index(), - 0 + fn runtime_options_default_preserves_semantic_hook_predicates() { + let atn = predicate_after_token_atn(); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("x"), + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 2, 1, 2), + ], + RejectingPredicateHooks::default(), ); - assert_eq!( - atn.add_state(AtnStateKind::BlockStart, Some(0)) - .expect("state") - .index(), - 1 + + let result = + parser.parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()); + + assert!( + result.is_err(), + "default runtime options must not bypass semantic hooks for predicate ATNs" ); assert_eq!( - atn.add_state(AtnStateKind::RuleStop, Some(0)) - .expect("state") - .index(), - 2 - ); - atn.set_rule_to_start_state(vec![0]) - .expect("rule start states"); - atn.set_rule_to_stop_state(vec![2]) - .expect("rule stop states"); - atn.add_transition(0, ParserTransitionSpec::Epsilon { target: 1 }) - .expect("transition"); - atn.add_transition( - 1, - ParserTransitionSpec::Atom { - target: 2, - label: 1, - }, - ) - .expect("transition"); - atn.add_transition( - 1, - ParserTransitionSpec::Atom { - target: 2, - label: 2, - }, - ) - .expect("transition"); - let atn = finish_atn(atn); - - let mut parser = mini_parser(vec![TestToken::eof("parser-test", 0, 1, 0)]); - parser.fast_recovery_enabled = false; - let mut visiting = FxHashSet::default(); - let mut memo = FxHashMap::default(); - let mut expected = ExpectedTokens::default(); - let outcomes = parser.recognize_state_fast( - &atn, - FastRecognizeRequest { - state_number: 1, - stop_state: 2, - index: 0, - rule_start_index: 0, - decision_start_index: None, - precedence: 0, - depth: 0, - recovery_symbols: parser.empty_recovery_symbols(), - recovery_state: None, - }, - FastRecognizeScratch { - predicate_context: None, - visiting: &mut visiting, - memo: &mut memo, - expected: &mut expected, - native_depth: 0, - }, + parser.semantic_hooks.predicates, + vec![(1, 0, 0, Some("y".to_owned()))] ); + assert_eq!(parser.fast_predicate_cache.get(&(1, 0, 0)), Some(&false)); + } - assert!(outcomes.is_empty()); - assert_eq!(memo.len(), 1); - assert!(memo.values().next().expect("memo entry").is_empty()); - - parser.clean_memo_mode = CleanMemoMode::Sparse; - visiting.clear(); - memo.clear(); - expected = ExpectedTokens::default(); - let sparse_outcomes = parser.recognize_state_fast( - &atn, - FastRecognizeRequest { - state_number: 1, - stop_state: 2, - index: 0, - rule_start_index: 0, - decision_start_index: None, - precedence: 0, - depth: 0, - recovery_symbols: parser.empty_recovery_symbols(), - recovery_state: None, - }, - FastRecognizeScratch { - predicate_context: None, - visiting: &mut visiting, - memo: &mut memo, - expected: &mut expected, - native_depth: 0, - }, + #[test] + fn committed_action_runs_before_later_predicate() { + let atn = committed_action_then_predicate_atn(); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ], + StatefulActionHooks::default(), ); - assert!(sparse_outcomes.is_empty()); - assert!(memo.is_empty()); - } + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(0, 7)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("the predicate should observe the preceding committed action"); - #[test] - fn wildcard_matches_non_eof_only() { - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - let matched = parser.match_wildcard().expect("wildcard"); - assert_eq!(parser.node(matched).text(), "x"); - assert!(parser.match_wildcard().is_err()); + assert_eq!(parser.node(tree).text(), "x"); + assert!(deferred_actions.is_empty()); + assert_eq!(parser.semantic_hooks.events, ["action:7", "predicate:true"]); } #[test] - fn add_parse_child_records_match_even_without_tree_building() { - // `sync_decision`'s "is the current context empty" flag must reflect real - // matches, not parse-tree children: when `build_parse_trees(false)`, - // `children` stays empty but `has_matched_child` must still flip so nested - // recovery does not wrongly suppress single-token deletion. - let mut parser = mini_parser(vec![TestToken::eof("parser-test", 1, 1, 1)]); - let token = TestToken::new(1).with_text("x"); + fn committed_action_hook_observes_parameterized_rule_argument() { + let atn = parameterized_child_action_eof_atn(); + let rule_args = [ParserRuleArg { + source_state: 0, + rule_index: 1, + value: 42, + inherit_local: false, + }]; + let mut parser = mini_parser_with_hooks( + vec![TestToken::eof("parser-test", 0, 1, 0)], + ActionContextHooks::default(), + ); - parser.set_build_parse_trees(false); - let mut ctx = ParserRuleContext::new(0, 0); - assert!(!ctx.has_matched_child()); - let child = parser.terminal_tree(token.id); - parser.add_parse_child(&mut ctx, child); - // Tree building is off, so no child is stored... - assert_eq!(ctx.child_count(), 0); - assert_eq!(parser.parse_tree_storage().node_count(), 0); - // ...but the match is recorded, so the context is no longer "empty". - assert!(ctx.has_matched_child()); + parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(1, 20), (4, 10)], + rule_args: &rule_args, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the parameterized child should parse"); - // With tree building on, the child is stored and the match is recorded. - parser.set_build_parse_trees(true); - let mut ctx = ParserRuleContext::new(0, 0); - let child = parser.terminal_tree(token.id); - parser.add_parse_child(&mut ctx, child); - assert_eq!(ctx.child_count(), 1); - assert!(ctx.has_matched_child()); + assert_eq!( + parser.semantic_hooks.actions[0], + (10, Some(42), None), + "the child action should observe its invocation argument" + ); } #[test] - fn disabled_tree_building_does_not_grow_flat_storage() { - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::new(1).with_text("y"), - TestToken::eof("parser-test", 2, 1, 2), - ]); - parser.set_build_parse_trees(false); - let mut context = ParserRuleContext::new(0, -1); + fn committed_parent_propagates_child_eof_consumption() { + let atn = parameterized_child_action_eof_atn(); + let mut parser = mini_parser_with_hooks( + vec![TestToken::eof("parser-test", 0, 1, 0)], + ActionContextHooks::default(), + ); - for _ in 0..2 { - let child = parser.match_token(1).expect("token should match"); - parser.add_parse_child(&mut context, child); - } - let current = parser.input.lt_id(1).expect("EOF token"); - let error = parser.error_tree(current); - parser.add_parse_child(&mut context, error); - let root = parser.rule_node(context); + let (tree, _) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(1, 20), (4, 10)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("the parent should retain its child's EOF boundary"); assert_eq!( - parser.parse_tree_storage().stats(), - ParseTreeStats::default() + parser.semantic_hooks.actions[1], + (20, None, Some(0)), + "the parent action should stop at EOF" ); - assert!( - parser - .parse_tree_storage() - .node(parser.token_store(), root) - .is_none(), - "the no-tree sentinel must not resolve to stored data" + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + assert_eq!(root.stop().map(|token| token.token_type()), Some(TOKEN_EOF)); + let child = root + .child_rules(1) + .next() + .expect("the parent should contain the child rule"); + assert_eq!( + child.stop().map(|token| token.token_type()), + Some(TOKEN_EOF) ); } #[test] - fn disabled_tree_building_skips_recognition_rule_node_storage() { - let atn = ordinary_star_loop_atn(); - let mut parser = mini_parser(repeated_x_tokens(3)); - parser.set_build_parse_trees(false); + fn committed_walker_does_not_run_action_in_losing_alternative() { + let atn = losing_alternative_action_atn(); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(2).with_text("y"), + TestToken::eof("parser-test", 1, 1, 1), + ], + StatefulActionHooks::default(), + ); - parser - .parse_atn_rule(&atn, 0) - .expect("ordinary repetition should parse without a tree"); + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(2, 0)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("the token-led second alternative should be selected"); - assert_eq!(parser.input.index(), 3); - assert!(parser.recognition_arena.nodes.is_empty()); - assert!(parser.recognition_arena.seq_links.is_empty()); - assert!(parser.recognition_arena.deferred_nodes.is_empty()); - assert!(parser.recognition_arena.deferred_rules.is_empty()); - assert!(!parser.fast_token_nodes_enabled); - assert!(parser.fast_recognize_scratch.memo.is_empty()); + assert_eq!(parser.node(tree).text(), "y"); + assert!(deferred_actions.is_empty()); + assert!(parser.semantic_hooks.events.is_empty()); } #[test] - fn parser_interprets_simple_atn_rule() { - let atn = token_then_eof_atn(); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - - let tree = parser - .parse_atn_rule(&atn, 0) - .expect("artificial parser rule should parse"); - assert_eq!(parser.node(tree).text(), "x"); - assert_eq!(parser.number_of_syntax_errors(), 0); - assert_eq!( - parser - .node(tree) - .first_rule_stop(0) - .expect("rule should stop at EOF") - .token_type(), - TOKEN_EOF + fn committed_walker_honors_decision_overrides() { + let atn = predicate_gated_same_lookahead_atn([0, 1]); + let predicates = [(0, 0, ParserPredicate::True), (0, 1, ParserPredicate::True)]; + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ], + ForcedSecondAlternativeHooks::default(), ); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), - ]); - let (tree, actions) = parser - .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) - .expect("runtime-option parser rule should parse"); - assert!(actions.is_empty()); - assert_eq!( - parser - .node(tree) - .first_rule_stop(0) - .expect("rule should stop at EOF") - .token_type(), - TOKEN_EOF - ); + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + track_alt_numbers: true, + predicates: &predicates, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the forced second alternative should parse"); + + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + assert_eq!(root.alt_number(), 2); + assert_eq!(root.text(), "x"); + assert!(deferred_actions.is_empty()); + assert_eq!(parser.semantic_hooks.decisions, [(0, 0, 2)]); + assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn runtime_options_default_ignores_noop_action_transitions() { - let atn = noop_action_then_token_then_eof_atn(); + fn committed_walker_sll_mode_does_not_report_full_context_diagnostics() { + let atn = predicate_gated_same_lookahead_atn([0, 1]); + let predicates = [(0, 0, ParserPredicate::True), (0, 1, ParserPredicate::True)]; + let diagnostics = Arc::new(Mutex::new(Vec::new())); let mut parser = mini_parser(vec![ TestToken::new(1).with_text("x"), TestToken::eof("parser-test", 1, 1, 1), ]); + parser.set_prediction_mode(PredictionMode::Sll); + parser.set_report_diagnostic_errors(true); + parser.remove_error_listeners(); + parser.add_error_listener(RecordingErrorListener { + diagnostics: Arc::clone(&diagnostics), + }); - let (tree, actions) = parser - .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) - .expect("no-op parser action should not force action replay"); + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + predicates: &predicates, + ..ParserRuntimeOptions::default() + }, + ) + .expect("SLL prediction should select the first viable alternative"); assert_eq!(parser.node(tree).text(), "x"); + assert!(deferred_actions.is_empty()); + assert_eq!(parser.number_of_syntax_errors(), 0); assert!( - actions.is_empty(), - "action_index=None transitions are ANTLR metadata, not replay actions" + diagnostics + .lock() + .expect("recorded diagnostics lock") + .is_empty(), + "SLL mode must not retry with full context or report LL diagnostics" ); - assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn parser_exposes_buffered_token_stream_after_parse() { - let atn = token_then_eof_atn(); + fn committed_walker_filters_diagnostics_after_semantic_selection() { + let atn = predicate_gated_same_lookahead_atn([0, 1]); + let predicates = [ + (0, 0, ParserPredicate::False), + (0, 1, ParserPredicate::True), + ]; + let diagnostics = Arc::new(Mutex::new(Vec::new())); let mut parser = mini_parser(vec![ TestToken::new(1).with_text("x"), TestToken::eof("parser-test", 1, 1, 1), ]); + parser.set_prediction_mode(PredictionMode::LlExactAmbigDetection); + parser.set_report_diagnostic_errors(true); + parser.remove_error_listeners(); + parser.add_error_listener(RecordingErrorListener { + diagnostics: Arc::clone(&diagnostics), + }); - let tree = parser - .parse_atn_rule(&atn, 0) - .expect("artificial parser rule should parse"); - assert_eq!(parser.node(tree).text(), "x"); + let (tree, _) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + track_alt_numbers: true, + predicates: &predicates, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the true predicate should make the second alternative unique"); - let stream = parser.token_stream(); - let source_index_after_parse = stream.token_source().index; - let buffered = stream.tokens().collect::>(); - assert_eq!(buffered.len(), 2); - assert_eq!(buffered[0].text(), Some("x")); - assert_eq!(buffered[0].token_id().index(), 0); - assert_eq!(buffered[1].token_type(), TOKEN_EOF); - assert_eq!(stream.token_source().index, source_index_after_parse); - drop(buffered); + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + assert_eq!(root.alt_number(), 2); + assert!( + diagnostics + .lock() + .expect("recorded diagnostics lock") + .is_empty(), + "predicate filtering made the decision unambiguous" + ); + } - let stream = parser.into_token_stream(); - assert_eq!(stream.token_source().index, source_index_after_parse); - assert_eq!( - stream.tokens().next().expect("first token").text(), - Some("x") + #[test] + fn committed_walker_skips_diagnostic_only_predicates_when_reporting_is_disabled() { + let atn = predicate_gated_same_lookahead_atn([0, 1]); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ], + RecordingHooks::default(), ); + parser.set_prediction_mode(PredictionMode::LlExactAmbigDetection); + + let (tree, _) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + track_alt_numbers: true, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the first predicate-bearing alternative should parse"); + + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + assert_eq!(root.alt_number(), 1); assert_eq!( - stream.tokens().nth(1).expect("EOF token").token_type(), - TOKEN_EOF + parser.semantic_hooks.predicates, + [ + (0, 0, 0, Some("x".to_owned())), + (0, 0, 0, Some("x".to_owned())), + ], + "diagnostic-only alternatives must not invoke semantic hooks" ); } #[test] - fn parsed_file_exposes_all_buffered_tokens() { - let atn = token_then_eof_atn(); + fn committed_walker_falls_back_only_to_simulator_viable_alternatives() { + let atn = semantic_fallback_viability_atn(); + let predicates = [ + (0, 0, ParserPredicate::False), + (0, 1, ParserPredicate::True), + ]; let mut parser = mini_parser(vec![ - TestToken::new(99) - .with_text(" comment") - .with_channel(HIDDEN_CHANNEL), - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 9, 1, 9), + TestToken::new(1).with_text("a"), + TestToken::new(3).with_text("c"), + TestToken::eof("parser-test", 2, 1, 2), ]); - let tree = parser - .parse_atn_rule(&atn, 0) - .expect("artificial parser rule should parse"); - let parsed = parser.into_parsed_file(tree); + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + track_alt_numbers: true, + predicates: &predicates, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the true A C alternative should survive semantic fallback"); - // Snapshot the full buffered stream — hidden-channel comment, default-channel token, EOF — - // as (type, channel, text) triples; contents make the count self-evident. - insta::assert_debug_snapshot!( - "parsed_file_exposes_all_buffered_tokens", - parsed - .tokens() - .iter() - .map(|token| (token.token_type(), token.channel(), token.text())) - .collect::>() - ); - assert_eq!(parsed.tokens().into_iter().count(), 3); + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + assert_eq!(root.alt_number(), 3); + assert_eq!(root.text(), "ac"); + assert!(deferred_actions.is_empty()); + assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn parser_syntax_error_count_tracks_interpreted_recovery() { - let atn = token_then_eof_atn(); + fn committed_walker_evaluates_predicates_reached_through_rule_calls() { + let atn = rule_call_predicate_decision_atn(); + let predicates = [(1, 0, ParserPredicate::False)]; let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 2, 1, 2), + TestToken::new(1).with_text("a"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + track_alt_numbers: true, + predicates: &predicates, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the direct caller alternative should survive the false callee predicate"); + + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + assert_eq!(root.alt_number(), 2); + assert_eq!(root.text(), "a"); + assert_eq!(root.child_rules(1).count(), 0); + assert!(deferred_actions.is_empty()); + assert_eq!(parser.number_of_syntax_errors(), 0); + } + + #[test] + fn committed_walker_uses_callee_argument_for_prediction_predicates() { + let atn = rule_call_predicate_decision_atn(); + let predicates = [(1, 0, ParserPredicate::LocalIntEquals { value: 1 })]; + let rule_args = [ParserRuleArg { + source_state: 2, + rule_index: 1, + value: 2, + inherit_local: false, + }]; + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("a"), + TestToken::eof("parser-test", 1, 1, 1), ]); - let tree = parser - .parse_atn_rule(&atn, 0) - .expect("invalid token should recover into an error node"); + let (tree, _) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + track_alt_numbers: true, + predicates: &predicates, + rule_args: &rule_args, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the direct alternative should survive the false callee predicate"); - assert_eq!(parser.number_of_syntax_errors(), 1); - assert_eq!( - parser - .node(tree) - .first_error_token() - .expect("recovery should embed an error token") - .text(), - Some("y") - ); + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + assert_eq!(root.alt_number(), 2); + assert_eq!(root.child_rules(1).count(), 0); + assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn failed_interpreted_parse_notifies_error_listener() { - let atn = token_then_eof_atn(); + fn committed_predicate_star_loop_uses_single_token_deletion() { + let atn = predicate_gated_star_loop_atn(); + let predicates = [(0, 0, ParserPredicate::True)]; + let diagnostics = Arc::new(Mutex::new(Vec::new())); let mut parser = mini_parser(vec![ - TestToken::new(2) - .with_text("y") - .with_span(0, 0) - .with_byte_span(0, 1) - .with_position(3, 5), - TestToken::eof("parser-test", 1, 1, 1), + TestToken::new(2).with_text("x"), + TestToken::new(1).with_text("a"), + TestToken::eof("parser-test", 2, 1, 2), ]); parser.remove_error_listeners(); - let diagnostics = Arc::new(Mutex::new(Vec::new())); parser.add_error_listener(RecordingErrorListener { diagnostics: Arc::clone(&diagnostics), }); - let error = parser - .parse_atn_rule(&atn, 0) - .expect_err("start-rule mismatch should remain a parser error"); + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + predicates: &predicates, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the loop decision should delete the extraneous token and continue"); + assert_eq!(parser.node(tree).text(), "xa"); + assert!(deferred_actions.is_empty()); assert_eq!(parser.number_of_syntax_errors(), 1); - assert!(matches!(&error, AntlrError::ParserError { .. })); insta::assert_debug_snapshot!( - "failed_interpreted_parse_notifies_error_listener", + "committed_predicate_star_loop_uses_single_token_deletion", *diagnostics.lock().expect("recorded diagnostics lock") ); } #[test] - fn adaptive_direct_rule_uses_simulator_decision() { - let atn = two_alt_decision_atn(); - let mut simulator = ParserAtnSimulator::new(&atn); - let mut parser = mini_parser(vec![ - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 1, 1, 1), - ]); + fn committed_walker_applies_legacy_and_semir_actions_before_indexed_hooks() { + let atn = committed_action_then_predicate_atn(); + let member_actions = [ParserMemberAction { + source_state: 0, + member: 0, + delta: 2, + }]; + let return_actions = [ParserReturnAction { + source_state: 0, + rule_index: 0, + name: "legacy", + value: 3, + }]; + let predicates = [( + 0, + 0, + ParserPredicate::MemberEquals { + member: 0, + value: 7, + equals: true, + }, + )]; + let mut ir = SemIr::new(); + let semantic_member = ParserMemberAction { + source_state: 0, + member: 0, + delta: 5, + } + .lower_into_semir(&mut ir); + let semantic_return = ParserReturnAction { + source_state: 0, + rule_index: 0, + name: "semantic", + value: 11, + } + .lower_into_semir(&mut ir); + let semantics = ParserSemantics { + ir, + predicates: Vec::new(), + actions: vec![semantic_member, semantic_return], + }; + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ], + StatefulActionHooks::default(), + ); - let tree = parser - .parse_atn_rule_adaptive_or_fallback(&atn, &mut simulator, 0) - .expect("direct adaptive rule should parse"); + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(0, 7)], + predicates: &predicates, + semantics: Some(&semantics), + member_actions: &member_actions, + return_actions: &return_actions, + ..ParserRuntimeOptions::default() + }, + ) + .expect("the predicate should observe both committed member actions"); - assert_eq!(parser.node(tree).text(), "y"); - assert_eq!(parser.input.index(), 1); + let root = parser.node(tree).as_rule().expect("entry result is a rule"); + assert_eq!(root.text(), "x"); + assert_eq!(root.int_return("legacy"), Some(3)); + assert_eq!(root.int_return("semantic"), Some(11)); + assert_eq!(parser.int_member(0), Some(7)); + assert!(deferred_actions.is_empty()); + assert_eq!(parser.semantic_hooks.events, ["action:7"]); + assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn adaptive_direct_rule_restores_input_on_fallback() { - let atn = predicate_after_token_atn(); - let mut simulator = ParserAtnSimulator::new(&atn); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 2, 1, 2), - ]); + fn committed_walker_runs_action_once_per_star_loop_iteration() { + let atn = committed_action_star_loop_atn(); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("a"), + TestToken::new(1).with_text("b"), + TestToken::eof("parser-test", 2, 1, 2), + ], + StatefulActionHooks::default(), + ); - let tree = parser - .parse_atn_rule_adaptive_or_fallback(&atn, &mut simulator, 0) - .expect("fallback recognizer should parse"); + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(2, 3)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("the committed star loop should parse"); - assert_eq!(parser.node(tree).text(), "xy"); - assert_eq!(parser.input.index(), 2); - let stats = parser.parse_tree_storage().stats(); - assert_eq!(stats.nodes, parser.node(tree).descendants().count()); - assert_eq!(stats.edges, stats.nodes.saturating_sub(1)); - assert_eq!(stats.scratch_links, 0); + assert_eq!(parser.node(tree).text(), "ab"); + assert!(deferred_actions.is_empty()); + assert_eq!(parser.semantic_hooks.events, ["action:3", "action:3"]); } #[test] - fn unknown_predicate_policy_defaults_to_assume_true() { - let atn = predicate_after_token_atn(); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 2, 1, 2), - ]); + fn committed_walker_has_no_total_step_cap() { + const TOKEN_COUNT: usize = RECOGNITION_DEPTH_LIMIT + 1; + let atn = committed_action_star_loop_atn(); + let mut parser = mini_parser(repeated_x_tokens(TOKEN_COUNT)); + parser.set_build_parse_trees(false); - let (tree, _) = parser - .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) - .expect("unknown predicate should pass under the default policy"); + parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("valid committed loops must not have a total-work cap"); - assert_eq!(parser.node(tree).text(), "xy"); + assert_eq!(parser.input.index(), TOKEN_COUNT); assert_eq!(parser.number_of_syntax_errors(), 0); } #[test] - fn private_context_alt_tracking_keeps_fast_predicate_recognition() { - let atn = predicate_gated_same_lookahead_atn([0, 1]); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), - ]); + fn committed_walker_rejects_non_consuming_cycles() { + let atn = committed_non_consuming_cycle_atn(); + let mut parser = mini_parser(vec![TestToken::eof("parser-test", 0, 1, 0)]); + parser.set_bail_on_error(true); - let (tree, _) = parser + let error = parser .parse_atn_rule_with_runtime_options( &atn, 0, ParserRuntimeOptions { - predicates: &[ - (0, 0, ParserPredicate::False), - (0, 1, ParserPredicate::True), - ], - track_context_alt_numbers: true, + action_indices: &[(usize::MAX, 0)], ..ParserRuntimeOptions::default() }, ) - .expect("the second predicate-gated alternative should match"); + .expect_err("a non-consuming cycle must not spin forever"); - let root = parser.node(tree).as_rule().expect("entry result is a rule"); - insta::assert_debug_snapshot!( - "private_context_alt_tracking_keeps_fast_predicate_recognition", - (root.alt_number(), root.context_alt_number(), root.text()) + assert!( + error.to_string().contains("non-consuming ATN cycle"), + "unexpected error: {error}" ); - assert_eq!(parser.number_of_syntax_errors(), 0); - assert_eq!(parser.fast_predicate_cache.get(&(0, 0, 0)), Some(&false)); - assert_eq!(parser.fast_predicate_cache.get(&(0, 0, 1)), Some(&true)); } #[test] - fn nested_interpreted_parse_preserves_prior_unknown_predicate_hits() { - // A generated parent may record an unknown-predicate coordinate, then - // descend into an interpreted child. The child's interpreter entry must - // not wipe the parent's recorded hit before the top-level surfaces it. - let atn = token_then_eof_atn(); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::eof("parser-test", 1, 1, 1), - ]); + fn deeply_nested_committed_rule_calls_grow_the_stack() { + const DEPTH: usize = 4_096; + const STACK_SIZE: usize = 256 * 1024; + let atn = nested_rule_chain_atn(DEPTH); + std::thread::Builder::new() + .name("nested-committed-rules".to_owned()) + .stack_size(STACK_SIZE) + .spawn(move || { + let mut parser = mini_parser(vec![TestToken::new(1).with_text("x")]); + parser.set_build_parse_trees(false); + parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(usize::MAX, 0)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("nested committed rules should grow the native stack"); + assert_eq!(parser.input.index(), 1); + }) + .expect("small-stack thread should start") + .join() + .expect("nested committed rules should not overflow their stack"); + } - // Simulate the parent having recorded a fail-loud coordinate. - parser.unknown_predicate_hits.push((7, 3)); + #[test] + fn committed_walker_runs_action_once_per_left_recursive_operator() { + let atn = committed_action_left_recursive_atn(); + let mut parser = mini_parser_with_hooks( + vec![ + TestToken::new(1).with_text("a"), + TestToken::new(3).with_text("+"), + TestToken::new(1).with_text("b"), + TestToken::new(3).with_text("+"), + TestToken::new(1).with_text("c"), + TestToken::eof("parser-test", 5, 1, 5), + ], + StatefulActionHooks::default(), + ); - // Run an interpreted child parse that records no coordinate of its own. - parser - .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) - .expect("child rule parses"); + let (tree, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(6, 11)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("the committed left-recursive rule should parse"); - // The parent's coordinate must still be present for the top-level entry. - let error = parser - .take_unknown_semantic_error() - .expect("parent's recorded coordinate must survive the nested interpreted parse"); - let AntlrError::Unsupported(message) = error else { - panic!("expected AntlrError::Unsupported, got {error:?}"); - }; - assert!(message.contains("pred_index=3"), "message: {message}"); + assert_eq!(parser.node(tree).text(), "a+b+c"); + assert!(deferred_actions.is_empty()); + assert_eq!(parser.semantic_hooks.events, ["action:11", "action:11"]); } #[test] - fn unknown_predicate_policy_assume_false_kills_the_guarded_path() { - let atn = predicate_after_token_atn(); + fn committed_left_recursive_depth_cap_keeps_listener_events_balanced() { + let atn = committed_action_left_recursive_atn(); + let events = Arc::new(Mutex::new(Vec::new())); let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 2, 1, 2), + TestToken::new(1).with_text("a"), + TestToken::new(3).with_text("+"), + TestToken::new(1).with_text("b"), + TestToken::eof("parser-test", 3, 1, 3), ]); + parser.set_max_rule_depth(Some(1)); + parser.add_parse_listener(RecordingParseListener { + events: Arc::clone(&events), + }); - let result = parser.parse_atn_rule_with_runtime_options( - &atn, - 0, - ParserRuntimeOptions { - unknown_predicate_policy: UnknownSemanticPolicy::AssumeFalse, - ..ParserRuntimeOptions::default() - }, - ); + let error = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(6, 11)], + ..ParserRuntimeOptions::default() + }, + ) + .expect_err("the left-recursive expansion should exceed the depth cap"); - assert!( - result.is_err(), - "the only path is predicate-guarded, so assume-false must fail the parse" + insta::assert_debug_snapshot!( + "committed_left_recursive_depth_cap_keeps_listener_events_balanced", + ( + error.to_string(), + events.lock().expect("parse-listener event lock").as_slice(), + ) ); } - #[test] - fn predicate_failure_message_keeps_semantic_recovery_path() { - let atn = predicate_after_token_atn(); + #[test] + fn committed_walker_preserves_nested_rule_listener_events() { + let atn = ordinary_star_loop_atn(); + let events = Arc::new(Mutex::new(Vec::new())); let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::new(2).with_text("y"), + TestToken::new(1).with_text("a"), + TestToken::new(1).with_text("b"), TestToken::eof("parser-test", 2, 1, 2), ]); + parser.add_parse_listener(RecordingParseListener { + events: Arc::clone(&events), + }); let (tree, _) = parser .parse_atn_rule_with_runtime_options( &atn, 0, ParserRuntimeOptions { - predicates: &[( - 0, - 0, - ParserPredicate::FalseWithMessage { - message: "predicate rejected input", - }, - )], + action_indices: &[(usize::MAX, 0)], ..ParserRuntimeOptions::default() }, ) - .expect("failure-message predicates recover through the semantic interpreter"); + .expect("the committed nested-rule path should parse"); - assert_eq!(parser.node(tree).text(), "xy"); - assert_eq!(parser.number_of_syntax_errors(), 1); - assert!( - parser.fast_predicate_cache.is_empty(), - "failure-message predicates need the semantic interpreter's recovery outcome" + assert_eq!(parser.node(tree).text(), "ab"); + assert_eq!( + *events.lock().expect("parse-listener event lock"), + [ + "enter:0", "enter:1", "exit:1", "enter:1", "exit:1", "exit:0", + ] ); } #[test] - fn unknown_predicate_policy_error_names_the_coordinate() { - let atn = predicate_after_token_atn(); + fn committed_walker_enforces_rule_depth_cap() { + let atn = ordinary_star_loop_atn(); let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 2, 1, 2), + TestToken::new(1).with_text("a"), + TestToken::eof("parser-test", 1, 1, 1), ]); + parser.set_max_rule_depth(Some(1)); let error = parser .parse_atn_rule_with_runtime_options( &atn, 0, ParserRuntimeOptions { - unknown_predicate_policy: UnknownSemanticPolicy::Error, + action_indices: &[(usize::MAX, 0)], ..ParserRuntimeOptions::default() }, ) - .expect_err("evaluating an unknown predicate under Error policy must fail"); + .expect_err("the nested rule should exceed the committed-path cap"); - let AntlrError::Unsupported(message) = error else { - panic!("expected AntlrError::Unsupported, got {error:?}"); - }; - assert!( - message.contains("unsupported semantic predicate"), - "message should name the failure class: {message}" - ); assert!( - message.contains("pred_index=0"), - "message should carry the coordinate: {message}" + error + .to_string() + .contains("rule nesting depth limit of 1 exceeded"), + "unexpected error: {error}" ); } #[test] - fn fail_loud_hits_do_not_leak_into_a_reused_interpreter_parse() { - // A parser reused after a fail-loud parse must not carry the old - // coordinates into a later parse. The fail-loud return keeps the hits - // (so a generated parent can surface a recovered child's coordinate), - // and the next parse's entry stashes/replaces them, so a subsequent - // clean parse surfaces no stale error. - let atn = predicate_after_token_atn(); - let mut parser = mini_parser(vec![ - TestToken::new(1).with_text("x"), - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 2, 1, 2), - ]); + fn committed_abort_precedes_and_clears_unhandled_action_error() { + let atn = action_then_nested_rule_atn(); + let mut parser = mini_parser_with_hooks( + vec![TestToken::eof("parser-test", 0, 1, 0)], + DecliningActionHooks::default(), + ); + parser.set_max_rule_depth(Some(1)); - parser + let error = parser .parse_atn_rule_with_runtime_options( &atn, 0, ParserRuntimeOptions { + action_indices: &[(0, 7)], unknown_predicate_policy: UnknownSemanticPolicy::Error, ..ParserRuntimeOptions::default() }, ) - .expect_err("first parse fails loud under the Error policy"); + .expect_err("the recovered child abort must outrank the earlier action miss"); - // The failed parse kept its coordinate on the parser (so a generated - // parent could surface a recovered child). A top-level reuse resets the - // hits — generated parsers call `reset_unknown_semantic_hits` at their - // public entry; direct interpreter-API callers do the same. - parser.reset_unknown_semantic_hits(); + assert_eq!(parser.semantic_hooks.actions, [0]); + assert!( + error + .to_string() + .contains("rule nesting depth limit of 1 exceeded"), + "unexpected error: {error}" + ); + assert!( + parser.take_parse_abort().is_none(), + "the returned abort must not remain sticky" + ); assert!( parser.take_unknown_semantic_error().is_none(), - "reset must drop stale unknown-predicate coordinates before a reused parse" + "the masked action miss must not poison parser reuse" ); } - #[derive(Debug, Default)] - struct RecordingHooks { - predicates: Vec<(usize, usize, usize, Option)>, - actions: Vec<(usize, String, Option)>, - action_trees: Vec>, - } - - impl SemanticHooks for RecordingHooks { - fn sempred( - &mut self, - ctx: &mut ParserSemCtx<'_, S>, - rule_index: usize, - pred_index: usize, - ) -> Option - where - S: TokenSource, - { - self.predicates.push(( - ctx.input_index(), - rule_index, - pred_index, - ctx.token_text(1) - .and_then(|token| token.text().map(str::to_owned)), - )); - Some(true) - } - - fn action(&mut self, ctx: &mut ParserSemCtx<'_, S>, action: ParserAction) -> bool - where - S: TokenSource, - { - self.actions.push(( - action.source_state(), - ctx.action_text(), - ctx.rule_name().map(str::to_owned), - )); - self.action_trees.push(ctx.tree().map(Node::text)); - true - } - } - - #[derive(Debug, Default)] - struct RejectingPredicateHooks { - predicates: Vec<(usize, usize, usize, Option)>, - } - - impl SemanticHooks for RejectingPredicateHooks { - fn sempred( - &mut self, - ctx: &mut ParserSemCtx<'_, S>, - rule_index: usize, - pred_index: usize, - ) -> Option - where - S: TokenSource, - { - self.predicates.push(( - ctx.input_index(), - rule_index, - pred_index, - ctx.token_text(1) - .and_then(|token| token.text().map(str::to_owned)), - )); - Some(false) - } - } - #[test] - fn fast_predicate_cache_replays_hook_once_per_coordinate_and_input() { - let atn = predicate_gated_same_lookahead_atn([0, 0]); + fn top_level_committed_semantic_error_does_not_poison_reuse() { + let atn = committed_action_then_predicate_atn(); + let predicates = [(0, 0, ParserPredicate::True)]; let mut parser = mini_parser_with_hooks( vec![ TestToken::new(1).with_text("x"), TestToken::eof("parser-test", 1, 1, 1), ], - RecordingHooks::default(), + DecliningActionHooks::default(), + ); + + let error = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(0, 7)], + predicates: &predicates, + unknown_predicate_policy: UnknownSemanticPolicy::Error, + ..ParserRuntimeOptions::default() + }, + ) + .expect_err("the declined committed action must fail loud"); + assert!( + error.to_string().contains("unhandled semantic action"), + "unexpected error: {error}" ); + parser.input.seek(0); let (tree, _) = parser - .parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()) - .expect("both alternatives share one replay-safe predicate result"); + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + predicates: &predicates, + ..ParserRuntimeOptions::default() + }, + ) + .expect("a clean interpreted reuse must not observe the prior action miss"); assert_eq!(parser.node(tree).text(), "x"); - assert_eq!( - parser.semantic_hooks.predicates, - vec![(0, 0, 0, Some("x".to_owned()))] + assert!( + parser.take_unknown_semantic_error().is_none(), + "the returned top-level semantic error must drain its recorded hit" ); - assert_eq!(parser.fast_predicate_cache.get(&(0, 0, 0)), Some(&true)); } #[test] - fn semantic_hook_handles_unknown_predicate_before_error_policy() { - let atn = predicate_after_token_atn(); + fn committed_walker_runs_handled_rule_init_before_indexed_action() { + let atn = committed_action_then_predicate_atn(); let mut parser = mini_parser_with_hooks( vec![ TestToken::new(1).with_text("x"), - TestToken::new(2).with_text("y"), - TestToken::eof("parser-test", 2, 1, 2), + TestToken::eof("parser-test", 1, 1, 1), ], - RecordingHooks::default(), + InitOrderingHooks::default(), ); - let (tree, _) = parser + let (_, deferred_actions) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + init_action_rules: &[0], + action_indices: &[(0, 7)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("the named action should observe rule-init state"); + + assert!(deferred_actions.is_empty()); + assert_eq!( + parser.semantic_hooks.events, + ["init", "action:7:initialized=true", "predicate:true",] + ); + } + + #[test] + fn committed_walker_defers_unhandled_rule_init_for_legacy_replay() { + let atn = token_then_eof_atn(); + let mut parser = mini_parser(vec![ + TestToken::new(1).with_text("x"), + TestToken::eof("parser-test", 1, 1, 1), + ]); + + let (_, deferred_actions) = parser .parse_atn_rule_with_runtime_options( &atn, 0, ParserRuntimeOptions { + init_action_rules: &[0], + action_indices: &[(usize::MAX, 0)], unknown_predicate_policy: UnknownSemanticPolicy::Error, ..ParserRuntimeOptions::default() }, ) - .expect("hook supplies the missing predicate result"); + .expect("a declined init should remain available for legacy replay"); - assert_eq!(parser.node(tree).text(), "xy"); assert_eq!( - parser.semantic_hooks.predicates, - vec![(1, 0, 0, Some("y".to_owned()))] + deferred_actions, + [ParserAction::new_rule_init(0, 0, Some(0))] ); - assert_eq!(parser.fast_predicate_cache.get(&(1, 0, 0)), Some(&true)); } #[test] - fn runtime_options_default_preserves_semantic_hook_predicates() { - let atn = predicate_after_token_atn(); + fn committed_walker_dispatches_recovery_diagnostics() { + let atn = noop_action_then_token_then_eof_atn(); + let diagnostics = Arc::new(Mutex::new(Vec::new())); let mut parser = mini_parser_with_hooks( vec![ TestToken::new(1).with_text("x"), TestToken::new(2).with_text("y"), TestToken::eof("parser-test", 2, 1, 2), ], - RejectingPredicateHooks::default(), + StatefulActionHooks::default(), ); + parser.remove_error_listeners(); + parser.add_error_listener(RecordingErrorListener { + diagnostics: Arc::clone(&diagnostics), + }); - let result = - parser.parse_atn_rule_with_runtime_options(&atn, 0, ParserRuntimeOptions::default()); + let (tree, _) = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(0, 5)], + ..ParserRuntimeOptions::default() + }, + ) + .expect("the committed rule should recover"); - assert!( - result.is_err(), - "default runtime options must not bypass semantic hooks for predicate ATNs" + assert_eq!(parser.node(tree).text(), "xy"); + assert_eq!(parser.number_of_syntax_errors(), 1); + insta::assert_debug_snapshot!( + "committed_walker_dispatches_recovery_diagnostics", + *diagnostics.lock().expect("recorded diagnostics lock") ); - assert_eq!( - parser.semantic_hooks.predicates, - vec![(1, 0, 0, Some("y".to_owned()))] + } + + #[test] + fn committed_bail_error_notifies_error_listener() { + let atn = noop_action_then_token_then_eof_atn(); + let diagnostics = Arc::new(Mutex::new(Vec::new())); + let mut parser = mini_parser(vec![ + TestToken::new(2) + .with_text("y") + .with_span(0, 0) + .with_byte_span(0, 1) + .with_position(3, 5), + TestToken::eof("parser-test", 1, 1, 1), + ]); + parser.set_bail_on_error(true); + parser.remove_error_listeners(); + parser.add_error_listener(RecordingErrorListener { + diagnostics: Arc::clone(&diagnostics), + }); + + let error = parser + .parse_atn_rule_with_runtime_options( + &atn, + 0, + ParserRuntimeOptions { + action_indices: &[(0, 5)], + ..ParserRuntimeOptions::default() + }, + ) + .expect_err("bail mode must return the committed token mismatch"); + let diagnostics = diagnostics + .lock() + .expect("recorded diagnostics lock") + .clone(); + + insta::assert_debug_snapshot!( + "committed_bail_error_notifies_error_listener", + (error, diagnostics) ); - assert_eq!(parser.fast_predicate_cache.get(&(1, 0, 0)), Some(&false)); } #[test] diff --git a/src/prediction.rs b/src/prediction.rs index d49c3fbf..410c9827 100644 --- a/src/prediction.rs +++ b/src/prediction.rs @@ -70,6 +70,12 @@ pub struct ContextId(u32); pub const EMPTY_CONTEXT: ContextId = ContextId(0); +impl ContextId { + pub(crate) const fn compact(self) -> u32 { + self.0 + } +} + #[derive(Clone, Copy, Debug, Eq, PartialEq)] enum ContextTag { Empty, @@ -848,6 +854,140 @@ fn combine_semantic_context( } } +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct PredictionRuleCall { + pub(crate) source_state: usize, + pub(crate) rule_index: usize, +} + +#[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct PredictionPredicateCall { + pub(crate) rule_index: usize, + pub(crate) pred_index: usize, + pub(crate) rule_calls: Vec, +} + +#[derive(Clone, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +struct PredictionSemanticProvenance { + active_rule_calls: Vec, + predicate_calls: Vec, +} + +#[repr(transparent)] +#[derive(Clone, Copy, Debug, Default, Eq, Hash, Ord, PartialEq, PartialOrd)] +pub(crate) struct PredictionSemanticProvenanceId(u32); + +#[derive(Debug, Default)] +pub(crate) struct PredictionSemanticProvenanceArena { + records: Vec, + interner_heads: FxHashMap, + interner_next: Vec>, +} + +impl PredictionSemanticProvenanceArena { + pub(crate) fn enter_rule( + &mut self, + id: PredictionSemanticProvenanceId, + source_state: usize, + rule_index: usize, + ) -> PredictionSemanticProvenanceId { + let mut provenance = self.get(id).cloned().unwrap_or_default(); + provenance.active_rule_calls.push(PredictionRuleCall { + source_state, + rule_index, + }); + self.intern(provenance) + } + + pub(crate) fn exit_rule( + &mut self, + id: PredictionSemanticProvenanceId, + ) -> PredictionSemanticProvenanceId { + let Some(mut provenance) = self.get(id).cloned() else { + return PredictionSemanticProvenanceId::default(); + }; + provenance.active_rule_calls.pop(); + self.intern(provenance) + } + + pub(crate) fn record_predicate( + &mut self, + id: PredictionSemanticProvenanceId, + rule_index: usize, + pred_index: usize, + ) -> PredictionSemanticProvenanceId { + let mut provenance = self.get(id).cloned().unwrap_or_default(); + let call = PredictionPredicateCall { + rule_index, + pred_index, + rule_calls: provenance.active_rule_calls.clone(), + }; + if !provenance.predicate_calls.contains(&call) { + provenance.predicate_calls.push(call); + } + self.intern(provenance) + } + + pub(crate) fn predicate_calls( + &self, + id: PredictionSemanticProvenanceId, + ) -> &[PredictionPredicateCall] { + self.get(id) + .map_or(&[], |provenance| provenance.predicate_calls.as_slice()) + } + + fn get(&self, id: PredictionSemanticProvenanceId) -> Option<&PredictionSemanticProvenance> { + let index = id.0.checked_sub(1)?; + self.records.get(usize::try_from(index).ok()?) + } + + fn find_interned( + &self, + cached_hash: u64, + provenance: &PredictionSemanticProvenance, + ) -> Option { + let mut candidate = self.interner_heads.get(&cached_hash).copied(); + while let Some(id) = candidate { + let index = usize::try_from(id.0.checked_sub(1)?).ok()?; + if self.records.get(index) == Some(provenance) { + return Some(id); + } + candidate = self.interner_next.get(index).copied().flatten(); + } + None + } + + fn intern( + &mut self, + provenance: PredictionSemanticProvenance, + ) -> PredictionSemanticProvenanceId { + if provenance.active_rule_calls.is_empty() && provenance.predicate_calls.is_empty() { + return PredictionSemanticProvenanceId::default(); + } + let mut hasher = PredictionFxHasher::default(); + provenance.hash(&mut hasher); + let cached_hash = hasher.finish(); + if let Some(id) = self.find_interned(cached_hash, &provenance) { + return id; + } + let id = PredictionSemanticProvenanceId( + u32::try_from(self.records.len() + 1) + .expect("prediction semantic provenance arena exhausted"), + ); + assert!( + id.0 <= ATN_CONFIG_PROVENANCE_MASK, + "prediction semantic provenance arena exhausted" + ); + let previous = self.interner_heads.insert(cached_hash, id); + self.records.push(provenance); + self.interner_next.push(previous); + id + } +} + +const ATN_CONFIG_PRECEDENCE_FILTER_SUPPRESSED: u32 = 1 << 31; +const ATN_CONFIG_PROVENANCE_MASK: u32 = !ATN_CONFIG_PRECEDENCE_FILTER_SUPPRESSED; + #[derive(Clone, Debug, Eq, Hash, Ord, PartialEq, PartialOrd)] pub(crate) struct AtnConfig { pub(crate) state: usize, @@ -855,7 +995,7 @@ pub(crate) struct AtnConfig { pub(crate) context: ContextId, pub(crate) semantic_context: SemanticContext, pub(crate) reaches_into_outer_context: usize, - pub(crate) precedence_filter_suppressed: bool, + semantic_provenance_and_flags: u32, #[cfg(debug_assertions)] context_generation: u64, } @@ -869,7 +1009,7 @@ impl AtnConfig { context, semantic_context: SemanticContext::None, reaches_into_outer_context: 0, - precedence_filter_suppressed: false, + semantic_provenance_and_flags: 0, #[cfg(debug_assertions)] context_generation: arena.generation(), } @@ -895,10 +1035,65 @@ impl AtnConfig { let mut moved = Self::new(state, self.alt, context, arena); moved.semantic_context = self.semantic_context.clone(); moved.reaches_into_outer_context = self.reaches_into_outer_context; - moved.precedence_filter_suppressed = self.precedence_filter_suppressed; + moved.semantic_provenance_and_flags = self.semantic_provenance_and_flags; moved } + pub(crate) fn enter_prediction_rule( + &mut self, + arena: &mut PredictionSemanticProvenanceArena, + source_state: usize, + rule_index: usize, + ) { + let id = arena.enter_rule(self.semantic_provenance_id(), source_state, rule_index); + self.set_semantic_provenance_id(id); + } + + pub(crate) fn exit_prediction_rule(&mut self, arena: &mut PredictionSemanticProvenanceArena) { + let id = arena.exit_rule(self.semantic_provenance_id()); + self.set_semantic_provenance_id(id); + } + + pub(crate) fn record_prediction_predicate( + &mut self, + arena: &mut PredictionSemanticProvenanceArena, + rule_index: usize, + pred_index: usize, + ) { + let id = arena.record_predicate(self.semantic_provenance_id(), rule_index, pred_index); + self.set_semantic_provenance_id(id); + } + + pub(crate) const fn semantic_provenance_id(&self) -> PredictionSemanticProvenanceId { + PredictionSemanticProvenanceId( + self.semantic_provenance_and_flags & ATN_CONFIG_PROVENANCE_MASK, + ) + } + + pub(crate) const fn semantic_provenance_and_flags(&self) -> u32 { + self.semantic_provenance_and_flags + } + + fn set_semantic_provenance_id(&mut self, id: PredictionSemanticProvenanceId) { + debug_assert_eq!(id.0 & ATN_CONFIG_PRECEDENCE_FILTER_SUPPRESSED, 0); + self.semantic_provenance_and_flags = + (self.semantic_provenance_and_flags & ATN_CONFIG_PRECEDENCE_FILTER_SUPPRESSED) | id.0; + } + + pub(crate) const fn precedence_filter_suppressed(&self) -> bool { + self.semantic_provenance_and_flags & ATN_CONFIG_PRECEDENCE_FILTER_SUPPRESSED != 0 + } + + pub(crate) const fn suppress_precedence_filter(&mut self) { + self.semantic_provenance_and_flags |= ATN_CONFIG_PRECEDENCE_FILTER_SUPPRESSED; + } + + pub(crate) const fn merge_precedence_filter_suppression(&mut self, other: &Self) { + if other.precedence_filter_suppressed() { + self.suppress_precedence_filter(); + } + } + pub(crate) fn assert_store(&self, arena: &ContextArena) { arena.assert_valid(self.context); #[cfg(debug_assertions)] @@ -972,7 +1167,7 @@ impl AtnConfigSet { existing.reaches_into_outer_context = existing .reaches_into_outer_context .max(config.reaches_into_outer_context); - existing.precedence_filter_suppressed |= config.precedence_filter_suppressed; + existing.merge_precedence_filter_suppression(&config); self.conflicting_alts.clear(); false } else { @@ -1114,6 +1309,7 @@ struct AtnConfigKey { state: usize, alt: usize, semantic_context: SemanticContext, + semantic_provenance: PredictionSemanticProvenanceId, } impl From<&AtnConfig> for AtnConfigKey { @@ -1122,6 +1318,7 @@ impl From<&AtnConfig> for AtnConfigKey { state: config.state, alt: config.alt, semantic_context: config.semantic_context.clone(), + semantic_provenance: config.semantic_provenance_id(), } } } @@ -1328,6 +1525,93 @@ mod tests { assert_eq!(arena.len(set.configs()[0].context), 2); } + #[test] + fn predicate_provenance_is_idempotent_per_rule_path() { + let arena = ContextArena::new(); + let mut provenance = PredictionSemanticProvenanceArena::default(); + let mut config = AtnConfig::new(1, 1, EMPTY_CONTEXT, &arena); + config.enter_prediction_rule(&mut provenance, 4, 2); + config.record_prediction_predicate(&mut provenance, 2, 3); + let after_first = provenance + .predicate_calls(config.semantic_provenance_id()) + .to_vec(); + + config.record_prediction_predicate(&mut provenance, 2, 3); + + assert_eq!( + provenance.predicate_calls(config.semantic_provenance_id()), + after_first, + "revisiting one predicate on the same rule path must not grow closure keys" + ); + } + + #[test] + fn provenance_arena_stores_records_once_and_verifies_hash_collisions() { + let mut arena = PredictionSemanticProvenanceArena::default(); + let first = PredictionSemanticProvenance { + active_rule_calls: vec![PredictionRuleCall { + source_state: 4, + rule_index: 2, + }], + predicate_calls: Vec::new(), + }; + let first_id = arena.intern(first.clone()); + + assert_eq!(arena.intern(first), first_id); + assert_eq!(arena.records.len(), 1); + assert_eq!(arena.interner_next.len(), 1); + + let second = PredictionSemanticProvenance { + active_rule_calls: vec![PredictionRuleCall { + source_state: 5, + rule_index: 3, + }], + predicate_calls: Vec::new(), + }; + let mut hasher = PredictionFxHasher::default(); + second.hash(&mut hasher); + arena.interner_heads.insert(hasher.finish(), first_id); + + let second_id = arena.intern(second.clone()); + assert_ne!(second_id, first_id); + assert_eq!(arena.intern(second), second_id); + assert_eq!(arena.records.len(), 2); + assert_eq!(arena.interner_next.len(), 2); + } + + #[test] + fn config_set_keeps_distinct_prediction_provenance() { + let mut arena = ContextArena::new(); + let mut provenance = PredictionSemanticProvenanceArena::default(); + let mut workspace = PredictionWorkspace::default(); + let mut first = AtnConfig::new(1, 1, EMPTY_CONTEXT, &arena); + first.enter_prediction_rule(&mut provenance, 4, 2); + let mut second = AtnConfig::new(1, 1, EMPTY_CONTEXT, &arena); + second.enter_prediction_rule(&mut provenance, 5, 3); + let mut set = AtnConfigSet::new(); + + assert!(set.add(first.clone(), &mut arena, &mut workspace)); + assert!(set.add(second, &mut arena, &mut workspace)); + assert!(!set.add(first, &mut arena, &mut workspace)); + assert_eq!(set.len(), 2); + assert_eq!(set.config_index.len(), set.len()); + + set.remap_contexts(&[EMPTY_CONTEXT], &arena); + assert_eq!(set.config_index.len(), set.len()); + } + + #[cfg(target_pointer_width = "64")] + #[test] + fn parser_config_hot_path_layout_stays_compact() { + let debug_generation = if cfg!(debug_assertions) { + size_of::() + } else { + 0 + }; + assert!(size_of::() <= 64 + debug_generation); + assert!(size_of::() <= 56); + } + #[test] fn workspace_drops_pathological_capacity() { let mut workspace = PredictionWorkspace::default(); diff --git a/src/snapshots/antlr4_runtime__parser__tests__committed_bail_error_notifies_error_listener.snap b/src/snapshots/antlr4_runtime__parser__tests__committed_bail_error_notifies_error_listener.snap new file mode 100644 index 00000000..e0e73a03 --- /dev/null +++ b/src/snapshots/antlr4_runtime__parser__tests__committed_bail_error_notifies_error_listener.snap @@ -0,0 +1,42 @@ +--- +source: src/parser.rs +expression: "(error, diagnostics)" +--- +( + ParserError { + line: 3, + column: 5, + message: "mismatched input 'y' expecting 'x'", + offending: Some( + TokenId( + 0, + ), + ), + }, + [ + RecordedDiagnostic { + grammar_file_name: "Mini.g4", + offending_text: Some( + "y", + ), + line: 3, + column: 5, + span: Some( + 0..1, + ), + message: "mismatched input 'y' expecting 'x'", + error: Some( + ParserError { + line: 3, + column: 5, + message: "mismatched input 'y' expecting 'x'", + offending: Some( + TokenId( + 0, + ), + ), + }, + ), + }, + ], +) diff --git a/src/snapshots/antlr4_runtime__parser__tests__committed_left_recursive_depth_cap_keeps_listener_events_balanced.snap b/src/snapshots/antlr4_runtime__parser__tests__committed_left_recursive_depth_cap_keeps_listener_events_balanced.snap new file mode 100644 index 00000000..6580483e --- /dev/null +++ b/src/snapshots/antlr4_runtime__parser__tests__committed_left_recursive_depth_cap_keeps_listener_events_balanced.snap @@ -0,0 +1,11 @@ +--- +source: src/parser.rs +expression: "(error.to_string(),\nevents.lock().expect(\"parse-listener event lock\").as_slice(),)" +--- +( + "parser error at 1:0: rule nesting depth limit of 1 exceeded", + [ + "enter:0", + "exit:0", + ], +) diff --git a/src/snapshots/antlr4_runtime__parser__tests__committed_predicate_star_loop_uses_single_token_deletion.snap b/src/snapshots/antlr4_runtime__parser__tests__committed_predicate_star_loop_uses_single_token_deletion.snap new file mode 100644 index 00000000..4dd9652b --- /dev/null +++ b/src/snapshots/antlr4_runtime__parser__tests__committed_predicate_star_loop_uses_single_token_deletion.snap @@ -0,0 +1,17 @@ +--- +source: src/parser.rs +expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" +--- +[ + RecordedDiagnostic { + grammar_file_name: "Mini.g4", + offending_text: Some( + "x", + ), + line: 1, + column: 0, + span: None, + message: "extraneous input 'x' expecting {, 'x'}", + error: None, + }, +] diff --git a/src/snapshots/antlr4_runtime__parser__tests__committed_walker_dispatches_recovery_diagnostics.snap b/src/snapshots/antlr4_runtime__parser__tests__committed_walker_dispatches_recovery_diagnostics.snap new file mode 100644 index 00000000..13cd2f4a --- /dev/null +++ b/src/snapshots/antlr4_runtime__parser__tests__committed_walker_dispatches_recovery_diagnostics.snap @@ -0,0 +1,17 @@ +--- +source: src/parser.rs +expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" +--- +[ + RecordedDiagnostic { + grammar_file_name: "Mini.g4", + offending_text: Some( + "y", + ), + line: 1, + column: 0, + span: None, + message: "extraneous input 'y' expecting ", + error: None, + }, +] diff --git a/src/xpath/generated/x_path_lexer.rs b/src/xpath/generated/x_path_lexer.rs index 873387d6..7a275bac 100644 --- a/src/xpath/generated/x_path_lexer.rs +++ b/src/xpath/generated/x_path_lexer.rs @@ -1,6 +1,6 @@ // @generated by antlr-rust-runtime v0.25.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(1, "0.25.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(3, "0.25.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index e4832cc1..698208bd 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -302,6 +302,17 @@ fn generated_modules_enforce_codegen_api_compatibility() { "__antlr4_rust_require_codegen_api!({},", antlr4_runtime::__ANTLR4_RUST_CODEGEN_API ); + for revision in [1, 2] { + let supported = format!("__antlr4_rust_require_codegen_api!({revision},"); + let mut supported_parser = parser.clone(); + let check_start = supported_parser + .find(¤t) + .expect("parser check should contain the current API revision"); + supported_parser.replace_range(check_start..check_start + current.len(), &supported); + fs::write(&parser_path, supported_parser).expect("parser should be writable"); + assert_generated_modules_compile(temp.path(), &modules); + } + let unsupported = "__antlr4_rust_require_codegen_api!(999,"; let mut incompatible_parser = parser; let check_start = incompatible_parser @@ -322,10 +333,9 @@ fn generated_modules_enforce_codegen_api_compatibility() { .take(2) .collect::>() .join("\n"); - let supported_revision = antlr4_runtime::__ANTLR4_RUST_CODEGEN_API; assert!( - diagnostic.contains(&format!("supports revision {supported_revision}")), - "diagnostic should name runtime revision {supported_revision}: {diagnostic}" + diagnostic.contains("supports revisions 1, 2, and 3"), + "diagnostic should name every supported revision: {diagnostic}" ); insta::assert_snapshot!( "generated_codegen_api_mismatch_diagnostic", @@ -4978,6 +4988,413 @@ mod recog_receiver_tests { ); } +#[allow(clippy::disallowed_methods)] // `insta` assertion macros unwrap internal I/O. +#[test] +fn named_parser_actions_run_at_committed_positions_on_both_parser_paths() { + let temp = temporary_directory("named-parser-actions"); + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/antlr4-rust-gen/parser-action-hooks"); + let out = temp.path().join("generated"); + + let output = run_antlr4_rust_gen(&[ + fixture.join("ActionTiming.g4").as_os_str(), + OsStr::new("--sem-patterns"), + fixture.join("patterns.toml").as_os_str(), + OsStr::new("--sem-unknown"), + OsStr::new("error"), + OsStr::new("--require-full-semantics"), + OsStr::new("--out-dir"), + out.as_os_str(), + ]); + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + utf8(&output.stdout), + utf8(&output.stderr) + ); + + let manifest = + fs::read_to_string(out.join("semantics.json")).expect("manifest should be emitted"); + insta::assert_snapshot!("named_parser_actions_semantics_manifest", manifest); + + let parser = + fs::read_to_string(out.join("action_timing_parser.rs")).expect("parser should be emitted"); + for expected in [ + "pub trait ActionTimingParserHooks", + "fn enter", + "fn enter_scope", + "fn exit_scope", + "fn tick", + "fn seed", + "fn reduce", + "fn middle", + "match (action.rule_index(), action.action_index())", + "parser_action_at_current_indexed", + "parser_action_hook_with_context", + "parser_action_hook_with_context_and_local", + "action_indices: &[(", + ] { + assert!(parser.contains(expected), "missing {expected:?}\n{parser}"); + } + + let test_source = r####" +#[cfg(test)] +mod named_action_tests { + use super::action_timing_lexer::ActionTimingLexer; + use super::action_timing_parser::{ + ActionTimingParser, ActionTimingParserHooks, ActionTimingParserTypedHooks, + }; + use antlr4_runtime::{ + AntlrError, CommonTokenStream, InputStream, Parser as _, ParserSemCtx, TokenSource, + }; + use std::cell::RefCell; + use std::rc::Rc; + + #[derive(Default)] + struct Hooks { + entered: usize, + events: Rc>>, + } + + impl ActionTimingParserHooks for Hooks { + fn enter( + &mut self, + _ctx: &mut ParserSemCtx<'_, L>, + name: &str, + level: i64, + enabled: bool, + ) where + L: TokenSource, + { + self.entered += 1; + self.events + .borrow_mut() + .push(format!("enter:{name}:{level}:{enabled}")); + } + + fn is_entered(&mut self, _ctx: &mut ParserSemCtx<'_, L>) -> bool + where + L: TokenSource, + { + let result = self.entered > 0; + self.events.borrow_mut().push(format!("predicate:{result}")); + result + } + + fn enter_scope(&mut self, _ctx: &mut ParserSemCtx<'_, L>) + where + L: TokenSource, + { + self.events.borrow_mut().push("scope+".to_owned()); + } + + fn exit_scope(&mut self, _ctx: &mut ParserSemCtx<'_, L>) + where + L: TokenSource, + { + self.events.borrow_mut().push("scope-".to_owned()); + } + + fn tick(&mut self, _ctx: &mut ParserSemCtx<'_, L>, value: i64) + where + L: TokenSource, + { + self.events.borrow_mut().push(format!("tick:{value}")); + } + + fn lose(&mut self, _ctx: &mut ParserSemCtx<'_, L>) + where + L: TokenSource, + { + self.events.borrow_mut().push("lose".to_owned()); + } + + fn seed(&mut self, _ctx: &mut ParserSemCtx<'_, L>) + where + L: TokenSource, + { + self.events.borrow_mut().push("seed".to_owned()); + } + + fn reduce(&mut self, _ctx: &mut ParserSemCtx<'_, L>) + where + L: TokenSource, + { + self.events.borrow_mut().push("reduce".to_owned()); + } + + fn exit(&mut self, _ctx: &mut ParserSemCtx<'_, L>, name: &str) + where + L: TokenSource, + { + self.entered -= 1; + self.events.borrow_mut().push(format!("exit:{name}")); + } + + fn middle(&mut self, _ctx: &mut ParserSemCtx<'_, L>, name: &str) + where + L: TokenSource, + { + self.events.borrow_mut().push(name.to_owned()); + } + + fn observe_argument(&mut self, ctx: &mut ParserSemCtx<'_, L>) + where + L: TokenSource, + { + let value = ctx + .local_int_arg() + .expect("the parameterized rule argument should be visible"); + self.events.borrow_mut().push(format!("argument:{value}")); + } + } + + type TestParser = ActionTimingParser< + ActionTimingLexer, + ActionTimingParserTypedHooks, + >; + type Entry = fn(&mut TestParser) -> Result; + + #[derive(Debug, Eq, PartialEq)] + struct Outcome { + events: Vec, + syntax_errors: usize, + text: String, + } + + fn run(input: &str, entry: Entry) -> Outcome { + let events = Rc::new(RefCell::new(Vec::new())); + let lexer = ActionTimingLexer::new(InputStream::new(input)); + let mut parser = ActionTimingParser::with_typed_hooks( + CommonTokenStream::new(lexer), + Hooks { + entered: 0, + events: Rc::clone(&events), + }, + ); + parser.remove_error_listeners(); + let root = entry(&mut parser).expect("fixture input should parse"); + let outcome = Outcome { + events: events.borrow().clone(), + syntax_errors: parser.number_of_syntax_errors(), + text: parser.node(root).text(), + }; + outcome + } + + #[test] + fn generated_and_interpreted_order_match() { + let input = "nest item more b x+y+z"; + let generated = run(input, TestParser::generated); + let interpreted = run(input, TestParser::interpreted); + + assert_eq!(generated, interpreted); + assert_eq!( + generated.events, + [ + "enter:outer:1:true", + "predicate:true", + "scope+", + "scope+", + "scope-", + "scope-", + "tick:7", + "tick:7", + "seed", + "reduce", + "reduce", + "exit:outer", + ] + ); + assert_eq!(generated.syntax_errors, 0); + assert_eq!(generated.text, "nestitemmorebx+y+z"); + assert!( + !generated.events.iter().any(|event| event == "lose"), + "the action in the losing alternative must not run" + ); + } + + #[test] + fn parameterized_rule_arguments_reach_both_action_paths() { + let generated = run("", TestParser::generated_argument); + let interpreted = run("", TestParser::interpreted_argument); + + assert_eq!(generated.events, ["argument:17"]); + assert_eq!(interpreted.events, ["argument:23"]); + assert_eq!(generated.text, ""); + assert_eq!(interpreted.text, ""); + } + + #[test] + fn recovery_before_and_after_an_action_matches() { + for input in ["c a b c", "a b a c"] { + let generated = run(input, TestParser::recover_generated); + let interpreted = run(input, TestParser::recover_interpreted); + + assert_eq!(generated, interpreted, "input {input:?}"); + assert_eq!(generated.events, ["middle"], "input {input:?}"); + assert_eq!(generated.syntax_errors, 1, "input {input:?}"); + } + } +} +"####; + + assert_generated_project( + temp.path(), + &["action_timing_lexer.rs", "action_timing_parser.rs"], + test_source, + ); +} + +#[test] +fn forwarded_parser_rule_arguments_reach_named_actions() { + let fixture = Path::new(env!("CARGO_MANIFEST_DIR")) + .join("tests/fixtures/antlr4-rust-gen/parser-action-forwarded-args"); + let temp = temporary_directory("parser-action-forwarded-args"); + let out = temp.path().join("generated"); + let output = run_antlr4_rust_gen(&[ + fixture.join("Forwarded.g4").as_os_str(), + OsStr::new("--sem-patterns"), + fixture.join("patterns.toml").as_os_str(), + OsStr::new("--sem-unknown"), + OsStr::new("error"), + OsStr::new("--require-full-semantics"), + OsStr::new("--out-dir"), + out.as_os_str(), + ]); + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + utf8(&output.stdout), + utf8(&output.stderr) + ); + + let parser = + fs::read_to_string(out.join("forwarded_parser.rs")).expect("parser should be emitted"); + assert!( + parser.contains("parser_action_hook_with_context_and_local"), + "{parser}" + ); + assert!(parser.contains("inherit_local: true"), "{parser}"); + + let test_source = r####" +#[cfg(test)] +mod forwarded_argument_tests { + use super::forwarded_lexer::ForwardedLexer; + use super::forwarded_parser::{ + ForwardedParser, ForwardedParserHooks, ForwardedParserTypedHooks, + }; + use antlr4_runtime::{ + AntlrError, CommonTokenStream, InputStream, ParserSemCtx, TokenSource, + }; + use std::cell::RefCell; + use std::rc::Rc; + + #[derive(Default)] + struct Hooks { + values: Rc>>, + } + + impl ForwardedParserHooks for Hooks { + fn observe_argument(&mut self, ctx: &mut ParserSemCtx<'_, L>) + where + L: TokenSource, + { + self.values.borrow_mut().push( + ctx.local_int_arg() + .expect("forwarded parser argument should be visible"), + ); + } + } + + type TestParser = + ForwardedParser, ForwardedParserTypedHooks>; + type Entry = fn(&mut TestParser) -> Result; + + fn run(entry: Entry) -> Vec { + let values = Rc::new(RefCell::new(Vec::new())); + let lexer = ForwardedLexer::new(InputStream::new("")); + let mut parser = ForwardedParser::with_typed_hooks( + CommonTokenStream::new(lexer), + Hooks { + values: Rc::clone(&values), + }, + ); + entry(&mut parser).expect("fixture input should parse"); + let result = values.borrow().clone(); + result + } + + #[test] + fn generated_and_interpreted_paths_forward_arguments() { + assert_eq!(run(TestParser::generated), [29]); + assert_eq!(run(TestParser::interpreted), [31]); + } +} +"####; + + assert_generated_project( + temp.path(), + &["forwarded_lexer.rs", "forwarded_parser.rs"], + test_source, + ); +} + +#[allow(clippy::disallowed_methods)] // `insta` assertion macros unwrap internal I/O. +#[test] +fn parser_action_hook_signatures_reject_normalized_conflicts() { + let temp = temporary_directory("parser-action-signature-conflict"); + let grammar = temp.path().join("Conflict.g4"); + let patterns = temp.path().join("patterns.toml"); + let out = temp.path().join("generated"); + fs::write( + &grammar, + "grammar Conflict;\n\ + start: {this.Mark(\"x\");} {this.Mark(1);} A EOF;\n\ + A: 'a';\n", + ) + .expect("grammar should be writable"); + fs::write( + &patterns, + "version = 1\n\ + [[helper]]\n\ + kind = \"parser-action\"\n\ + name = \"Mark\"\n\ + arguments = \"string\"\n\ + returns = \"unit\"\n\ + lower = \"hook\"\n\ + [[helper]]\n\ + kind = \"parser-action\"\n\ + name = \"Mark\"\n\ + arguments = \"integer\"\n\ + returns = \"unit\"\n\ + lower = \"hook\"\n", + ) + .expect("semantic patterns should be writable"); + + let output = run_antlr4_rust_gen(&[ + grammar.as_os_str(), + OsStr::new("--sem-patterns"), + patterns.as_os_str(), + OsStr::new("--out-dir"), + out.as_os_str(), + ]); + assert!(!output.status.success(), "conflicting hooks should fail"); + let stderr = utf8(&output.stderr); + assert!( + !stderr.contains(&temp.path().display().to_string()), + "diagnostic must not expose temporary paths: {stderr}" + ); + insta::assert_snapshot!( + "parser_action_hook_signature_conflict_diagnostic", + normalize_current_package_version(stderr) + ); + assert!( + !out.exists(), + "failed generation must not leave partial output" + ); +} + #[test] fn imported_lexer_action_generates_typed_hook_from_structural_body() { let temp = temporary_directory("imported-lexer-hook"); diff --git a/tests/fixtures/antlr4-rust-gen/parser-action-forwarded-args/Forwarded.g4 b/tests/fixtures/antlr4-rust-gen/parser-action-forwarded-args/Forwarded.g4 new file mode 100644 index 00000000..2ed7479e --- /dev/null +++ b/tests/fixtures/antlr4-rust-gen/parser-action-forwarded-args/Forwarded.g4 @@ -0,0 +1,23 @@ +grammar Forwarded; + +generated + : forwarding[29] EOF + ; + +interpreted + : force[2147483648] forwarding[31] EOF + ; + +forwarding[int parent_arg] + : forwarded[parent_arg] + ; + +forwarded[int value] + : {ObserveArgument();} + ; + +force[int ignored] + : + ; + +WS: [ \t\r\n]+ -> skip; diff --git a/tests/fixtures/antlr4-rust-gen/parser-action-forwarded-args/patterns.toml b/tests/fixtures/antlr4-rust-gen/parser-action-forwarded-args/patterns.toml new file mode 100644 index 00000000..2ff853e1 --- /dev/null +++ b/tests/fixtures/antlr4-rust-gen/parser-action-forwarded-args/patterns.toml @@ -0,0 +1,7 @@ +version = 1 + +[[helper]] +kind = "parser-action" +lower = "hook" +name = "ObserveArgument" +returns = "unit" diff --git a/tests/fixtures/antlr4-rust-gen/parser-action-hooks/ActionTiming.g4 b/tests/fixtures/antlr4-rust-gen/parser-action-hooks/ActionTiming.g4 new file mode 100644 index 00000000..4c2533cc --- /dev/null +++ b/tests/fixtures/antlr4-rust-gen/parser-action-hooks/ActionTiming.g4 @@ -0,0 +1,67 @@ +grammar ActionTiming; + +generated + : body EOF + ; + +interpreted + : force[2147483648] body EOF + ; + +recoverGenerated + : recoveryBody EOF + ; + +recoverInterpreted + : force[2147483648] recoveryBody EOF + ; + +generatedArgument + : parameterized[17] EOF + ; + +interpretedArgument + : force[2147483648] parameterized[23] EOF + ; + +parameterized[int value] + : {ObserveArgument();} + ; + +body + : {recog.Enter("outer", 1, true);} + {this.IsEntered()}? + nested + ({Tick(7);} ID)+ + (({Lose();} A) | B) + expression + {self.Exit("outer");} + ; + +nested + : {EnterScope();} child {ExitScope();} + ; + +child + : {EnterScope();} ID {ExitScope();} + ; + +expression + : expression PLUS ID {Reduce();} + | ID {Seed();} + ; + +recoveryBody + : A {Middle("middle");} B C + ; + +force[int ignored] + : + ; + +A: 'a'; +B: 'b'; +C: 'c'; +PLUS: '+'; +ID: [d-z]+; +WS: [ \t\r\n]+ -> skip; diff --git a/tests/fixtures/antlr4-rust-gen/parser-action-hooks/patterns.toml b/tests/fixtures/antlr4-rust-gen/parser-action-hooks/patterns.toml new file mode 100644 index 00000000..b875ab09 --- /dev/null +++ b/tests/fixtures/antlr4-rust-gen/parser-action-hooks/patterns.toml @@ -0,0 +1,72 @@ +version = 1 + +[[helper]] +arguments = "string, integer, bool" +kind = "parser-action" +lower = "hook" +name = "Enter" +receiver = "recog" +returns = "unit" + +[[helper]] +kind = "parser-predicate" +lower = "hook" +name = "IsEntered" +returns = "bool" + +[[helper]] +kind = "parser-action" +lower = "hook" +name = "EnterScope" +returns = "unit" + +[[helper]] +kind = "parser-action" +lower = "hook" +name = "ExitScope" +returns = "unit" + +[[helper]] +arguments = "integer" +kind = "parser-action" +lower = "hook" +name = "Tick" +returns = "unit" + +[[helper]] +kind = "parser-action" +lower = "hook" +name = "Lose" +returns = "unit" + +[[helper]] +kind = "parser-action" +lower = "hook" +name = "Seed" +returns = "unit" + +[[helper]] +kind = "parser-action" +lower = "hook" +name = "Reduce" +returns = "unit" + +[[helper]] +arguments = "string" +kind = "parser-action" +lower = "hook" +name = "Exit" +returns = "unit" + +[[helper]] +arguments = "string" +kind = "parser-action" +lower = "hook" +name = "Middle" +returns = "unit" + +[[helper]] +kind = "parser-action" +lower = "hook" +name = "ObserveArgument" +returns = "unit" diff --git a/tests/fixtures/antlr4-rust-gen/unsupported-rule-argument/T.g4 b/tests/fixtures/antlr4-rust-gen/unsupported-rule-argument/T.g4 new file mode 100644 index 00000000..fbad7dd9 --- /dev/null +++ b/tests/fixtures/antlr4-rust-gen/unsupported-rule-argument/T.g4 @@ -0,0 +1,11 @@ +grammar T; + +start + : child[1 + 2] EOF + ; + +child[int value] + : ID + ; + +ID: 'x'; diff --git a/tests/snapshots/antlr4_rust_gen_cli__antlr4rust_compat_semantics_manifest.snap b/tests/snapshots/antlr4_rust_gen_cli__antlr4rust_compat_semantics_manifest.snap index b4ca7457..03c82044 100644 --- a/tests/snapshots/antlr4_rust_gen_cli__antlr4rust_compat_semantics_manifest.snap +++ b/tests/snapshots/antlr4_rust_gen_cli__antlr4rust_compat_semantics_manifest.snap @@ -19,10 +19,10 @@ expression: manifest "coordinates": [ {"kind": "parser-predicate", "rule": "assignment", "rule_index": 0, "index": 0, "atn_state": null, "line": 6, "column": 6, "body": "let __not_code = r#\"recog.input.peek(1) _localctx.context()\"#; // recog.output.write(\"not code\")...", "disposition": "translated", "template": "Embedded"}, {"kind": "parser-predicate", "rule": "history", "rule_index": 1, "index": 1, "atn_state": null, "line": 27, "column": 28, "body": "let mut __bk: isize = 1; let (__last_type, __last_text) = recog.input.lt(-__bk) .map(|t| (t.get_...", "disposition": "translated", "template": "Embedded"}, - {"kind": "parser-action", "rule": "inlineAction", "rule_index": 2, "index": null, "atn_state": 24, "line": 44, "column": 17, "body": "let __ek: isize = 1; let __text = recog.input.lt(-__ek) .map(|t| t.get_text().to_owned()) .unwra...", "disposition": "translated", "template": "Embedded"}, + {"kind": "parser-action", "rule": "inlineAction", "rule_index": 2, "index": 0, "atn_state": 24, "line": 44, "column": 17, "body": "let __ek: isize = 1; let __text = recog.input.lt(-__ek) .map(|t| t.get_text().to_owned()) .unwra...", "disposition": "translated", "template": "Embedded"}, {"kind": "parser-predicate", "rule": "nativeAssignment", "rule_index": 3, "index": 2, "atn_state": null, "line": 55, "column": 6, "body": "let __first = self.base.token_stream().lt(1); let __first_type = __first.map(|t| t.token_type())...", "disposition": "translated", "template": "Embedded"}, {"kind": "parser-predicate", "rule": "nativeHistory", "rule_index": 4, "index": 3, "atn_state": null, "line": 79, "column": 28, "body": "let mut __bk: isize = 1; let (__last_type, __last_text) = self.base.token_stream().lt(-__bk) .ma...", "disposition": "translated", "template": "Embedded"}, - {"kind": "parser-action", "rule": "nativeInlineAction", "rule_index": 5, "index": null, "atn_state": 39, "line": 96, "column": 17, "body": "let __ek: isize = 1; let __text = self.base.token_stream().lt(-__ek) .map(|t| t.text_or_empty()....", "disposition": "translated", "template": "Embedded"} + {"kind": "parser-action", "rule": "nativeInlineAction", "rule_index": 5, "index": 1, "atn_state": 39, "line": 96, "column": 17, "body": "let __ek: isize = 1; let __text = self.base.token_stream().lt(-__ek) .map(|t| t.text_or_empty()....", "disposition": "translated", "template": "Embedded"} ] }, { diff --git a/tests/snapshots/antlr4_rust_gen_cli__generated_codegen_api_checks.snap b/tests/snapshots/antlr4_rust_gen_cli__generated_codegen_api_checks.snap index dbb77be2..901d825b 100644 --- a/tests/snapshots/antlr4_rust_gen_cli__generated_codegen_api_checks.snap +++ b/tests/snapshots/antlr4_rust_gen_cli__generated_codegen_api_checks.snap @@ -2,5 +2,5 @@ source: tests/antlr4_rust_gen_cli.rs expression: normalize_current_package_version(&checks) --- -lexer: antlr4_runtime::__antlr4_rust_require_codegen_api!(1, ""); -parser: antlr4_runtime::__antlr4_rust_require_codegen_api!(1, ""); +lexer: antlr4_runtime::__antlr4_rust_require_codegen_api!(3, ""); +parser: antlr4_runtime::__antlr4_rust_require_codegen_api!(3, ""); diff --git a/tests/snapshots/antlr4_rust_gen_cli__generated_codegen_api_mismatch_diagnostic.snap b/tests/snapshots/antlr4_rust_gen_cli__generated_codegen_api_mismatch_diagnostic.snap index 007c5171..b2d79f05 100644 --- a/tests/snapshots/antlr4_rust_gen_cli__generated_codegen_api_mismatch_diagnostic.snap +++ b/tests/snapshots/antlr4_rust_gen_cli__generated_codegen_api_mismatch_diagnostic.snap @@ -2,5 +2,5 @@ source: tests/antlr4_rust_gen_cli.rs expression: normalize_current_package_version(&diagnostic) --- -error: antlr4-rust generated-code API mismatch: antlr4-rust-gen v emitted generated-code API revision 999, but the selected antlr-rust-runtime supports revision 1; regenerate this recognizer with a compatible antlr4-rust-gen or select a compatible antlr-rust-runtime dependency +error: antlr4-rust generated-code API mismatch: antlr4-rust-gen v emitted generated-code API revision 999, but the selected antlr-rust-runtime supports revisions 1, 2, and 3; regenerate this recognizer with a compatible antlr4-rust-gen or select a compatible antlr-rust-runtime dependency --> src/codegen_api_parser.rs:3:1 diff --git a/tests/snapshots/antlr4_rust_gen_cli__named_parser_actions_semantics_manifest.snap b/tests/snapshots/antlr4_rust_gen_cli__named_parser_actions_semantics_manifest.snap new file mode 100644 index 00000000..de06a69f --- /dev/null +++ b/tests/snapshots/antlr4_rust_gen_cli__named_parser_actions_semantics_manifest.snap @@ -0,0 +1,37 @@ +--- +source: tests/antlr4_rust_gen_cli.rs +expression: manifest +--- +{ + "version": 2, + "policy": "error", + "note": "unknown coordinates currently default to assume-true; a future minor release changes the default to error", + "options": [], + "grammars": [ + { + "kind": "lexer", + "name": "ActionTimingLexer", + "coordinates": [] + }, + { + "kind": "parser", + "name": "ActionTimingParser", + "coordinates": [ + {"kind": "parser-action", "rule": "parameterized", "rule_index": 6, "index": 0, "atn_state": 47, "line": 28, "column": 6, "body": "ObserveArgument();", "disposition": "hooked", "template": "Hook(observe_argument)"}, + {"kind": "parser-predicate", "rule": "body", "rule_index": 7, "index": 0, "atn_state": null, "line": 33, "column": 6, "body": "this.IsEntered()", "disposition": "hooked", "template": "Hook"}, + {"kind": "parser-action", "rule": "body", "rule_index": 7, "index": 1, "atn_state": 49, "line": 32, "column": 6, "body": "recog.Enter(\"outer\", 1, true);", "disposition": "hooked", "template": "Hook(enter)"}, + {"kind": "parser-action", "rule": "body", "rule_index": 7, "index": 2, "atn_state": 52, "line": 35, "column": 7, "body": "Tick(7);", "disposition": "hooked", "template": "Hook(tick)"}, + {"kind": "parser-action", "rule": "body", "rule_index": 7, "index": 3, "atn_state": 58, "line": 36, "column": 8, "body": "Lose();", "disposition": "hooked", "template": "Hook(lose)"}, + {"kind": "parser-action", "rule": "body", "rule_index": 7, "index": 4, "atn_state": 64, "line": 38, "column": 6, "body": "self.Exit(\"outer\");", "disposition": "hooked", "template": "Hook(exit)"}, + {"kind": "parser-action", "rule": "nested", "rule_index": 8, "index": 5, "atn_state": 66, "line": 42, "column": 6, "body": "EnterScope();", "disposition": "hooked", "template": "Hook(enter_scope)"}, + {"kind": "parser-action", "rule": "nested", "rule_index": 8, "index": 6, "atn_state": 68, "line": 42, "column": 28, "body": "ExitScope();", "disposition": "hooked", "template": "Hook(exit_scope)"}, + {"kind": "parser-action", "rule": "child", "rule_index": 9, "index": 7, "atn_state": 70, "line": 46, "column": 6, "body": "EnterScope();", "disposition": "hooked", "template": "Hook(enter_scope)"}, + {"kind": "parser-action", "rule": "child", "rule_index": 9, "index": 8, "atn_state": 72, "line": 46, "column": 25, "body": "ExitScope();", "disposition": "hooked", "template": "Hook(exit_scope)"}, + {"kind": "parser-action", "rule": "expression", "rule_index": 10, "index": 9, "atn_state": 74, "line": null, "column": null, "body": null, "disposition": "synthetic", "template": null}, + {"kind": "parser-action", "rule": "expression", "rule_index": 10, "index": 10, "atn_state": 76, "line": 51, "column": 9, "body": "Seed();", "disposition": "hooked", "template": "Hook(seed)"}, + {"kind": "parser-action", "rule": "expression", "rule_index": 10, "index": 11, "atn_state": 81, "line": 50, "column": 25, "body": "Reduce();", "disposition": "hooked", "template": "Hook(reduce)"}, + {"kind": "parser-action", "rule": "recoveryBody", "rule_index": 11, "index": 12, "atn_state": 88, "line": 55, "column": 8, "body": "Middle(\"middle\");", "disposition": "hooked", "template": "Hook(middle)"} + ] + } + ] +} diff --git a/tests/snapshots/antlr4_rust_gen_cli__parser_action_hook_signature_conflict_diagnostic.snap b/tests/snapshots/antlr4_rust_gen_cli__parser_action_hook_signature_conflict_diagnostic.snap new file mode 100644 index 00000000..a9da6f3c --- /dev/null +++ b/tests/snapshots/antlr4_rust_gen_cli__parser_action_hook_signature_conflict_diagnostic.snap @@ -0,0 +1,5 @@ +--- +source: tests/antlr4_rust_gen_cli.rs +expression: normalize_current_package_version(&stderr) +--- +Error: Custom { kind: InvalidData, error: "typed semantic helper Mark has conflicting literal signatures [String] and [Integer]" } diff --git a/third_party/antlr-v4-grammar/self-hosted.sha256 b/third_party/antlr-v4-grammar/self-hosted.sha256 index 9a68d599..b70c5a0f 100644 --- a/third_party/antlr-v4-grammar/self-hosted.sha256 +++ b/third_party/antlr-v4-grammar/self-hosted.sha256 @@ -2,5 +2,5 @@ 1286e542499e4480b3ab5ff60e4a4a7faf21134ca4c4f8f1f468f30095fa25cb third_party/antlr-v4-grammar/ANTLRv4Parser.g4 c7114545a75ab294215819962e92e570383dc830fd5768463dab04e6733bcb80 third_party/antlr-v4-grammar/predefined.tokens 5803594bd2c8dd2d5180f1ca08fc70dfc80308479d18a7c4a1b743fa523b55ec third_party/antlr-v4-grammar/antlr-v4.toml -9e496a3961320a136499b6caf44c8291425efb846bc6ab8f0edc47ae184957bb src/bin_support/grammar/generated/antlr_v4_lexer.rs -229a7934eba57fd50ac872f0cdd09fe4896554531186a837ef12cedb39fb1f92 src/bin_support/grammar/generated/antlr_v4_parser.rs +68ad6c4d7c21830c8082adf992da06f906f65a46c1c1f1e0f6d15cf65decfad4 src/bin_support/grammar/generated/antlr_v4_lexer.rs +b3389cda35b4f4438e6d6ead4a127ddd8d9c8839346f5394e11baec3d05c9efd src/bin_support/grammar/generated/antlr_v4_parser.rs