diff --git a/src/bin/antlr4-runtime-testsuite.rs b/src/bin/antlr4-runtime-testsuite.rs index cff6cf81..a63ca324 100644 --- a/src/bin/antlr4-runtime-testsuite.rs +++ b/src/bin/antlr4-runtime-testsuite.rs @@ -1071,7 +1071,7 @@ fn parser_smoke_main(descriptor: &Descriptor) -> String { "" }; format!( - "pub mod generated {{\n pub mod {lexer_module};\n pub mod {parser_module};\n}}\n\nuse antlr4_runtime::{{AntlrError, CommonTokenStream, InputStream, Parser}};\nuse generated::{lexer_module}::{lexer_type};\nuse generated::{parser_module}::{parser_type};\n\nfn main() {{\n let handle = std::thread::Builder::new()\n // Runtime-suite smoke crates run deeply nested generated parser paths;\n // this is harness-only and does not change the runtime's default stack.\n .stack_size(128 * 1024 * 1024)\n .spawn(|| {{\n let lexer = {lexer_type}::new(InputStream::new(\"{}\"));\n let tokens = CommonTokenStream::new(lexer);\n let mut parser = {parser_type}::new(tokens);\n parser.set_build_parse_trees({build_parse_trees});\n parser.set_report_diagnostic_errors({report_diagnostic_errors});\n{prediction_mode} if let Err(error) = parser.{start_rule}() {{\n match error {{\n AntlrError::ParserError {{ line, column, message }} => eprintln!(\"line {{line}}:{{column}} {{message}}\"),\n other => eprintln!(\"{{other}}\"),\n }}\n }}\n }})\n .expect(\"parser smoke thread should start\");\n handle.join().expect(\"parser smoke thread should finish\");\n}}\n", + "pub mod generated {{\n pub mod {lexer_module};\n pub mod {parser_module};\n}}\n\nuse antlr4_runtime::{{AntlrError, CommonTokenStream, InputStream, Parser}};\nuse generated::{lexer_module}::{lexer_type};\nuse generated::{parser_module}::{parser_type};\n\nfn main() {{\n let handle = std::thread::Builder::new()\n // Runtime-suite smoke crates run deeply nested generated parser paths;\n // this is harness-only and does not change the runtime's default stack.\n .stack_size(128 * 1024 * 1024)\n .spawn(|| {{\n let lexer = {lexer_type}::new(InputStream::new(\"{}\"));\n let tokens = CommonTokenStream::new(lexer);\n let mut parser = {parser_type}::new(tokens);\n parser.set_build_parse_trees({build_parse_trees});\n parser.set_report_diagnostic_errors({report_diagnostic_errors});\n{prediction_mode} if let Err(error) = parser.{start_rule}() {{\n match error {{\n AntlrError::ParserError {{ line, column, message, .. }} => eprintln!(\"line {{line}}:{{column}} {{message}}\"),\n other => eprintln!(\"{{other}}\"),\n }}\n }}\n }})\n .expect(\"parser smoke thread should start\");\n handle.join().expect(\"parser smoke thread should finish\");\n}}\n", rust_string(&descriptor.input) ) } diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index d6b6f245..8258761c 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -3306,6 +3306,7 @@ where fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool {{ antlr4_runtime::Recognizer::notify_error_listeners( self, + None, source_error.line, source_error.column, &source_error.message, @@ -5184,12 +5185,13 @@ fn render_generated_rule_dispatch_with_rule_names( .copied() .unwrap_or_default() { - // The interpreted fast path never consults the depth cap, so a - // configured bound overrides the ATN preference: correctness of - // the resource limit beats the long-call-chain optimization. + // The interpreted fast path never consults the depth cap nor + // fires parse-listener events, so either feature overrides the + // ATN preference: correctness of the resource limit and listener + // coverage beat the long-call-chain optimization. writeln!( out, - " {index} if self.generated_only() || self.base.has_rule_depth_cap() => Some(self.parse_generated_rule_{index}_dispatch(precedence, allow_fallback))," + " {index} if self.generated_only() || self.base.has_rule_depth_cap() || self.base.has_parse_listeners() => Some(self.parse_generated_rule_{index}_dispatch(precedence, allow_fallback))," ) .expect("writing to a string cannot fail"); } else { @@ -5228,19 +5230,27 @@ fn render_generated_rule_dispatch_with_rule_names( // Rule nesting maps onto native call depth; sample remaining stack // capacity at the shared dispatch boundary so deeply nested input // grows onto a segmented stack instead of aborting the process. The - // optional depth-cap probe stays inline-cheap (one `Option` check - // when unset) and its error, while absorbed by rule-level recovery - // like any rule failure, stays sticky until the top-level entry - // drains it — that pairing is what actually enforces the abort. - // Plain `if let` keeps generated output edition-2021 compatible. + // optional depth-cap and parse-listener probes stay inline-cheap + // (one `Option`/emptiness check each when unused) and their errors, + // while absorbed by rule-level recovery like any rule failure, stay + // sticky until the top-level entry drains them — that pairing is + // what actually enforces the abort. The matching listener exit event + // fires from the rule body's exit paths (`finish_rule`/recovery), so + // enter/exit stay balanced. Plain `if let` keeps generated output + // edition-2021 compatible. writeln!( out, " if let Some(error) = self.base.rule_depth_cap_violation() {{\n \ return Err(GeneratedRuleError::Fatal(error));\n \ }}\n \ - if self.base.generated_rule_stack_check_due() {{\n \ + if let Some(error) = self.base.parse_listener_enter_rule({index}) {{\n \ + return Err(GeneratedRuleError::Fatal(error));\n \ + }}\n \ + let __listener_result = if self.base.generated_rule_stack_check_due() {{\n \ antlr4_runtime::grow_generated_rule_stack(|| {target_call})\n \ - }} else {{\n {target_call}\n }}" + }} else {{\n {target_call}\n }};\n \ + self.base.parse_listener_exit_rule({index});\n \ + __listener_result" ) .expect("writing to a string cannot fail"); writeln!(out, " }}").expect("writing to a string cannot fail"); @@ -5847,11 +5857,12 @@ fn render_generated_step( { // ATN-preferred child: route through `parse_rule_precedence_from_generated`. // The rule's `parse_generated_rule` dispatch arm is guarded by - // `generated_only() || has_rule_depth_cap()`: in normal uncapped - // mode the generated probe returns `None` and the wrapper parses - // the child on the INTERPRETED path (preserving the ATN-preferred - // optimization); a configured depth cap flips it to the generated - // body, which is the only path that enforces the cap. + // `generated_only() || has_rule_depth_cap() || has_parse_listeners()`: + // in the default configuration the generated probe returns `None` + // and the wrapper parses the child on the INTERPRETED path + // (preserving the ATN-preferred optimization); a configured depth + // cap or a registered parse listener flips it to the generated + // body, the only path that enforces the cap and fires events. from_generated_call } else { generated_child_call @@ -6941,12 +6952,23 @@ fn render_generated_left_recursive_loop( ) .expect("writing to a string cannot fail"); } + // Upstream fires `triggerExitRuleEvent` at the TOP of each operator-loop + // pass (`recRuleSetPrevCtx`, a StarBlock iteration op) — the outgoing + // iteration's context exits before the next expansion enters, so live + // listener depth never accumulates across a flat operator chain. The + // dispatch wrapper's single exit then plays `unrollRecursionContexts`' + // one-link walk when the rule finishes. + writeln!( + out, + "{pad} self.base.parse_listener_exit_rule({rule_index});" + ) + .expect("writing to a string cannot fail"); // Each operator iteration deepens the tree without a rule frame; probe // the depth cap BEFORE the expansion push so the boundary matches the // dispatch site (which checks before its rule-frame push): frames and // expansions are both admitted up to the cap, and token-only operator - // alternatives (no nested rule dispatch) still abort promptly instead of - // at the top-level drain. + // alternatives (no nested rule dispatch) still abort promptly instead + // of at the top-level drain. writeln!( out, "{pad} if let Some(__depth_error) = self.base.rule_depth_cap_violation() {{\n\ @@ -6959,6 +6981,17 @@ fn render_generated_left_recursive_loop( "{pad} self.base.push_new_recursion_context_with_previous({entry_state}isize, {rule_index}, &mut __ctx);" ) .expect("writing to a string cannot fail"); + // The listener enter probe fires AFTER the expansion push, mirroring + // upstream (`pushNewRecursionContext` assigns `_ctx` before + // `triggerEnterRuleEvent`): an abort here propagates through the rule's + // error paths, and the dispatch wrapper's exit keeps the pair balanced. + writeln!( + out, + "{pad} if let Some(__listener_error) = self.base.parse_listener_enter_rule({rule_index}) {{\n\ + {pad} return Err(__listener_error);\n\ + {pad} }}" + ) + .expect("writing to a string cannot fail"); render_generated_steps(out, body, indent + 3, render_context); writeln!(out, "{pad} }}").expect("writing to a string cannot fail"); writeln!(out, "{pad} {exit_alt} => break,").expect("writing to a string cannot fail"); @@ -7114,6 +7147,8 @@ const GENERATED_PARSER_RESERVED_RULE_METHODS: &[&str] = &[ "clear_dfa", "add_error_listener", "remove_error_listeners", + "add_parse_listener", + "remove_parse_listeners", "node", "into_token_stream", "into_token_store", @@ -9254,6 +9289,30 @@ fn embedded_render_slots( /// Kept as a standalone literal (single braces, no format placeholders) so the /// large parser template stays under the line-length lint and this ANTLR /// `Parser.compileParseTreePattern` analog reads as ordinary code. +/// Renders the parse-listener registration facade on the generated parser. +/// +/// Kept as a standalone literal (single braces, no format placeholders) so the +/// large parser template stays under the line-length lint. +const fn render_parse_listener_facade() -> &'static str { + r" + /// Registers a listener for committed rule enter/exit events during + /// recognition (ANTLR's `addParseListener`). See + /// [`antlr4_runtime::ParseListener`] for the delivery contract. + pub fn add_parse_listener(&mut self, listener: T) + where + T: antlr4_runtime::ParseListener + 'static, + { + self.base.add_parse_listener(listener); + } + + /// Removes every registered parse listener and returns them, dropping + /// any sticky abort a removed listener had requested. + pub fn remove_parse_listeners(&mut self) -> Vec> { + self.base.remove_parse_listeners() + } +" +} + const fn render_compile_parse_tree_pattern_method() -> &'static str { r#" /// Compiles a tree pattern rooted at parser rule `rule_index`. @@ -9323,6 +9382,7 @@ fn render_parser_with_options( let patterns = options.patterns.unwrap_or(&empty_patterns); let type_name = rust_type_name(grammar_name); let compile_pattern_method = render_compile_parse_tree_pattern_method(); + let parse_listener_facade = render_parse_listener_facade(); let metadata = render_parser_metadata(grammar_name, data); let parser_atn = data.parser_atn()?; let parser_atn_data = render_u32_slice(parser_atn.packed_words()); @@ -9657,6 +9717,7 @@ where self.base.remove_error_listeners(); }} +{parse_listener_facade} /// Fully resets parser-owned state and rewinds the current token stream. pub fn reset(&mut self) {{ self.base.reset(); @@ -9772,10 +9833,11 @@ where // are preserved so a generated parent can surface a recovered child's // fail-loud coordinate at this boundary. self.base.reset_unknown_semantic_hits(); - // Likewise drop a stale depth-cap violation: entry rules share one - // parser instance, and the sticky flag must not poison the next - // parse when the previous one exited through an error path. - let _ = self.base.take_rule_depth_error(); + // Likewise drop stale sticky aborts (depth-cap violation, + // parse-listener abort): entry rules share one parser instance, + // and the flags must not poison the next parse when the previous + // one exited through an error path. + let _ = self.base.take_parse_abort(); }} let __rule_start = antlr4_runtime::IntStream::index(self.base.input()); let __generated_only = self.generated_only(); @@ -9796,13 +9858,13 @@ where if let Some(semantic_error) = self.base.take_unknown_semantic_error() {{ return Err(semantic_error); }} - // The depth-cap violation wins over an error derived - // from it (e.g. a sync failure after recovery absorbed - // the capped rule): the caller must learn the resource - // bound was hit, and draining un-poisons the instance - // for the next entry-rule call. - if let Some(depth_error) = self.base.take_rule_depth_error() {{ - return Err(depth_error); + // A sticky abort (depth cap, listener) wins over an + // error derived from it (e.g. a sync failure after + // recovery absorbed the aborted rule): the caller must + // learn the real cause, and draining un-poisons the + // instance for the next entry-rule call. + if let Some(abort) = self.base.take_parse_abort() {{ + return Err(abort); }} }} return Err(error.into_error()); @@ -9828,10 +9890,11 @@ where if let Some(error) = self.base.take_unknown_semantic_error() {{ return Err(error); }} - // A depth-cap violation is a resource bound, not a syntax error: - // rule-level recovery may have produced a tree anyway, but the - // parse must still fail (and a reused parser must start clean). - if let Some(error) = self.base.take_rule_depth_error() {{ + // A sticky abort (depth-cap violation, listener abort) is not a + // syntax error: rule-level recovery may have produced a tree + // anyway, but the parse must still fail (and a reused parser + // must start clean). + if let Some(error) = self.base.take_parse_abort() {{ return Err(error); }} }} @@ -9908,6 +9971,10 @@ where fn set_prediction_mode(&mut self, mode: antlr4_runtime::PredictionMode) {{ self.base.set_prediction_mode(mode); }} fn max_rule_depth(&self) -> Option {{ self.base.max_rule_depth() }} fn set_max_rule_depth(&mut self, depth: Option) {{ self.base.set_max_rule_depth(depth); }} + // Route through the trait impl: BaseParser's inherent generic method + // would re-box the already-boxed listener. + fn add_parse_listener(&mut self, listener: Box) {{ antlr4_runtime::Parser::add_parse_listener(&mut self.base, listener); }} + fn remove_parse_listeners(&mut self) -> Vec> {{ self.base.remove_parse_listeners() }} }} {generated_footer}"# )) @@ -12036,6 +12103,8 @@ mod tests { "clearDfa".to_owned(), "addErrorListener".to_owned(), "removeErrorListeners".to_owned(), + "addParseListener".to_owned(), + "removeParseListeners".to_owned(), "compileParseTreePattern".to_owned(), "regularRule".to_owned(), ]; @@ -12051,6 +12120,8 @@ mod tests { "clear_dfa_rule", "add_error_listener_rule", "remove_error_listeners_rule", + "add_parse_listener_rule", + "remove_parse_listeners_rule", "compile_parse_tree_pattern_rule", "regular_rule" ] @@ -12601,7 +12672,7 @@ mod tests { // A configured depth cap overrides the ATN preference: only generated // bodies enforce the bound, so the guard admits either trigger. assert!(rendered.contains( - "0 if self.generated_only() || self.base.has_rule_depth_cap() => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))" + "0 if self.generated_only() || self.base.has_rule_depth_cap() || self.base.has_parse_listeners() => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))" )); assert!(!rendered.contains( "0 => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))" diff --git a/src/bin_support/grammar/frontend.rs b/src/bin_support/grammar/frontend.rs index 71e2bebc..31bb3ed1 100644 --- a/src/bin_support/grammar/frontend.rs +++ b/src/bin_support/grammar/frontend.rs @@ -959,6 +959,7 @@ where fn syntax_error( &mut self, _recognizer: &R, + _offending: Option>, line: usize, column: usize, message: &str, diff --git a/src/bin_support/grammar/generated/antlr_v4_lexer.rs b/src/bin_support/grammar/generated/antlr_v4_lexer.rs index 985f706c..0a5fe693 100644 --- a/src/bin_support/grammar/generated/antlr_v4_lexer.rs +++ b/src/bin_support/grammar/generated/antlr_v4_lexer.rs @@ -390,6 +390,7 @@ where fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool { antlr4_runtime::Recognizer::notify_error_listeners( self, + None, source_error.line, source_error.column, &source_error.message, diff --git a/src/errors.rs b/src/errors.rs index 64ae9f0e..25bab40c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,4 +1,5 @@ use crate::recognizer::Recognizer; +use crate::token::{TokenId, TokenView}; use thiserror::Error; #[derive(Debug, Error, Clone, Eq, PartialEq)] @@ -18,6 +19,12 @@ pub enum AntlrError { line: usize, column: usize, message: String, + /// Token the error is anchored to, when one exists. The anchor must + /// be captured where the error is built: prediction restores the + /// input cursor, so the current lookahead at reporting time is not + /// necessarily the offending token (`no viable alternative` anchors + /// at the error index while the cursor sits at the decision start). + offending: Option, }, #[error("unsupported runtime feature: {0}")] Unsupported(String), @@ -30,9 +37,15 @@ pub enum AntlrError { /// generically, as [`ConsoleErrorListener`] does, when a listener will be /// registered. pub trait ErrorListener { + /// `offending` carries the token the diagnostic points at, matching + /// ANTLR's `syntaxError(recognizer, offendingSymbol, ...)`. It is `None` + /// for lexer errors (no token was produced) and for diagnostics that are + /// not anchored to a specific token. + #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature fn syntax_error( &mut self, recognizer: &R, + offending: Option>, line: usize, column: usize, message: &str, @@ -48,6 +61,7 @@ impl ErrorListener for ConsoleErrorListener { fn syntax_error( &mut self, _recognizer: &R, + _offending: Option>, line: usize, column: usize, message: &str, diff --git a/src/lib.rs b/src/lib.rs index 50f64da5..f7d02cc9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,10 +34,11 @@ pub use lexer::{ BaseLexer, Lexer, LexerCustomAction, LexerLifecycleCtx, LexerMode, LexerPredicate, LexerSemCtx, }; pub use parser::{ - BailErrorStrategy, BaseParser, ExpectedTokenSet, NoSemanticHooks, Parser, ParserAction, - ParserMemberAction, ParserPredicate, ParserReturnAction, ParserRuleArg, ParserRuntimeOptions, - ParserSemCtx, ParserSemanticAction, ParserSemanticPredicate, ParserSemantics, PredictionMode, - RecognitionArenaStats, SemanticHooks, UnknownSemanticPolicy, grow_generated_rule_stack, + BailErrorStrategy, BaseParser, EnterRuleEvent, ExpectedTokenSet, NoSemanticHooks, + ParseListener, Parser, ParserAction, ParserMemberAction, ParserPredicate, ParserReturnAction, + ParserRuleArg, ParserRuntimeOptions, ParserSemCtx, ParserSemanticAction, + ParserSemanticPredicate, ParserSemantics, PredictionMode, RecognitionArenaStats, SemanticHooks, + UnknownSemanticPolicy, grow_generated_rule_stack, }; #[cfg(feature = "perf-counters")] pub use perf::{dump as dump_prediction_perf_counters, reset as reset_prediction_perf_counters}; diff --git a/src/parser.rs b/src/parser.rs index 322b6fb3..afdced86 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -128,6 +128,104 @@ const ADAPTIVE_DIRECT_STEP_LIMIT: usize = RECOGNITION_DEPTH_LIMIT; pub fn grow_generated_rule_stack(body: impl FnOnce() -> R) -> R { stacker::maybe_grow(FAST_RECOGNIZE_RED_ZONE, FAST_RECOGNIZE_STACK_SIZE, body) } + +/// Receives committed rule enter/exit events during recognition, matching +/// ANTLR's `addParseListener` contract ([`Parser::add_parse_listener`], +/// also inherent on [`BaseParser`] and generated parsers). +/// +/// Events fire on the generated recursive-descent path as rules are entered +/// and exited, with left-recursive operator loops following upstream's +/// timing exactly: each loop pass first exits the outgoing iteration +/// (`recRuleSetPrevCtx`) and then enters the new expansion +/// (`pushNewRecursionContext` firing `triggerEnterRuleEvent`), so live +/// listener depth never accumulates across a flat operator chain — +/// `a + a + … + a` peaks at depth 2 like every ANTLR target. On expansion +/// events, [`EnterRuleEvent::current`] anchors at the operator-side +/// lookahead (the token the expansion starts at), whereas Java's +/// `ctx.start` reaches back to the whole expression's first token — anchor +/// diagnostics accordingly. Enter events fire in registration order and +/// exit events in reverse registration order, matching upstream. Enter/exit +/// calls balance on every completed path, including error recovery and +/// aborts inside operator loops — with one exception shared with Java: an +/// ordinary rule's enter that returns `Err` receives no matching exit +/// (upstream calls `enterRule` outside the generated `try`/`finally`, so a +/// throwing listener skips `exitRule` the same way). Listener state shared +/// across parses via `Arc` should be reset after an abort (the unmatched +/// ordinary-rule enter leaves counters one high). +/// +/// Divergence from Java to know about: upstream generated rule methods run +/// only on the committed parse, while this runtime may re-enter a rule while +/// recovering from a syntax error — such retries deliver additional balanced +/// enter/exit pairs. Depth counters and resource bounds (the primary use +/// case) are unaffected; exact once-per-node collectors should prefer the +/// post-parse tree walker. +/// +/// `enter_every_rule` is fallible: returning `Err` aborts the parse with +/// that error. The abort is sticky through rule-level recovery — the parse +/// fails even when recovery could have produced a tree, mirroring how a +/// thrown exception escapes ANTLR's `triggerEnterRuleEvent`. Rules the +/// generator emitted no body for (interpreter-only fallback) do not fire +/// events; when any parse listener is registered, generated dispatch routes +/// ATN-preferred rules through their generated bodies so real grammars +/// observe every rule. +/// +/// Cost: with no listener registered, dispatch pays one emptiness check per +/// rule boundary (benchmarked at baseline). With one registered, dispatch +/// itself is a few percent; on grammars where the generator classified rules +/// ATN-preferred, the dominant cost is the routing override above — the same +/// one [`Parser::set_max_rule_depth`] takes — which trades that fast path +/// for observability. Grammars without ATN-preferred rules (most small DSLs) +/// pay only the dispatch. +pub trait ParseListener: Send { + /// Called when a generated rule is entered, before its body runs, and + /// once per left-recursive operator expansion. + /// + /// Returning `Err` aborts the parse with the given error. + fn enter_every_rule(&mut self, event: &EnterRuleEvent<'_>) -> Result<(), AntlrError>; + + /// Called when a generated rule exits, after its body (and any rule-level + /// error recovery) finished, and once per left-recursive operator + /// expansion as the rule unrolls. + fn exit_every_rule(&mut self, rule_index: usize) { + let _ = rule_index; + } +} + +/// Boxed listeners forward to their inner implementation, so the boxes +/// returned by [`Parser::remove_parse_listeners`] can be re-registered +/// through [`Parser::add_parse_listener`] unchanged. +impl ParseListener for Box { + fn enter_every_rule(&mut self, event: &EnterRuleEvent<'_>) -> Result<(), AntlrError> { + (**self).enter_every_rule(event) + } + + fn exit_every_rule(&mut self, rule_index: usize) { + (**self).exit_every_rule(rule_index); + } +} + +/// A rule-entry event delivered to [`ParseListener::enter_every_rule`]. +/// +/// Non-exhaustive so future fields (alt number, invoking state, a context +/// handle) extend the event without breaking implementors. +#[derive(Debug)] +#[non_exhaustive] +pub struct EnterRuleEvent<'a> { + /// Index of the rule being entered (compare against the generated + /// `RULE_*` constants). + pub rule_index: usize, + /// The lookahead token the rule starts at — its line/column/offsets + /// anchor listener diagnostics — or `None` at end of input. + pub current: Option>, +} + +struct ParseListenerSlot(Box); + +impl std::fmt::Debug for ParseListenerSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ParseListener") + } +} /// Probe window for deciding whether clean-pass memo entries are reusable /// enough to keep caching. High-cardinality parses mostly produce one-shot /// entries; compact ambiguous loops repeatedly hit the same keys. @@ -1158,6 +1256,18 @@ pub trait Parser: Recognizer { /// Rules the generator emitted no body for (interpreter-only fallback) /// do not check the cap. fn set_max_rule_depth(&mut self, _depth: Option) {} + + /// Registers a listener for committed rule enter/exit events during + /// recognition (ANTLR's `addParseListener`). See [`ParseListener`] for + /// the delivery contract. The default implementation drops the listener; + /// [`BaseParser`] and generated parsers deliver events. + fn add_parse_listener(&mut self, _listener: Box) {} + + /// Removes every registered parse listener and returns them, dropping + /// any sticky abort a removed listener had requested. + fn remove_parse_listeners(&mut self) -> Vec> { + Vec::new() + } } #[derive(Debug)] @@ -1203,6 +1313,16 @@ pub struct BaseParser { /// recovering (ANTLR's `BailErrorStrategy`). Generated recognizers set it /// through `set_error_handler(BailErrorStrategy::new())`. bail_on_error: bool, + /// Parse listeners receiving committed rule enter/exit events during + /// recognition (ANTLR's `addParseListener`). Empty in the default + /// configuration, and every dispatch site is gated on emptiness so the + /// unused feature costs one predictable branch per rule boundary. + parse_listeners: Vec, + /// Sticky abort requested by a parse listener's `enter_every_rule`. + /// Mirrors `rule_depth_error`: rule-level recovery absorbs the error like + /// any rule failure, so the flag stays set until the top-level entry + /// drains it and fails the parse. + parse_listener_abort: Option, /// Optional cap on rule-nesting depth for adversarial-input hardening. /// `None` (default) parses unbounded nesting; `Some(n)` aborts the parse /// with a positioned syntax error once `n` rule frames are exceeded. @@ -2185,6 +2305,10 @@ struct ParserDiagnostic { line: usize, column: usize, message: String, + /// Token the diagnostic is anchored to, resolved to a view when the + /// diagnostic is dispatched to error listeners. `None` when no token + /// exists (synthetic positions, lexer-originated messages). + offending: Option, } #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -4835,6 +4959,8 @@ where precedence_stack: vec![0], invoked_predicates: Vec::new(), bail_on_error: false, + parse_listeners: Vec::new(), + parse_listener_abort: None, max_rule_depth: None, rule_depth_error: None, recursion_expansions: 0, @@ -4898,6 +5024,7 @@ where self.decision_override_generation = 0; self.unknown_predicate_hits.clear(); self.unhandled_action_hits.clear(); + self.parse_listener_abort = None; self.rule_depth_error = None; self.recursion_expansions = 0; self.recursion_expansion_marks.clear(); @@ -5066,7 +5193,11 @@ where } fn dispatch_parser_diagnostic(&self, diagnostic: &ParserDiagnostic) { + let offending = diagnostic + .offending + .and_then(|token| self.token_store().view(token)); self.notify_error_listeners( + offending, diagnostic.line, diagnostic.column, &diagnostic.message, @@ -5087,7 +5218,10 @@ where if self.input.token_source().report_error(source_error) { return; } + // Lexer errors have no offending token: the failure is that no token + // could be produced, matching ANTLR's null offendingSymbol. self.notify_error_listeners( + None, source_error.line, source_error.column, &source_error.message, @@ -5359,6 +5493,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if current_type == token_type { @@ -5386,6 +5521,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if current_type == token_type { @@ -5419,6 +5555,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if interval_set_contains(intervals, current_type) { @@ -5451,6 +5588,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if set.contains(current_type) { @@ -5484,6 +5622,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if (min_vocabulary..=max_vocabulary).contains(¤t_type) @@ -5524,6 +5663,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if (min_vocabulary..=max_vocabulary).contains(¤t_type) && !set.contains(current_type) @@ -5576,6 +5716,7 @@ where line: current_line, column: current_column, message: format!("mismatched input {current_display} expecting {expected_display}"), + offending: Some(current), }); } if current_type != TOKEN_EOF @@ -5588,6 +5729,7 @@ where line: current_line, column: current_column, message, + offending: Some(current), }); self.record_syntax_errors(1); self.generated_sync_expected = None; @@ -5629,6 +5771,7 @@ where line: current_line, column: current_column, message, + offending: Some(current), }); self.record_syntax_errors(1); self.generated_sync_expected = None; @@ -5661,6 +5804,7 @@ where message: format!( "mismatched input {current_display} expecting {mismatch_expected_display}" ), + offending: Some(current), }) } @@ -5708,6 +5852,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if matches(current_type) { @@ -5799,14 +5944,15 @@ where if let Some(error) = &self.rule_depth_error { return error.clone(); } - let (line, column) = self - .input - .lt(1) + let current = self.input.lt(1); + let (line, column) = current + .as_ref() .map_or((0, 0), |token| (token.line(), token.column())); let error = AntlrError::ParserError { line, column, message: format!("rule nesting depth limit of {max} exceeded"), + offending: current.as_ref().map(Token::token_id), }; self.rule_depth_error = Some(error.clone()); error @@ -5833,6 +5979,125 @@ where self.max_rule_depth.is_some() } + /// Registers a listener for committed rule enter/exit events during + /// recognition (ANTLR's `addParseListener`). See [`ParseListener`] for + /// the delivery contract. + pub fn add_parse_listener(&mut self, listener: L) + where + L: ParseListener + 'static, + { + self.parse_listeners + .push(ParseListenerSlot(Box::new(listener))); + } + + /// Removes every registered parse listener and returns them, dropping any + /// sticky abort a removed listener had requested. + /// + /// Returning the boxed listeners gives callers back the state they + /// accumulated (depth counters, collected events) without threading + /// shared handles through the listener. + pub fn remove_parse_listeners(&mut self) -> Vec> { + self.parse_listener_abort = None; + self.parse_listeners.drain(..).map(|slot| slot.0).collect() + } + + /// Reports whether any parse listener is registered. + /// + /// Generated dispatch consults this alongside [`Self::has_rule_depth_cap`] + /// when choosing between the generated body (which fires events) and the + /// ATN-preferred interpreted fast path (which does not). + #[must_use] + pub const fn has_parse_listeners(&self) -> bool { + !self.parse_listeners.is_empty() + } + + /// Fires `enter_every_rule` on registered parse listeners, returning the + /// abort error if any listener requested one. + /// + /// Generated rule dispatch calls this after the depth-cap probe and + /// before the rule body runs; the generated left-recursive loop calls it + /// once per operator expansion, mirroring upstream ANTLR's simulated + /// rule-entry event for `pushNewRecursionContext`. A listener abort is + /// sticky exactly like a depth-cap violation: rule-level recovery absorbs + /// the returned error, so the flag holds until the top-level entry drains + /// it via [`Self::take_parse_listener_abort`] and fails the parse. + pub fn parse_listener_enter_rule(&mut self, rule_index: usize) -> Option { + if self.parse_listeners.is_empty() { + return None; + } + self.parse_listener_enter_rule_dispatch(rule_index) + } + + fn parse_listener_enter_rule_dispatch(&mut self, rule_index: usize) -> Option { + if let Some(error) = &self.parse_listener_abort { + return Some(error.clone()); + } + let event = EnterRuleEvent { + rule_index, + current: self.input.lt(1), + }; + // Split borrows: the token view borrows the input while listeners + // need `&mut`, so listeners are taken out for the dispatch. Listener + // methods have no parser access and cannot observe the absence. + let mut listeners = std::mem::take(&mut self.parse_listeners); + let mut abort = None; + for slot in &mut listeners { + if let Err(error) = slot.0.enter_every_rule(&event) { + abort = Some(error); + break; + } + } + self.parse_listeners = listeners; + if let Some(error) = abort { + self.parse_listener_abort = Some(error.clone()); + return Some(error); + } + None + } + + /// Fires `exit_every_rule` on registered parse listeners. + /// + /// Generated rule bodies call this on every exit path — success and + /// recovery alike — keeping enter/exit pairs balanced, and the generated + /// left-recursive loop calls it once per operator expansion when the rule + /// finishes unrolling. + pub fn parse_listener_exit_rule(&mut self, rule_index: usize) { + if self.parse_listeners.is_empty() { + return; + } + // Reverse registration order, matching upstream ANTLR + // (`Parser.triggerExitRuleEvent` walks listeners back to front). + for slot in self.parse_listeners.iter_mut().rev() { + slot.0.exit_every_rule(rule_index); + } + } + + /// Drains the sticky parse-listener abort recorded by + /// [`Self::parse_listener_enter_rule`], if any. + /// + /// Generated top-level rule entries call this after recognition so an + /// aborted parse fails even when recovery produced a tree, and so a + /// reused parser starts its next parse clean. + pub const fn take_parse_listener_abort(&mut self) -> Option { + self.parse_listener_abort.take() + } + + /// Drains every sticky parse abort — the depth-cap violation and the + /// parse-listener abort — returning the depth error preferentially. + /// + /// Generated top-level rule entries call this on both exit paths: the + /// recorded abort wins over errors derived from it (recovery may have + /// absorbed the aborted rule and failed differently later), a recovered + /// `Ok` tree still fails when an abort was recorded, and draining leaves + /// the instance clean for the next entry-rule call. + pub fn take_parse_abort(&mut self) -> Option { + if let Some(error) = self.rule_depth_error.take() { + self.parse_listener_abort = None; + return Some(error); + } + self.parse_listener_abort.take() + } + /// Enters a generated parser rule and returns the context object the /// generated method should populate. pub fn enter_rule(&mut self, state: isize, rule_index: usize) -> ParserRuleContext { @@ -6009,14 +6274,19 @@ where fn generated_rule_error_diagnostic(&self, error: AntlrError) -> ParserDiagnostic { match error { + // The anchor recorded where the error was built wins over the + // current lookahead: prediction restores the cursor, so lt(1) + // here can point at the decision start rather than the error. AntlrError::ParserError { line, column, message, + offending, } => ParserDiagnostic { line, column, message, + offending, }, AntlrError::MismatchedInput { expected, found } => diagnostic_for_token( self.input.lt(1), @@ -6034,6 +6304,7 @@ where line, column, message, + offending: None, }, AntlrError::Unsupported(message) => diagnostic_for_token(self.input.lt(1), message), } @@ -6092,7 +6363,9 @@ where self.set_state(state); // Counts toward the depth cap: each operator iteration deepens the // parse tree one level without pushing a rule frame, and upstream - // fires a rule-entry listener event for it. + // fires a rule-entry listener event for it. The parse-listener enter + // event for this expansion fires from the generated loop's probe + // just before this call, where a listener abort can propagate. self.recursion_expansions += 1; if let Some(stop) = self .rule_stop_token_index(self.input.index(), false) @@ -6118,6 +6391,11 @@ where if self.precedence_stack.len() > 1 { self.precedence_stack.pop(); } + // Parse-listener exits for expansions fire inside the generated + // operator loop (top of each pass, upstream's `recRuleSetPrevCtx`), + // and the dispatch wrapper's exit covers the final live context — + // upstream's `unrollRecursionContexts` walks exactly one link, so no + // batched exits happen here. Only the depth-cap accounting rewinds. if let Some(mark) = self.recursion_expansion_marks.pop() { self.recursion_expansions = mark; } @@ -6359,6 +6637,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; if self.token_type_for_id(current) == TOKEN_EOF { return Err(AntlrError::MismatchedInput { @@ -6540,6 +6819,7 @@ where .map_or_else(|| "''".to_owned(), token_input_display), self.expected_symbols_display(&expected_symbols) ), + offending: current.as_ref().map(Token::token_id), }) } @@ -6674,6 +6954,7 @@ where line: diagnostic.line, column: diagnostic.column, message: diagnostic.message, + offending: diagnostic.offending, } } @@ -6684,6 +6965,7 @@ where line: current.as_ref().map(Token::line).unwrap_or_default(), column: current.as_ref().map(Token::column).unwrap_or_default(), message: format!("rule failed predicate: {}", message.into()), + offending: current.as_ref().map(Token::token_id), } } @@ -6703,6 +6985,7 @@ where line: current.as_ref().map(Token::line).unwrap_or_default(), column: current.as_ref().map(Token::column).unwrap_or_default(), message: format!("rule {rule_name} {}", message.into()), + offending: current.as_ref().map(Token::token_id), } } @@ -7306,6 +7589,7 @@ where line: 0, column: 0, message: format!("missing token at index {index}"), + offending: None, })?; let is_eof = self.token_type_for_id(token) == TOKEN_EOF; let child = self.terminal_tree(token); @@ -7638,6 +7922,7 @@ where line, column, message, + offending: current.as_ref().map(Token::token_id), } } @@ -11959,11 +12244,14 @@ fn display_input_text(text: &str) -> String { } fn diagnostic_for_token(token: Option, message: String) -> ParserDiagnostic { - let (line, column) = token.map_or((0, 0), |token| (token.line(), token.column())); + let (line, column, offending) = token.map_or((0, 0, None), |token| { + (token.line(), token.column(), Some(token.token_id())) + }); ParserDiagnostic { line, column, message, + offending, } } @@ -12650,6 +12938,14 @@ where fn set_max_rule_depth(&mut self, depth: Option) { self.max_rule_depth = depth; } + + fn add_parse_listener(&mut self, listener: Box) { + self.parse_listeners.push(ParseListenerSlot(listener)); + } + + fn remove_parse_listeners(&mut self) -> Vec> { + Self::remove_parse_listeners(self) + } } #[cfg(test)] @@ -12660,7 +12956,9 @@ mod tests { ParserAtnPredictionDiagnostic, ParserAtnPredictionDiagnosticKind, ParserAtnSimulator, }; use crate::atn::serialized::{AtnDeserializer, SerializedAtn}; - use crate::token::{HIDDEN_CHANNEL, Token, TokenId, TokenSink, TokenSpec, TokenStoreError}; + use crate::token::{ + HIDDEN_CHANNEL, Token, TokenId, TokenSink, TokenSpec, TokenStoreError, TokenView, + }; use crate::token_stream::CommonTokenStream; use crate::tree::{NodeKind, ParseTreeStats}; use crate::vocabulary::Vocabulary; @@ -12820,6 +13118,7 @@ mod tests { #[derive(Clone, Debug, Eq, PartialEq)] struct RecordedDiagnostic { grammar_file_name: String, + offending_text: Option, line: usize, column: usize, message: String, @@ -12838,6 +13137,7 @@ mod tests { fn syntax_error( &mut self, recognizer: &R, + offending: Option>, line: usize, column: usize, message: &str, @@ -12848,6 +13148,7 @@ mod tests { .expect("recorded diagnostics lock") .push(RecordedDiagnostic { grammar_file_name: recognizer.grammar_file_name().to_owned(), + offending_text: offending.and_then(|token| token.text().map(str::to_owned)), line, column, message: message.to_owned(), @@ -12921,6 +13222,7 @@ mod tests { line: 1, column: 2, message: "missing 'x' at 'y'".to_owned(), + offending: None, }]; let token_errors = [ TokenSourceError::new(1, 1, "token recognition error at: '@'"), @@ -12944,6 +13246,44 @@ mod tests { ); } + #[test] + fn recovery_diagnostics_expose_the_offending_token_to_listeners() { + let mut parser = mini_parser(vec![ + TestToken::new(7) + .with_text("oops") + .with_span(0, 3) + .with_position(1, 2), + TestToken::eof("parser-test", 4, 1, 6), + ]); + parser.remove_error_listeners(); + let diagnostics = Arc::new(Mutex::new(Vec::new())); + parser.add_error_listener(RecordingErrorListener { + diagnostics: Arc::clone(&diagnostics), + }); + let offending = parser.input.lt_id(1); + assert!(offending.is_some(), "current token should be buffered"); + let parser_diagnostics = [ParserDiagnostic { + line: 1, + column: 2, + message: "extraneous input 'oops'".to_owned(), + offending, + }]; + + parser.dispatch_generated_diagnostics(&parser_diagnostics, &[]); + + // Listeners receive a resolvable view of the offending token — the + // ANTLR offendingSymbol contract downstream span-building error + // reporters (miette-style byte-offset underlines) rely on. + let recorded = diagnostics + .lock() + .expect("recorded diagnostics lock") + .clone(); + insta::assert_debug_snapshot!( + "recovery_diagnostics_expose_the_offending_token_to_listeners", + recorded + ); + } + #[test] fn parser_leaves_token_errors_to_source_owned_listeners() { let source_diagnostics = Rc::new(RefCell::new(Vec::new())); @@ -15579,6 +15919,7 @@ mod tests { line: 1, column: 3, message: "missing 'Y' at ''".to_owned(), + offending: parser.input.lt_id(1), }] ); } @@ -15807,6 +16148,7 @@ mod tests { line: 1, column: 1, message: "missing {} at ''".to_owned(), + offending: parser.input.lt_id(1), }] ); } @@ -15855,6 +16197,7 @@ mod tests { line: 1, column: 1, message: "missing 'x' at ''".to_owned(), + offending: parser.input.lt_id(1), }] ); } @@ -15885,6 +16228,10 @@ mod tests { let mut child = parser.enter_rule(4, 1); parser.discard_invoking_state(marker); + // The anchor recorded where the error was built must survive into the + // dispatched diagnostic even though recovery consumes past it below. + let offending = parser.input.lt_id(1); + assert!(offending.is_some(), "the 'z' token should be buffered"); parser.recover_generated_rule( &mut child, &atn, @@ -15892,6 +16239,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), + offending, }, ); let tree = parser.finish_rule(child, false); @@ -15908,6 +16256,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), + offending, }] ); parser.exit_rule(); @@ -17708,6 +18057,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'x'".to_owned(), + offending: None, }]), deferred_nodes: FastDeferredNodeId::EMPTY, nodes: NodeSeqId::EMPTY, @@ -17766,6 +18116,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'x' expecting 'a'".to_owned(), + offending: None, }]), deferred_nodes: FastDeferredNodeId::EMPTY, nodes: NodeSeqId::EMPTY, @@ -17777,6 +18128,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'x' expecting 'b'".to_owned(), + offending: None, }]), deferred_nodes: FastDeferredNodeId::EMPTY, nodes: NodeSeqId::EMPTY, @@ -17788,6 +18140,7 @@ mod tests { line: 1, column: 0, message: "missing 'a' at 'x'".to_owned(), + offending: None, }]), deferred_nodes: FastDeferredNodeId::EMPTY, nodes: NodeSeqId::EMPTY, @@ -18157,11 +18510,13 @@ mod tests { line: 1, column: 0, message: "missing X".to_owned(), + offending: None, }]); let _discarded_diagnostics = arena.diagnostic_sequence([ParserDiagnostic { line: 1, column: 1, message: "discarded".to_owned(), + offending: None, }]); let deferred_children = arena.deferred_fragment(live); let _deferred_rule = arena.deferred_rule_node(FastDeferredRule { @@ -18277,17 +18632,20 @@ mod tests { line: 1, column: 0, message: "first".to_owned(), + offending: None, }, ParserDiagnostic { line: 1, column: 1, message: "second".to_owned(), + offending: None, }, ]); let suffix = arena.diagnostic_sequence([ParserDiagnostic { line: 1, column: 2, message: "third".to_owned(), + offending: None, }]); let extras_before = arena.extras.len(); @@ -18445,6 +18803,7 @@ mod tests { line: 1, column: 3, message: "missing 'Y' at ''".to_owned(), + offending: None, }]); let first_alt = RecognizeOutcome { index: 2, diff --git a/src/recognizer.rs b/src/recognizer.rs index 521f848a..caf2ee04 100644 --- a/src/recognizer.rs +++ b/src/recognizer.rs @@ -2,6 +2,7 @@ use std::fmt; use std::sync::{Arc, Mutex}; use crate::errors::{AntlrError, ConsoleErrorListener, ErrorListener}; +use crate::token::TokenView; use crate::vocabulary::Vocabulary; #[derive(Clone)] @@ -15,9 +16,11 @@ impl ErrorListenerSlot { Self(Arc::new(Mutex::new(listener))) } + #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature fn syntax_error( &self, recognizer: &(dyn Recognizer + '_), + offending: Option>, line: usize, column: usize, message: &str, @@ -26,7 +29,7 @@ impl ErrorListenerSlot { self.0 .lock() .expect("error listener lock poisoned") - .syntax_error(recognizer, line, column, message, error); + .syntax_error(recognizer, offending, line, column, message, error); } } @@ -122,16 +125,18 @@ impl RecognizerData { self.error_listeners.clear(); } + #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature fn notify_error_listeners( &self, recognizer: &dyn Recognizer, + offending: Option>, line: usize, column: usize, message: &str, error: Option<&AntlrError>, ) { for listener in &self.error_listeners { - listener.syntax_error(recognizer, line, column, message, error); + listener.syntax_error(recognizer, offending, line, column, message, error); } } } @@ -187,8 +192,13 @@ pub trait Recognizer { } /// Sends one diagnostic to every registered error listener. + /// + /// `offending` is the token the diagnostic is anchored to, when one + /// exists — parser diagnostics resolve it from the token store; lexer + /// diagnostics pass `None` because no token was produced. fn notify_error_listeners( &self, + offending: Option>, line: usize, column: usize, message: &str, @@ -197,7 +207,7 @@ pub trait Recognizer { Self: Sized, { self.data() - .notify_error_listeners(self, line, column, message, error); + .notify_error_listeners(self, offending, line, column, message, error); } fn sempred(&mut self, _rule_index: usize, _pred_index: usize) -> bool { @@ -215,6 +225,7 @@ mod tests { #[derive(Clone, Debug, Eq, PartialEq)] struct RecordedError { grammar_file_name: String, + offending_text: Option, line: usize, column: usize, message: String, @@ -233,6 +244,7 @@ mod tests { fn syntax_error( &mut self, recognizer: &R, + offending: Option>, line: usize, column: usize, message: &str, @@ -243,6 +255,7 @@ mod tests { .expect("recorded errors lock") .push(RecordedError { grammar_file_name: recognizer.grammar_file_name().to_owned(), + offending_text: offending.and_then(|token| token.text().map(str::to_owned)), line, column, message: message.to_owned(), @@ -295,8 +308,9 @@ mod tests { line: 3, column: 5, message: "unexpected token".to_owned(), + offending: None, }; - recognizer.notify_error_listeners(3, 5, "unexpected token", Some(&error)); + recognizer.notify_error_listeners(None, 3, 5, "unexpected token", Some(&error)); insta::assert_debug_snapshot!( "recognizers_replace_the_default_console_error_listener", diff --git a/src/snapshots/antlr4_runtime__parser__tests__generated_prediction_diagnostics_use_adaptive_context.snap b/src/snapshots/antlr4_runtime__parser__tests__generated_prediction_diagnostics_use_adaptive_context.snap index 320c0736..be8586a9 100644 --- a/src/snapshots/antlr4_runtime__parser__tests__generated_prediction_diagnostics_use_adaptive_context.snap +++ b/src/snapshots/antlr4_runtime__parser__tests__generated_prediction_diagnostics_use_adaptive_context.snap @@ -7,15 +7,30 @@ expression: parser.generated_parser_diagnostics line: 1, column: 2, message: "reportAttemptingFullContext d=0 (s), input='xy'", + offending: Some( + TokenId( + 1, + ), + ), }, ParserDiagnostic { line: 1, column: 0, message: "reportContextSensitivity d=0 (s), input='x'", + offending: Some( + TokenId( + 0, + ), + ), }, ParserDiagnostic { line: 1, column: 2, message: "reportAttemptingFullContext d=0 (s), input='xy'", + offending: Some( + TokenId( + 1, + ), + ), }, ] diff --git a/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap b/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap index f2bef13b..f04bf717 100644 --- a/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap +++ b/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap @@ -5,6 +5,7 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" [ RecordedDiagnostic { grammar_file_name: "Mini.g4", + offending_text: None, line: 1, column: 1, message: "token recognition error at: '@'", @@ -12,6 +13,7 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" }, RecordedDiagnostic { grammar_file_name: "Mini.g4", + offending_text: None, line: 1, column: 2, message: "missing 'x' at 'y'", @@ -19,6 +21,7 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" }, RecordedDiagnostic { grammar_file_name: "Mini.g4", + offending_text: None, line: 1, column: 3, message: "token recognition error at: '#'", diff --git a/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap b/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap new file mode 100644 index 00000000..a81afa7a --- /dev/null +++ b/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap @@ -0,0 +1,16 @@ +--- +source: src/parser.rs +expression: recorded +--- +[ + RecordedDiagnostic { + grammar_file_name: "Mini.g4", + offending_text: Some( + "oops", + ), + line: 1, + column: 2, + message: "extraneous input 'oops'", + error: None, + }, +] diff --git a/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap b/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap index f9d3a929..a658bce3 100644 --- a/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap +++ b/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap @@ -5,6 +5,7 @@ expression: "*errors.lock().expect(\"recorded errors lock\")" [ RecordedError { grammar_file_name: "Test.g4", + offending_text: None, line: 3, column: 5, message: "unexpected token", @@ -13,6 +14,7 @@ expression: "*errors.lock().expect(\"recorded errors lock\")" line: 3, column: 5, message: "unexpected token", + offending: None, }, ), }, diff --git a/src/xpath/generated/x_path_lexer.rs b/src/xpath/generated/x_path_lexer.rs index 5cde72fd..53ec07a0 100644 --- a/src/xpath/generated/x_path_lexer.rs +++ b/src/xpath/generated/x_path_lexer.rs @@ -226,6 +226,7 @@ where fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool { antlr4_runtime::Recognizer::notify_error_listeners( self, + None, source_error.line, source_error.column, &source_error.message, diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index dd26c38d..de5cf4ee 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -1716,6 +1716,320 @@ mod deep_nesting_tests { "a different entry rule on the same instance starts clean" ); } + + /// The cel-rust `RecursionListener` shape: count `expr` nesting live, + /// abort the parse past a limit — ported verbatim onto + /// `add_parse_listener` (issue #202). + struct RecursionListener { + max: u16, + depth: u16, + high_water: std::sync::Arc, + } + + impl antlr4_runtime::ParseListener for RecursionListener { + fn enter_every_rule( + &mut self, + event: &antlr4_runtime::EnterRuleEvent<'_>, + ) -> Result<(), antlr4_runtime::AntlrError> { + if event.rule_index == super::nest_parser::RULE_EXPR { + self.depth += 1; + self.high_water + .fetch_max(self.depth, std::sync::atomic::Ordering::Relaxed); + } + if self.depth > self.max { + use antlr4_runtime::Token as _; + let (line, column) = event + .current + .as_ref() + .map_or((0, 0), |token| (token.line(), token.column())); + return Err(antlr4_runtime::AntlrError::ParserError { + line, + column, + message: format!("Recursion limit of {} exceeded", self.max), + offending: event.current.as_ref().map(antlr4_runtime::Token::token_id), + }); + } + Ok(()) + } + + fn exit_every_rule(&mut self, rule_index: usize) { + if rule_index == super::nest_parser::RULE_EXPR { + self.depth -= 1; + } + } + } + + /// Records the event stream for order/balance assertions. + struct TracingListener { + tag: &'static str, + events: std::sync::Arc>>, + } + + impl antlr4_runtime::ParseListener for TracingListener { + fn enter_every_rule( + &mut self, + event: &antlr4_runtime::EnterRuleEvent<'_>, + ) -> Result<(), antlr4_runtime::AntlrError> { + self.events + .lock() + .expect("trace lock") + .push(format!("enter{}:{}", self.tag, event.rule_index)); + Ok(()) + } + + fn exit_every_rule(&mut self, rule_index: usize) { + self.events + .lock() + .expect("trace lock") + .push(format!("exit{}:{}", self.tag, rule_index)); + } + } + + #[test] + fn parse_listener_counts_rules_and_aborts_past_a_limit() { + use std::sync::Arc; + use std::sync::atomic::{AtomicU16, Ordering}; + + // Under the limit: events fire, parse succeeds, enter/exit balance + // (depth returns to zero, so high-water == max nesting seen). + let high_water = Arc::new(AtomicU16::new(0)); + let lexer = NestLexer::new(InputStream::new(&nested(3))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 32, + depth: 0, + high_water: Arc::clone(&high_water), + }); + assert!(parser.s().is_ok(), "shallow input parses under the limit"); + assert_eq!( + high_water.load(Ordering::Relaxed), + 4, + "one expr per bracket level plus the outermost expr" + ); + + // Successful left-recursive chain: the live depth counter returns to + // its starting value. Proven through the public API: parse the same + // under-limit chain twice with one listener instance — any residual + // depth from parse one would raise parse two's high-water mark. + let high_water = Arc::new(AtomicU16::new(0)); + let chain = vec!["a"; 20].join("+"); + let lexer = NestLexer::new(InputStream::new(&chain)); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 1_000, + depth: 0, + high_water: Arc::clone(&high_water), + }); + assert!(parser.s().is_ok(), "under-limit operator chain parses"); + let first_peak = high_water.load(Ordering::Relaxed); + assert!(first_peak > 0, "the chain nests expr rules"); + let lexer = NestLexer::new(InputStream::new(&chain)); + parser.set_token_stream(CommonTokenStream::new(lexer)); + assert!(parser.s().is_ok(), "same chain parses again"); + assert_eq!( + high_water.load(Ordering::Relaxed), + first_peak, + "depth returned to zero after the successful LR parse" + ); + + // Past the limit: the listener aborts with its own positioned error, + // sticky through recovery. + let lexer = NestLexer::new(InputStream::new(&nested(64))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 8, + depth: 0, + high_water: Arc::new(AtomicU16::new(0)), + }); + let error = parser.s().expect_err("listener abort must fail the parse"); + assert!( + error.to_string().contains("Recursion limit of 8 exceeded"), + "unexpected error: {error}" + ); + + // Flat operator chains do NOT accumulate live listener depth: each + // loop pass exits the outgoing iteration before the next expansion + // enters (upstream recRuleSetPrevCtx), so a 40-term `a+a+...` chain + // peaks at expr depth 2 in every ANTLR target — including this one — + // and parses fine under a limit of 8. + let chain = vec!["a"; 40].join("+"); + let high_water = Arc::new(AtomicU16::new(0)); + let lexer = NestLexer::new(InputStream::new(&chain)); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 8, + depth: 0, + high_water: Arc::clone(&high_water), + }); + assert!( + parser.s().is_ok(), + "flat operator chain stays at Java's live depth" + ); + assert_eq!( + high_water.load(Ordering::Relaxed), + 2, + "operator chain peaks at depth 2, matching the Java oracle" + ); + + // The abort does not poison the instance: clearing listeners (which + // also returns them and drops the sticky abort) and reusing the + // parser parses clean input. + let lexer = NestLexer::new(InputStream::new(&nested(64))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 8, + depth: 0, + high_water: Arc::new(AtomicU16::new(0)), + }); + let error = parser.s().expect_err("nested input exceeds the limit"); + assert!( + error.to_string().contains("Recursion limit of 8 exceeded"), + "unexpected error: {error}" + ); + let lexer = NestLexer::new(InputStream::new("a")); + parser.set_token_stream(CommonTokenStream::new(lexer)); + let mut removed = parser.remove_parse_listeners(); + assert_eq!(removed.len(), 1, "removed listeners are handed back"); + assert!(parser.s().is_ok(), "reused parser starts clean"); + + // Returned boxes re-register as-is (ParseListener is implemented for + // Box), preserving accumulated listener state. + let boxed = removed.pop().expect("one listener was removed"); + let lexer = NestLexer::new(InputStream::new(&nested(64))); + parser.set_token_stream(CommonTokenStream::new(lexer)); + parser.add_parse_listener(boxed); + let error = parser + .s() + .expect_err("re-registered listener still enforces its limit"); + assert!( + error.to_string().contains("Recursion limit of 8 exceeded"), + "unexpected error: {error}" + ); + } + + #[test] + fn parse_listener_event_order_matches_upstream() { + use std::sync::{Arc, Mutex}; + + // Two listeners: enters fire in registration order, exits in reverse + // (upstream Parser.triggerExitRuleEvent walks back to front), and + // pairs balance across recovery on malformed input. + let events = Arc::new(Mutex::new(Vec::new())); + let lexer = NestLexer::new(InputStream::new("[a]")); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(TracingListener { + tag: "A", + events: Arc::clone(&events), + }); + parser.add_parse_listener(TracingListener { + tag: "B", + events: Arc::clone(&events), + }); + assert!(parser.s().is_ok()); + let trace = events.lock().expect("trace lock").clone(); + let s_rule = super::nest_parser::RULE_S; + assert_eq!(trace.first().map(String::as_str), Some(format!("enterA:{s_rule}").as_str())); + assert_eq!(trace.get(1).map(String::as_str), Some(format!("enterB:{s_rule}").as_str())); + // Last two events close the entry rule: B exits before A. + assert_eq!( + trace.last().map(String::as_str), + Some(format!("exitA:{s_rule}").as_str()) + ); + assert_eq!( + trace.get(trace.len() - 2).map(String::as_str), + Some(format!("exitB:{s_rule}").as_str()) + ); + // Balance: every rule index enters exactly as often as it exits, + // for both listeners. + let count = |needle: &str| trace.iter().filter(|event| event.starts_with(needle)).count(); + assert_eq!(count("enterA:"), count("exitA:")); + assert_eq!(count("enterB:"), count("exitB:")); + + // Recovery path: malformed input still balances. + let events = Arc::new(Mutex::new(Vec::new())); + let lexer = NestLexer::new(InputStream::new("[a")); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(TracingListener { + tag: "R", + events: Arc::clone(&events), + }); + let _ = parser.s(); + let trace = events.lock().expect("trace lock").clone(); + let count = |needle: &str| trace.iter().filter(|event| event.starts_with(needle)).count(); + assert_eq!( + count("enterR:"), + count("exitR:"), + "recovery keeps pairs balanced: {trace:?}" + ); + + // Successful left-recursive operator chain: each expansion fires a + // simulated enter (upstream triggerEnterRuleEvent parity) and the + // unroll fires the matching exits. `a+a+a+a` yields exactly 7 + // RULE_EXPR pairs: 1 rule dispatch + 3 expansions + 3 right-operand + // dispatches. + let events = Arc::new(Mutex::new(Vec::new())); + let lexer = NestLexer::new(InputStream::new("a+a+a+a")); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(TracingListener { + tag: "L", + events: Arc::clone(&events), + }); + assert!(parser.s().is_ok(), "operator chain parses"); + let trace = events.lock().expect("trace lock").clone(); + let expr_rule = super::nest_parser::RULE_EXPR; + let count = |needle: String| trace.iter().filter(|event| **event == needle).count(); + assert_eq!( + count(format!("enterL:{expr_rule}")), + 7, + "expr enters = dispatch + expansions + operands: {trace:?}" + ); + assert_eq!( + count(format!("enterL:{expr_rule}")), + count(format!("exitL:{expr_rule}")), + "successful LR unroll balances expansion exits: {trace:?}" + ); + } + + #[test] + fn depth_cap_and_listener_abort_coexist() { + // Whichever bound trips first surfaces; the other never fires because + // the sticky abort stops rule entries (and thus stack growth). With a + // tight listener limit the listener error wins the race... + let lexer = NestLexer::new(InputStream::new(&nested(64))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.set_max_rule_depth(Some(64)); + parser.add_parse_listener(RecursionListener { + max: 1, + depth: 0, + high_water: std::sync::Arc::new(std::sync::atomic::AtomicU16::new(0)), + }); + let error = parser.s().expect_err("listener limit must fail the parse"); + assert!( + error.to_string().contains("Recursion limit of 1 exceeded"), + "listener abort surfaces when it trips first: {error}" + ); + + // ...and with a tight cap the depth violation wins the race. + let lexer = NestLexer::new(InputStream::new(&nested(64))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.set_max_rule_depth(Some(8)); + parser.add_parse_listener(RecursionListener { + max: 1_000, + depth: 0, + high_water: std::sync::Arc::new(std::sync::atomic::AtomicU16::new(0)), + }); + let error = parser.s().expect_err("depth cap must fail the parse"); + assert!( + error.to_string().contains("rule nesting depth limit of 8"), + "depth-cap violation surfaces when it trips first: {error}" + ); + + let lexer = NestLexer::new(InputStream::new("a")); + parser.set_token_stream(CommonTokenStream::new(lexer)); + parser.set_max_rule_depth(None); + let _ = parser.remove_parse_listeners(); + assert!(parser.s().is_ok(), "instance is clean after either abort"); + } } "#, );