From cbefcfd9570d87d065bc35ee10f47b1f8ffce215 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sat, 25 Jul 2026 15:10:50 +0200 Subject: [PATCH 1/9] feat(runtime): pass the offending token to ErrorListener::syntax_error ErrorListener::syntax_error now receives offending: Option right after the recognizer, matching ANTLR's canonical syntaxError(recognizer, offendingSymbol, ...) contract. Every reference runtime passes the offending symbol; span-building error reporters (e.g. avdl's miette diagnostics with byte-offset underlines) need start_byte/stop_byte from the token, not just (line, column). ParserDiagnostic records the anchoring TokenId at each creation site (diagnostic_for_token already had the token in hand; the extraneous/ missing recovery paths record the current token) and dispatch resolves it to a TokenView from the token store. Lexer-originated diagnostics pass None, matching ANTLR's null offendingSymbol for lexer errors. Fixes #195 --- src/bin/antlr4-rust-gen.rs | 1 + src/bin_support/grammar/frontend.rs | 1 + .../grammar/generated/antlr_v4_lexer.rs | 1 + src/errors.rs | 8 ++ src/parser.rs | 78 ++++++++++++++++++- src/recognizer.rs | 21 ++++- ...tion_diagnostics_use_adaptive_context.snap | 15 ++++ ...gnostics_through_registered_listeners.snap | 3 + ...ce_the_default_console_error_listener.snap | 1 + src/xpath/generated/x_path_lexer.rs | 1 + 10 files changed, 124 insertions(+), 6 deletions(-) diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index b8c71f95..2af87f1b 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -3306,6 +3306,7 @@ where fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool {{ antlr4_runtime::Recognizer::notify_error_listeners( self, + None, source_error.line, source_error.column, &source_error.message, diff --git a/src/bin_support/grammar/frontend.rs b/src/bin_support/grammar/frontend.rs index 71e2bebc..31bb3ed1 100644 --- a/src/bin_support/grammar/frontend.rs +++ b/src/bin_support/grammar/frontend.rs @@ -959,6 +959,7 @@ where fn syntax_error( &mut self, _recognizer: &R, + _offending: Option>, line: usize, column: usize, message: &str, diff --git a/src/bin_support/grammar/generated/antlr_v4_lexer.rs b/src/bin_support/grammar/generated/antlr_v4_lexer.rs index 985f706c..0a5fe693 100644 --- a/src/bin_support/grammar/generated/antlr_v4_lexer.rs +++ b/src/bin_support/grammar/generated/antlr_v4_lexer.rs @@ -390,6 +390,7 @@ where fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool { antlr4_runtime::Recognizer::notify_error_listeners( self, + None, source_error.line, source_error.column, &source_error.message, diff --git a/src/errors.rs b/src/errors.rs index 64ae9f0e..646c8e96 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,4 +1,5 @@ use crate::recognizer::Recognizer; +use crate::token::TokenView; use thiserror::Error; #[derive(Debug, Error, Clone, Eq, PartialEq)] @@ -30,9 +31,15 @@ pub enum AntlrError { /// generically, as [`ConsoleErrorListener`] does, when a listener will be /// registered. pub trait ErrorListener { + /// `offending` carries the token the diagnostic points at, matching + /// ANTLR's `syntaxError(recognizer, offendingSymbol, ...)`. It is `None` + /// for lexer errors (no token was produced) and for diagnostics that are + /// not anchored to a specific token. + #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature fn syntax_error( &mut self, recognizer: &R, + offending: Option>, line: usize, column: usize, message: &str, @@ -48,6 +55,7 @@ impl ErrorListener for ConsoleErrorListener { fn syntax_error( &mut self, _recognizer: &R, + _offending: Option>, line: usize, column: usize, message: &str, diff --git a/src/parser.rs b/src/parser.rs index 3d771114..a20b5a53 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -2143,6 +2143,10 @@ struct ParserDiagnostic { line: usize, column: usize, message: String, + /// Token the diagnostic is anchored to, resolved to a view when the + /// diagnostic is dispatched to error listeners. `None` when no token + /// exists (synthetic positions, lexer-originated messages). + offending: Option, } #[derive(Clone, Debug, Default, Eq, PartialEq)] @@ -5017,7 +5021,11 @@ where } fn dispatch_parser_diagnostic(&self, diagnostic: &ParserDiagnostic) { + let offending = diagnostic + .offending + .and_then(|token| self.token_store().view(token)); self.notify_error_listeners( + offending, diagnostic.line, diagnostic.column, &diagnostic.message, @@ -5038,7 +5046,10 @@ where if self.input.token_source().report_error(source_error) { return; } + // Lexer errors have no offending token: the failure is that no token + // could be produced, matching ANTLR's null offendingSymbol. self.notify_error_listeners( + None, source_error.line, source_error.column, &source_error.message, @@ -5539,6 +5550,7 @@ where line: current_line, column: current_column, message, + offending: Some(current), }); self.record_syntax_errors(1); self.generated_sync_expected = None; @@ -5580,6 +5592,7 @@ where line: current_line, column: current_column, message, + offending: Some(current), }); self.record_syntax_errors(1); self.generated_sync_expected = None; @@ -5899,6 +5912,7 @@ where line, column, message, + offending: None, }, AntlrError::MismatchedInput { expected, found } => diagnostic_for_token( self.input.lt(1), @@ -5916,6 +5930,7 @@ where line, column, message, + offending: None, }, AntlrError::Unsupported(message) => diagnostic_for_token(self.input.lt(1), message), } @@ -11829,11 +11844,14 @@ fn display_input_text(text: &str) -> String { } fn diagnostic_for_token(token: Option, message: String) -> ParserDiagnostic { - let (line, column) = token.map_or((0, 0), |token| (token.line(), token.column())); + let (line, column, offending) = token.map_or((0, 0, None), |token| { + (token.line(), token.column(), Some(token.token_id())) + }); ParserDiagnostic { line, column, message, + offending, } } @@ -12522,7 +12540,9 @@ mod tests { ParserAtnPredictionDiagnostic, ParserAtnPredictionDiagnosticKind, ParserAtnSimulator, }; use crate::atn::serialized::{AtnDeserializer, SerializedAtn}; - use crate::token::{HIDDEN_CHANNEL, Token, TokenId, TokenSink, TokenSpec, TokenStoreError}; + use crate::token::{ + HIDDEN_CHANNEL, Token, TokenId, TokenSink, TokenSpec, TokenStoreError, TokenView, + }; use crate::token_stream::CommonTokenStream; use crate::tree::{NodeKind, ParseTreeStats}; use crate::vocabulary::Vocabulary; @@ -12682,6 +12702,7 @@ mod tests { #[derive(Clone, Debug, Eq, PartialEq)] struct RecordedDiagnostic { grammar_file_name: String, + offending_text: Option, line: usize, column: usize, message: String, @@ -12700,6 +12721,7 @@ mod tests { fn syntax_error( &mut self, recognizer: &R, + offending: Option>, line: usize, column: usize, message: &str, @@ -12710,6 +12732,7 @@ mod tests { .expect("recorded diagnostics lock") .push(RecordedDiagnostic { grammar_file_name: recognizer.grammar_file_name().to_owned(), + offending_text: offending.and_then(|token| token.text().map(str::to_owned)), line, column, message: message.to_owned(), @@ -12783,6 +12806,7 @@ mod tests { line: 1, column: 2, message: "missing 'x' at 'y'".to_owned(), + offending: None, }]; let token_errors = [ TokenSourceError::new(1, 1, "token recognition error at: '@'"), @@ -12806,6 +12830,42 @@ mod tests { ); } + #[test] + fn recovery_diagnostics_expose_the_offending_token_to_listeners() { + let mut parser = mini_parser(vec![ + TestToken::new(7) + .with_text("oops") + .with_span(0, 3) + .with_position(1, 2), + TestToken::eof("parser-test", 4, 1, 6), + ]); + parser.remove_error_listeners(); + let diagnostics = Arc::new(Mutex::new(Vec::new())); + parser.add_error_listener(RecordingErrorListener { + diagnostics: Arc::clone(&diagnostics), + }); + let offending = parser.input.lt_id(1); + assert!(offending.is_some(), "current token should be buffered"); + let parser_diagnostics = [ParserDiagnostic { + line: 1, + column: 2, + message: "extraneous input 'oops'".to_owned(), + offending, + }]; + + parser.dispatch_generated_diagnostics(&parser_diagnostics, &[]); + + // Listeners receive a resolvable view of the offending token — the + // ANTLR offendingSymbol contract downstream span-building error + // reporters (miette-style byte-offset underlines) rely on. + let recorded = diagnostics + .lock() + .expect("recorded diagnostics lock") + .clone(); + assert_eq!(recorded.len(), 1); + assert_eq!(recorded[0].offending_text.as_deref(), Some("oops")); + } + #[test] fn parser_leaves_token_errors_to_source_owned_listeners() { let source_diagnostics = Rc::new(RefCell::new(Vec::new())); @@ -15441,6 +15501,7 @@ mod tests { line: 1, column: 3, message: "missing 'Y' at ''".to_owned(), + offending: parser.input.lt_id(1), }] ); } @@ -15669,6 +15730,7 @@ mod tests { line: 1, column: 1, message: "missing {} at ''".to_owned(), + offending: parser.input.lt_id(1), }] ); } @@ -15717,6 +15779,7 @@ mod tests { line: 1, column: 1, message: "missing 'x' at ''".to_owned(), + offending: parser.input.lt_id(1), }] ); } @@ -15770,6 +15833,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), + offending: None, }] ); parser.exit_rule(); @@ -17570,6 +17634,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'x'".to_owned(), + offending: None, }]), deferred_nodes: FastDeferredNodeId::EMPTY, nodes: NodeSeqId::EMPTY, @@ -17628,6 +17693,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'x' expecting 'a'".to_owned(), + offending: None, }]), deferred_nodes: FastDeferredNodeId::EMPTY, nodes: NodeSeqId::EMPTY, @@ -17639,6 +17705,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'x' expecting 'b'".to_owned(), + offending: None, }]), deferred_nodes: FastDeferredNodeId::EMPTY, nodes: NodeSeqId::EMPTY, @@ -17650,6 +17717,7 @@ mod tests { line: 1, column: 0, message: "missing 'a' at 'x'".to_owned(), + offending: None, }]), deferred_nodes: FastDeferredNodeId::EMPTY, nodes: NodeSeqId::EMPTY, @@ -18019,11 +18087,13 @@ mod tests { line: 1, column: 0, message: "missing X".to_owned(), + offending: None, }]); let _discarded_diagnostics = arena.diagnostic_sequence([ParserDiagnostic { line: 1, column: 1, message: "discarded".to_owned(), + offending: None, }]); let deferred_children = arena.deferred_fragment(live); let _deferred_rule = arena.deferred_rule_node(FastDeferredRule { @@ -18139,17 +18209,20 @@ mod tests { line: 1, column: 0, message: "first".to_owned(), + offending: None, }, ParserDiagnostic { line: 1, column: 1, message: "second".to_owned(), + offending: None, }, ]); let suffix = arena.diagnostic_sequence([ParserDiagnostic { line: 1, column: 2, message: "third".to_owned(), + offending: None, }]); let extras_before = arena.extras.len(); @@ -18307,6 +18380,7 @@ mod tests { line: 1, column: 3, message: "missing 'Y' at ''".to_owned(), + offending: None, }]); let first_alt = RecognizeOutcome { index: 2, diff --git a/src/recognizer.rs b/src/recognizer.rs index 521f848a..70324607 100644 --- a/src/recognizer.rs +++ b/src/recognizer.rs @@ -2,6 +2,7 @@ use std::fmt; use std::sync::{Arc, Mutex}; use crate::errors::{AntlrError, ConsoleErrorListener, ErrorListener}; +use crate::token::TokenView; use crate::vocabulary::Vocabulary; #[derive(Clone)] @@ -15,9 +16,11 @@ impl ErrorListenerSlot { Self(Arc::new(Mutex::new(listener))) } + #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature fn syntax_error( &self, recognizer: &(dyn Recognizer + '_), + offending: Option>, line: usize, column: usize, message: &str, @@ -26,7 +29,7 @@ impl ErrorListenerSlot { self.0 .lock() .expect("error listener lock poisoned") - .syntax_error(recognizer, line, column, message, error); + .syntax_error(recognizer, offending, line, column, message, error); } } @@ -122,16 +125,18 @@ impl RecognizerData { self.error_listeners.clear(); } + #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature fn notify_error_listeners( &self, recognizer: &dyn Recognizer, + offending: Option>, line: usize, column: usize, message: &str, error: Option<&AntlrError>, ) { for listener in &self.error_listeners { - listener.syntax_error(recognizer, line, column, message, error); + listener.syntax_error(recognizer, offending, line, column, message, error); } } } @@ -187,8 +192,13 @@ pub trait Recognizer { } /// Sends one diagnostic to every registered error listener. + /// + /// `offending` is the token the diagnostic is anchored to, when one + /// exists — parser diagnostics resolve it from the token store; lexer + /// diagnostics pass `None` because no token was produced. fn notify_error_listeners( &self, + offending: Option>, line: usize, column: usize, message: &str, @@ -197,7 +207,7 @@ pub trait Recognizer { Self: Sized, { self.data() - .notify_error_listeners(self, line, column, message, error); + .notify_error_listeners(self, offending, line, column, message, error); } fn sempred(&mut self, _rule_index: usize, _pred_index: usize) -> bool { @@ -215,6 +225,7 @@ mod tests { #[derive(Clone, Debug, Eq, PartialEq)] struct RecordedError { grammar_file_name: String, + offending_text: Option, line: usize, column: usize, message: String, @@ -233,6 +244,7 @@ mod tests { fn syntax_error( &mut self, recognizer: &R, + offending: Option>, line: usize, column: usize, message: &str, @@ -243,6 +255,7 @@ mod tests { .expect("recorded errors lock") .push(RecordedError { grammar_file_name: recognizer.grammar_file_name().to_owned(), + offending_text: offending.and_then(|token| token.text().map(str::to_owned)), line, column, message: message.to_owned(), @@ -296,7 +309,7 @@ mod tests { column: 5, message: "unexpected token".to_owned(), }; - recognizer.notify_error_listeners(3, 5, "unexpected token", Some(&error)); + recognizer.notify_error_listeners(None, 3, 5, "unexpected token", Some(&error)); insta::assert_debug_snapshot!( "recognizers_replace_the_default_console_error_listener", diff --git a/src/snapshots/antlr4_runtime__parser__tests__generated_prediction_diagnostics_use_adaptive_context.snap b/src/snapshots/antlr4_runtime__parser__tests__generated_prediction_diagnostics_use_adaptive_context.snap index 320c0736..be8586a9 100644 --- a/src/snapshots/antlr4_runtime__parser__tests__generated_prediction_diagnostics_use_adaptive_context.snap +++ b/src/snapshots/antlr4_runtime__parser__tests__generated_prediction_diagnostics_use_adaptive_context.snap @@ -7,15 +7,30 @@ expression: parser.generated_parser_diagnostics line: 1, column: 2, message: "reportAttemptingFullContext d=0 (s), input='xy'", + offending: Some( + TokenId( + 1, + ), + ), }, ParserDiagnostic { line: 1, column: 0, message: "reportContextSensitivity d=0 (s), input='x'", + offending: Some( + TokenId( + 0, + ), + ), }, ParserDiagnostic { line: 1, column: 2, message: "reportAttemptingFullContext d=0 (s), input='xy'", + offending: Some( + TokenId( + 1, + ), + ), }, ] diff --git a/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap b/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap index f2bef13b..f04bf717 100644 --- a/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap +++ b/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap @@ -5,6 +5,7 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" [ RecordedDiagnostic { grammar_file_name: "Mini.g4", + offending_text: None, line: 1, column: 1, message: "token recognition error at: '@'", @@ -12,6 +13,7 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" }, RecordedDiagnostic { grammar_file_name: "Mini.g4", + offending_text: None, line: 1, column: 2, message: "missing 'x' at 'y'", @@ -19,6 +21,7 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" }, RecordedDiagnostic { grammar_file_name: "Mini.g4", + offending_text: None, line: 1, column: 3, message: "token recognition error at: '#'", diff --git a/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap b/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap index f9d3a929..044a7fbd 100644 --- a/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap +++ b/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap @@ -5,6 +5,7 @@ expression: "*errors.lock().expect(\"recorded errors lock\")" [ RecordedError { grammar_file_name: "Test.g4", + offending_text: None, line: 3, column: 5, message: "unexpected token", diff --git a/src/xpath/generated/x_path_lexer.rs b/src/xpath/generated/x_path_lexer.rs index 5cde72fd..53ec07a0 100644 --- a/src/xpath/generated/x_path_lexer.rs +++ b/src/xpath/generated/x_path_lexer.rs @@ -226,6 +226,7 @@ where fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool { antlr4_runtime::Recognizer::notify_error_listeners( self, + None, source_error.line, source_error.column, &source_error.message, From 9c2cda29ed1548596b54f5e8e98396d75356b845 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sat, 25 Jul 2026 16:30:22 +0200 Subject: [PATCH 2/9] fix(runtime): carry the offending token on AntlrError::ParserError MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Review on #196 found the offending-token contract held only for extraneous-input/missing-token recovery and prediction diagnostics: generated parsers route ordinary mismatched-input, no-viable-alternative, failed-predicate, and sync errors through AntlrError::ParserError, whose diagnostic arm hard-coded offending: None — exactly the errors a real recognizer reports most. ParserError gains offending: Option, recorded where each error is built (recover_generated_match, generated sync, failed-predicate builders, recognition_error) rather than resolved at reporting time: prediction restores the input cursor, so lt(1) at dispatch can point at the decision start instead of the error index — no_viable_alternative_error_at now forwards the anchor its diagnostic already computed. Also converts the new listener test to a named insta snapshot per house style. --- src/bin/antlr4-runtime-testsuite.rs | 2 +- src/errors.rs | 8 ++++- src/parser.rs | 35 ++++++++++++++++--- src/recognizer.rs | 1 + ...pose_the_offending_token_to_listeners.snap | 16 +++++++++ ...ce_the_default_console_error_listener.snap | 1 + 6 files changed, 57 insertions(+), 6 deletions(-) create mode 100644 src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap diff --git a/src/bin/antlr4-runtime-testsuite.rs b/src/bin/antlr4-runtime-testsuite.rs index cff6cf81..a63ca324 100644 --- a/src/bin/antlr4-runtime-testsuite.rs +++ b/src/bin/antlr4-runtime-testsuite.rs @@ -1071,7 +1071,7 @@ fn parser_smoke_main(descriptor: &Descriptor) -> String { "" }; format!( - "pub mod generated {{\n pub mod {lexer_module};\n pub mod {parser_module};\n}}\n\nuse antlr4_runtime::{{AntlrError, CommonTokenStream, InputStream, Parser}};\nuse generated::{lexer_module}::{lexer_type};\nuse generated::{parser_module}::{parser_type};\n\nfn main() {{\n let handle = std::thread::Builder::new()\n // Runtime-suite smoke crates run deeply nested generated parser paths;\n // this is harness-only and does not change the runtime's default stack.\n .stack_size(128 * 1024 * 1024)\n .spawn(|| {{\n let lexer = {lexer_type}::new(InputStream::new(\"{}\"));\n let tokens = CommonTokenStream::new(lexer);\n let mut parser = {parser_type}::new(tokens);\n parser.set_build_parse_trees({build_parse_trees});\n parser.set_report_diagnostic_errors({report_diagnostic_errors});\n{prediction_mode} if let Err(error) = parser.{start_rule}() {{\n match error {{\n AntlrError::ParserError {{ line, column, message }} => eprintln!(\"line {{line}}:{{column}} {{message}}\"),\n other => eprintln!(\"{{other}}\"),\n }}\n }}\n }})\n .expect(\"parser smoke thread should start\");\n handle.join().expect(\"parser smoke thread should finish\");\n}}\n", + "pub mod generated {{\n pub mod {lexer_module};\n pub mod {parser_module};\n}}\n\nuse antlr4_runtime::{{AntlrError, CommonTokenStream, InputStream, Parser}};\nuse generated::{lexer_module}::{lexer_type};\nuse generated::{parser_module}::{parser_type};\n\nfn main() {{\n let handle = std::thread::Builder::new()\n // Runtime-suite smoke crates run deeply nested generated parser paths;\n // this is harness-only and does not change the runtime's default stack.\n .stack_size(128 * 1024 * 1024)\n .spawn(|| {{\n let lexer = {lexer_type}::new(InputStream::new(\"{}\"));\n let tokens = CommonTokenStream::new(lexer);\n let mut parser = {parser_type}::new(tokens);\n parser.set_build_parse_trees({build_parse_trees});\n parser.set_report_diagnostic_errors({report_diagnostic_errors});\n{prediction_mode} if let Err(error) = parser.{start_rule}() {{\n match error {{\n AntlrError::ParserError {{ line, column, message, .. }} => eprintln!(\"line {{line}}:{{column}} {{message}}\"),\n other => eprintln!(\"{{other}}\"),\n }}\n }}\n }})\n .expect(\"parser smoke thread should start\");\n handle.join().expect(\"parser smoke thread should finish\");\n}}\n", rust_string(&descriptor.input) ) } diff --git a/src/errors.rs b/src/errors.rs index 646c8e96..25bab40c 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,5 +1,5 @@ use crate::recognizer::Recognizer; -use crate::token::TokenView; +use crate::token::{TokenId, TokenView}; use thiserror::Error; #[derive(Debug, Error, Clone, Eq, PartialEq)] @@ -19,6 +19,12 @@ pub enum AntlrError { line: usize, column: usize, message: String, + /// Token the error is anchored to, when one exists. The anchor must + /// be captured where the error is built: prediction restores the + /// input cursor, so the current lookahead at reporting time is not + /// necessarily the offending token (`no viable alternative` anchors + /// at the error index while the cursor sits at the decision start). + offending: Option, }, #[error("unsupported runtime feature: {0}")] Unsupported(String), diff --git a/src/parser.rs b/src/parser.rs index a20b5a53..844239ee 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -5321,6 +5321,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if current_type == token_type { @@ -5348,6 +5349,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if current_type == token_type { @@ -5381,6 +5383,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if interval_set_contains(intervals, current_type) { @@ -5413,6 +5416,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if set.contains(current_type) { @@ -5446,6 +5450,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if (min_vocabulary..=max_vocabulary).contains(¤t_type) @@ -5486,6 +5491,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if (min_vocabulary..=max_vocabulary).contains(¤t_type) && !set.contains(current_type) @@ -5538,6 +5544,7 @@ where line: current_line, column: current_column, message: format!("mismatched input {current_display} expecting {expected_display}"), + offending: Some(current), }); } if current_type != TOKEN_EOF @@ -5625,6 +5632,7 @@ where message: format!( "mismatched input {current_display} expecting {mismatch_expected_display}" ), + offending: Some(current), }) } @@ -5672,6 +5680,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; let current_type = self.token_type_for_id(current); if matches(current_type) { @@ -5904,15 +5913,19 @@ where fn generated_rule_error_diagnostic(&self, error: AntlrError) -> ParserDiagnostic { match error { + // The anchor recorded where the error was built wins over the + // current lookahead: prediction restores the cursor, so lt(1) + // here can point at the decision start rather than the error. AntlrError::ParserError { line, column, message, + offending, } => ParserDiagnostic { line, column, message, - offending: None, + offending, }, AntlrError::MismatchedInput { expected, found } => diagnostic_for_token( self.input.lt(1), @@ -6244,6 +6257,7 @@ where line: 0, column: 0, message: "missing current token".to_owned(), + offending: None, })?; if self.token_type_for_id(current) == TOKEN_EOF { return Err(AntlrError::MismatchedInput { @@ -6425,6 +6439,7 @@ where .map_or_else(|| "''".to_owned(), token_input_display), self.expected_symbols_display(&expected_symbols) ), + offending: current.as_ref().map(Token::token_id), }) } @@ -6559,6 +6574,7 @@ where line: diagnostic.line, column: diagnostic.column, message: diagnostic.message, + offending: diagnostic.offending, } } @@ -6569,6 +6585,7 @@ where line: current.as_ref().map(Token::line).unwrap_or_default(), column: current.as_ref().map(Token::column).unwrap_or_default(), message: format!("rule failed predicate: {}", message.into()), + offending: current.as_ref().map(Token::token_id), } } @@ -6588,6 +6605,7 @@ where line: current.as_ref().map(Token::line).unwrap_or_default(), column: current.as_ref().map(Token::column).unwrap_or_default(), message: format!("rule {rule_name} {}", message.into()), + offending: current.as_ref().map(Token::token_id), } } @@ -7191,6 +7209,7 @@ where line: 0, column: 0, message: format!("missing token at index {index}"), + offending: None, })?; let is_eof = self.token_type_for_id(token) == TOKEN_EOF; let child = self.terminal_tree(token); @@ -7523,6 +7542,7 @@ where line, column, message, + offending: current.as_ref().map(Token::token_id), } } @@ -12862,8 +12882,10 @@ mod tests { .lock() .expect("recorded diagnostics lock") .clone(); - assert_eq!(recorded.len(), 1); - assert_eq!(recorded[0].offending_text.as_deref(), Some("oops")); + insta::assert_debug_snapshot!( + "recovery_diagnostics_expose_the_offending_token_to_listeners", + recorded + ); } #[test] @@ -15810,6 +15832,10 @@ mod tests { let mut child = parser.enter_rule(4, 1); parser.discard_invoking_state(marker); + // The anchor recorded where the error was built must survive into the + // dispatched diagnostic even though recovery consumes past it below. + let offending = parser.input.lt_id(1); + assert!(offending.is_some(), "the 'z' token should be buffered"); parser.recover_generated_rule( &mut child, &atn, @@ -15817,6 +15843,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), + offending, }, ); let tree = parser.finish_rule(child, false); @@ -15833,7 +15860,7 @@ mod tests { line: 1, column: 0, message: "mismatched input 'z' expecting {'X', 'Y'}".to_owned(), - offending: None, + offending, }] ); parser.exit_rule(); diff --git a/src/recognizer.rs b/src/recognizer.rs index 70324607..caf2ee04 100644 --- a/src/recognizer.rs +++ b/src/recognizer.rs @@ -308,6 +308,7 @@ mod tests { line: 3, column: 5, message: "unexpected token".to_owned(), + offending: None, }; recognizer.notify_error_listeners(None, 3, 5, "unexpected token", Some(&error)); diff --git a/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap b/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap new file mode 100644 index 00000000..a81afa7a --- /dev/null +++ b/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap @@ -0,0 +1,16 @@ +--- +source: src/parser.rs +expression: recorded +--- +[ + RecordedDiagnostic { + grammar_file_name: "Mini.g4", + offending_text: Some( + "oops", + ), + line: 1, + column: 2, + message: "extraneous input 'oops'", + error: None, + }, +] diff --git a/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap b/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap index 044a7fbd..a658bce3 100644 --- a/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap +++ b/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap @@ -14,6 +14,7 @@ expression: "*errors.lock().expect(\"recorded errors lock\")" line: 3, column: 5, message: "unexpected token", + offending: None, }, ), }, From 8e7884cb6da165b9ee47045837246fa5ab2fe438 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sat, 25 Jul 2026 17:08:58 +0200 Subject: [PATCH 3/9] ci: displace wedged Copy/Paste Detection run From 91c871e8f607fc565d4eea2a66b5d5e8bf4b629c Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sat, 25 Jul 2026 23:07:19 +0200 Subject: [PATCH 4/9] merge: combine offending-token (#195) with main incl. depth cap (#199) The depth-cap violation error now carries its offending token like every other ParserError built at a known input position. --- src/parser.rs | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/parser.rs b/src/parser.rs index 2b338ac9..0b65c24f 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -5821,14 +5821,15 @@ where if let Some(error) = &self.rule_depth_error { return error.clone(); } - let (line, column) = self - .input - .lt(1) + let current = self.input.lt(1); + let (line, column) = current + .as_ref() .map_or((0, 0), |token| (token.line(), token.column())); let error = AntlrError::ParserError { line, column, message: format!("rule nesting depth limit of {max} exceeded"), + offending: current.as_ref().map(Token::token_id), }; self.rule_depth_error = Some(error.clone()); error From d259246bc171334fca89f9d0a223027c9e1bb723 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sat, 25 Jul 2026 23:53:36 +0200 Subject: [PATCH 5/9] =?UTF-8?q?feat(runtime):=20add=5Fparse=5Flistener=20?= =?UTF-8?q?=E2=80=94=20parse-time=20rule=20enter/exit=20events?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ANTLR's addParseListener delivers enterEveryRule/exitEveryRule during recognition; cel-rust's RecursionListener (live expr-depth counting with parse abort) is built on it. Our runtime only had post-parse listeners, forcing ports to rework such listeners into post-parse tree walks. ParseListener (enter fallible for aborts, exit infallible) dispatches from generated rule bodies: enter before the body after the depth-cap probe, exit on every exit path, one simulated enter per left-recursive operator expansion (upstream Parser.pushNewRecursionContext fires triggerEnterRuleEvent) with matching exits as the rule unrolls — pairs always balance. A listener abort is sticky through rule-level recovery, drained at the top-level entry via take_parse_abort() (unified with the depth-cap violation, depth error preferred), and cleared at entry so instances never poison the next parse. Costs nothing when unused: dispatch sites gate on list emptiness (one predictable branch; Kotlin parse timings unchanged, ktor parse-bench fixture at baseline). When a listener is registered, generated dispatch routes ATN-preferred rules through their generated bodies so real grammars observe every rule; interpreter-only rules do not fire events (documented divergence, matching the depth cap). Emitted probes are plain if-let — generated output stays edition-2021 clean. e2e: cel-rust's RecursionListener ported verbatim in the deep-nesting fixture test — live counting, positioned abort error, LR expansion entries counted, clean reuse after abort. Closes #202 --- src/bin/antlr4-rust-gen.rs | 105 +++++++++++++------ src/lib.rs | 9 +- src/parser.rs | 191 ++++++++++++++++++++++++++++++++++- tests/antlr4_rust_gen_cli.rs | 106 +++++++++++++++++++ 4 files changed, 374 insertions(+), 37 deletions(-) diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index ab0847e7..7de959ea 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -5185,12 +5185,13 @@ fn render_generated_rule_dispatch_with_rule_names( .copied() .unwrap_or_default() { - // The interpreted fast path never consults the depth cap, so a - // configured bound overrides the ATN preference: correctness of - // the resource limit beats the long-call-chain optimization. + // The interpreted fast path never consults the depth cap nor + // fires parse-listener events, so either feature overrides the + // ATN preference: correctness of the resource limit and listener + // coverage beat the long-call-chain optimization. writeln!( out, - " {index} if self.generated_only() || self.base.has_rule_depth_cap() => Some(self.parse_generated_rule_{index}_dispatch(precedence, allow_fallback))," + " {index} if self.generated_only() || self.base.has_rule_depth_cap() || self.base.has_parse_listeners() => Some(self.parse_generated_rule_{index}_dispatch(precedence, allow_fallback))," ) .expect("writing to a string cannot fail"); } else { @@ -5229,19 +5230,27 @@ fn render_generated_rule_dispatch_with_rule_names( // Rule nesting maps onto native call depth; sample remaining stack // capacity at the shared dispatch boundary so deeply nested input // grows onto a segmented stack instead of aborting the process. The - // optional depth-cap probe stays inline-cheap (one `Option` check - // when unset) and its error, while absorbed by rule-level recovery - // like any rule failure, stays sticky until the top-level entry - // drains it — that pairing is what actually enforces the abort. - // Plain `if let` keeps generated output edition-2021 compatible. + // optional depth-cap and parse-listener probes stay inline-cheap + // (one `Option`/emptiness check each when unused) and their errors, + // while absorbed by rule-level recovery like any rule failure, stay + // sticky until the top-level entry drains them — that pairing is + // what actually enforces the abort. The matching listener exit event + // fires from the rule body's exit paths (`finish_rule`/recovery), so + // enter/exit stay balanced. Plain `if let` keeps generated output + // edition-2021 compatible. writeln!( out, " if let Some(error) = self.base.rule_depth_cap_violation() {{\n \ return Err(GeneratedRuleError::Fatal(error));\n \ }}\n \ - if self.base.generated_rule_stack_check_due() {{\n \ + if let Some(error) = self.base.parse_listener_enter_rule({index}) {{\n \ + return Err(GeneratedRuleError::Fatal(error));\n \ + }}\n \ + let __listener_result = if self.base.generated_rule_stack_check_due() {{\n \ antlr4_runtime::grow_generated_rule_stack(|| {target_call})\n \ - }} else {{\n {target_call}\n }}" + }} else {{\n {target_call}\n }};\n \ + self.base.parse_listener_exit_rule({index});\n \ + __listener_result" ) .expect("writing to a string cannot fail"); writeln!(out, " }}").expect("writing to a string cannot fail"); @@ -6943,15 +6952,20 @@ fn render_generated_left_recursive_loop( .expect("writing to a string cannot fail"); } // Each operator iteration deepens the tree without a rule frame; probe - // the depth cap BEFORE the expansion push so the boundary matches the - // dispatch site (which checks before its rule-frame push): frames and - // expansions are both admitted up to the cap, and token-only operator - // alternatives (no nested rule dispatch) still abort promptly instead of - // at the top-level drain. + // the depth cap and the parse-listener enter event BEFORE the expansion + // push so the boundary matches the dispatch site (which checks before + // its rule-frame push): frames and expansions are both admitted up to + // the cap, listeners see the simulated rule entry upstream fires for + // pushNewRecursionContext, and token-only operator alternatives (no + // nested rule dispatch) still abort promptly instead of at the + // top-level drain. The matching exit events fire as the rule unrolls. writeln!( out, "{pad} if let Some(__depth_error) = self.base.rule_depth_cap_violation() {{\n\ {pad} return Err(__depth_error);\n\ + {pad} }}\n\ + {pad} if let Some(__listener_error) = self.base.parse_listener_enter_rule({rule_index}) {{\n\ + {pad} return Err(__listener_error);\n\ {pad} }}" ) .expect("writing to a string cannot fail"); @@ -9255,6 +9269,29 @@ fn embedded_render_slots( /// Kept as a standalone literal (single braces, no format placeholders) so the /// large parser template stays under the line-length lint and this ANTLR /// `Parser.compileParseTreePattern` analog reads as ordinary code. +/// Renders the parse-listener registration facade on the generated parser. +/// +/// Kept as a standalone literal (single braces, no format placeholders) so the +/// large parser template stays under the line-length lint. +const fn render_parse_listener_facade() -> &'static str { + r" + /// Registers a listener for committed rule enter/exit events during + /// recognition (ANTLR's `addParseListener`). See + /// [`antlr4_runtime::ParseListener`] for the delivery contract. + pub fn add_parse_listener(&mut self, listener: T) + where + T: antlr4_runtime::ParseListener + 'static, + { + self.base.add_parse_listener(listener); + } + + /// Removes every registered parse listener. + pub fn remove_parse_listeners(&mut self) { + self.base.remove_parse_listeners(); + } +" +} + const fn render_compile_parse_tree_pattern_method() -> &'static str { r#" /// Compiles a tree pattern rooted at parser rule `rule_index`. @@ -9324,6 +9361,7 @@ fn render_parser_with_options( let patterns = options.patterns.unwrap_or(&empty_patterns); let type_name = rust_type_name(grammar_name); let compile_pattern_method = render_compile_parse_tree_pattern_method(); + let parse_listener_facade = render_parse_listener_facade(); let metadata = render_parser_metadata(grammar_name, data); let parser_atn = data.parser_atn()?; let parser_atn_data = render_u32_slice(parser_atn.packed_words()); @@ -9658,6 +9696,7 @@ where self.base.remove_error_listeners(); }} +{parse_listener_facade} /// Fully resets parser-owned state and rewinds the current token stream. pub fn reset(&mut self) {{ self.base.reset(); @@ -9773,10 +9812,11 @@ where // are preserved so a generated parent can surface a recovered child's // fail-loud coordinate at this boundary. self.base.reset_unknown_semantic_hits(); - // Likewise drop a stale depth-cap violation: entry rules share one - // parser instance, and the sticky flag must not poison the next - // parse when the previous one exited through an error path. - let _ = self.base.take_rule_depth_error(); + // Likewise drop stale sticky aborts (depth-cap violation, + // parse-listener abort): entry rules share one parser instance, + // and the flags must not poison the next parse when the previous + // one exited through an error path. + let _ = self.base.take_parse_abort(); }} let __rule_start = antlr4_runtime::IntStream::index(self.base.input()); let __generated_only = self.generated_only(); @@ -9797,13 +9837,13 @@ where if let Some(semantic_error) = self.base.take_unknown_semantic_error() {{ return Err(semantic_error); }} - // The depth-cap violation wins over an error derived - // from it (e.g. a sync failure after recovery absorbed - // the capped rule): the caller must learn the resource - // bound was hit, and draining un-poisons the instance - // for the next entry-rule call. - if let Some(depth_error) = self.base.take_rule_depth_error() {{ - return Err(depth_error); + // A sticky abort (depth cap, listener) wins over an + // error derived from it (e.g. a sync failure after + // recovery absorbed the aborted rule): the caller must + // learn the real cause, and draining un-poisons the + // instance for the next entry-rule call. + if let Some(abort) = self.base.take_parse_abort() {{ + return Err(abort); }} }} return Err(error.into_error()); @@ -9829,10 +9869,11 @@ where if let Some(error) = self.base.take_unknown_semantic_error() {{ return Err(error); }} - // A depth-cap violation is a resource bound, not a syntax error: - // rule-level recovery may have produced a tree anyway, but the - // parse must still fail (and a reused parser must start clean). - if let Some(error) = self.base.take_rule_depth_error() {{ + // A sticky abort (depth-cap violation, listener abort) is not a + // syntax error: rule-level recovery may have produced a tree + // anyway, but the parse must still fail (and a reused parser + // must start clean). + if let Some(error) = self.base.take_parse_abort() {{ return Err(error); }} }} @@ -12602,7 +12643,7 @@ mod tests { // A configured depth cap overrides the ATN preference: only generated // bodies enforce the bound, so the guard admits either trigger. assert!(rendered.contains( - "0 if self.generated_only() || self.base.has_rule_depth_cap() => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))" + "0 if self.generated_only() || self.base.has_rule_depth_cap() || self.base.has_parse_listeners() => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))" )); assert!(!rendered.contains( "0 => Some(self.parse_generated_rule_0_dispatch(precedence, allow_fallback))" diff --git a/src/lib.rs b/src/lib.rs index 50f64da5..f792efb3 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,10 +34,11 @@ pub use lexer::{ BaseLexer, Lexer, LexerCustomAction, LexerLifecycleCtx, LexerMode, LexerPredicate, LexerSemCtx, }; pub use parser::{ - BailErrorStrategy, BaseParser, ExpectedTokenSet, NoSemanticHooks, Parser, ParserAction, - ParserMemberAction, ParserPredicate, ParserReturnAction, ParserRuleArg, ParserRuntimeOptions, - ParserSemCtx, ParserSemanticAction, ParserSemanticPredicate, ParserSemantics, PredictionMode, - RecognitionArenaStats, SemanticHooks, UnknownSemanticPolicy, grow_generated_rule_stack, + BailErrorStrategy, BaseParser, ExpectedTokenSet, NoSemanticHooks, ParseListener, Parser, + ParserAction, ParserMemberAction, ParserPredicate, ParserReturnAction, ParserRuleArg, + ParserRuntimeOptions, ParserSemCtx, ParserSemanticAction, ParserSemanticPredicate, + ParserSemantics, PredictionMode, RecognitionArenaStats, SemanticHooks, UnknownSemanticPolicy, + grow_generated_rule_stack, }; #[cfg(feature = "perf-counters")] pub use perf::{dump as dump_prediction_perf_counters, reset as reset_prediction_perf_counters}; diff --git a/src/parser.rs b/src/parser.rs index 0b65c24f..bd828977 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -128,6 +128,59 @@ const ADAPTIVE_DIRECT_STEP_LIMIT: usize = RECOGNITION_DEPTH_LIMIT; pub fn grow_generated_rule_stack(body: impl FnOnce() -> R) -> R { stacker::maybe_grow(FAST_RECOGNIZE_RED_ZONE, FAST_RECOGNIZE_STACK_SIZE, body) } + +/// Receives committed rule enter/exit events during recognition, matching +/// ANTLR's `addParseListener` contract ([`Parser::add_parse_listener`]). +/// +/// Events fire on the generated recursive-descent path as rules are entered +/// and exited, including one simulated entry per left-recursive operator +/// expansion (upstream `Parser.pushNewRecursionContext` fires +/// `triggerEnterRuleEvent` for exactly that case). Enter/exit calls are +/// always balanced, including on error-recovery paths. +/// +/// Divergence from Java to know about: upstream generated rule methods run +/// only on the committed parse, while this runtime may re-enter a rule while +/// recovering from a syntax error — such retries deliver additional balanced +/// enter/exit pairs. Depth counters and resource bounds (the primary use +/// case) are unaffected; exact once-per-node collectors should prefer the +/// post-parse tree walker. +/// +/// `enter_every_rule` is fallible: returning `Err` aborts the parse with +/// that error. The abort is sticky through rule-level recovery — the parse +/// fails even when recovery could have produced a tree, mirroring how a +/// thrown exception escapes ANTLR's `triggerEnterRuleEvent`. Rules the +/// generator emitted no body for (interpreter-only fallback) do not fire +/// events; when any parse listener is registered, generated dispatch routes +/// ATN-preferred rules through their generated bodies so real grammars +/// observe every rule. +pub trait ParseListener: Send { + /// Called when a generated rule is entered, before its body runs, and + /// once per left-recursive operator expansion. `current` is the lookahead + /// token the rule starts at (its line/column/offsets anchor listener + /// diagnostics), or `None` at end of input. + /// + /// Returning `Err` aborts the parse with the given error. + fn enter_every_rule( + &mut self, + rule_index: usize, + current: Option>, + ) -> Result<(), AntlrError>; + + /// Called when a generated rule exits, after its body (and any rule-level + /// error recovery) finished, and once per left-recursive operator + /// expansion as the rule unrolls. + fn exit_every_rule(&mut self, rule_index: usize) { + let _ = rule_index; + } +} + +struct ParseListenerSlot(Box); + +impl std::fmt::Debug for ParseListenerSlot { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("ParseListener") + } +} /// Probe window for deciding whether clean-pass memo entries are reusable /// enough to keep caching. High-cardinality parses mostly produce one-shot /// entries; compact ambiguous loops repeatedly hit the same keys. @@ -1203,6 +1256,16 @@ pub struct BaseParser { /// recovering (ANTLR's `BailErrorStrategy`). Generated recognizers set it /// through `set_error_handler(BailErrorStrategy::new())`. bail_on_error: bool, + /// Parse listeners receiving committed rule enter/exit events during + /// recognition (ANTLR's `addParseListener`). Empty in the default + /// configuration, and every dispatch site is gated on emptiness so the + /// unused feature costs one predictable branch per rule boundary. + parse_listeners: Vec, + /// Sticky abort requested by a parse listener's `enter_every_rule`. + /// Mirrors `rule_depth_error`: rule-level recovery absorbs the error like + /// any rule failure, so the flag stays set until the top-level entry + /// drains it and fails the parse. + parse_listener_abort: Option, /// Optional cap on rule-nesting depth for adversarial-input hardening. /// `None` (default) parses unbounded nesting; `Some(n)` aborts the parse /// with a positioned syntax error once `n` rule frames are exceeded. @@ -4839,6 +4902,8 @@ where precedence_stack: vec![0], invoked_predicates: Vec::new(), bail_on_error: false, + parse_listeners: Vec::new(), + parse_listener_abort: None, max_rule_depth: None, rule_depth_error: None, recursion_expansions: 0, @@ -4902,6 +4967,7 @@ where self.decision_override_generation = 0; self.unknown_predicate_hits.clear(); self.unhandled_action_hits.clear(); + self.parse_listener_abort = None; self.rule_depth_error = None; self.recursion_expansions = 0; self.recursion_expansion_marks.clear(); @@ -5856,6 +5922,115 @@ where self.max_rule_depth.is_some() } + /// Registers a listener for committed rule enter/exit events during + /// recognition (ANTLR's `addParseListener`). See [`ParseListener`] for + /// the delivery contract. + pub fn add_parse_listener(&mut self, listener: L) + where + L: ParseListener + 'static, + { + self.parse_listeners + .push(ParseListenerSlot(Box::new(listener))); + } + + /// Removes every registered parse listener. + pub fn remove_parse_listeners(&mut self) { + self.parse_listeners.clear(); + } + + /// Reports whether any parse listener is registered. + /// + /// Generated dispatch consults this alongside [`Self::has_rule_depth_cap`] + /// when choosing between the generated body (which fires events) and the + /// ATN-preferred interpreted fast path (which does not). + #[must_use] + pub const fn has_parse_listeners(&self) -> bool { + !self.parse_listeners.is_empty() + } + + /// Fires `enter_every_rule` on registered parse listeners, returning the + /// abort error if any listener requested one. + /// + /// Generated rule dispatch calls this after the depth-cap probe and + /// before the rule body runs; the generated left-recursive loop calls it + /// once per operator expansion, mirroring upstream ANTLR's simulated + /// rule-entry event for `pushNewRecursionContext`. A listener abort is + /// sticky exactly like a depth-cap violation: rule-level recovery absorbs + /// the returned error, so the flag holds until the top-level entry drains + /// it via [`Self::take_parse_listener_abort`] and fails the parse. + pub fn parse_listener_enter_rule(&mut self, rule_index: usize) -> Option { + if self.parse_listeners.is_empty() { + return None; + } + self.parse_listener_enter_rule_cold(rule_index) + } + + #[cold] + fn parse_listener_enter_rule_cold(&mut self, rule_index: usize) -> Option { + if let Some(error) = &self.parse_listener_abort { + return Some(error.clone()); + } + let current = self.input.lt(1); + // Split borrows: the token view borrows the input while listeners + // need `&mut`, so listeners are taken out for the dispatch. Listener + // methods have no parser access and cannot observe the absence. + let mut listeners = std::mem::take(&mut self.parse_listeners); + let mut abort = None; + for slot in &mut listeners { + if let Err(error) = slot.0.enter_every_rule(rule_index, current) { + abort = Some(error); + break; + } + } + self.parse_listeners = listeners; + if let Some(error) = abort { + self.parse_listener_abort = Some(error.clone()); + return Some(error); + } + None + } + + /// Fires `exit_every_rule` on registered parse listeners. + /// + /// Generated rule bodies call this on every exit path — success and + /// recovery alike — keeping enter/exit pairs balanced, and the generated + /// left-recursive loop calls it once per operator expansion when the rule + /// finishes unrolling. + pub fn parse_listener_exit_rule(&mut self, rule_index: usize) { + if self.parse_listeners.is_empty() { + return; + } + for slot in &mut self.parse_listeners { + slot.0.exit_every_rule(rule_index); + } + } + + /// Drains the sticky parse-listener abort recorded by + /// [`Self::parse_listener_enter_rule`], if any. + /// + /// Generated top-level rule entries call this after recognition so an + /// aborted parse fails even when recovery produced a tree, and so a + /// reused parser starts its next parse clean. + pub const fn take_parse_listener_abort(&mut self) -> Option { + self.parse_listener_abort.take() + } + + /// Drains every sticky parse abort — the depth-cap violation and the + /// parse-listener abort — returning the depth error preferentially. + /// + /// Generated top-level rule entries call this on both exit paths: the + /// recorded abort wins over errors derived from it (recovery may have + /// absorbed the aborted rule and failed differently later), a recovered + /// `Ok` tree still fails when an abort was recorded, and draining leaves + /// the instance clean for the next entry-rule call. + pub fn take_parse_abort(&mut self) -> Option { + if let Some(error) = self.rule_depth_error.take() { + self.parse_listener_abort = None; + return Some(error); + } + self.parse_listener_abort.take() + } + /// Enters a generated parser rule and returns the context object the /// generated method should populate. pub fn enter_rule(&mut self, state: isize, rule_index: usize) -> ParserRuleContext { @@ -6121,7 +6296,9 @@ where self.set_state(state); // Counts toward the depth cap: each operator iteration deepens the // parse tree one level without pushing a rule frame, and upstream - // fires a rule-entry listener event for it. + // fires a rule-entry listener event for it. The parse-listener enter + // event for this expansion fires from the generated loop's probe + // just before this call, where a listener abort can propagate. self.recursion_expansions += 1; if let Some(stop) = self .rule_stop_token_index(self.input.index(), false) @@ -6148,6 +6325,18 @@ where self.precedence_stack.pop(); } if let Some(mark) = self.recursion_expansion_marks.pop() { + // Each expansion fired a simulated rule-entry event; fire the + // matching exits as the rule unrolls so parse listeners see + // balanced pairs (upstream unrolls via exitRule per context). + if !self.parse_listeners.is_empty() { + let rule_index = self + .rule_context_stack + .last() + .map_or(usize::MAX, |frame| frame.rule_index); + for _ in mark..self.recursion_expansions { + self.parse_listener_exit_rule(rule_index); + } + } self.recursion_expansions = mark; } self.exit_rule(); diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index dd26c38d..9761ebe9 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -1716,6 +1716,112 @@ mod deep_nesting_tests { "a different entry rule on the same instance starts clean" ); } + + /// The cel-rust `RecursionListener` shape: count `expr` nesting live, + /// abort the parse past a limit — ported verbatim onto + /// `add_parse_listener` (issue #202). + struct RecursionListener { + max: u16, + depth: u16, + high_water: std::sync::Arc, + } + + impl antlr4_runtime::ParseListener for RecursionListener { + fn enter_every_rule( + &mut self, + rule_index: usize, + current: Option>, + ) -> Result<(), antlr4_runtime::AntlrError> { + if rule_index == super::nest_parser::RULE_EXPR { + self.depth += 1; + self.high_water + .fetch_max(self.depth, std::sync::atomic::Ordering::Relaxed); + } + if self.depth > self.max { + use antlr4_runtime::Token as _; + let (line, column) = current + .as_ref() + .map_or((0, 0), |token| (token.line(), token.column())); + return Err(antlr4_runtime::AntlrError::ParserError { + line, + column, + message: format!("Recursion limit of {} exceeded", self.max), + offending: current.as_ref().map(antlr4_runtime::Token::token_id), + }); + } + Ok(()) + } + + fn exit_every_rule(&mut self, rule_index: usize) { + if rule_index == super::nest_parser::RULE_EXPR { + self.depth -= 1; + } + } + } + + #[test] + fn parse_listener_counts_rules_and_aborts_past_a_limit() { + use std::sync::Arc; + use std::sync::atomic::{AtomicU16, Ordering}; + + // Under the limit: events fire, parse succeeds, enter/exit balance + // (depth returns to zero, so high-water == max nesting seen). + let high_water = Arc::new(AtomicU16::new(0)); + let lexer = NestLexer::new(InputStream::new(&nested(3))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 32, + depth: 0, + high_water: Arc::clone(&high_water), + }); + assert!(parser.s().is_ok(), "shallow input parses under the limit"); + assert_eq!( + high_water.load(Ordering::Relaxed), + 4, + "one expr per bracket level plus the outermost expr" + ); + + // Past the limit: the listener aborts with its own positioned error, + // sticky through recovery. + let lexer = NestLexer::new(InputStream::new(&nested(64))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 8, + depth: 0, + high_water: Arc::new(AtomicU16::new(0)), + }); + let error = parser.s().expect_err("listener abort must fail the parse"); + assert!( + error.to_string().contains("Recursion limit of 8 exceeded"), + "unexpected error: {error}" + ); + + // Left-recursive operator expansions fire simulated rule entries + // (upstream triggerEnterRuleEvent parity): a 40-term `a+a+...` chain + // exceeds an expr limit of 8 even though rule frames barely nest. + let chain = vec!["a"; 40].join("+"); + let lexer = NestLexer::new(InputStream::new(&chain)); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 8, + depth: 0, + high_water: Arc::new(AtomicU16::new(0)), + }); + let error = parser + .s() + .expect_err("operator expansions must count as rule entries"); + assert!( + error.to_string().contains("Recursion limit of 8 exceeded"), + "unexpected error: {error}" + ); + + // The abort does not poison the instance: clearing listeners and + // reusing the parser parses clean input. + let lexer = NestLexer::new(InputStream::new("a")); + parser.set_token_stream(CommonTokenStream::new(lexer)); + parser.remove_parse_listeners(); + assert!(parser.s().is_ok(), "reused parser starts clean"); + } } "#, ); From 4155572ef084adb441f3182e4f2716770ded42b9 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sun, 26 Jul 2026 00:41:59 +0200 Subject: [PATCH 6/9] fix(runtime): address parse-listener review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Reserved-name collision: add_parse_listener/remove_parse_listeners join GENERATED_PARSER_RESERVED_RULE_METHODS so a grammar rule named addParseListener renames to add_parse_listener_rule instead of colliding with the facade (E0592). 2. Exit events now fire in reverse registration order, matching upstream Parser.triggerExitRuleEvent; new e2e test registers two tracing listeners and pins enter A,B / exit B,A plus pair balance on both the success and recovery paths. 3. Doc correction: the aborting enter_every_rule receives no matching exit (same as Java, where enterRule throws before try/finally); listener state shared across parses must be reset after an abort. 4. remove_parse_listeners clears the sticky abort and returns the boxed listeners, giving callers their accumulated state back without external shared handles. 5. enter_every_rule takes #[non_exhaustive] EnterRuleEvent so future fields extend the event without breaking implementors; the dispatch helper loses its #[cold] (it is the hot path once registered). 6. New coverage: multi-listener order, recovery balance, depth-cap + listener coexistence (either bound trips first and surfaces). Measured with-listener cost (finding 7): on the CEL grammar — the migration target, no ATN-preferred rules — a counting listener adds ~5% (0.0433 -> 0.0453 ms on cel-rust's criterion stress expression). On grammars with ATN-preferred rules the dominant cost is the routing override those rules take (Kotlin ktor fixture 10.2 -> 86 ms), shared byte-for-byte with the depth cap (cap-only run measures identically); listener dispatch on top of it is noise. Documented on the trait. --- src/bin/antlr4-rust-gen.rs | 13 +++- src/lib.rs | 10 +-- src/parser.rs | 70 ++++++++++++----- tests/antlr4_rust_gen_cli.rs | 141 +++++++++++++++++++++++++++++++++-- 4 files changed, 199 insertions(+), 35 deletions(-) diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index 7de959ea..e07da062 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -7129,6 +7129,8 @@ const GENERATED_PARSER_RESERVED_RULE_METHODS: &[&str] = &[ "clear_dfa", "add_error_listener", "remove_error_listeners", + "add_parse_listener", + "remove_parse_listeners", "node", "into_token_stream", "into_token_store", @@ -9285,9 +9287,10 @@ const fn render_parse_listener_facade() -> &'static str { self.base.add_parse_listener(listener); } - /// Removes every registered parse listener. - pub fn remove_parse_listeners(&mut self) { - self.base.remove_parse_listeners(); + /// Removes every registered parse listener and returns them, dropping + /// any sticky abort a removed listener had requested. + pub fn remove_parse_listeners(&mut self) -> Vec> { + self.base.remove_parse_listeners() } " } @@ -12078,6 +12081,8 @@ mod tests { "clearDfa".to_owned(), "addErrorListener".to_owned(), "removeErrorListeners".to_owned(), + "addParseListener".to_owned(), + "removeParseListeners".to_owned(), "compileParseTreePattern".to_owned(), "regularRule".to_owned(), ]; @@ -12093,6 +12098,8 @@ mod tests { "clear_dfa_rule", "add_error_listener_rule", "remove_error_listeners_rule", + "add_parse_listener_rule", + "remove_parse_listeners_rule", "compile_parse_tree_pattern_rule", "regular_rule" ] diff --git a/src/lib.rs b/src/lib.rs index f792efb3..f7d02cc9 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -34,11 +34,11 @@ pub use lexer::{ BaseLexer, Lexer, LexerCustomAction, LexerLifecycleCtx, LexerMode, LexerPredicate, LexerSemCtx, }; pub use parser::{ - BailErrorStrategy, BaseParser, ExpectedTokenSet, NoSemanticHooks, ParseListener, Parser, - ParserAction, ParserMemberAction, ParserPredicate, ParserReturnAction, ParserRuleArg, - ParserRuntimeOptions, ParserSemCtx, ParserSemanticAction, ParserSemanticPredicate, - ParserSemantics, PredictionMode, RecognitionArenaStats, SemanticHooks, UnknownSemanticPolicy, - grow_generated_rule_stack, + BailErrorStrategy, BaseParser, EnterRuleEvent, ExpectedTokenSet, NoSemanticHooks, + ParseListener, Parser, ParserAction, ParserMemberAction, ParserPredicate, ParserReturnAction, + ParserRuleArg, ParserRuntimeOptions, ParserSemCtx, ParserSemanticAction, + ParserSemanticPredicate, ParserSemantics, PredictionMode, RecognitionArenaStats, SemanticHooks, + UnknownSemanticPolicy, grow_generated_rule_stack, }; #[cfg(feature = "perf-counters")] pub use perf::{dump as dump_prediction_perf_counters, reset as reset_prediction_perf_counters}; diff --git a/src/parser.rs b/src/parser.rs index bd828977..35243f45 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -135,8 +135,13 @@ pub fn grow_generated_rule_stack(body: impl FnOnce() -> R) -> R { /// Events fire on the generated recursive-descent path as rules are entered /// and exited, including one simulated entry per left-recursive operator /// expansion (upstream `Parser.pushNewRecursionContext` fires -/// `triggerEnterRuleEvent` for exactly that case). Enter/exit calls are -/// always balanced, including on error-recovery paths. +/// `triggerEnterRuleEvent` for exactly that case). Enter events fire in +/// registration order and exit events in reverse registration order, +/// matching upstream. Enter/exit calls balance on every completed path, +/// including error recovery — with one exception shared with Java: the +/// enter that returns `Err` (or whose Java analog throws) receives no +/// matching exit, so listener state shared across parses via `Arc` must be +/// reset after an abort. /// /// Divergence from Java to know about: upstream generated rule methods run /// only on the committed parse, while this runtime may re-enter a rule while @@ -153,18 +158,20 @@ pub fn grow_generated_rule_stack(body: impl FnOnce() -> R) -> R { /// events; when any parse listener is registered, generated dispatch routes /// ATN-preferred rules through their generated bodies so real grammars /// observe every rule. +/// +/// Cost: with no listener registered, dispatch pays one emptiness check per +/// rule boundary (benchmarked at baseline). With one registered, dispatch +/// itself is a few percent; on grammars where the generator classified rules +/// ATN-preferred, the dominant cost is the routing override above — the same +/// one [`Parser::set_max_rule_depth`] takes — which trades that fast path +/// for observability. Grammars without ATN-preferred rules (most small DSLs) +/// pay only the dispatch. pub trait ParseListener: Send { /// Called when a generated rule is entered, before its body runs, and - /// once per left-recursive operator expansion. `current` is the lookahead - /// token the rule starts at (its line/column/offsets anchor listener - /// diagnostics), or `None` at end of input. + /// once per left-recursive operator expansion. /// /// Returning `Err` aborts the parse with the given error. - fn enter_every_rule( - &mut self, - rule_index: usize, - current: Option>, - ) -> Result<(), AntlrError>; + fn enter_every_rule(&mut self, event: &EnterRuleEvent<'_>) -> Result<(), AntlrError>; /// Called when a generated rule exits, after its body (and any rule-level /// error recovery) finished, and once per left-recursive operator @@ -174,6 +181,21 @@ pub trait ParseListener: Send { } } +/// A rule-entry event delivered to [`ParseListener::enter_every_rule`]. +/// +/// Non-exhaustive so future fields (alt number, invoking state, a context +/// handle) extend the event without breaking implementors. +#[derive(Debug)] +#[non_exhaustive] +pub struct EnterRuleEvent<'a> { + /// Index of the rule being entered (compare against the generated + /// `RULE_*` constants). + pub rule_index: usize, + /// The lookahead token the rule starts at — its line/column/offsets + /// anchor listener diagnostics — or `None` at end of input. + pub current: Option>, +} + struct ParseListenerSlot(Box); impl std::fmt::Debug for ParseListenerSlot { @@ -5933,9 +5955,15 @@ where .push(ParseListenerSlot(Box::new(listener))); } - /// Removes every registered parse listener. - pub fn remove_parse_listeners(&mut self) { - self.parse_listeners.clear(); + /// Removes every registered parse listener and returns them, dropping any + /// sticky abort a removed listener had requested. + /// + /// Returning the boxed listeners gives callers back the state they + /// accumulated (depth counters, collected events) without threading + /// shared handles through the listener. + pub fn remove_parse_listeners(&mut self) -> Vec> { + self.parse_listener_abort = None; + self.parse_listeners.drain(..).map(|slot| slot.0).collect() } /// Reports whether any parse listener is registered. @@ -5962,22 +5990,24 @@ where if self.parse_listeners.is_empty() { return None; } - self.parse_listener_enter_rule_cold(rule_index) + self.parse_listener_enter_rule_dispatch(rule_index) } - #[cold] - fn parse_listener_enter_rule_cold(&mut self, rule_index: usize) -> Option { + fn parse_listener_enter_rule_dispatch(&mut self, rule_index: usize) -> Option { if let Some(error) = &self.parse_listener_abort { return Some(error.clone()); } - let current = self.input.lt(1); + let event = EnterRuleEvent { + rule_index, + current: self.input.lt(1), + }; // Split borrows: the token view borrows the input while listeners // need `&mut`, so listeners are taken out for the dispatch. Listener // methods have no parser access and cannot observe the absence. let mut listeners = std::mem::take(&mut self.parse_listeners); let mut abort = None; for slot in &mut listeners { - if let Err(error) = slot.0.enter_every_rule(rule_index, current) { + if let Err(error) = slot.0.enter_every_rule(&event) { abort = Some(error); break; } @@ -6000,7 +6030,9 @@ where if self.parse_listeners.is_empty() { return; } - for slot in &mut self.parse_listeners { + // Reverse registration order, matching upstream ANTLR + // (`Parser.triggerExitRuleEvent` walks listeners back to front). + for slot in self.parse_listeners.iter_mut().rev() { slot.0.exit_every_rule(rule_index); } } diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index 9761ebe9..64faf013 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -1729,24 +1729,24 @@ mod deep_nesting_tests { impl antlr4_runtime::ParseListener for RecursionListener { fn enter_every_rule( &mut self, - rule_index: usize, - current: Option>, + event: &antlr4_runtime::EnterRuleEvent<'_>, ) -> Result<(), antlr4_runtime::AntlrError> { - if rule_index == super::nest_parser::RULE_EXPR { + if event.rule_index == super::nest_parser::RULE_EXPR { self.depth += 1; self.high_water .fetch_max(self.depth, std::sync::atomic::Ordering::Relaxed); } if self.depth > self.max { use antlr4_runtime::Token as _; - let (line, column) = current + let (line, column) = event + .current .as_ref() .map_or((0, 0), |token| (token.line(), token.column())); return Err(antlr4_runtime::AntlrError::ParserError { line, column, message: format!("Recursion limit of {} exceeded", self.max), - offending: current.as_ref().map(antlr4_runtime::Token::token_id), + offending: event.current.as_ref().map(antlr4_runtime::Token::token_id), }); } Ok(()) @@ -1759,6 +1759,32 @@ mod deep_nesting_tests { } } + /// Records the event stream for order/balance assertions. + struct TracingListener { + tag: &'static str, + events: std::sync::Arc>>, + } + + impl antlr4_runtime::ParseListener for TracingListener { + fn enter_every_rule( + &mut self, + event: &antlr4_runtime::EnterRuleEvent<'_>, + ) -> Result<(), antlr4_runtime::AntlrError> { + self.events + .lock() + .expect("trace lock") + .push(format!("enter{}:{}", self.tag, event.rule_index)); + Ok(()) + } + + fn exit_every_rule(&mut self, rule_index: usize) { + self.events + .lock() + .expect("trace lock") + .push(format!("exit{}:{}", self.tag, rule_index)); + } + } + #[test] fn parse_listener_counts_rules_and_aborts_past_a_limit() { use std::sync::Arc; @@ -1815,13 +1841,112 @@ mod deep_nesting_tests { "unexpected error: {error}" ); - // The abort does not poison the instance: clearing listeners and - // reusing the parser parses clean input. + // The abort does not poison the instance: clearing listeners (which + // also returns them and drops the sticky abort) and reusing the + // parser parses clean input. let lexer = NestLexer::new(InputStream::new("a")); parser.set_token_stream(CommonTokenStream::new(lexer)); - parser.remove_parse_listeners(); + let removed = parser.remove_parse_listeners(); + assert_eq!(removed.len(), 1, "removed listeners are handed back"); assert!(parser.s().is_ok(), "reused parser starts clean"); } + + #[test] + fn parse_listener_event_order_matches_upstream() { + use std::sync::{Arc, Mutex}; + + // Two listeners: enters fire in registration order, exits in reverse + // (upstream Parser.triggerExitRuleEvent walks back to front), and + // pairs balance across recovery on malformed input. + let events = Arc::new(Mutex::new(Vec::new())); + let lexer = NestLexer::new(InputStream::new("[a]")); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(TracingListener { + tag: "A", + events: Arc::clone(&events), + }); + parser.add_parse_listener(TracingListener { + tag: "B", + events: Arc::clone(&events), + }); + assert!(parser.s().is_ok()); + let trace = events.lock().expect("trace lock").clone(); + let s_rule = super::nest_parser::RULE_S; + assert_eq!(trace.first().map(String::as_str), Some(format!("enterA:{s_rule}").as_str())); + assert_eq!(trace.get(1).map(String::as_str), Some(format!("enterB:{s_rule}").as_str())); + // Last two events close the entry rule: B exits before A. + assert_eq!( + trace.last().map(String::as_str), + Some(format!("exitA:{s_rule}").as_str()) + ); + assert_eq!( + trace.get(trace.len() - 2).map(String::as_str), + Some(format!("exitB:{s_rule}").as_str()) + ); + // Balance: every rule index enters exactly as often as it exits, + // for both listeners. + let count = |needle: &str| trace.iter().filter(|event| event.starts_with(needle)).count(); + assert_eq!(count("enterA:"), count("exitA:")); + assert_eq!(count("enterB:"), count("exitB:")); + + // Recovery path: malformed input still balances. + let events = Arc::new(Mutex::new(Vec::new())); + let lexer = NestLexer::new(InputStream::new("[a")); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(TracingListener { + tag: "R", + events: Arc::clone(&events), + }); + let _ = parser.s(); + let trace = events.lock().expect("trace lock").clone(); + let count = |needle: &str| trace.iter().filter(|event| event.starts_with(needle)).count(); + assert_eq!( + count("enterR:"), + count("exitR:"), + "recovery keeps pairs balanced: {trace:?}" + ); + } + + #[test] + fn depth_cap_and_listener_abort_coexist() { + // Whichever bound trips first surfaces; the other never fires because + // the sticky abort stops rule entries (and thus stack growth). With a + // tight listener limit the listener error wins the race... + let lexer = NestLexer::new(InputStream::new(&nested(64))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.set_max_rule_depth(Some(64)); + parser.add_parse_listener(RecursionListener { + max: 1, + depth: 0, + high_water: std::sync::Arc::new(std::sync::atomic::AtomicU16::new(0)), + }); + let error = parser.s().expect_err("listener limit must fail the parse"); + assert!( + error.to_string().contains("Recursion limit of 1 exceeded"), + "listener abort surfaces when it trips first: {error}" + ); + + // ...and with a tight cap the depth violation wins the race. + let lexer = NestLexer::new(InputStream::new(&nested(64))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.set_max_rule_depth(Some(8)); + parser.add_parse_listener(RecursionListener { + max: 1_000, + depth: 0, + high_water: std::sync::Arc::new(std::sync::atomic::AtomicU16::new(0)), + }); + let error = parser.s().expect_err("depth cap must fail the parse"); + assert!( + error.to_string().contains("rule nesting depth limit of 8"), + "depth-cap violation surfaces when it trips first: {error}" + ); + + let lexer = NestLexer::new(InputStream::new("a")); + parser.set_token_stream(CommonTokenStream::new(lexer)); + parser.set_max_rule_depth(None); + let _ = parser.remove_parse_listeners(); + assert!(parser.s().is_ok(), "instance is clean after either abort"); + } } "#, ); From e87fb3a26c1bcd5099117cd0fedc13b130b11275 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sun, 26 Jul 2026 01:12:58 +0200 Subject: [PATCH 7/9] fix(runtime): parse-listener round-2 review findings MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 1. Parser trait gains add_parse_listener(Box) / remove_parse_listeners() (mirroring set_max_rule_depth), fixing the broken intra-doc link and letting code generic over P: Parser reach listener registration. 2. LR-expansion aborts now match Java exactly: the listener enter probe fires AFTER push_new_recursion_context_with_previous (upstream assigns _ctx before triggerEnterRuleEvent), so an aborting expansion is already counted and the unroll fires its matching exit — Java parity via the finally-driven unrollRecursionContexts. Ordinary-rule aborts keep the documented no-exit behavior (also Java parity). Trait doc states both cases precisely. 3. Success-path LR coverage: a+a+a+a under a tracing listener pins 7 RULE_EXPR enters (1 dispatch + 3 expansions + 3 operands) and full enter/exit balance through the unroll loop. 4. Stale ATN-preferred routing comment updated for the listener guard. --- src/bin/antlr4-rust-gen.rs | 39 ++++++++++++++++++++++-------------- src/parser.rs | 36 ++++++++++++++++++++++++++++----- tests/antlr4_rust_gen_cli.rs | 27 +++++++++++++++++++++++++ 3 files changed, 82 insertions(+), 20 deletions(-) diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index e07da062..cc3258a2 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -5857,11 +5857,12 @@ fn render_generated_step( { // ATN-preferred child: route through `parse_rule_precedence_from_generated`. // The rule's `parse_generated_rule` dispatch arm is guarded by - // `generated_only() || has_rule_depth_cap()`: in normal uncapped - // mode the generated probe returns `None` and the wrapper parses - // the child on the INTERPRETED path (preserving the ATN-preferred - // optimization); a configured depth cap flips it to the generated - // body, which is the only path that enforces the cap. + // `generated_only() || has_rule_depth_cap() || has_parse_listeners()`: + // in the default configuration the generated probe returns `None` + // and the wrapper parses the child on the INTERPRETED path + // (preserving the ATN-preferred optimization); a configured depth + // cap or a registered parse listener flips it to the generated + // body, the only path that enforces the cap and fires events. from_generated_call } else { generated_child_call @@ -6952,20 +6953,15 @@ fn render_generated_left_recursive_loop( .expect("writing to a string cannot fail"); } // Each operator iteration deepens the tree without a rule frame; probe - // the depth cap and the parse-listener enter event BEFORE the expansion - // push so the boundary matches the dispatch site (which checks before - // its rule-frame push): frames and expansions are both admitted up to - // the cap, listeners see the simulated rule entry upstream fires for - // pushNewRecursionContext, and token-only operator alternatives (no - // nested rule dispatch) still abort promptly instead of at the - // top-level drain. The matching exit events fire as the rule unrolls. + // the depth cap BEFORE the expansion push so the boundary matches the + // dispatch site (which checks before its rule-frame push): frames and + // expansions are both admitted up to the cap, and token-only operator + // alternatives (no nested rule dispatch) still abort promptly instead + // of at the top-level drain. writeln!( out, "{pad} if let Some(__depth_error) = self.base.rule_depth_cap_violation() {{\n\ {pad} return Err(__depth_error);\n\ - {pad} }}\n\ - {pad} if let Some(__listener_error) = self.base.parse_listener_enter_rule({rule_index}) {{\n\ - {pad} return Err(__listener_error);\n\ {pad} }}" ) .expect("writing to a string cannot fail"); @@ -6974,6 +6970,19 @@ fn render_generated_left_recursive_loop( "{pad} self.base.push_new_recursion_context_with_previous({entry_state}isize, {rule_index}, &mut __ctx);" ) .expect("writing to a string cannot fail"); + // The listener enter probe fires AFTER the expansion push, mirroring + // upstream (`pushNewRecursionContext` assigns `_ctx` before + // `triggerEnterRuleEvent`): an abort here propagates through the rule's + // error paths, whose `unroll_recursion_context` fires the matching exit + // for the already-counted expansion — Java's `finally` unroll does the + // same, so aborting expansions stay enter/exit balanced. + writeln!( + out, + "{pad} if let Some(__listener_error) = self.base.parse_listener_enter_rule({rule_index}) {{\n\ + {pad} return Err(__listener_error);\n\ + {pad} }}" + ) + .expect("writing to a string cannot fail"); render_generated_steps(out, body, indent + 3, render_context); writeln!(out, "{pad} }}").expect("writing to a string cannot fail"); writeln!(out, "{pad} {exit_alt} => break,").expect("writing to a string cannot fail"); diff --git a/src/parser.rs b/src/parser.rs index 35243f45..c29190f8 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -130,7 +130,8 @@ pub fn grow_generated_rule_stack(body: impl FnOnce() -> R) -> R { } /// Receives committed rule enter/exit events during recognition, matching -/// ANTLR's `addParseListener` contract ([`Parser::add_parse_listener`]). +/// ANTLR's `addParseListener` contract ([`Parser::add_parse_listener`], +/// also inherent on [`BaseParser`] and generated parsers). /// /// Events fire on the generated recursive-descent path as rules are entered /// and exited, including one simulated entry per left-recursive operator @@ -138,10 +139,15 @@ pub fn grow_generated_rule_stack(body: impl FnOnce() -> R) -> R { /// `triggerEnterRuleEvent` for exactly that case). Enter events fire in /// registration order and exit events in reverse registration order, /// matching upstream. Enter/exit calls balance on every completed path, -/// including error recovery — with one exception shared with Java: the -/// enter that returns `Err` (or whose Java analog throws) receives no -/// matching exit, so listener state shared across parses via `Arc` must be -/// reset after an abort. +/// including error recovery — with one exception shared with Java: an +/// ordinary rule's enter that returns `Err` receives no matching exit +/// (upstream calls `enterRule` outside the generated `try`/`finally`, so a +/// throwing listener skips `exitRule` the same way). Left-recursive +/// expansion enters DO receive their exit even on abort — the expansion is +/// already pushed when the probe fires, and the unroll emits its exit, +/// mirroring Java's `finally`-driven `unrollRecursionContexts`. Listener +/// state shared across parses via `Arc` should still be reset after an +/// abort (the unmatched ordinary-rule enter leaves counters one high). /// /// Divergence from Java to know about: upstream generated rule methods run /// only on the committed parse, while this runtime may re-enter a rule while @@ -1233,6 +1239,18 @@ pub trait Parser: Recognizer { /// Rules the generator emitted no body for (interpreter-only fallback) /// do not check the cap. fn set_max_rule_depth(&mut self, _depth: Option) {} + + /// Registers a listener for committed rule enter/exit events during + /// recognition (ANTLR's `addParseListener`). See [`ParseListener`] for + /// the delivery contract. The default implementation drops the listener; + /// [`BaseParser`] and generated parsers deliver events. + fn add_parse_listener(&mut self, _listener: Box) {} + + /// Removes every registered parse listener and returns them, dropping + /// any sticky abort a removed listener had requested. + fn remove_parse_listeners(&mut self) -> Vec> { + Vec::new() + } } #[derive(Debug)] @@ -12910,6 +12928,14 @@ where fn set_max_rule_depth(&mut self, depth: Option) { self.max_rule_depth = depth; } + + fn add_parse_listener(&mut self, listener: Box) { + self.parse_listeners.push(ParseListenerSlot(listener)); + } + + fn remove_parse_listeners(&mut self) -> Vec> { + Self::remove_parse_listeners(self) + } } #[cfg(test)] diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index 64faf013..d7d97067 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -1905,6 +1905,33 @@ mod deep_nesting_tests { count("exitR:"), "recovery keeps pairs balanced: {trace:?}" ); + + // Successful left-recursive operator chain: each expansion fires a + // simulated enter (upstream triggerEnterRuleEvent parity) and the + // unroll fires the matching exits. `a+a+a+a` yields exactly 7 + // RULE_EXPR pairs: 1 rule dispatch + 3 expansions + 3 right-operand + // dispatches. + let events = Arc::new(Mutex::new(Vec::new())); + let lexer = NestLexer::new(InputStream::new("a+a+a+a")); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(TracingListener { + tag: "L", + events: Arc::clone(&events), + }); + assert!(parser.s().is_ok(), "operator chain parses"); + let trace = events.lock().expect("trace lock").clone(); + let expr_rule = super::nest_parser::RULE_EXPR; + let count = |needle: String| trace.iter().filter(|event| **event == needle).count(); + assert_eq!( + count(format!("enterL:{expr_rule}")), + 7, + "expr enters = dispatch + expansions + operands: {trace:?}" + ); + assert_eq!( + count(format!("enterL:{expr_rule}")), + count(format!("exitL:{expr_rule}")), + "successful LR unroll balances expansion exits: {trace:?}" + ); } #[test] From 55cd6245bea25fd27bfed7a37f76e37a5df3e125 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sun, 26 Jul 2026 01:45:12 +0200 Subject: [PATCH 8/9] fix(runtime): make removed parse listeners re-registrable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CodeRabbit round 3: Box returned by remove_parse_listeners could not be passed back to the inherent add_parse_listener (Box did not implement ParseListener). Add the forwarding impl for Box and pin the round-trip in the e2e test — a removed listener re-registers with its accumulated state and still enforces its limit. Also cover the left-recursive success path: one listener instance parses the same under-limit operator chain twice; identical high-water marks prove the live depth counter returned to zero after the first LR unroll (balanced enter/exit through expansions). --- src/parser.rs | 13 ++++++++++++ tests/antlr4_rust_gen_cli.rs | 41 +++++++++++++++++++++++++++++++++++- 2 files changed, 53 insertions(+), 1 deletion(-) diff --git a/src/parser.rs b/src/parser.rs index c29190f8..af197b6f 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -187,6 +187,19 @@ pub trait ParseListener: Send { } } +/// Boxed listeners forward to their inner implementation, so the boxes +/// returned by [`Parser::remove_parse_listeners`] can be re-registered +/// through [`Parser::add_parse_listener`] unchanged. +impl ParseListener for Box { + fn enter_every_rule(&mut self, event: &EnterRuleEvent<'_>) -> Result<(), AntlrError> { + (**self).enter_every_rule(event) + } + + fn exit_every_rule(&mut self, rule_index: usize) { + (**self).exit_every_rule(rule_index); + } +} + /// A rule-entry event delivered to [`ParseListener::enter_every_rule`]. /// /// Non-exhaustive so future fields (alt number, invoking state, a context diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index d7d97067..b885d865 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -1807,6 +1807,31 @@ mod deep_nesting_tests { "one expr per bracket level plus the outermost expr" ); + // Successful left-recursive chain: the live depth counter returns to + // its starting value. Proven through the public API: parse the same + // under-limit chain twice with one listener instance — any residual + // depth from parse one would raise parse two's high-water mark. + let high_water = Arc::new(AtomicU16::new(0)); + let chain = vec!["a"; 20].join("+"); + let lexer = NestLexer::new(InputStream::new(&chain)); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 1_000, + depth: 0, + high_water: Arc::clone(&high_water), + }); + assert!(parser.s().is_ok(), "under-limit operator chain parses"); + let first_peak = high_water.load(Ordering::Relaxed); + assert!(first_peak > 0, "the chain nests expr rules"); + let lexer = NestLexer::new(InputStream::new(&chain)); + parser.set_token_stream(CommonTokenStream::new(lexer)); + assert!(parser.s().is_ok(), "same chain parses again"); + assert_eq!( + high_water.load(Ordering::Relaxed), + first_peak, + "depth returned to zero after the successful LR parse" + ); + // Past the limit: the listener aborts with its own positioned error, // sticky through recovery. let lexer = NestLexer::new(InputStream::new(&nested(64))); @@ -1846,9 +1871,23 @@ mod deep_nesting_tests { // parser parses clean input. let lexer = NestLexer::new(InputStream::new("a")); parser.set_token_stream(CommonTokenStream::new(lexer)); - let removed = parser.remove_parse_listeners(); + let mut removed = parser.remove_parse_listeners(); assert_eq!(removed.len(), 1, "removed listeners are handed back"); assert!(parser.s().is_ok(), "reused parser starts clean"); + + // Returned boxes re-register as-is (ParseListener is implemented for + // Box), preserving accumulated listener state. + let boxed = removed.pop().expect("one listener was removed"); + let lexer = NestLexer::new(InputStream::new(&nested(64))); + parser.set_token_stream(CommonTokenStream::new(lexer)); + parser.add_parse_listener(boxed); + let error = parser + .s() + .expect_err("re-registered listener still enforces its limit"); + assert!( + error.to_string().contains("Recursion limit of 8 exceeded"), + "unexpected error: {error}" + ); } #[test] From 3e4aaa9db29f38a47327807e1b6e9d3d1e080549 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Sun, 26 Jul 2026 02:12:38 +0200 Subject: [PATCH 9/9] fix(runtime): match upstream left-recursive listener exit timing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Round-4 review built a Java oracle (ANTLR 4.13.2 on this PR's own Nest.g4) and caught that batching expansion exits at unroll diverges from upstream on live depth: recRuleSetPrevCtx fires triggerExitRuleEvent at the TOP of each operator-loop pass, so the outgoing iteration exits before the next expansion enters and flat chains never accumulate depth (a+a+...+a peaks at 2 in every ANTLR target; ours peaked at chain width). The motivating cel-rust RecursionListener would have rejected wide flat CEL expressions every other target accepts. The generated operator loop now exits the outgoing iteration first, and unroll_recursion_context drops the batched exits (upstream's unrollRecursionContexts walks exactly one link — the dispatch wrapper's single exit plays it). Event counts unchanged; timing now matches the oracle: 7/7 depth 2, 79/79 depth 2, bracket cases unchanged. The e2e that encoded the divergence now pins the Java-oracle depth of 2 for a 40-term chain. Also from review: generated impl Parser blocks forward add_parse_listener/remove_parse_listeners (they inherited the no-op trait defaults — generic registration silently dropped listeners and trait-removal left them firing with the sticky abort uncleared), and the trait doc gains the expansion-anchor note (EnterRuleEvent::current is the operator-side lookahead, not Java's whole-expression start). --- src/bin/antlr4-rust-gen.rs | 19 ++++++++++++--- src/parser.rs | 45 +++++++++++++++++------------------- tests/antlr4_rust_gen_cli.rs | 35 ++++++++++++++++++++-------- 3 files changed, 63 insertions(+), 36 deletions(-) diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index cc3258a2..8258761c 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -6952,6 +6952,17 @@ fn render_generated_left_recursive_loop( ) .expect("writing to a string cannot fail"); } + // Upstream fires `triggerExitRuleEvent` at the TOP of each operator-loop + // pass (`recRuleSetPrevCtx`, a StarBlock iteration op) — the outgoing + // iteration's context exits before the next expansion enters, so live + // listener depth never accumulates across a flat operator chain. The + // dispatch wrapper's single exit then plays `unrollRecursionContexts`' + // one-link walk when the rule finishes. + writeln!( + out, + "{pad} self.base.parse_listener_exit_rule({rule_index});" + ) + .expect("writing to a string cannot fail"); // Each operator iteration deepens the tree without a rule frame; probe // the depth cap BEFORE the expansion push so the boundary matches the // dispatch site (which checks before its rule-frame push): frames and @@ -6973,9 +6984,7 @@ fn render_generated_left_recursive_loop( // The listener enter probe fires AFTER the expansion push, mirroring // upstream (`pushNewRecursionContext` assigns `_ctx` before // `triggerEnterRuleEvent`): an abort here propagates through the rule's - // error paths, whose `unroll_recursion_context` fires the matching exit - // for the already-counted expansion — Java's `finally` unroll does the - // same, so aborting expansions stay enter/exit balanced. + // error paths, and the dispatch wrapper's exit keeps the pair balanced. writeln!( out, "{pad} if let Some(__listener_error) = self.base.parse_listener_enter_rule({rule_index}) {{\n\ @@ -9962,6 +9971,10 @@ where fn set_prediction_mode(&mut self, mode: antlr4_runtime::PredictionMode) {{ self.base.set_prediction_mode(mode); }} fn max_rule_depth(&self) -> Option {{ self.base.max_rule_depth() }} fn set_max_rule_depth(&mut self, depth: Option) {{ self.base.set_max_rule_depth(depth); }} + // Route through the trait impl: BaseParser's inherent generic method + // would re-box the already-boxed listener. + fn add_parse_listener(&mut self, listener: Box) {{ antlr4_runtime::Parser::add_parse_listener(&mut self.base, listener); }} + fn remove_parse_listeners(&mut self) -> Vec> {{ self.base.remove_parse_listeners() }} }} {generated_footer}"# )) diff --git a/src/parser.rs b/src/parser.rs index af197b6f..afdced86 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -134,20 +134,24 @@ pub fn grow_generated_rule_stack(body: impl FnOnce() -> R) -> R { /// also inherent on [`BaseParser`] and generated parsers). /// /// Events fire on the generated recursive-descent path as rules are entered -/// and exited, including one simulated entry per left-recursive operator -/// expansion (upstream `Parser.pushNewRecursionContext` fires -/// `triggerEnterRuleEvent` for exactly that case). Enter events fire in -/// registration order and exit events in reverse registration order, -/// matching upstream. Enter/exit calls balance on every completed path, -/// including error recovery — with one exception shared with Java: an +/// and exited, with left-recursive operator loops following upstream's +/// timing exactly: each loop pass first exits the outgoing iteration +/// (`recRuleSetPrevCtx`) and then enters the new expansion +/// (`pushNewRecursionContext` firing `triggerEnterRuleEvent`), so live +/// listener depth never accumulates across a flat operator chain — +/// `a + a + … + a` peaks at depth 2 like every ANTLR target. On expansion +/// events, [`EnterRuleEvent::current`] anchors at the operator-side +/// lookahead (the token the expansion starts at), whereas Java's +/// `ctx.start` reaches back to the whole expression's first token — anchor +/// diagnostics accordingly. Enter events fire in registration order and +/// exit events in reverse registration order, matching upstream. Enter/exit +/// calls balance on every completed path, including error recovery and +/// aborts inside operator loops — with one exception shared with Java: an /// ordinary rule's enter that returns `Err` receives no matching exit /// (upstream calls `enterRule` outside the generated `try`/`finally`, so a -/// throwing listener skips `exitRule` the same way). Left-recursive -/// expansion enters DO receive their exit even on abort — the expansion is -/// already pushed when the probe fires, and the unroll emits its exit, -/// mirroring Java's `finally`-driven `unrollRecursionContexts`. Listener -/// state shared across parses via `Arc` should still be reset after an -/// abort (the unmatched ordinary-rule enter leaves counters one high). +/// throwing listener skips `exitRule` the same way). Listener state shared +/// across parses via `Arc` should be reset after an abort (the unmatched +/// ordinary-rule enter leaves counters one high). /// /// Divergence from Java to know about: upstream generated rule methods run /// only on the committed parse, while this runtime may re-enter a rule while @@ -6387,19 +6391,12 @@ where if self.precedence_stack.len() > 1 { self.precedence_stack.pop(); } + // Parse-listener exits for expansions fire inside the generated + // operator loop (top of each pass, upstream's `recRuleSetPrevCtx`), + // and the dispatch wrapper's exit covers the final live context — + // upstream's `unrollRecursionContexts` walks exactly one link, so no + // batched exits happen here. Only the depth-cap accounting rewinds. if let Some(mark) = self.recursion_expansion_marks.pop() { - // Each expansion fired a simulated rule-entry event; fire the - // matching exits as the rule unrolls so parse listeners see - // balanced pairs (upstream unrolls via exitRule per context). - if !self.parse_listeners.is_empty() { - let rule_index = self - .rule_context_stack - .last() - .map_or(usize::MAX, |frame| frame.rule_index); - for _ in mark..self.recursion_expansions { - self.parse_listener_exit_rule(rule_index); - } - } self.recursion_expansions = mark; } self.exit_rule(); diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index b885d865..de5cf4ee 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -1847,28 +1847,45 @@ mod deep_nesting_tests { "unexpected error: {error}" ); - // Left-recursive operator expansions fire simulated rule entries - // (upstream triggerEnterRuleEvent parity): a 40-term `a+a+...` chain - // exceeds an expr limit of 8 even though rule frames barely nest. + // Flat operator chains do NOT accumulate live listener depth: each + // loop pass exits the outgoing iteration before the next expansion + // enters (upstream recRuleSetPrevCtx), so a 40-term `a+a+...` chain + // peaks at expr depth 2 in every ANTLR target — including this one — + // and parses fine under a limit of 8. let chain = vec!["a"; 40].join("+"); + let high_water = Arc::new(AtomicU16::new(0)); let lexer = NestLexer::new(InputStream::new(&chain)); let mut parser = NestParser::new(CommonTokenStream::new(lexer)); parser.add_parse_listener(RecursionListener { max: 8, depth: 0, - high_water: Arc::new(AtomicU16::new(0)), + high_water: Arc::clone(&high_water), }); - let error = parser - .s() - .expect_err("operator expansions must count as rule entries"); assert!( - error.to_string().contains("Recursion limit of 8 exceeded"), - "unexpected error: {error}" + parser.s().is_ok(), + "flat operator chain stays at Java's live depth" + ); + assert_eq!( + high_water.load(Ordering::Relaxed), + 2, + "operator chain peaks at depth 2, matching the Java oracle" ); // The abort does not poison the instance: clearing listeners (which // also returns them and drops the sticky abort) and reusing the // parser parses clean input. + let lexer = NestLexer::new(InputStream::new(&nested(64))); + let mut parser = NestParser::new(CommonTokenStream::new(lexer)); + parser.add_parse_listener(RecursionListener { + max: 8, + depth: 0, + high_water: Arc::new(AtomicU16::new(0)), + }); + let error = parser.s().expect_err("nested input exceeds the limit"); + assert!( + error.to_string().contains("Recursion limit of 8 exceeded"), + "unexpected error: {error}" + ); let lexer = NestLexer::new(InputStream::new("a")); parser.set_token_stream(CommonTokenStream::new(lexer)); let mut removed = parser.remove_parse_listeners();