diff --git a/README.md b/README.md index 71acb044..cff6adb5 100644 --- a/README.md +++ b/README.md @@ -227,6 +227,9 @@ fn main() -> Result<(), antlr4_runtime::AntlrError> { Generated recognizers install a `ConsoleErrorListener` by default. Remove it from both the lexer and parser to suppress recovery output, as above, or call `add_error_listener` after removal to redirect diagnostics to a replacement. +`ErrorListener::syntax_error` receives a `SyntaxErrorEvent`; its `span` is the +resolved half-open UTF-8 byte range for parser tokens and lexer failures, when +the input stream can provide byte offsets. ### Reusing Recognizers diff --git a/docs/migration.md b/docs/migration.md index 6d7c935d..e1d79b86 100644 --- a/docs/migration.md +++ b/docs/migration.md @@ -6,6 +6,49 @@ generator changes. Generated lexers and parsers must use the same release of `antlr4-rust-gen --version` and compare the reported release with the `antlr-rust-runtime` dependency version. +## Structured Syntax Error Events and Byte Spans + +`ErrorListener::syntax_error` now receives one `&SyntaxErrorEvent<'_>` instead +of separate offending-token, line, column, message, and error arguments: + +```rust +// Before +fn syntax_error( + &mut self, + recognizer: &R, + offending: Option>, + line: usize, + column: usize, + message: &str, + error: Option<&AntlrError>, +); + +// After +fn syntax_error(&mut self, recognizer: &R, event: &SyntaxErrorEvent<'_>); +``` + +Read `event.span` for the resolved half-open UTF-8 byte range. Lexer failures +and parser diagnostics use the same event shape; streams and token sources that +cannot resolve byte offsets leave the span as `None`. + +`Token::start_byte()` and `stop_byte()` now return `Option`, while +`byte_span()` returns `Option>`. `None` means the token source +could not resolve exact byte offsets. Custom token sources must set +Unicode-scalar and UTF-8 byte positions independently: + +```rust +TokenSpec::explicit(token_type, text) + .with_span(scalar_start, scalar_stop) + .with_byte_span(byte_start, byte_end) +``` + +`TokenSpec::with_span` no longer assumes scalar indexes are byte offsets. +Omit `with_byte_span` when no exact mapping exists. + +`TokenSourceError` gained an optional `span` and, like `SyntaxErrorEvent`, is +non-exhaustive. Construct token-source diagnostics with +`TokenSourceError::new(...).with_span(...)` instead of a struct literal. + ## Recognizer Reuse Method Names Generated parsers now reserve `reset`, `set_token_stream`, diff --git a/src/atn/lexer.rs b/src/atn/lexer.rs index 18cbb789..07e093fa 100644 --- a/src/atn/lexer.rs +++ b/src/atn/lexer.rs @@ -2214,12 +2214,16 @@ fn record_token_recognition_error(lexer: &BaseLexer, start: usize, stop: u where I: CharStream, { - let stop = stop.saturating_sub(1); - let text = display_error_text(&lexer.input().text(TextInterval::new(start, stop))); - lexer.record_error( + let inclusive_stop = stop.saturating_sub(1); + let text = display_error_text(&lexer.input().text(TextInterval::new(start, inclusive_stop))); + // Defensive callers that do not advance `stop` still identify one failing + // scalar; normal non-EOF recognition errors already pass `stop > start`. + let scalar_end = stop.max(start.saturating_add(1)); + lexer.record_error_for_scalar_span( lexer.line(), lexer.column(), format!("token recognition error at: '{text}'"), + start..scalar_end, ); } @@ -2257,6 +2261,8 @@ where #[cfg(test)] mod tests { + use std::sync::{Arc, Mutex}; + use super::*; use crate::atn::lexer_dfa::{ CompiledLexerActionTrace, CompiledLexerConfig, CompiledLexerContext, @@ -2264,7 +2270,8 @@ mod tests { use crate::atn::serialized::{AtnDeserializer, SerializedAtn}; use crate::atn::{LexerAtnState, LexerTransition}; use crate::char_stream::InputStream; - use crate::recognizer::RecognizerData; + use crate::errors::{ErrorListener, SyntaxErrorEvent}; + use crate::recognizer::{Recognizer, RecognizerData}; use crate::token::{DEFAULT_CHANNEL, HIDDEN_CHANNEL, TOKEN_EOF, Token, TokenStore, TokenView}; use crate::vocabulary::Vocabulary; @@ -2283,6 +2290,38 @@ mod tests { ) } + #[derive(Clone, Debug)] + struct SpanListener(Arc>>>>); + + impl ErrorListener for SpanListener + where + R: Recognizer + ?Sized, + { + fn syntax_error(&mut self, _recognizer: &R, event: &SyntaxErrorEvent<'_>) { + self.0 + .lock() + .expect("recorded spans lock") + .push(event.span.clone()); + } + } + + #[test] + fn lexer_error_listener_receives_utf8_byte_span() { + let spans = Arc::new(Mutex::new(Vec::new())); + let mut lexer = BaseLexer::new(InputStream::new("aβz"), recognizer_data()); + lexer.remove_error_listeners(); + lexer.add_error_listener(SpanListener(Arc::clone(&spans))); + lexer.commit_position(0, 1); + lexer.begin_token(); + + record_token_recognition_error(&lexer, 1, 2); + let errors = lexer.drain_errors(); + assert_eq!(errors.len(), 1); + lexer.notify_error_listeners(SyntaxErrorEvent::from(&errors[0])); + + assert_eq!(*spans.lock().expect("recorded spans lock"), [Some(1..3)]); + } + fn predicate_atn() -> LexerAtn { let mut atn = LexerAtn::new(1); @@ -2861,7 +2900,7 @@ mod tests { assert_eq!(dot.text(), Some(".")); assert_eq!(dot.start(), 0); assert_eq!(dot.stop(), 0); - assert_eq!(dot.byte_span(), 0..1); + assert_eq!(dot.byte_span(), Some(0..1)); assert_eq!((dot.line(), dot.column()), (1, 0)); let identifier = sink @@ -2871,7 +2910,7 @@ mod tests { assert_eq!(identifier.text(), Some("β")); assert_eq!(identifier.start(), 1); assert_eq!(identifier.stop(), 1); - assert_eq!(identifier.byte_span(), 1..3); + assert_eq!(identifier.byte_span(), Some(1..3)); assert_eq!((identifier.line(), identifier.column()), (1, 1)); assert_eq!( diff --git a/src/atn/lexer_dfa.rs b/src/atn/lexer_dfa.rs index 5995ad2b..e8131bac 100644 --- a/src/atn/lexer_dfa.rs +++ b/src/atn/lexer_dfa.rs @@ -1641,8 +1641,8 @@ mod tests { channel: i32, start: usize, stop: usize, - start_byte: usize, - stop_byte: usize, + start_byte: Option, + stop_byte: Option, line: usize, column: usize, } diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index c38c146b..9c3c23d3 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -4011,14 +4011,7 @@ where self.base.drain_errors() }} fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool {{ - antlr4_runtime::Recognizer::notify_error_listeners( - self, - None, - source_error.line, - source_error.column, - &source_error.message, - None, - ); + antlr4_runtime::Recognizer::notify_error_listeners(self, source_error.into()); true }} fn lexer_dfa_string(&self) -> String {{ @@ -20574,7 +20567,7 @@ dispose = "hook" assert!(module.contains( "fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool" )); - assert!(module.contains("Recognizer::notify_error_listeners(")); + assert!(module.contains("Recognizer::notify_error_listeners(self, source_error.into());")); assert!(!module.contains("CommonToken")); assert!(!module.contains("TokenFactory")); } diff --git a/src/bin_support/grammar/atn/interp_test.rs b/src/bin_support/grammar/atn/interp_test.rs index c3b2e679..1a73b5c5 100644 --- a/src/bin_support/grammar/atn/interp_test.rs +++ b/src/bin_support/grammar/atn/interp_test.rs @@ -1091,8 +1091,8 @@ mod tests { channel: i32, start: usize, stop: usize, - byte_start: usize, - byte_stop: usize, + byte_start: Option, + byte_stop: Option, line: usize, column: usize, text: String, diff --git a/src/bin_support/grammar/frontend.rs b/src/bin_support/grammar/frontend.rs index 36ef887c..4794bc2c 100644 --- a/src/bin_support/grammar/frontend.rs +++ b/src/bin_support/grammar/frontend.rs @@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex}; use antlr4_runtime::{ AsRuleNode, CharStream as _, CommonTokenStream, ErrorListener, InputStream, Node, NodeId, - NodeKind, Parser, Recognizer, TOKEN_EOF as RUNTIME_TOKEN_EOF, Token, + NodeKind, Parser, Recognizer, SyntaxErrorEvent, TOKEN_EOF as RUNTIME_TOKEN_EOF, Token, }; use super::generated::antlr_v4_lexer::{ @@ -534,8 +534,14 @@ where token_stream .tokens() .map(|token| { - let start = u32::try_from(token.start_byte()); - let end = u32::try_from(token.stop_byte()); + let Some(bytes) = token.byte_span() else { + return Err(invalid_span( + source, + "token source did not provide a byte span", + )); + }; + let start = u32::try_from(bytes.start); + let end = u32::try_from(bytes.end); let (Ok(start), Ok(end)) = (start, end) else { return Err(invalid_span(source, "token byte span exceeds 4 GiB")); }; @@ -980,22 +986,14 @@ impl ErrorListener for DiagnosticCollector where R: Recognizer + ?Sized, { - fn syntax_error( - &mut self, - _recognizer: &R, - _offending: Option>, - line: usize, - column: usize, - message: &str, - _error: Option<&antlr4_runtime::AntlrError>, - ) { + fn syntax_error(&mut self, _recognizer: &R, event: &SyntaxErrorEvent<'_>) { self.0 .lock() .expect("grammar diagnostic collector mutex poisoned") .push(ReportedDiagnostic { - line, - column, - message: message.to_owned(), + line: event.line, + column: event.column, + message: event.message.to_owned(), }); } } diff --git a/src/bin_support/grammar/generated/antlr_v4_lexer.rs b/src/bin_support/grammar/generated/antlr_v4_lexer.rs index 02e92d9f..49f96f45 100644 --- a/src/bin_support/grammar/generated/antlr_v4_lexer.rs +++ b/src/bin_support/grammar/generated/antlr_v4_lexer.rs @@ -381,14 +381,7 @@ where self.base.drain_errors() } fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool { - antlr4_runtime::Recognizer::notify_error_listeners( - self, - None, - source_error.line, - source_error.column, - &source_error.message, - None, - ); + antlr4_runtime::Recognizer::notify_error_listeners(self, source_error.into()); true } fn lexer_dfa_string(&self) -> String { diff --git a/src/errors.rs b/src/errors.rs index 25bab40c..23f8720d 100644 --- a/src/errors.rs +++ b/src/errors.rs @@ -1,5 +1,7 @@ +use std::ops::Range; + use crate::recognizer::Recognizer; -use crate::token::{TokenId, TokenView}; +use crate::token::{TokenId, TokenSourceError, TokenView}; use thiserror::Error; #[derive(Debug, Error, Clone, Eq, PartialEq)] @@ -30,6 +32,43 @@ pub enum AntlrError { Unsupported(String), } +/// Structured context for one recognizer diagnostic. +#[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] +pub struct SyntaxErrorEvent<'a> { + /// Token the diagnostic is anchored to, when one exists. + /// + /// Lexer errors have no offending token because the failed match did not + /// produce one. + pub offending: Option>, + /// One-based input line where the diagnostic starts. + pub line: usize, + /// Zero-based column within `line` where the diagnostic starts. + pub column: usize, + /// Half-open UTF-8 byte span of the offending source text. + /// + /// Custom streams and token sources that cannot resolve byte offsets leave + /// this as `None`. + pub span: Option>, + /// ANTLR-compatible diagnostic message without the leading line/column. + pub message: &'a str, + /// Recognition error that caused the diagnostic, when one exists. + pub error: Option<&'a AntlrError>, +} + +impl<'a> From<&'a TokenSourceError> for SyntaxErrorEvent<'a> { + fn from(error: &'a TokenSourceError) -> Self { + Self { + offending: None, + line: error.line, + column: error.column, + span: error.span.clone(), + message: &error.message, + error: None, + } + } +} + /// Receives recognizer diagnostics. /// /// Listeners registered through [`Recognizer::add_error_listener`] must be @@ -37,20 +76,8 @@ pub enum AntlrError { /// generically, as [`ConsoleErrorListener`] does, when a listener will be /// registered. pub trait ErrorListener { - /// `offending` carries the token the diagnostic points at, matching - /// ANTLR's `syntaxError(recognizer, offendingSymbol, ...)`. It is `None` - /// for lexer errors (no token was produced) and for diagnostics that are - /// not anchored to a specific token. - #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature - fn syntax_error( - &mut self, - recognizer: &R, - offending: Option>, - line: usize, - column: usize, - message: &str, - error: Option<&AntlrError>, - ); + /// Receives one diagnostic with its ANTLR position and resolved byte span. + fn syntax_error(&mut self, recognizer: &R, event: &SyntaxErrorEvent<'_>); } #[derive(Debug, Default)] @@ -58,15 +85,7 @@ pub struct ConsoleErrorListener; impl ErrorListener for ConsoleErrorListener { #[allow(clippy::print_stderr)] - fn syntax_error( - &mut self, - _recognizer: &R, - _offending: Option>, - line: usize, - column: usize, - message: &str, - _error: Option<&AntlrError>, - ) { - eprintln!("line {line}:{column} {message}"); + fn syntax_error(&mut self, _recognizer: &R, event: &SyntaxErrorEvent<'_>) { + eprintln!("line {}:{} {}", event.line, event.column, event.message); } } diff --git a/src/lexer.rs b/src/lexer.rs index e988b710..7257c6a8 100644 --- a/src/lexer.rs +++ b/src/lexer.rs @@ -1,6 +1,7 @@ use std::cell::{RefCell, RefMut}; use std::collections::{BTreeSet, HashMap, VecDeque}; use std::hash::BuildHasherDefault; +use std::ops::Range; use std::rc::Rc; use crate::atn::LexerAtn; @@ -1491,7 +1492,7 @@ where }); let (start_byte, stop_byte) = source_interval .or_else(|| self.token_byte_span(stop)) - .unwrap_or((self.token_start, self.token_start)); + .unwrap_or((usize::MAX, usize::MAX)); TokenSpec { token_type, channel, @@ -1631,7 +1632,7 @@ where } fn eof_token_spec(&self) -> TokenSpec { - let byte_offset = self.eof_byte_offset().unwrap_or_else(|| self.input.index()); + let byte_offset = self.eof_byte_offset().unwrap_or(usize::MAX); TokenSpec::eof(self.input.index(), byte_offset, self.line, self.column) } @@ -1659,6 +1660,20 @@ where }; Some(byte_offset) } + + fn byte_span_for_scalar_range(&self, span: Range) -> Option> { + if span.start > span.end { + return None; + } + if span.is_empty() { + let offset = self.byte_offset_at(span.start)?; + return Some(offset..offset); + } + let (start, end) = self + .input + .byte_interval(TextInterval::new(span.start, span.end - 1))?; + Some(start..end) + } } impl Recognizer for BaseLexer @@ -1742,12 +1757,32 @@ where self.force_interpreted } - /// Buffers a lexer diagnostic until the token stream consumer is ready to - /// emit errors in parser-compatible order. + /// Buffers a lexer diagnostic for the current token span until the token + /// stream consumer can emit it in parser-compatible order. + /// + /// `line` and `column` should identify the current token start. Use + /// [`Self::record_error_for_scalar_span`] when the diagnostic covers a + /// different input range. pub fn record_error(&self, line: usize, column: usize, message: impl Into) { - self.errors - .borrow_mut() - .push(TokenSourceError::new(line, column, message)); + let scalar_span = self.token_start..self.input.index().max(self.token_start); + self.record_error_for_scalar_span(line, column, message, scalar_span); + } + + /// Buffers a lexer diagnostic for an explicit half-open Unicode-scalar span. + /// + /// The span is converted through [`CharStream::byte_interval`]. Streams + /// without an exact UTF-8 byte mapping leave the diagnostic byte span + /// unknown. + pub fn record_error_for_scalar_span( + &self, + line: usize, + column: usize, + message: impl Into, + scalar_span: Range, + ) { + let mut error = TokenSourceError::new(line, column, message); + error.span = self.byte_span_for_scalar_range(scalar_span); + self.errors.borrow_mut().push(error); } /// Records one fail-loud semantic-hook miss per coordinate and token start. @@ -1952,41 +1987,64 @@ mod tests { use crate::vocabulary::Vocabulary; #[derive(Clone, Debug)] - struct UnsharedInput(InputStream); + struct UnsharedInput { + input: InputStream, + maps_bytes: bool, + } + + impl UnsharedInput { + fn mapped(input: InputStream) -> Self { + Self { + input, + maps_bytes: true, + } + } + + fn scalar_only(input: InputStream) -> Self { + Self { + input, + maps_bytes: false, + } + } + } impl IntStream for UnsharedInput { fn consume(&mut self) { - self.0.consume(); + self.input.consume(); } fn la(&mut self, offset: isize) -> i32 { - self.0.la(offset) + self.input.la(offset) } fn index(&self) -> usize { - self.0.index() + self.input.index() } fn seek(&mut self, index: usize) { - self.0.seek(index); + self.input.seek(index); } fn size(&self) -> usize { - self.0.size() + self.input.size() } fn source_name(&self) -> &str { - self.0.source_name() + self.input.source_name() } } impl CharStream for UnsharedInput { fn text(&self, interval: TextInterval) -> String { - self.0.text(interval) + self.input.text(interval) } fn byte_interval(&self, interval: TextInterval) -> Option<(usize, usize)> { - self.0.byte_interval(interval) + if self.maps_bytes { + self.input.byte_interval(interval) + } else { + None + } } } @@ -2012,7 +2070,31 @@ mod tests { // snapshot the explicit (start, stop, text, byte_span) record rather than the token. insta::assert_compact_debug_snapshot!( (token.start(), token.stop(), token.text(), token.byte_span()), - @r#"(1, 0, Some(""), 2..2)"# + @r#"(1, 0, Some(""), Some(2..2))"# + ); + } + + #[test] + fn eof_token_has_no_byte_span_without_byte_mapping() { + let data = RecognizerData::new( + "T", + Vocabulary::new( + std::iter::empty::>(), + std::iter::empty::>(), + std::iter::empty::>(), + ), + ); + let mut lexer = BaseLexer::new(UnsharedInput::scalar_only(InputStream::new("β")), data); + lexer.consume_char(); + + let mut store = TokenStore::new(lexer.source_text(), lexer.source_name()); + let mut sink = TokenSink::new(&mut store); + let id = lexer.eof_token(&mut sink).expect("test token should fit"); + let token = sink.view(id).expect("emitted token should exist"); + + insta::assert_compact_debug_snapshot!( + (token.start(), token.stop(), token.text(), token.byte_span()), + @r#"(1, 0, Some(""), None)"# ); } @@ -2041,7 +2123,7 @@ mod tests { // snapshot the explicit (start, stop, text, byte_span) record rather than the token. insta::assert_compact_debug_snapshot!( (token.start(), token.stop(), token.text(), token.byte_span()), - @r#"(1, 0, Some(""), 2..2)"# + @r#"(1, 0, Some(""), Some(2..2))"# ); } @@ -2070,7 +2152,7 @@ mod tests { // snapshot the explicit (start, stop, text, byte_span) record rather than the token. insta::assert_compact_debug_snapshot!( (token.start(), token.stop(), token.text(), token.byte_span()), - @r#"(0, 0, Some("β"), 0..2)"# + @r#"(0, 0, Some("β"), Some(0..2))"# ); } @@ -2084,7 +2166,7 @@ mod tests { std::iter::empty::>(), ), ); - let mut lexer = BaseLexer::new(UnsharedInput(InputStream::new("β")), data); + let mut lexer = BaseLexer::new(UnsharedInput::mapped(InputStream::new("β")), data); lexer.begin_token(); lexer.consume_char(); @@ -2096,7 +2178,7 @@ mod tests { let token = sink.view(id).expect("emitted token should exist"); assert_eq!(token.text(), Some("β")); - assert_eq!(token.byte_span(), 0..2); + assert_eq!(token.byte_span(), Some(0..2)); } #[test] @@ -2133,7 +2215,7 @@ mod tests { std::iter::empty::>(), ), ); - let mut lexer = BaseLexer::new(UnsharedInput(InputStream::new("a\nb")), data); + let mut lexer = BaseLexer::new(UnsharedInput::mapped(InputStream::new("a\nb")), data); lexer.begin_token(); lexer.commit_position(0, 3); @@ -2157,7 +2239,7 @@ mod tests { lexer.record_semantic_error(false, 3, 7); let errors = lexer.drain_errors(); - insta::assert_compact_debug_snapshot!(errors, @r#"[TokenSourceError { line: 1, column: 0, message: "unhandled lexer semantic predicate: rule=3 index=7" }]"#); + insta::assert_compact_debug_snapshot!(errors, @r#"[TokenSourceError { line: 1, column: 0, span: Some(0..0), message: "unhandled lexer semantic predicate: rule=3 index=7" }]"#); lexer.begin_token(); lexer.record_semantic_error(false, 3, 7); diff --git a/src/lib.rs b/src/lib.rs index ff80f8ca..9f604754 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -27,7 +27,7 @@ pub use atn::parser::{ParserAtnPrediction, ParserAtnSimulator, ParserAtnSimulato pub use byte_stream::ByteStream; pub use char_stream::{CharStream, InputStream, PositionSummary, TextInterval}; pub use dfa::{DfaStateId, DfaTransition, ParserDfa, ParserDfaStateView, ParserDfaStats}; -pub use errors::{AntlrError, ConsoleErrorListener, ErrorListener}; +pub use errors::{AntlrError, ConsoleErrorListener, ErrorListener, SyntaxErrorEvent}; pub use generated::{GeneratedLexer, GeneratedParser, GrammarMetadata}; pub use int_stream::{EOF, IntStream, UNKNOWN_SOURCE_NAME}; pub use lexer::{ diff --git a/src/parser.rs b/src/parser.rs index 354b3049..5e6d97d8 100644 --- a/src/parser.rs +++ b/src/parser.rs @@ -83,7 +83,7 @@ use crate::atn::parser_atn::{ #[cfg(test)] use crate::atn::parser_atn::{ParserAtnBuilder, ParserTransitionSpec}; use crate::char_stream::CharStream; -use crate::errors::AntlrError; +use crate::errors::{AntlrError, SyntaxErrorEvent}; use crate::int_stream::IntStream; use crate::lexer::{LexerCustomAction, LexerLifecycleCtx, LexerSemCtx}; use crate::recognizer::{Recognizer, RecognizerData}; @@ -5243,6 +5243,25 @@ where self.dispatch_generated_diagnostics(&parser_diagnostics, &token_errors); } + fn syntax_error_event<'a>( + &'a self, + offending: Option, + line: usize, + column: usize, + message: &'a str, + error: Option<&'a AntlrError>, + ) -> SyntaxErrorEvent<'a> { + let offending = offending.and_then(|token| self.token_store().view(token)); + SyntaxErrorEvent { + offending, + line, + column, + span: offending.and_then(|token| token.byte_span()), + message, + error, + } + } + /// Emits a fatal parser error after an entry-rule parse commits to returning it. /// /// Generated parsers call this only at their public entry boundary. Nested @@ -5257,21 +5276,23 @@ where else { return; }; - let offending = offending.and_then(|token| self.token_store().view(token)); - self.notify_error_listeners(offending, *line, *column, message, Some(error)); + self.notify_error_listeners(self.syntax_error_event( + *offending, + *line, + *column, + message, + Some(error), + )); } fn dispatch_parser_diagnostic(&self, diagnostic: &ParserDiagnostic) { - let offending = diagnostic - .offending - .and_then(|token| self.token_store().view(token)); - self.notify_error_listeners( - offending, + self.notify_error_listeners(self.syntax_error_event( + diagnostic.offending, diagnostic.line, diagnostic.column, &diagnostic.message, None, - ); + )); } fn dispatch_parser_diagnostics<'a>( @@ -5289,13 +5310,7 @@ where } // Lexer errors have no offending token: the failure is that no token // could be produced, matching ANTLR's null offendingSymbol. - self.notify_error_listeners( - None, - source_error.line, - source_error.column, - &source_error.message, - None, - ); + self.notify_error_listeners(source_error.into()); } fn dispatch_token_source_errors(&self, errors: &[TokenSourceError]) { @@ -5581,7 +5596,6 @@ where .insert( TokenSpec::explicit(token_type, text) .with_span(usize::MAX, usize::MAX) - .with_byte_span(0, 0) .with_position(line, column), ) .map_err(|error| AntlrError::Unsupported(error.to_string())) @@ -13079,9 +13093,7 @@ mod tests { ParserAtnPredictionDiagnostic, ParserAtnPredictionDiagnosticKind, ParserAtnSimulator, }; use crate::atn::serialized::{AtnDeserializer, SerializedAtn}; - use crate::token::{ - HIDDEN_CHANNEL, Token, TokenId, TokenSink, TokenSpec, TokenStoreError, TokenView, - }; + use crate::token::{HIDDEN_CHANNEL, Token, TokenId, TokenSink, TokenSpec, TokenStoreError}; use crate::token_stream::CommonTokenStream; use crate::tree::{NodeKind, ParseTreeStats}; use crate::vocabulary::Vocabulary; @@ -13140,14 +13152,13 @@ mod tests { self } - const fn with_span(mut self, start: usize, stop: usize) -> Self { - self.spec.start = start; - self.spec.stop = stop; - self.spec.start_byte = start; - self.spec.stop_byte = match stop.checked_add(1) { - Some(end) if end >= start => end, - Some(_) | None => start, - }; + fn with_span(mut self, start: usize, stop: usize) -> Self { + self.spec = self.spec.with_span(start, stop); + self + } + + fn with_byte_span(mut self, start: usize, stop: usize) -> Self { + self.spec = self.spec.with_byte_span(start, stop); self } @@ -13199,12 +13210,12 @@ mod tests { &self.source_name } - fn start_byte(&self) -> usize { - self.spec.start_byte + fn start_byte(&self) -> Option { + (self.spec.start_byte != usize::MAX).then_some(self.spec.start_byte) } - fn stop_byte(&self) -> usize { - self.spec.stop_byte + fn stop_byte(&self) -> Option { + (self.spec.stop_byte != usize::MAX).then_some(self.spec.stop_byte) } } @@ -13244,6 +13255,7 @@ mod tests { offending_text: Option, line: usize, column: usize, + span: Option>, message: String, error: Option, } @@ -13257,25 +13269,20 @@ mod tests { where R: Recognizer + ?Sized, { - fn syntax_error( - &mut self, - recognizer: &R, - offending: Option>, - line: usize, - column: usize, - message: &str, - error: Option<&AntlrError>, - ) { + fn syntax_error(&mut self, recognizer: &R, event: &SyntaxErrorEvent<'_>) { self.diagnostics .lock() .expect("recorded diagnostics lock") .push(RecordedDiagnostic { grammar_file_name: recognizer.grammar_file_name().to_owned(), - offending_text: offending.and_then(|token| token.text().map(str::to_owned)), - line, - column, - message: message.to_owned(), - error: error.cloned(), + offending_text: event + .offending + .and_then(|token| token.text().map(str::to_owned)), + line: event.line, + column: event.column, + span: event.span.clone(), + message: event.message.to_owned(), + error: event.error.cloned(), }); } } @@ -13348,8 +13355,8 @@ mod tests { offending: None, }]; let token_errors = [ - TokenSourceError::new(1, 1, "token recognition error at: '@'"), - TokenSourceError::new(1, 3, "token recognition error at: '#'"), + TokenSourceError::new(1, 1, "token recognition error at: '@'").with_span(1..2), + TokenSourceError::new(1, 3, "token recognition error at: '#'").with_span(3..4), ]; parser.dispatch_generated_diagnostics(&parser_diagnostics, &token_errors); @@ -13375,6 +13382,7 @@ mod tests { TestToken::new(7) .with_text("oops") .with_span(0, 3) + .with_byte_span(0, 4) .with_position(1, 2), TestToken::eof("parser-test", 4, 1, 6), ]); @@ -13407,6 +13415,38 @@ mod tests { ); } + #[test] + fn recovery_diagnostics_preserve_unknown_custom_token_span() { + let mut parser = mini_parser(vec![ + TestToken::new(7) + .with_text("oops") + .with_span(0, 3) + .with_position(1, 2), + TestToken::eof("parser-test", 4, 1, 6), + ]); + parser.remove_error_listeners(); + let diagnostics = Arc::new(Mutex::new(Vec::new())); + parser.add_error_listener(RecordingErrorListener { + diagnostics: Arc::clone(&diagnostics), + }); + let offending = parser.input.lt_id(1); + assert!(offending.is_some(), "current token should be buffered"); + + parser.dispatch_parser_diagnostic(&ParserDiagnostic { + line: 1, + column: 2, + message: "extraneous input 'oops'".to_owned(), + offending, + }); + + let span = { + let diagnostics = diagnostics.lock().expect("recorded diagnostics lock"); + assert_eq!(diagnostics.len(), 1); + diagnostics[0].span.clone() + }; + assert_eq!(span, None); + } + #[test] fn parser_leaves_token_errors_to_source_owned_listeners() { let source_diagnostics = Rc::new(RefCell::new(Vec::new())); @@ -17292,6 +17332,7 @@ mod tests { TestToken::new(2) .with_text("y") .with_span(0, 0) + .with_byte_span(0, 1) .with_position(3, 5), TestToken::eof("parser-test", 1, 1, 1), ]); diff --git a/src/recognizer.rs b/src/recognizer.rs index f0f54729..4e9f9527 100644 --- a/src/recognizer.rs +++ b/src/recognizer.rs @@ -1,8 +1,7 @@ use std::fmt; use std::sync::{Arc, Mutex}; -use crate::errors::{AntlrError, ConsoleErrorListener, ErrorListener}; -use crate::token::TokenView; +use crate::errors::{ConsoleErrorListener, ErrorListener, SyntaxErrorEvent}; use crate::vocabulary::Vocabulary; #[derive(Clone)] @@ -16,20 +15,11 @@ impl ErrorListenerSlot { Self(Arc::new(Mutex::new(listener))) } - #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature - fn syntax_error( - &self, - recognizer: &(dyn Recognizer + '_), - offending: Option>, - line: usize, - column: usize, - message: &str, - error: Option<&AntlrError>, - ) { + fn syntax_error(&self, recognizer: &(dyn Recognizer + '_), event: &SyntaxErrorEvent<'_>) { self.0 .lock() .expect("error listener lock poisoned") - .syntax_error(recognizer, offending, line, column, message, error); + .syntax_error(recognizer, event); } } @@ -168,21 +158,12 @@ impl RecognizerData { self.error_listeners.clear(); } - #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature - fn notify_error_listeners( - &self, - recognizer: &dyn Recognizer, - offending: Option>, - line: usize, - column: usize, - message: &str, - error: Option<&AntlrError>, - ) { + fn notify_error_listeners(&self, recognizer: &dyn Recognizer, event: &SyntaxErrorEvent<'_>) { if self.console_error_listener { - ConsoleErrorListener.syntax_error(recognizer, offending, line, column, message, error); + ConsoleErrorListener.syntax_error(recognizer, event); } for listener in &self.error_listeners { - listener.syntax_error(recognizer, offending, line, column, message, error); + listener.syntax_error(recognizer, event); } } } @@ -238,22 +219,11 @@ pub trait Recognizer { } /// Sends one diagnostic to every registered error listener. - /// - /// `offending` is the token the diagnostic is anchored to, when one - /// exists — parser diagnostics resolve it from the token store; lexer - /// diagnostics pass `None` because no token was produced. - fn notify_error_listeners( - &self, - offending: Option>, - line: usize, - column: usize, - message: &str, - error: Option<&AntlrError>, - ) where + fn notify_error_listeners(&self, event: SyntaxErrorEvent<'_>) + where Self: Sized, { - self.data() - .notify_error_listeners(self, offending, line, column, message, error); + self.data().notify_error_listeners(self, &event); } fn sempred(&mut self, _rule_index: usize, _pred_index: usize) -> bool { @@ -269,6 +239,7 @@ mod tests { use std::mem::size_of; use super::*; + use crate::errors::AntlrError; use crate::generated::GrammarMetadata; static SHARED_METADATA: GrammarMetadata = GrammarMetadata::new( @@ -288,6 +259,7 @@ mod tests { offending_text: Option, line: usize, column: usize, + span: Option>, message: String, error: Option, } @@ -301,25 +273,20 @@ mod tests { where R: Recognizer + ?Sized, { - fn syntax_error( - &mut self, - recognizer: &R, - offending: Option>, - line: usize, - column: usize, - message: &str, - error: Option<&AntlrError>, - ) { + fn syntax_error(&mut self, recognizer: &R, event: &SyntaxErrorEvent<'_>) { self.errors .lock() .expect("recorded errors lock") .push(RecordedError { grammar_file_name: recognizer.grammar_file_name().to_owned(), - offending_text: offending.and_then(|token| token.text().map(str::to_owned)), - line, - column, - message: message.to_owned(), - error: error.cloned(), + offending_text: event + .offending + .and_then(|token| token.text().map(str::to_owned)), + line: event.line, + column: event.column, + span: event.span.clone(), + message: event.message.to_owned(), + error: event.error.cloned(), }); } } @@ -372,7 +339,14 @@ mod tests { message: "unexpected token".to_owned(), offending: None, }; - recognizer.notify_error_listeners(None, 3, 5, "unexpected token", Some(&error)); + recognizer.notify_error_listeners(SyntaxErrorEvent { + offending: None, + line: 3, + column: 5, + span: Some(17..27), + message: "unexpected token", + error: Some(&error), + }); insta::assert_debug_snapshot!( "recognizers_replace_the_default_console_error_listener", diff --git a/src/snapshots/antlr4_runtime__parser__tests__failed_interpreted_parse_notifies_error_listener.snap b/src/snapshots/antlr4_runtime__parser__tests__failed_interpreted_parse_notifies_error_listener.snap index c3272c87..44722873 100644 --- a/src/snapshots/antlr4_runtime__parser__tests__failed_interpreted_parse_notifies_error_listener.snap +++ b/src/snapshots/antlr4_runtime__parser__tests__failed_interpreted_parse_notifies_error_listener.snap @@ -10,6 +10,9 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" ), line: 3, column: 5, + span: Some( + 0..1, + ), message: "mismatched input 'y' expecting 'x'", error: Some( ParserError { diff --git a/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap b/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap index f04bf717..f1d99651 100644 --- a/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap +++ b/src/snapshots/antlr4_runtime__parser__tests__parser_dispatches_recovery_diagnostics_through_registered_listeners.snap @@ -8,6 +8,9 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" offending_text: None, line: 1, column: 1, + span: Some( + 1..2, + ), message: "token recognition error at: '@'", error: None, }, @@ -16,6 +19,7 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" offending_text: None, line: 1, column: 2, + span: None, message: "missing 'x' at 'y'", error: None, }, @@ -24,6 +28,9 @@ expression: "*diagnostics.lock().expect(\"recorded diagnostics lock\")" offending_text: None, line: 1, column: 3, + span: Some( + 3..4, + ), message: "token recognition error at: '#'", error: None, }, diff --git a/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap b/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap index a81afa7a..970e2668 100644 --- a/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap +++ b/src/snapshots/antlr4_runtime__parser__tests__recovery_diagnostics_expose_the_offending_token_to_listeners.snap @@ -10,6 +10,9 @@ expression: recorded ), line: 1, column: 2, + span: Some( + 0..4, + ), message: "extraneous input 'oops'", error: None, }, diff --git a/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap b/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap index a658bce3..f5d9bf51 100644 --- a/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap +++ b/src/snapshots/antlr4_runtime__recognizer__tests__recognizers_replace_the_default_console_error_listener.snap @@ -8,6 +8,9 @@ expression: "*errors.lock().expect(\"recorded errors lock\")" offending_text: None, line: 3, column: 5, + span: Some( + 17..27, + ), message: "unexpected token", error: Some( ParserError { diff --git a/src/token.rs b/src/token.rs index 847d259a..074c5339 100644 --- a/src/token.rs +++ b/src/token.rs @@ -10,7 +10,8 @@ pub const HIDDEN_CHANNEL: i32 = 1; /// Largest source or location offset accepted by the compact token store. /// -/// `u32::MAX` is reserved for ANTLR's synthetic `-1` source boundary. +/// `u32::MAX` is reserved for unknown and ANTLR synthetic `-1` source +/// boundaries. pub const MAX_TOKEN_OFFSET: usize = (u32::MAX - 1) as usize; #[repr(transparent)] @@ -87,15 +88,15 @@ pub trait Token: fmt::Debug { TextInterval::new(self.start(), self.stop()) } - /// Zero-based absolute start offset measured in UTF-8 bytes. - fn start_byte(&self) -> usize; + /// Zero-based absolute start offset measured in UTF-8 bytes, when available. + fn start_byte(&self) -> Option; - /// Zero-based exclusive end offset measured in UTF-8 bytes. - fn stop_byte(&self) -> usize; + /// Zero-based exclusive end offset measured in UTF-8 bytes, when available. + fn stop_byte(&self) -> Option; - /// Zero-based UTF-8 byte span for the token text. - fn byte_span(&self) -> Range { - self.start_byte()..self.stop_byte() + /// Zero-based UTF-8 byte span for the token text, when available. + fn byte_span(&self) -> Option> { + Some(self.start_byte()?..self.stop_byte()?) } } @@ -136,11 +137,11 @@ impl Token for &T { (**self).source_name() } - fn start_byte(&self) -> usize { + fn start_byte(&self) -> Option { (**self).start_byte() } - fn stop_byte(&self) -> usize { + fn stop_byte(&self) -> Option { (**self).stop_byte() } } @@ -171,8 +172,8 @@ impl TokenSpec { channel: DEFAULT_CHANNEL, start: 0, stop: 0, - start_byte: 0, - stop_byte: 1, + start_byte: usize::MAX, + stop_byte: usize::MAX, line: 1, column: 0, text: Some(text.into()), @@ -203,15 +204,18 @@ impl TokenSpec { } #[must_use] + /// Sets the inclusive Unicode-scalar span without inferring byte offsets. + /// + /// Call [`Self::with_byte_span`] separately when the token source can + /// resolve exact UTF-8 byte boundaries. pub const fn with_span(mut self, start: usize, stop: usize) -> Self { self.start = start; self.stop = stop; - self.start_byte = start; - self.stop_byte = default_stop_byte(start, stop); self } #[must_use] + /// Sets the half-open UTF-8 byte span resolved by the token source. pub const fn with_byte_span(mut self, start_byte: usize, stop_byte: usize) -> Self { self.start_byte = start_byte; self.stop_byte = stop_byte; @@ -372,8 +376,8 @@ impl TokenStore { let id = TokenId(raw_id); let scalar_start = compact_boundary("start offset", spec.start)?; let scalar_stop = compact_boundary("stop offset", spec.stop)?; - let byte_start = compact_offset("start byte", spec.start_byte)?; - let byte_stop = compact_offset("stop byte", spec.stop_byte)?; + let byte_start = compact_boundary("start byte", spec.start_byte)?; + let byte_stop = compact_boundary("stop byte", spec.stop_byte)?; let line = compact_offset("line", spec.line)?; let column = compact_offset("column", spec.column)?; @@ -470,19 +474,33 @@ impl TokenStore { } /// Returns the token's zero-based UTF-8 byte start offset. + /// + /// Returns `None` when `id` is absent or its token source did not provide + /// an exact byte offset. #[must_use] pub fn start_byte(&self, id: TokenId) -> Option { self.byte_starts .get(id.index()) - .map(|offset| *offset as usize) + .copied() + .and_then(expand_byte_boundary) } /// Returns the token's zero-based exclusive UTF-8 byte stop offset. + /// + /// Returns `None` when `id` is absent or its token source did not provide + /// an exact byte offset. #[must_use] pub fn stop_byte(&self, id: TokenId) -> Option { self.byte_stops .get(id.index()) - .map(|offset| *offset as usize) + .copied() + .and_then(expand_byte_boundary) + } + + /// Returns the token's half-open UTF-8 byte span, when available. + #[must_use] + pub fn byte_span(&self, id: TokenId) -> Option> { + Some(self.start_byte(id)?..self.stop_byte(id)?) } fn explicit_text(&self, id: TokenId) -> Option<&str> { @@ -556,13 +574,6 @@ impl DoubleEndedIterator for TokenIter<'_> { impl ExactSizeIterator for TokenIter<'_> {} -const fn default_stop_byte(start: usize, stop: usize) -> usize { - match stop.checked_add(1) { - Some(end) if end >= start => end, - Some(_) | None => start, - } -} - const fn compact_boundary(field: &'static str, value: usize) -> Result { if value == usize::MAX { return Ok(u32::MAX); @@ -664,12 +675,12 @@ impl Token for TokenView<'_> { self.store.source_name.as_ref() } - fn start_byte(&self) -> usize { - self.store.byte_starts[self.id.index()] as usize + fn start_byte(&self) -> Option { + self.store.start_byte(self.id) } - fn stop_byte(&self) -> usize { - self.store.byte_stops[self.id.index()] as usize + fn stop_byte(&self) -> Option { + self.store.stop_byte(self.id) } } @@ -709,6 +720,14 @@ const fn expand_boundary(value: u32) -> usize { } } +const fn expand_byte_boundary(value: u32) -> Option { + if value == u32::MAX { + None + } else { + Some(value as usize) + } +} + /// Mutable append-only view used by a token source. #[derive(Debug)] pub struct TokenSink<'a> { @@ -735,11 +754,14 @@ impl<'a> TokenSink<'a> { /// A diagnostic buffered by a token source while it was producing tokens. #[derive(Clone, Debug, Eq, PartialEq)] +#[non_exhaustive] pub struct TokenSourceError { /// One-based input line where the diagnostic starts. pub line: usize, /// Zero-based column within `line` where the diagnostic starts. pub column: usize, + /// Half-open UTF-8 byte span of the offending input, when available. + pub span: Option>, /// ANTLR-compatible diagnostic message without the leading line/column. pub message: String, } @@ -750,9 +772,17 @@ impl TokenSourceError { Self { line, column, + span: None, message: message.into(), } } + + /// Attaches the resolved half-open UTF-8 byte span. + #[must_use] + pub const fn with_span(mut self, span: Range) -> Self { + self.span = Some(span); + self + } } pub trait TokenSource { @@ -845,7 +875,6 @@ mod tests { let store = one_token( TokenSpec::explicit(7, "") .with_span(usize::MAX, usize::MAX) - .with_byte_span(0, 0) .with_position(3, 9), ); assert_eq!( @@ -875,10 +904,25 @@ mod tests { assert_eq!(token.start(), 1); assert_eq!(token.stop(), 1); - assert_eq!(token.byte_span(), 2..4); + assert_eq!(token.start_byte(), Some(2)); + assert_eq!(token.stop_byte(), Some(4)); + assert_eq!(token.byte_span(), Some(2..4)); + assert_eq!(store.byte_span(id), Some(2..4)); assert_eq!(token.text(), Some("β")); } + #[test] + fn scalar_span_without_byte_offsets_remains_unknown() { + let store = one_token(TokenSpec::explicit(1, "β").with_span(0, 0)); + let token = store.view(TokenId(0)).expect("token"); + + assert_eq!((token.start(), token.stop()), (0, 0)); + assert_eq!(token.start_byte(), None); + assert_eq!(token.stop_byte(), None); + assert_eq!(token.byte_span(), None); + assert_eq!(store.byte_span(TokenId(0)), None); + } + #[test] fn source_backed_token_rejects_non_utf8_boundaries() { for (start_byte, stop_byte) in [(1, 2), (0, 1)] { diff --git a/src/tree.rs b/src/tree.rs index 85cfa13d..b9b1f069 100644 --- a/src/tree.rs +++ b/src/tree.rs @@ -1555,11 +1555,7 @@ mod tests { let mut tokens = TokenStore::new(None, ""); let deleted = token(&mut tokens, 1, "x"); let inserted = tokens - .push( - TokenSpec::explicit(2, "") - .with_span(usize::MAX, usize::MAX) - .with_byte_span(0, 0), - ) + .push(TokenSpec::explicit(2, "").with_span(usize::MAX, usize::MAX)) .expect("test token should fit"); let kept = token(&mut tokens, 3, "c"); let mut storage = ParseTreeStorage::new(); diff --git a/src/xpath/generated/x_path_lexer.rs b/src/xpath/generated/x_path_lexer.rs index ee398cdd..db16cdf3 100644 --- a/src/xpath/generated/x_path_lexer.rs +++ b/src/xpath/generated/x_path_lexer.rs @@ -217,14 +217,7 @@ where self.base.drain_errors() } fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool { - antlr4_runtime::Recognizer::notify_error_listeners( - self, - None, - source_error.line, - source_error.column, - &source_error.message, - None, - ); + antlr4_runtime::Recognizer::notify_error_listeners(self, source_error.into()); true } fn lexer_dfa_string(&self) -> String { diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index 4b883190..a3b94515 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -906,7 +906,7 @@ mod fatal_error_listener_tests { use super::fatal_parser::FatalParser; use antlr4_runtime::{ AntlrError, CommonTokenStream, ErrorListener, InputStream, Parser as _, Recognizer, - TokenView, + SyntaxErrorEvent, }; #[derive(Clone, Debug, Eq, PartialEq)] @@ -914,6 +914,7 @@ mod fatal_error_listener_tests { offending_text: Option, line: usize, column: usize, + span: Option>, message: String, error: Option, } @@ -935,21 +936,16 @@ mod fatal_error_listener_tests { where R: Recognizer + ?Sized, { - fn syntax_error( - &mut self, - _recognizer: &R, - offending: Option>, - line: usize, - column: usize, - message: &str, - error: Option<&AntlrError>, - ) { + fn syntax_error(&mut self, _recognizer: &R, event: &SyntaxErrorEvent<'_>) { self.events.lock().expect("events lock").push(Event { - offending_text: offending.and_then(|token| token.text().map(str::to_owned)), - line, - column, - message: message.to_owned(), - error: error.cloned(), + offending_text: event + .offending + .and_then(|token| token.text().map(str::to_owned)), + line: event.line, + column: event.column, + span: event.span.clone(), + message: event.message.to_owned(), + error: event.error.cloned(), }); } } @@ -1286,6 +1282,14 @@ mod combined_literal_tests { ) .expect("byte stream should parse through the generic helper"); assert_eq!(parsed.tokens().len(), 4); + assert_eq!( + parsed + .tokens() + .iter() + .map(|token| token.byte_span()) + .collect::>(), + [Some(0..5), Some(6..11), Some(12..17), Some(17..17)] + ); } } "#, diff --git a/tests/snapshots/antlr4_rust_gen_cli__fatal_entry_preserves_prior_recovery_diagnostics.snap b/tests/snapshots/antlr4_rust_gen_cli__fatal_entry_preserves_prior_recovery_diagnostics.snap index f216ef97..649aeaeb 100644 --- a/tests/snapshots/antlr4_rust_gen_cli__fatal_entry_preserves_prior_recovery_diagnostics.snap +++ b/tests/snapshots/antlr4_rust_gen_cli__fatal_entry_preserves_prior_recovery_diagnostics.snap @@ -21,6 +21,9 @@ EntrySnapshot { ), line: 1, column: 1, + span: Some( + 1..2, + ), message: "mismatched input 'd' expecting {'b', 'c'}", error: None, }, @@ -30,6 +33,9 @@ EntrySnapshot { ), line: 1, column: 3, + span: Some( + 3..4, + ), message: "mismatched input 'd' expecting {'b', 'c'}", error: Some( ParserError { diff --git a/tests/snapshots/antlr4_rust_gen_cli__semantic_override_does_not_leak_prior_recovery_diagnostics.snap b/tests/snapshots/antlr4_rust_gen_cli__semantic_override_does_not_leak_prior_recovery_diagnostics.snap index 29e82d5a..b46c69bf 100644 --- a/tests/snapshots/antlr4_rust_gen_cli__semantic_override_does_not_leak_prior_recovery_diagnostics.snap +++ b/tests/snapshots/antlr4_rust_gen_cli__semantic_override_does_not_leak_prior_recovery_diagnostics.snap @@ -14,6 +14,9 @@ EntrySnapshot { ), line: 1, column: 1, + span: Some( + 1..2, + ), message: "mismatched input 'd' expecting {'b', 'c'}", error: None, }, @@ -23,6 +26,9 @@ EntrySnapshot { ), line: 1, column: 2, + span: Some( + 2..3, + ), message: "rule failed predicate: semantic predicate", error: None, },