feat(runtime)!: expose resolved byte spans to error listeners

README.md

@@ -227,6 +227,9 @@ fn main() -> Result<(), antlr4_runtime::AntlrError> {
 Generated recognizers install a `ConsoleErrorListener` by default. Remove it
 from both the lexer and parser to suppress recovery output, as above, or call
 `add_error_listener` after removal to redirect diagnostics to a replacement.
+`ErrorListener::syntax_error` receives a `SyntaxErrorEvent`; its `span` is the
+resolved half-open UTF-8 byte range for parser tokens and lexer failures, when
+the input stream can provide byte offsets.
 
 ### Reusing Recognizers
 

docs/migration.md

@@ -6,6 +6,49 @@ generator changes. Generated lexers and parsers must use the same release of
 `antlr4-rust-gen --version` and compare the reported release with the
 `antlr-rust-runtime` dependency version.
 
+## Structured Syntax Error Events and Byte Spans
+
+`ErrorListener::syntax_error` now receives one `&SyntaxErrorEvent<'_>` instead
+of separate offending-token, line, column, message, and error arguments:
+
+```rust
+// Before
+fn syntax_error(
+    &mut self,
+    recognizer: &R,
+    offending: Option<TokenView<'_>>,
+    line: usize,
+    column: usize,
+    message: &str,
+    error: Option<&AntlrError>,
+);
+
+// After
+fn syntax_error(&mut self, recognizer: &R, event: &SyntaxErrorEvent<'_>);
+```
+
+Read `event.span` for the resolved half-open UTF-8 byte range. Lexer failures
+and parser diagnostics use the same event shape; streams and token sources that
+cannot resolve byte offsets leave the span as `None`.
+
+`Token::start_byte()` and `stop_byte()` now return `Option<usize>`, while
+`byte_span()` returns `Option<Range<usize>>`. `None` means the token source
+could not resolve exact byte offsets. Custom token sources must set
+Unicode-scalar and UTF-8 byte positions independently:
+
+```rust
+TokenSpec::explicit(token_type, text)
+    .with_span(scalar_start, scalar_stop)
+    .with_byte_span(byte_start, byte_end)
+```
+
+`TokenSpec::with_span` no longer assumes scalar indexes are byte offsets.
+Omit `with_byte_span` when no exact mapping exists.
+
+`TokenSourceError` gained an optional `span` and, like `SyntaxErrorEvent`, is
+non-exhaustive. Construct token-source diagnostics with
+`TokenSourceError::new(...).with_span(...)` instead of a struct literal.
+
 ## Recognizer Reuse Method Names
 
 Generated parsers now reserve `reset`, `set_token_stream`,

src/atn/lexer.rs

@@ -2214,12 +2214,16 @@ fn record_token_recognition_error<I>(lexer: &BaseLexer<I>, start: usize, stop: u
 where
     I: CharStream,
 {
-    let stop = stop.saturating_sub(1);
-    let text = display_error_text(&lexer.input().text(TextInterval::new(start, stop)));
-    lexer.record_error(
+    let inclusive_stop = stop.saturating_sub(1);
+    let text = display_error_text(&lexer.input().text(TextInterval::new(start, inclusive_stop)));
+    // Defensive callers that do not advance `stop` still identify one failing
+    // scalar; normal non-EOF recognition errors already pass `stop > start`.
+    let scalar_end = stop.max(start.saturating_add(1));
+    lexer.record_error_for_scalar_span(
         lexer.line(),
         lexer.column(),
         format!("token recognition error at: '{text}'"),
+        start..scalar_end,
     );
 }
 
@@ -2257,14 +2261,17 @@ where
 
 #[cfg(test)]
 mod tests {
+    use std::sync::{Arc, Mutex};
+
     use super::*;
     use crate::atn::lexer_dfa::{
         CompiledLexerActionTrace, CompiledLexerConfig, CompiledLexerContext,
     };
     use crate::atn::serialized::{AtnDeserializer, SerializedAtn};
     use crate::atn::{LexerAtnState, LexerTransition};
     use crate::char_stream::InputStream;
-    use crate::recognizer::RecognizerData;
+    use crate::errors::{ErrorListener, SyntaxErrorEvent};
+    use crate::recognizer::{Recognizer, RecognizerData};
     use crate::token::{DEFAULT_CHANNEL, HIDDEN_CHANNEL, TOKEN_EOF, Token, TokenStore, TokenView};
     use crate::vocabulary::Vocabulary;
 
@@ -2283,6 +2290,38 @@ mod tests {
         )
     }
 
+    #[derive(Clone, Debug)]
+    struct SpanListener(Arc<Mutex<Vec<Option<std::ops::Range<usize>>>>>);
+
+    impl<R> ErrorListener<R> for SpanListener
+    where
+        R: Recognizer + ?Sized,
+    {
+        fn syntax_error(&mut self, _recognizer: &R, event: &SyntaxErrorEvent<'_>) {
+            self.0
+                .lock()
+                .expect("recorded spans lock")
+                .push(event.span.clone());
+        }
+    }
+
+    #[test]
+    fn lexer_error_listener_receives_utf8_byte_span() {
+        let spans = Arc::new(Mutex::new(Vec::new()));
+        let mut lexer = BaseLexer::new(InputStream::new("aβz"), recognizer_data());
+        lexer.remove_error_listeners();
+        lexer.add_error_listener(SpanListener(Arc::clone(&spans)));
+        lexer.commit_position(0, 1);
+        lexer.begin_token();
+
+        record_token_recognition_error(&lexer, 1, 2);
+        let errors = lexer.drain_errors();
+        assert_eq!(errors.len(), 1);
+        lexer.notify_error_listeners(SyntaxErrorEvent::from(&errors[0]));
+
+        assert_eq!(*spans.lock().expect("recorded spans lock"), [Some(1..3)]);
+    }
+
     fn predicate_atn() -> LexerAtn {
         let mut atn = LexerAtn::new(1);
 
@@ -2861,7 +2900,7 @@ mod tests {
         assert_eq!(dot.text(), Some("."));
         assert_eq!(dot.start(), 0);
         assert_eq!(dot.stop(), 0);
-        assert_eq!(dot.byte_span(), 0..1);
+        assert_eq!(dot.byte_span(), Some(0..1));
         assert_eq!((dot.line(), dot.column()), (1, 0));
 
         let identifier = sink
@@ -2871,7 +2910,7 @@ mod tests {
         assert_eq!(identifier.text(), Some("β"));
         assert_eq!(identifier.start(), 1);
         assert_eq!(identifier.stop(), 1);
-        assert_eq!(identifier.byte_span(), 1..3);
+        assert_eq!(identifier.byte_span(), Some(1..3));
         assert_eq!((identifier.line(), identifier.column()), (1, 1));
 
         assert_eq!(

src/atn/lexer_dfa.rs

@@ -1641,8 +1641,8 @@ mod tests {
         channel: i32,
         start: usize,
         stop: usize,
-        start_byte: usize,
-        stop_byte: usize,
+        start_byte: Option<usize>,
+        stop_byte: Option<usize>,
         line: usize,
         column: usize,
     }

src/bin/antlr4-rust-gen.rs

@@ -4011,14 +4011,7 @@ where
         self.base.drain_errors()
     }}
     fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool {{
-        antlr4_runtime::Recognizer::notify_error_listeners(
-            self,
-            None,
-            source_error.line,
-            source_error.column,
-            &source_error.message,
-            None,
-        );
+        antlr4_runtime::Recognizer::notify_error_listeners(self, source_error.into());
         true
     }}
     fn lexer_dfa_string(&self) -> String {{
@@ -20574,7 +20567,7 @@ dispose = "hook"
         assert!(module.contains(
             "fn report_error(&self, source_error: &antlr4_runtime::token::TokenSourceError) -> bool"
         ));
-        assert!(module.contains("Recognizer::notify_error_listeners("));
+        assert!(module.contains("Recognizer::notify_error_listeners(self, source_error.into());"));
         assert!(!module.contains("CommonToken"));
         assert!(!module.contains("TokenFactory"));
     }

src/bin_support/grammar/atn/interp_test.rs

@@ -1091,8 +1091,8 @@ mod tests {
         channel: i32,
         start: usize,
         stop: usize,
-        byte_start: usize,
-        byte_stop: usize,
+        byte_start: Option<usize>,
+        byte_stop: Option<usize>,
         line: usize,
         column: usize,
         text: String,

src/bin_support/grammar/frontend.rs

@@ -6,7 +6,7 @@ use std::sync::{Arc, Mutex};
 
 use antlr4_runtime::{
     AsRuleNode, CharStream as _, CommonTokenStream, ErrorListener, InputStream, Node, NodeId,
-    NodeKind, Parser, Recognizer, TOKEN_EOF as RUNTIME_TOKEN_EOF, Token,
+    NodeKind, Parser, Recognizer, SyntaxErrorEvent, TOKEN_EOF as RUNTIME_TOKEN_EOF, Token,
 };
 
 use super::generated::antlr_v4_lexer::{
@@ -534,8 +534,14 @@ where
     token_stream
         .tokens()
         .map(|token| {
-            let start = u32::try_from(token.start_byte());
-            let end = u32::try_from(token.stop_byte());
+            let Some(bytes) = token.byte_span() else {
+                return Err(invalid_span(
+                    source,
+                    "token source did not provide a byte span",
+                ));
+            };
+            let start = u32::try_from(bytes.start);
+            let end = u32::try_from(bytes.end);
             let (Ok(start), Ok(end)) = (start, end) else {
                 return Err(invalid_span(source, "token byte span exceeds 4 GiB"));
             };
@@ -980,22 +986,14 @@ impl<R> ErrorListener<R> for DiagnosticCollector
 where
     R: Recognizer + ?Sized,
 {
-    fn syntax_error(
-        &mut self,
-        _recognizer: &R,
-        _offending: Option<antlr4_runtime::TokenView<'_>>,
-        line: usize,
-        column: usize,
-        message: &str,
-        _error: Option<&antlr4_runtime::AntlrError>,
-    ) {
+    fn syntax_error(&mut self, _recognizer: &R, event: &SyntaxErrorEvent<'_>) {
         self.0
             .lock()
             .expect("grammar diagnostic collector mutex poisoned")
             .push(ReportedDiagnostic {
-                line,
-                column,
-                message: message.to_owned(),
+                line: event.line,
+                column: event.column,
+                message: event.message.to_owned(),
             });
     }
 }

src/errors.rs

@@ -1,5 +1,7 @@
+use std::ops::Range;
+
 use crate::recognizer::Recognizer;
-use crate::token::{TokenId, TokenView};
+use crate::token::{TokenId, TokenSourceError, TokenView};
 use thiserror::Error;
 
 #[derive(Debug, Error, Clone, Eq, PartialEq)]
@@ -30,43 +32,60 @@ pub enum AntlrError {
     Unsupported(String),
 }
 
+/// Structured context for one recognizer diagnostic.
+#[derive(Clone, Debug, Eq, PartialEq)]
+#[non_exhaustive]
+pub struct SyntaxErrorEvent<'a> {
+    /// Token the diagnostic is anchored to, when one exists.
+    ///
+    /// Lexer errors have no offending token because the failed match did not
+    /// produce one.
+    pub offending: Option<TokenView<'a>>,
+    /// One-based input line where the diagnostic starts.
+    pub line: usize,
+    /// Zero-based column within `line` where the diagnostic starts.
+    pub column: usize,
+    /// Half-open UTF-8 byte span of the offending source text.
+    ///
+    /// Custom streams and token sources that cannot resolve byte offsets leave
+    /// this as `None`.
+    pub span: Option<Range<usize>>,
+    /// ANTLR-compatible diagnostic message without the leading line/column.
+    pub message: &'a str,
+    /// Recognition error that caused the diagnostic, when one exists.
+    pub error: Option<&'a AntlrError>,
+}
+
+impl<'a> From<&'a TokenSourceError> for SyntaxErrorEvent<'a> {
+    fn from(error: &'a TokenSourceError) -> Self {
+        Self {
+            offending: None,
+            line: error.line,
+            column: error.column,
+            span: error.span.clone(),
+            message: &error.message,
+            error: None,
+        }
+    }
+}
+
 /// Receives recognizer diagnostics.
 ///
 /// Listeners registered through [`Recognizer::add_error_listener`] must be
 /// [`Send`] and work with every recognizer type. Implement the trait
 /// generically, as [`ConsoleErrorListener`] does, when a listener will be
 /// registered.
 pub trait ErrorListener<R: Recognizer + ?Sized> {
-    /// `offending` carries the token the diagnostic points at, matching
-    /// ANTLR's `syntaxError(recognizer, offendingSymbol, ...)`. It is `None`
-    /// for lexer errors (no token was produced) and for diagnostics that are
-    /// not anchored to a specific token.
-    #[allow(clippy::too_many_arguments)] // mirrors ANTLR's canonical syntaxError signature
-    fn syntax_error(
-        &mut self,
-        recognizer: &R,
-        offending: Option<TokenView<'_>>,
-        line: usize,
-        column: usize,
-        message: &str,
-        error: Option<&AntlrError>,
-    );
+    /// Receives one diagnostic with its ANTLR position and resolved byte span.
+    fn syntax_error(&mut self, recognizer: &R, event: &SyntaxErrorEvent<'_>);
 }
 
 #[derive(Debug, Default)]
 pub struct ConsoleErrorListener;
 
 impl<R: Recognizer + ?Sized> ErrorListener<R> for ConsoleErrorListener {
     #[allow(clippy::print_stderr)]
-    fn syntax_error(
-        &mut self,
-        _recognizer: &R,
-        _offending: Option<TokenView<'_>>,
-        line: usize,
-        column: usize,
-        message: &str,
-        _error: Option<&AntlrError>,
-    ) {
-        eprintln!("line {line}:{column} {message}");
+    fn syntax_error(&mut self, _recognizer: &R, event: &SyntaxErrorEvent<'_>) {
+        eprintln!("line {}:{} {}", event.line, event.column, event.message);
     }
 }