diff --git a/README.md b/README.md index 3628fdd7..9c0aa771 100644 --- a/README.md +++ b/README.md @@ -183,8 +183,28 @@ fn main() -> Result<(), antlr4_runtime::AntlrError> { } ``` -Or construct each layer explicitly when you need to set source names, parser -options, or custom error handling before invoking the entry rule: +Use `parse_stream` with a preconstructed `CharStream` when parsing a file, +preserving its source name, or supplying a custom stream implementation: + +```rust +use std::fs::File; +use antlr4_runtime::InputStream; +use generated::json_lexer::JsonLexer; +use generated::json_parser::{self, JsonParser}; + +let input = InputStream::from_reader_with_source_name( + File::open(path)?, + path.display().to_string(), +)?; +let parsed = json_parser::parse_stream(input, JsonLexer::new, JsonParser::json)?; +``` + +`InputStream::from_reader` accepts any `std::io::Read` and validates UTF-8. +`parse_stream_with_parser` is the corresponding stream-based helper when the +caller also needs the parser afterward. + +Construct each layer explicitly when you need parser options or custom error +handling before invoking the entry rule: ```rust use antlr4_runtime::{CommonTokenStream, InputStream}; @@ -593,7 +613,13 @@ byte value (`0..=255`). It is generic over the backing store — `ByteStream::new(vec)` owns, `ByteStream::new(&buf[..])` borrows a network buffer zero-copy, and `ByteStream::from_reader(file)?` drains any `std::io::Read`. Because the bytes are not text, `text()` renders a matched span -as lowercase hex. +as lowercase hex. Generated parser modules accept it through `parse_stream`, so +binary inputs retain the same compact lexer/token-stream/parser setup as text: + +```rust +let input = ByteStream::from_reader(file)?; +let parsed = midi_parser::parse_stream(input, MidiLexer::new, MidiParser::file)?; +``` Length-prefixed formats ("read N, then consume N bytes") are data-dependent, so a pure grammar cannot frame them alone — the same constraint ANTLR's `bencoding` diff --git a/docs/kotlin-build.md b/docs/kotlin-build.md index 45b73804..bb5c8a94 100644 --- a/docs/kotlin-build.md +++ b/docs/kotlin-build.md @@ -89,9 +89,32 @@ assert!(tree.text().contains("fun")); assert!(!tokens.tokens().is_empty()); ``` -The generated helper is additive. The explicit path is still available when the -caller needs to name the input source, adjust parser options, or attach custom -error handling before the entry rule: +Use `parse_stream` to parse directly from a Rust reader while retaining the +input's source name: + +```rust +use std::fs::File; +use antlr4_runtime::InputStream; +use generated::kotlin_lexer::KotlinLexer; +use generated::kotlin_parser::{self, KotlinParser}; + +let path = std::path::Path::new("Main.kt"); +let input = InputStream::from_reader_with_source_name( + File::open(path).expect("Kotlin source should open"), + path.display().to_string(), +) +.expect("Kotlin source should be UTF-8"); +let parsed = kotlin_parser::parse_stream(input, KotlinLexer::new, KotlinParser::kotlin_file) + .expect("entry rule parses"); +assert!(parsed.tree().text().contains("fun")); +``` + +`InputStream::from_reader` also accepts stdin, sockets, cursors, and other +`std::io::Read` implementations when no explicit source name is needed. + +The generated helpers are additive. The explicit path remains available when +the caller needs to adjust parser options or attach custom error handling before +the entry rule: ```rust use antlr4_runtime::{CommonTokenStream, InputStream}; @@ -105,4 +128,5 @@ let tree = parser.kotlin_file().expect("entry rule parses"); assert!(tree.text().contains("fun")); ``` -Validated locally: the generated Kotlin lexer emits real tokens and the generated parser recognizes the `parser.kotlin_file()` entry rule for `fun main() {}`. +Validated locally: the generated Kotlin lexer emits real tokens and the generated parser recognizes the `parser.kotlin_file()` entry rule for +`fun main() {}`. diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index 9a40d0b0..9cec56bc 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -13512,12 +13512,12 @@ fn render_parser_base_initialization( out } -/// Renders the parser-module convenience that wires text input through the -/// caller-selected lexer, token stream, parser, and entry rule in one call. +/// Renders parser-module conveniences that wire text or a caller-provided +/// character stream through the lexer, token stream, parser, and entry rule. fn render_parser_parse_convenience(type_name: &str) -> String { let output_type_name = format!("{type_name}ParseOutput"); format!( - r#"/// Result from [`parse_with_parser`]. + r#"/// Result from [`parse_with_parser`] or [`parse_stream_with_parser`]. /// /// Keeps the generated parser available after the entry rule runs so callers /// can inspect diagnostics or recover the parser-owned token stream. @@ -13546,8 +13546,7 @@ pub fn parse( entry: impl FnOnce(&mut {type_name}) -> Result, ) -> Result {{ - let {output_type_name} {{ result, parser }} = parse_with_parser(input, lexer, entry)?; - Ok(parser.into_parsed_file(result)) + parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry) }} /// Parses UTF-8 text like [`parse`] while returning the parser after the entry @@ -13561,7 +13560,39 @@ pub fn parse_with_parser( entry: impl FnOnce(&mut {type_name}) -> Result, ) -> Result<{output_type_name}, antlr4_runtime::AntlrError> {{ - let lexer = lexer(antlr4_runtime::InputStream::new(input.as_ref())); + parse_stream_with_parser( + antlr4_runtime::InputStream::new(input.as_ref()), + lexer, + entry, + ) +}} + +/// Parses a caller-provided character stream by constructing the lexer, token +/// stream, parser, and caller-selected entry rule in one call. +/// +/// Unlike [`parse`], this accepts any [`antlr4_runtime::CharStream`], including +/// a named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn parse_stream( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut {type_name}) -> Result, +) -> Result +{{ + let {output_type_name} {{ result, parser }} = + parse_stream_with_parser(input, lexer, entry)?; + Ok(parser.into_parsed_file(result)) +}} + +/// Parses a caller-provided character stream like [`parse_stream`] while +/// returning the parser after the entry rule has run. +pub fn parse_stream_with_parser( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut {type_name}) -> Result, +) -> Result<{output_type_name}, antlr4_runtime::AntlrError> +{{ + let lexer = lexer(input); let tokens = CommonTokenStream::new(lexer); let mut parser = {type_name}::new(tokens); let result = entry(&mut parser)?; @@ -16156,11 +16187,22 @@ mod tests { let rendered = render_parser("TParser", &minimal_parser_data()).expect("parser should render"); + insta::assert_snapshot!( + "parser_parse_convenience", + render_parser_parse_convenience("TParser") + ); assert!(rendered.contains("pub struct TParserParseOutput")); assert!(rendered.contains("pub result: R,")); assert!(rendered.contains("pub parser: TParser,")); assert!(rendered.contains("pub fn parse(")); assert!(rendered.contains("pub fn parse_with_parser(")); + assert!( + rendered + .contains("pub fn parse_stream(") + ); + assert!(rendered.contains( + "pub fn parse_stream_with_parser(" + )); assert!( !rendered .contains(") -> Result\nwhere\n L: TokenSource,") @@ -16169,12 +16211,15 @@ mod tests { ") -> Result, antlr4_runtime::AntlrError>\nwhere\n L: TokenSource," )); assert!(rendered.contains("lexer: impl FnOnce(antlr4_runtime::InputStream) -> L")); - assert!(rendered.contains("antlr4_runtime::InputStream::new(input.as_ref())")); + assert!(rendered.contains( + "parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry)" + )); + assert!(rendered.contains("let lexer = lexer(input);")); assert!(rendered.contains("let tokens = CommonTokenStream::new(lexer);")); assert!(rendered.contains("let result = entry(&mut parser)?;")); assert!(rendered.contains("Ok(TParserParseOutput { result, parser })")); assert!(rendered.contains( - "let TParserParseOutput { result, parser } = parse_with_parser(input, lexer, entry)?;" + "parse_stream_with_parser(\n antlr4_runtime::InputStream::new(input.as_ref())," )); assert!(rendered.contains("Ok(parser.into_parsed_file(result))")); assert!(rendered.contains("pub fn new(input: CommonTokenStream) -> Self")); diff --git a/src/bin/snapshots/antlr4_rust_gen__tests__parser_parse_convenience.snap b/src/bin/snapshots/antlr4_rust_gen__tests__parser_parse_convenience.snap new file mode 100644 index 00000000..ddcd217b --- /dev/null +++ b/src/bin/snapshots/antlr4_rust_gen__tests__parser_parse_convenience.snap @@ -0,0 +1,85 @@ +--- +source: src/bin/antlr4-rust-gen.rs +expression: "render_parser_parse_convenience(\"TParser\")" +--- +/// Result from [`parse_with_parser`] or [`parse_stream_with_parser`]. +/// +/// Keeps the generated parser available after the entry rule runs so callers +/// can inspect diagnostics or recover the parser-owned token stream. +#[derive(Debug)] +pub struct TParserParseOutput +where + L: TokenSource, +{ + pub result: R, + pub parser: TParser, +} + +/// Parses UTF-8 text by constructing the lexer, token stream, parser, and +/// caller-selected entry rule in one call. +/// +/// Pass the generated lexer constructor and a parser entry rule, for example +/// `parse(src, MyGrammarLexer::new, TParser::file)`. +/// +/// The returned [`antlr4_runtime::ParsedFile`] owns the canonical token store, +/// flat CST storage, and entry-rule root. +/// Use [`parse_with_parser`] instead when the caller also needs parser +/// diagnostics after the entry rule runs. +pub fn parse( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, + entry: impl FnOnce(&mut TParser) -> Result, +) -> Result +{ + parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry) +} + +/// Parses UTF-8 text like [`parse`] while returning the parser after the entry +/// rule has run. +/// +/// This keeps the compact generated setup path available for callers that also +/// need `Parser::number_of_syntax_errors()` or `TParser::into_token_stream()`. +pub fn parse_with_parser( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, + entry: impl FnOnce(&mut TParser) -> Result, +) -> Result, antlr4_runtime::AntlrError> +{ + parse_stream_with_parser( + antlr4_runtime::InputStream::new(input.as_ref()), + lexer, + entry, + ) +} + +/// Parses a caller-provided character stream by constructing the lexer, token +/// stream, parser, and caller-selected entry rule in one call. +/// +/// Unlike [`parse`], this accepts any [`antlr4_runtime::CharStream`], including +/// a named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn parse_stream( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut TParser) -> Result, +) -> Result +{ + let TParserParseOutput { result, parser } = + parse_stream_with_parser(input, lexer, entry)?; + Ok(parser.into_parsed_file(result)) +} + +/// Parses a caller-provided character stream like [`parse_stream`] while +/// returning the parser after the entry rule has run. +pub fn parse_stream_with_parser( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut TParser) -> Result, +) -> Result, antlr4_runtime::AntlrError> +{ + let lexer = lexer(input); + let tokens = CommonTokenStream::new(lexer); + let mut parser = TParser::new(tokens); + let result = entry(&mut parser)?; + Ok(TParserParseOutput { result, parser }) +} diff --git a/src/bin_support/grammar/frontend.rs b/src/bin_support/grammar/frontend.rs index 56d20cd9..36ef887c 100644 --- a/src/bin_support/grammar/frontend.rs +++ b/src/bin_support/grammar/frontend.rs @@ -1,11 +1,12 @@ use std::fmt; use std::ops::Range; use std::path::{Path, PathBuf}; +use std::rc::Rc; use std::sync::{Arc, Mutex}; use antlr4_runtime::{ - AsRuleNode, CommonTokenStream, ErrorListener, InputStream, Node, NodeId, NodeKind, Parser, - Recognizer, TOKEN_EOF as RUNTIME_TOKEN_EOF, Token, + AsRuleNode, CharStream as _, CommonTokenStream, ErrorListener, InputStream, Node, NodeId, + NodeKind, Parser, Recognizer, TOKEN_EOF as RUNTIME_TOKEN_EOF, Token, }; use super::generated::antlr_v4_lexer::{ @@ -143,7 +144,7 @@ impl Iterator for CstDescendants<'_> { pub(crate) struct SourceFile { id: SourceId, logical_path: PathBuf, - text: Box, + text: Rc, line_starts: Box<[u32]>, tokens: Box<[SyntaxToken]>, trivia: Box<[u32]>, @@ -265,7 +266,18 @@ pub(crate) fn parse_source( logical_path: impl Into, text: impl Into>, ) -> Result { - let recovered = parse_source_recovering(source, logical_path, text)?; + let logical_path = logical_path.into(); + let text = text.into(); + let input = InputStream::with_source_name(&text, logical_path.to_string_lossy()); + parse_input_stream(source, logical_path, input) +} + +pub(crate) fn parse_input_stream( + source: SourceId, + logical_path: impl Into, + input: InputStream, +) -> Result { + let recovered = parse_input_stream_recovering(source, logical_path, input)?; if recovered.diagnostics.is_empty() { Ok(recovered.file) } else { @@ -282,8 +294,20 @@ pub(crate) fn parse_source_recovering( ) -> Result { let logical_path = logical_path.into(); let text = text.into(); - let line_starts = line_starts(source, &text)?; let input = InputStream::with_source_name(&text, logical_path.to_string_lossy()); + parse_input_stream_recovering(source, logical_path, input) +} + +pub(crate) fn parse_input_stream_recovering( + source: SourceId, + logical_path: impl Into, + input: InputStream, +) -> Result { + let logical_path = logical_path.into(); + let text = input + .source_text() + .expect("InputStream always exposes its complete source text"); + let line_starts = line_starts(source, &text)?; let mut lexer = AntlRv4Lexer::with_hooks(input, LexerAdaptor::default()); lexer.remove_error_listeners(); let mut token_stream = CommonTokenStream::try_new(lexer).map_err(|error| FrontendError { diff --git a/src/bin_support/grammar/generated/antlr_v4_lexer.rs b/src/bin_support/grammar/generated/antlr_v4_lexer.rs index a67f3a02..02e92d9f 100644 --- a/src/bin_support/grammar/generated/antlr_v4_lexer.rs +++ b/src/bin_support/grammar/generated/antlr_v4_lexer.rs @@ -1,4 +1,4 @@ -// @generated by antlr-rust-runtime v0.19.1 - do not edit +// @generated by antlr-rust-runtime v0.21.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] diff --git a/src/bin_support/grammar/generated/antlr_v4_parser.rs b/src/bin_support/grammar/generated/antlr_v4_parser.rs index 170616b5..2ca07a10 100644 --- a/src/bin_support/grammar/generated/antlr_v4_parser.rs +++ b/src/bin_support/grammar/generated/antlr_v4_parser.rs @@ -1,4 +1,4 @@ -// @generated by antlr-rust-runtime v0.19.1 - do not edit +// @generated by antlr-rust-runtime v0.21.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] @@ -10572,7 +10572,7 @@ pub fn parser_atn() -> &'static ParserAtn { atn() } -/// Result from [`parse_with_parser`]. +/// Result from [`parse_with_parser`] or [`parse_stream_with_parser`]. /// /// Keeps the generated parser available after the entry rule runs so callers /// can inspect diagnostics or recover the parser-owned token stream. @@ -10601,8 +10601,7 @@ pub fn parse( entry: impl FnOnce(&mut AntlRv4Parser) -> Result, ) -> Result { - let AntlRv4ParserParseOutput { result, parser } = parse_with_parser(input, lexer, entry)?; - Ok(parser.into_parsed_file(result)) + parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry) } /// Parses UTF-8 text like [`parse`] while returning the parser after the entry @@ -10616,7 +10615,39 @@ pub fn parse_with_parser( entry: impl FnOnce(&mut AntlRv4Parser) -> Result, ) -> Result, antlr4_runtime::AntlrError> { - let lexer = lexer(antlr4_runtime::InputStream::new(input.as_ref())); + parse_stream_with_parser( + antlr4_runtime::InputStream::new(input.as_ref()), + lexer, + entry, + ) +} + +/// Parses a caller-provided character stream by constructing the lexer, token +/// stream, parser, and caller-selected entry rule in one call. +/// +/// Unlike [`parse`], this accepts any [`antlr4_runtime::CharStream`], including +/// a named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn parse_stream( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut AntlRv4Parser) -> Result, +) -> Result +{ + let AntlRv4ParserParseOutput { result, parser } = + parse_stream_with_parser(input, lexer, entry)?; + Ok(parser.into_parsed_file(result)) +} + +/// Parses a caller-provided character stream like [`parse_stream`] while +/// returning the parser after the entry rule has run. +pub fn parse_stream_with_parser( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut AntlRv4Parser) -> Result, +) -> Result, antlr4_runtime::AntlrError> +{ + let lexer = lexer(input); let tokens = CommonTokenStream::new(lexer); let mut parser = AntlRv4Parser::new(tokens); let result = entry(&mut parser)?; diff --git a/src/bin_support/grammar/loader.rs b/src/bin_support/grammar/loader.rs index 09e78cb2..bd04cd28 100644 --- a/src/bin_support/grammar/loader.rs +++ b/src/bin_support/grammar/loader.rs @@ -1,9 +1,12 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::{Path, PathBuf}; +use std::rc::Rc; + +use antlr4_runtime::{CharStream as _, InputStream}; use super::diagnostic::{CompilationError, Diagnostic, Severity}; -use super::frontend::{SourceId, SourceSpan, parse_source, parse_source_recovering}; +use super::frontend::{SourceId, SourceSpan, parse_input_stream, parse_input_stream_recovering}; use super::model::{ GrammarId, ImportEdge, LoadedGrammarSet, LookupKind, LookupRecord, ParsedGrammarUnit, VocabularyEdge, VocabularySource, @@ -150,8 +153,11 @@ impl Loader { if let Some(source) = self.sources.id_for_canonical_path(&canonical) { return self.grammar_for_source.get(&source).copied(); } - let text = match fs::read_to_string(&canonical) { - Ok(text) => text, + let logical_path = user_spelling.unwrap_or(path).to_path_buf(); + let input = match fs::File::open(&canonical).and_then(|file| { + InputStream::from_reader_with_source_name(file, logical_path.to_string_lossy()) + }) { + Ok(input) => input, Err(error) => { self.diagnostics.push(Diagnostic::error( "G4L002", @@ -161,10 +167,12 @@ impl Loader { return None; } }; + let source_text = input + .source_text() + .expect("InputStream always exposes its complete source text"); let source = self.sources.next_id(); - let logical_path = user_spelling.unwrap_or(path).to_path_buf(); let file = if recover_syntax { - match parse_source_recovering(source, logical_path.clone(), text) { + match parse_input_stream_recovering(source, logical_path.clone(), input) { Ok(recovered) => { let has_diagnostics = !recovered.diagnostics.is_empty(); self.diagnostics.extend( @@ -182,16 +190,16 @@ impl Loader { } Err(error) => { self.record_frontend_error(&error); - self.retain_failed_source(canonical, source, logical_path); + self.retain_failed_source(canonical, source, logical_path, source_text); return None; } } } else { - match parse_source(source, logical_path.clone(), text) { + match parse_input_stream(source, logical_path.clone(), input) { Ok(file) => file, Err(error) => { self.record_frontend_error(&error); - self.retain_failed_source(canonical, source, logical_path); + self.retain_failed_source(canonical, source, logical_path, source_text); return None; } } @@ -215,8 +223,8 @@ impl Loader { canonical_path: PathBuf, source: SourceId, logical_path: PathBuf, + text: Rc, ) { - let text = fs::read_to_string(&canonical_path).unwrap_or_default(); self.sources .insert_failed(canonical_path, source, logical_path, text) .expect("canonical path checked before parsing"); diff --git a/src/bin_support/grammar/source.rs b/src/bin_support/grammar/source.rs index 0d3c2dfd..bcc9ba58 100644 --- a/src/bin_support/grammar/source.rs +++ b/src/bin_support/grammar/source.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +use std::rc::Rc; use super::frontend::{SourceFile, SourceId}; @@ -14,7 +15,7 @@ pub(crate) struct SourceSet { #[derive(Debug)] struct FailedSource { logical_path: PathBuf, - text: Box, + text: Rc, } impl SourceSet { @@ -39,7 +40,7 @@ impl SourceSet { canonical_path: PathBuf, source: SourceId, logical_path: PathBuf, - text: String, + text: Rc, ) -> Result<(), SourceId> { if let Some(id) = self.by_canonical_path.get(&canonical_path) { return Err(*id); @@ -49,13 +50,8 @@ impl SourceSet { .insert(canonical_path.clone(), source); self.canonical_paths.push(canonical_path); self.files.push(None); - self.failed_sources.insert( - source, - FailedSource { - logical_path, - text: text.into_boxed_str(), - }, - ); + self.failed_sources + .insert(source, FailedSource { logical_path, text }); Ok(()) } diff --git a/src/byte_stream.rs b/src/byte_stream.rs index 7bd02cc1..14d0bad5 100644 --- a/src/byte_stream.rs +++ b/src/byte_stream.rs @@ -38,11 +38,9 @@ //! // Zero-copy from an in-memory buffer (e.g. bytes read off a socket): //! let stream = ByteStream::new(&packet[..]); //! -//! // Feed it to any generated lexer built from a byte-oriented grammar. -//! let lexer = FooLexer::new(stream); -//! let tokens = CommonTokenStream::new(lexer); -//! let mut parser = FooParser::new(tokens); -//! let tree = parser.entry_rule()?; +//! // Feed it to any generated parser built from a byte-oriented grammar. +//! let parsed = +//! foo_parser::parse_stream(stream, FooLexer::new, FooParser::entry_rule)?; //! ``` //! //! Write lexer rules against the byte range, e.g. `BYTE : ' ' .. 'ÿ';`. A diff --git a/src/char_stream.rs b/src/char_stream.rs index 1e58e23c..ee995219 100644 --- a/src/char_stream.rs +++ b/src/char_stream.rs @@ -1,4 +1,5 @@ use crate::int_stream::{EOF, IntStream, UNKNOWN_SOURCE_NAME}; +use std::io; use std::rc::Rc; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -147,6 +148,33 @@ impl InputData { } impl InputStream { + /// Creates a character stream by draining UTF-8 text from a + /// [`std::io::Read`], using ANTLR's unknown source name placeholder. + /// + /// # Errors + /// + /// Returns any I/O error produced while reading, including + /// [`io::ErrorKind::InvalidData`] when the input is not valid UTF-8. + pub fn from_reader(reader: impl io::Read) -> io::Result { + Self::from_reader_with_source_name(reader, UNKNOWN_SOURCE_NAME) + } + + /// Creates a named character stream by draining UTF-8 text from a + /// [`std::io::Read`]. + /// + /// # Errors + /// + /// Returns any I/O error produced while reading, including + /// [`io::ErrorKind::InvalidData`] when the input is not valid UTF-8. + pub fn from_reader_with_source_name( + mut reader: impl io::Read, + source_name: impl Into, + ) -> io::Result { + let mut input = String::new(); + reader.read_to_string(&mut input)?; + Ok(Self::with_source_name(input, source_name)) + } + /// Creates a character stream from UTF-8 text using ANTLR's unknown source /// name placeholder. pub fn new(input: impl AsRef) -> Self { @@ -372,4 +400,28 @@ mod tests { (6, 3) ); } + + #[test] + fn reader_constructors_decode_utf8_and_preserve_source_names() { + let mut named = InputStream::from_reader_with_source_name( + io::Cursor::new("aβ\n".as_bytes()), + "sample.txt", + ) + .expect("in-memory UTF-8 should be readable"); + assert_eq!(named.source_name(), "sample.txt"); + assert_eq!(named.size(), 3); + assert_eq!(named.la(2), 'β' as i32); + + let unnamed = InputStream::from_reader(io::Cursor::new(b"text")) + .expect("in-memory UTF-8 should be readable"); + assert_eq!(unnamed.source_name(), UNKNOWN_SOURCE_NAME); + assert_eq!(unnamed.text(TextInterval::new(0, 3)), "text"); + } + + #[test] + fn reader_constructor_rejects_invalid_utf8() { + let error = InputStream::from_reader(io::Cursor::new([0xFF])) + .expect_err("invalid UTF-8 must not produce a character stream"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } } diff --git a/src/xpath/generated/x_path_lexer.rs b/src/xpath/generated/x_path_lexer.rs index c24dbfb6..ee398cdd 100644 --- a/src/xpath/generated/x_path_lexer.rs +++ b/src/xpath/generated/x_path_lexer.rs @@ -1,4 +1,4 @@ -// @generated by antlr-rust-runtime v0.19.1 - do not edit +// @generated by antlr-rust-runtime v0.21.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index a620dc50..362258a3 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -697,8 +697,11 @@ fn combined_literal_tokens_are_public_and_lexable() { #[cfg(test)] mod combined_literal_tests { use super::t_lexer::TLexer; - use super::t_parser::TParser; - use antlr4_runtime::{CommonTokenStream, InputStream, Parser as _}; + use super::t_parser::{self, TParser}; + use antlr4_runtime::{ + ByteStream, CommonTokenStream, InputStream, Parser as _, Token as _, + }; + use std::io::Cursor; #[test] fn recognizes_implicit_literal_rules() { @@ -708,6 +711,34 @@ mod combined_literal_tests { parser.greeting().expect("literal input should parse"); assert_eq!(parser.number_of_syntax_errors(), 0); } + + #[test] + fn generated_helpers_accept_named_text_and_byte_streams() { + let input = InputStream::from_reader_with_source_name( + Cursor::new(b"hello Alice world"), + "greeting.txt", + ) + .expect("in-memory UTF-8 should be readable"); + let output = + t_parser::parse_stream_with_parser(input, TLexer::new, TParser::greeting) + .expect("named text stream should parse"); + assert_eq!(output.parser.number_of_syntax_errors(), 0); + assert!( + output + .parser + .token_store() + .iter() + .all(|token| token.source_name() == "greeting.txt") + ); + + let parsed = t_parser::parse_stream( + ByteStream::new(b"hello Alice world".to_vec()), + TLexer::new, + TParser::greeting, + ) + .expect("byte stream should parse through the generic helper"); + assert_eq!(parsed.tokens().len(), 4); + } } "#, ); diff --git a/third_party/antlr-v4-grammar/self-hosted.sha256 b/third_party/antlr-v4-grammar/self-hosted.sha256 index ff50c340..748b68c3 100644 --- a/third_party/antlr-v4-grammar/self-hosted.sha256 +++ b/third_party/antlr-v4-grammar/self-hosted.sha256 @@ -2,5 +2,5 @@ d1c01af37d665bd94f318c25265bbbf07620689effeb562b460cf75d361d2ee9 third_party/an 1286e542499e4480b3ab5ff60e4a4a7faf21134ca4c4f8f1f468f30095fa25cb third_party/antlr-v4-grammar/ANTLRv4Parser.g4 c7114545a75ab294215819962e92e570383dc830fd5768463dab04e6733bcb80 third_party/antlr-v4-grammar/predefined.tokens 5803594bd2c8dd2d5180f1ca08fc70dfc80308479d18a7c4a1b743fa523b55ec third_party/antlr-v4-grammar/antlr-v4.toml -d0450a71c4af4ebbb35c635b2e59592f1b1a4d289e6747e8c5f9133f6d15eaeb src/bin_support/grammar/generated/antlr_v4_lexer.rs -00f507f7a00c3fb77ecc9524d865c03c196d6da2c1b0af20e56f5a4364a9f0ea src/bin_support/grammar/generated/antlr_v4_parser.rs +2f591094f325a6ad25dbe0ba880ab5625caecfa5364e10fe1acff7019f8469a2 src/bin_support/grammar/generated/antlr_v4_lexer.rs +ddb5a6d5c78e38a4a3084fa24f1efb343e16e224b953db3a9e4e7750adbffb7c src/bin_support/grammar/generated/antlr_v4_parser.rs