From 109cbd240687f28947126a56e43385b7ac21f9c9 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Tue, 28 Jul 2026 13:51:23 +0200 Subject: [PATCH 1/3] feat(runtime): add stream-based parse convenience Add fallible UTF-8 reader constructors to InputStream, including a source-named variant for file and diagnostic workflows. Generate parse_stream and parse_stream_with_parser helpers for any CharStream, and route the existing text helpers through the same construction path. This lets named InputStream and ByteStream inputs keep the compact generated parser setup API. Cover UTF-8 validation, source-name propagation, and generated InputStream/ByteStream consumers, and document the text-file and binary-file workflows. --- README.md | 32 ++++++- docs/kotlin-build.md | 32 ++++++- src/bin/antlr4-rust-gen.rs | 61 +++++++++++-- ..._gen__tests__parser_parse_convenience.snap | 85 +++++++++++++++++++ src/byte_stream.rs | 8 +- src/char_stream.rs | 52 ++++++++++++ tests/antlr4_rust_gen_cli.rs | 35 +++++++- 7 files changed, 283 insertions(+), 22 deletions(-) create mode 100644 src/bin/snapshots/antlr4_rust_gen__tests__parser_parse_convenience.snap diff --git a/README.md b/README.md index 3628fdd7..9c0aa771 100644 --- a/README.md +++ b/README.md @@ -183,8 +183,28 @@ fn main() -> Result<(), antlr4_runtime::AntlrError> { } ``` -Or construct each layer explicitly when you need to set source names, parser -options, or custom error handling before invoking the entry rule: +Use `parse_stream` with a preconstructed `CharStream` when parsing a file, +preserving its source name, or supplying a custom stream implementation: + +```rust +use std::fs::File; +use antlr4_runtime::InputStream; +use generated::json_lexer::JsonLexer; +use generated::json_parser::{self, JsonParser}; + +let input = InputStream::from_reader_with_source_name( + File::open(path)?, + path.display().to_string(), +)?; +let parsed = json_parser::parse_stream(input, JsonLexer::new, JsonParser::json)?; +``` + +`InputStream::from_reader` accepts any `std::io::Read` and validates UTF-8. +`parse_stream_with_parser` is the corresponding stream-based helper when the +caller also needs the parser afterward. + +Construct each layer explicitly when you need parser options or custom error +handling before invoking the entry rule: ```rust use antlr4_runtime::{CommonTokenStream, InputStream}; @@ -593,7 +613,13 @@ byte value (`0..=255`). It is generic over the backing store — `ByteStream::new(vec)` owns, `ByteStream::new(&buf[..])` borrows a network buffer zero-copy, and `ByteStream::from_reader(file)?` drains any `std::io::Read`. Because the bytes are not text, `text()` renders a matched span -as lowercase hex. +as lowercase hex. Generated parser modules accept it through `parse_stream`, so +binary inputs retain the same compact lexer/token-stream/parser setup as text: + +```rust +let input = ByteStream::from_reader(file)?; +let parsed = midi_parser::parse_stream(input, MidiLexer::new, MidiParser::file)?; +``` Length-prefixed formats ("read N, then consume N bytes") are data-dependent, so a pure grammar cannot frame them alone — the same constraint ANTLR's `bencoding` diff --git a/docs/kotlin-build.md b/docs/kotlin-build.md index 45b73804..bb5c8a94 100644 --- a/docs/kotlin-build.md +++ b/docs/kotlin-build.md @@ -89,9 +89,32 @@ assert!(tree.text().contains("fun")); assert!(!tokens.tokens().is_empty()); ``` -The generated helper is additive. The explicit path is still available when the -caller needs to name the input source, adjust parser options, or attach custom -error handling before the entry rule: +Use `parse_stream` to parse directly from a Rust reader while retaining the +input's source name: + +```rust +use std::fs::File; +use antlr4_runtime::InputStream; +use generated::kotlin_lexer::KotlinLexer; +use generated::kotlin_parser::{self, KotlinParser}; + +let path = std::path::Path::new("Main.kt"); +let input = InputStream::from_reader_with_source_name( + File::open(path).expect("Kotlin source should open"), + path.display().to_string(), +) +.expect("Kotlin source should be UTF-8"); +let parsed = kotlin_parser::parse_stream(input, KotlinLexer::new, KotlinParser::kotlin_file) + .expect("entry rule parses"); +assert!(parsed.tree().text().contains("fun")); +``` + +`InputStream::from_reader` also accepts stdin, sockets, cursors, and other +`std::io::Read` implementations when no explicit source name is needed. + +The generated helpers are additive. The explicit path remains available when +the caller needs to adjust parser options or attach custom error handling before +the entry rule: ```rust use antlr4_runtime::{CommonTokenStream, InputStream}; @@ -105,4 +128,5 @@ let tree = parser.kotlin_file().expect("entry rule parses"); assert!(tree.text().contains("fun")); ``` -Validated locally: the generated Kotlin lexer emits real tokens and the generated parser recognizes the `parser.kotlin_file()` entry rule for `fun main() {}`. +Validated locally: the generated Kotlin lexer emits real tokens and the generated parser recognizes the `parser.kotlin_file()` entry rule for +`fun main() {}`. diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index 9a40d0b0..9cec56bc 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -13512,12 +13512,12 @@ fn render_parser_base_initialization( out } -/// Renders the parser-module convenience that wires text input through the -/// caller-selected lexer, token stream, parser, and entry rule in one call. +/// Renders parser-module conveniences that wire text or a caller-provided +/// character stream through the lexer, token stream, parser, and entry rule. fn render_parser_parse_convenience(type_name: &str) -> String { let output_type_name = format!("{type_name}ParseOutput"); format!( - r#"/// Result from [`parse_with_parser`]. + r#"/// Result from [`parse_with_parser`] or [`parse_stream_with_parser`]. /// /// Keeps the generated parser available after the entry rule runs so callers /// can inspect diagnostics or recover the parser-owned token stream. @@ -13546,8 +13546,7 @@ pub fn parse( entry: impl FnOnce(&mut {type_name}) -> Result, ) -> Result {{ - let {output_type_name} {{ result, parser }} = parse_with_parser(input, lexer, entry)?; - Ok(parser.into_parsed_file(result)) + parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry) }} /// Parses UTF-8 text like [`parse`] while returning the parser after the entry @@ -13561,7 +13560,39 @@ pub fn parse_with_parser( entry: impl FnOnce(&mut {type_name}) -> Result, ) -> Result<{output_type_name}, antlr4_runtime::AntlrError> {{ - let lexer = lexer(antlr4_runtime::InputStream::new(input.as_ref())); + parse_stream_with_parser( + antlr4_runtime::InputStream::new(input.as_ref()), + lexer, + entry, + ) +}} + +/// Parses a caller-provided character stream by constructing the lexer, token +/// stream, parser, and caller-selected entry rule in one call. +/// +/// Unlike [`parse`], this accepts any [`antlr4_runtime::CharStream`], including +/// a named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn parse_stream( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut {type_name}) -> Result, +) -> Result +{{ + let {output_type_name} {{ result, parser }} = + parse_stream_with_parser(input, lexer, entry)?; + Ok(parser.into_parsed_file(result)) +}} + +/// Parses a caller-provided character stream like [`parse_stream`] while +/// returning the parser after the entry rule has run. +pub fn parse_stream_with_parser( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut {type_name}) -> Result, +) -> Result<{output_type_name}, antlr4_runtime::AntlrError> +{{ + let lexer = lexer(input); let tokens = CommonTokenStream::new(lexer); let mut parser = {type_name}::new(tokens); let result = entry(&mut parser)?; @@ -16156,11 +16187,22 @@ mod tests { let rendered = render_parser("TParser", &minimal_parser_data()).expect("parser should render"); + insta::assert_snapshot!( + "parser_parse_convenience", + render_parser_parse_convenience("TParser") + ); assert!(rendered.contains("pub struct TParserParseOutput")); assert!(rendered.contains("pub result: R,")); assert!(rendered.contains("pub parser: TParser,")); assert!(rendered.contains("pub fn parse(")); assert!(rendered.contains("pub fn parse_with_parser(")); + assert!( + rendered + .contains("pub fn parse_stream(") + ); + assert!(rendered.contains( + "pub fn parse_stream_with_parser(" + )); assert!( !rendered .contains(") -> Result\nwhere\n L: TokenSource,") @@ -16169,12 +16211,15 @@ mod tests { ") -> Result, antlr4_runtime::AntlrError>\nwhere\n L: TokenSource," )); assert!(rendered.contains("lexer: impl FnOnce(antlr4_runtime::InputStream) -> L")); - assert!(rendered.contains("antlr4_runtime::InputStream::new(input.as_ref())")); + assert!(rendered.contains( + "parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry)" + )); + assert!(rendered.contains("let lexer = lexer(input);")); assert!(rendered.contains("let tokens = CommonTokenStream::new(lexer);")); assert!(rendered.contains("let result = entry(&mut parser)?;")); assert!(rendered.contains("Ok(TParserParseOutput { result, parser })")); assert!(rendered.contains( - "let TParserParseOutput { result, parser } = parse_with_parser(input, lexer, entry)?;" + "parse_stream_with_parser(\n antlr4_runtime::InputStream::new(input.as_ref())," )); assert!(rendered.contains("Ok(parser.into_parsed_file(result))")); assert!(rendered.contains("pub fn new(input: CommonTokenStream) -> Self")); diff --git a/src/bin/snapshots/antlr4_rust_gen__tests__parser_parse_convenience.snap b/src/bin/snapshots/antlr4_rust_gen__tests__parser_parse_convenience.snap new file mode 100644 index 00000000..ddcd217b --- /dev/null +++ b/src/bin/snapshots/antlr4_rust_gen__tests__parser_parse_convenience.snap @@ -0,0 +1,85 @@ +--- +source: src/bin/antlr4-rust-gen.rs +expression: "render_parser_parse_convenience(\"TParser\")" +--- +/// Result from [`parse_with_parser`] or [`parse_stream_with_parser`]. +/// +/// Keeps the generated parser available after the entry rule runs so callers +/// can inspect diagnostics or recover the parser-owned token stream. +#[derive(Debug)] +pub struct TParserParseOutput +where + L: TokenSource, +{ + pub result: R, + pub parser: TParser, +} + +/// Parses UTF-8 text by constructing the lexer, token stream, parser, and +/// caller-selected entry rule in one call. +/// +/// Pass the generated lexer constructor and a parser entry rule, for example +/// `parse(src, MyGrammarLexer::new, TParser::file)`. +/// +/// The returned [`antlr4_runtime::ParsedFile`] owns the canonical token store, +/// flat CST storage, and entry-rule root. +/// Use [`parse_with_parser`] instead when the caller also needs parser +/// diagnostics after the entry rule runs. +pub fn parse( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, + entry: impl FnOnce(&mut TParser) -> Result, +) -> Result +{ + parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry) +} + +/// Parses UTF-8 text like [`parse`] while returning the parser after the entry +/// rule has run. +/// +/// This keeps the compact generated setup path available for callers that also +/// need `Parser::number_of_syntax_errors()` or `TParser::into_token_stream()`. +pub fn parse_with_parser( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, + entry: impl FnOnce(&mut TParser) -> Result, +) -> Result, antlr4_runtime::AntlrError> +{ + parse_stream_with_parser( + antlr4_runtime::InputStream::new(input.as_ref()), + lexer, + entry, + ) +} + +/// Parses a caller-provided character stream by constructing the lexer, token +/// stream, parser, and caller-selected entry rule in one call. +/// +/// Unlike [`parse`], this accepts any [`antlr4_runtime::CharStream`], including +/// a named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn parse_stream( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut TParser) -> Result, +) -> Result +{ + let TParserParseOutput { result, parser } = + parse_stream_with_parser(input, lexer, entry)?; + Ok(parser.into_parsed_file(result)) +} + +/// Parses a caller-provided character stream like [`parse_stream`] while +/// returning the parser after the entry rule has run. +pub fn parse_stream_with_parser( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut TParser) -> Result, +) -> Result, antlr4_runtime::AntlrError> +{ + let lexer = lexer(input); + let tokens = CommonTokenStream::new(lexer); + let mut parser = TParser::new(tokens); + let result = entry(&mut parser)?; + Ok(TParserParseOutput { result, parser }) +} diff --git a/src/byte_stream.rs b/src/byte_stream.rs index 7bd02cc1..14d0bad5 100644 --- a/src/byte_stream.rs +++ b/src/byte_stream.rs @@ -38,11 +38,9 @@ //! // Zero-copy from an in-memory buffer (e.g. bytes read off a socket): //! let stream = ByteStream::new(&packet[..]); //! -//! // Feed it to any generated lexer built from a byte-oriented grammar. -//! let lexer = FooLexer::new(stream); -//! let tokens = CommonTokenStream::new(lexer); -//! let mut parser = FooParser::new(tokens); -//! let tree = parser.entry_rule()?; +//! // Feed it to any generated parser built from a byte-oriented grammar. +//! let parsed = +//! foo_parser::parse_stream(stream, FooLexer::new, FooParser::entry_rule)?; //! ``` //! //! Write lexer rules against the byte range, e.g. `BYTE : ' ' .. 'ÿ';`. A diff --git a/src/char_stream.rs b/src/char_stream.rs index 1e58e23c..ee995219 100644 --- a/src/char_stream.rs +++ b/src/char_stream.rs @@ -1,4 +1,5 @@ use crate::int_stream::{EOF, IntStream, UNKNOWN_SOURCE_NAME}; +use std::io; use std::rc::Rc; #[derive(Clone, Copy, Debug, Eq, PartialEq)] @@ -147,6 +148,33 @@ impl InputData { } impl InputStream { + /// Creates a character stream by draining UTF-8 text from a + /// [`std::io::Read`], using ANTLR's unknown source name placeholder. + /// + /// # Errors + /// + /// Returns any I/O error produced while reading, including + /// [`io::ErrorKind::InvalidData`] when the input is not valid UTF-8. + pub fn from_reader(reader: impl io::Read) -> io::Result { + Self::from_reader_with_source_name(reader, UNKNOWN_SOURCE_NAME) + } + + /// Creates a named character stream by draining UTF-8 text from a + /// [`std::io::Read`]. + /// + /// # Errors + /// + /// Returns any I/O error produced while reading, including + /// [`io::ErrorKind::InvalidData`] when the input is not valid UTF-8. + pub fn from_reader_with_source_name( + mut reader: impl io::Read, + source_name: impl Into, + ) -> io::Result { + let mut input = String::new(); + reader.read_to_string(&mut input)?; + Ok(Self::with_source_name(input, source_name)) + } + /// Creates a character stream from UTF-8 text using ANTLR's unknown source /// name placeholder. pub fn new(input: impl AsRef) -> Self { @@ -372,4 +400,28 @@ mod tests { (6, 3) ); } + + #[test] + fn reader_constructors_decode_utf8_and_preserve_source_names() { + let mut named = InputStream::from_reader_with_source_name( + io::Cursor::new("aβ\n".as_bytes()), + "sample.txt", + ) + .expect("in-memory UTF-8 should be readable"); + assert_eq!(named.source_name(), "sample.txt"); + assert_eq!(named.size(), 3); + assert_eq!(named.la(2), 'β' as i32); + + let unnamed = InputStream::from_reader(io::Cursor::new(b"text")) + .expect("in-memory UTF-8 should be readable"); + assert_eq!(unnamed.source_name(), UNKNOWN_SOURCE_NAME); + assert_eq!(unnamed.text(TextInterval::new(0, 3)), "text"); + } + + #[test] + fn reader_constructor_rejects_invalid_utf8() { + let error = InputStream::from_reader(io::Cursor::new([0xFF])) + .expect_err("invalid UTF-8 must not produce a character stream"); + assert_eq!(error.kind(), io::ErrorKind::InvalidData); + } } diff --git a/tests/antlr4_rust_gen_cli.rs b/tests/antlr4_rust_gen_cli.rs index a620dc50..362258a3 100644 --- a/tests/antlr4_rust_gen_cli.rs +++ b/tests/antlr4_rust_gen_cli.rs @@ -697,8 +697,11 @@ fn combined_literal_tokens_are_public_and_lexable() { #[cfg(test)] mod combined_literal_tests { use super::t_lexer::TLexer; - use super::t_parser::TParser; - use antlr4_runtime::{CommonTokenStream, InputStream, Parser as _}; + use super::t_parser::{self, TParser}; + use antlr4_runtime::{ + ByteStream, CommonTokenStream, InputStream, Parser as _, Token as _, + }; + use std::io::Cursor; #[test] fn recognizes_implicit_literal_rules() { @@ -708,6 +711,34 @@ mod combined_literal_tests { parser.greeting().expect("literal input should parse"); assert_eq!(parser.number_of_syntax_errors(), 0); } + + #[test] + fn generated_helpers_accept_named_text_and_byte_streams() { + let input = InputStream::from_reader_with_source_name( + Cursor::new(b"hello Alice world"), + "greeting.txt", + ) + .expect("in-memory UTF-8 should be readable"); + let output = + t_parser::parse_stream_with_parser(input, TLexer::new, TParser::greeting) + .expect("named text stream should parse"); + assert_eq!(output.parser.number_of_syntax_errors(), 0); + assert!( + output + .parser + .token_store() + .iter() + .all(|token| token.source_name() == "greeting.txt") + ); + + let parsed = t_parser::parse_stream( + ByteStream::new(b"hello Alice world".to_vec()), + TLexer::new, + TParser::greeting, + ) + .expect("byte stream should parse through the generic helper"); + assert_eq!(parsed.tokens().len(), 4); + } } "#, ); From b9997afdf92308701513980fb68216df37b4718a Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Tue, 28 Jul 2026 14:08:57 +0200 Subject: [PATCH 2/3] chore(codegen): refresh self-hosted frontend Regenerate the checked-in ANTLRv4 frontend after extending parser output with stream-based convenience helpers. Refresh the generated-version banners and pinned hashes after proving the Stage 1 and Stage 2 fixed point. --- .../grammar/generated/antlr_v4_lexer.rs | 2 +- .../grammar/generated/antlr_v4_parser.rs | 41 ++++++++++++++++--- .../antlr-v4-grammar/self-hosted.sha256 | 4 +- 3 files changed, 39 insertions(+), 8 deletions(-) diff --git a/src/bin_support/grammar/generated/antlr_v4_lexer.rs b/src/bin_support/grammar/generated/antlr_v4_lexer.rs index a67f3a02..02e92d9f 100644 --- a/src/bin_support/grammar/generated/antlr_v4_lexer.rs +++ b/src/bin_support/grammar/generated/antlr_v4_lexer.rs @@ -1,4 +1,4 @@ -// @generated by antlr-rust-runtime v0.19.1 - do not edit +// @generated by antlr-rust-runtime v0.21.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] diff --git a/src/bin_support/grammar/generated/antlr_v4_parser.rs b/src/bin_support/grammar/generated/antlr_v4_parser.rs index 170616b5..2ca07a10 100644 --- a/src/bin_support/grammar/generated/antlr_v4_parser.rs +++ b/src/bin_support/grammar/generated/antlr_v4_parser.rs @@ -1,4 +1,4 @@ -// @generated by antlr-rust-runtime v0.19.1 - do not edit +// @generated by antlr-rust-runtime v0.21.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] @@ -10572,7 +10572,7 @@ pub fn parser_atn() -> &'static ParserAtn { atn() } -/// Result from [`parse_with_parser`]. +/// Result from [`parse_with_parser`] or [`parse_stream_with_parser`]. /// /// Keeps the generated parser available after the entry rule runs so callers /// can inspect diagnostics or recover the parser-owned token stream. @@ -10601,8 +10601,7 @@ pub fn parse( entry: impl FnOnce(&mut AntlRv4Parser) -> Result, ) -> Result { - let AntlRv4ParserParseOutput { result, parser } = parse_with_parser(input, lexer, entry)?; - Ok(parser.into_parsed_file(result)) + parse_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer, entry) } /// Parses UTF-8 text like [`parse`] while returning the parser after the entry @@ -10616,7 +10615,39 @@ pub fn parse_with_parser( entry: impl FnOnce(&mut AntlRv4Parser) -> Result, ) -> Result, antlr4_runtime::AntlrError> { - let lexer = lexer(antlr4_runtime::InputStream::new(input.as_ref())); + parse_stream_with_parser( + antlr4_runtime::InputStream::new(input.as_ref()), + lexer, + entry, + ) +} + +/// Parses a caller-provided character stream by constructing the lexer, token +/// stream, parser, and caller-selected entry rule in one call. +/// +/// Unlike [`parse`], this accepts any [`antlr4_runtime::CharStream`], including +/// a named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn parse_stream( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut AntlRv4Parser) -> Result, +) -> Result +{ + let AntlRv4ParserParseOutput { result, parser } = + parse_stream_with_parser(input, lexer, entry)?; + Ok(parser.into_parsed_file(result)) +} + +/// Parses a caller-provided character stream like [`parse_stream`] while +/// returning the parser after the entry rule has run. +pub fn parse_stream_with_parser( + input: I, + lexer: impl FnOnce(I) -> L, + entry: impl FnOnce(&mut AntlRv4Parser) -> Result, +) -> Result, antlr4_runtime::AntlrError> +{ + let lexer = lexer(input); let tokens = CommonTokenStream::new(lexer); let mut parser = AntlRv4Parser::new(tokens); let result = entry(&mut parser)?; diff --git a/third_party/antlr-v4-grammar/self-hosted.sha256 b/third_party/antlr-v4-grammar/self-hosted.sha256 index ff50c340..748b68c3 100644 --- a/third_party/antlr-v4-grammar/self-hosted.sha256 +++ b/third_party/antlr-v4-grammar/self-hosted.sha256 @@ -2,5 +2,5 @@ d1c01af37d665bd94f318c25265bbbf07620689effeb562b460cf75d361d2ee9 third_party/an 1286e542499e4480b3ab5ff60e4a4a7faf21134ca4c4f8f1f468f30095fa25cb third_party/antlr-v4-grammar/ANTLRv4Parser.g4 c7114545a75ab294215819962e92e570383dc830fd5768463dab04e6733bcb80 third_party/antlr-v4-grammar/predefined.tokens 5803594bd2c8dd2d5180f1ca08fc70dfc80308479d18a7c4a1b743fa523b55ec third_party/antlr-v4-grammar/antlr-v4.toml -d0450a71c4af4ebbb35c635b2e59592f1b1a4d289e6747e8c5f9133f6d15eaeb src/bin_support/grammar/generated/antlr_v4_lexer.rs -00f507f7a00c3fb77ecc9524d865c03c196d6da2c1b0af20e56f5a4364a9f0ea src/bin_support/grammar/generated/antlr_v4_parser.rs +2f591094f325a6ad25dbe0ba880ab5625caecfa5364e10fe1acff7019f8469a2 src/bin_support/grammar/generated/antlr_v4_lexer.rs +ddb5a6d5c78e38a4a3084fa24f1efb343e16e224b953db3a9e4e7750adbffb7c src/bin_support/grammar/generated/antlr_v4_parser.rs From f5c64b296fb880394da8768af9bc1fd393d7e981 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Tue, 28 Jul 2026 14:17:20 +0200 Subject: [PATCH 3/3] refactor(codegen): dogfood stream input convenience Load grammar sources through InputStream::from_reader_with_source_name and pass the resulting stream directly into the self-hosted frontend. Share its source text with successful and failed source records so diagnostics no longer require reopening malformed grammars. Regenerate the checked-in XPath lexer from its lexer-only grammar, confirming that the current generator changes only the version banner. --- src/bin_support/grammar/frontend.rs | 34 ++++++++++++++++++++++++----- src/bin_support/grammar/loader.rs | 26 ++++++++++++++-------- src/bin_support/grammar/source.rs | 14 +++++------- src/xpath/generated/x_path_lexer.rs | 2 +- 4 files changed, 52 insertions(+), 24 deletions(-) diff --git a/src/bin_support/grammar/frontend.rs b/src/bin_support/grammar/frontend.rs index 56d20cd9..36ef887c 100644 --- a/src/bin_support/grammar/frontend.rs +++ b/src/bin_support/grammar/frontend.rs @@ -1,11 +1,12 @@ use std::fmt; use std::ops::Range; use std::path::{Path, PathBuf}; +use std::rc::Rc; use std::sync::{Arc, Mutex}; use antlr4_runtime::{ - AsRuleNode, CommonTokenStream, ErrorListener, InputStream, Node, NodeId, NodeKind, Parser, - Recognizer, TOKEN_EOF as RUNTIME_TOKEN_EOF, Token, + AsRuleNode, CharStream as _, CommonTokenStream, ErrorListener, InputStream, Node, NodeId, + NodeKind, Parser, Recognizer, TOKEN_EOF as RUNTIME_TOKEN_EOF, Token, }; use super::generated::antlr_v4_lexer::{ @@ -143,7 +144,7 @@ impl Iterator for CstDescendants<'_> { pub(crate) struct SourceFile { id: SourceId, logical_path: PathBuf, - text: Box, + text: Rc, line_starts: Box<[u32]>, tokens: Box<[SyntaxToken]>, trivia: Box<[u32]>, @@ -265,7 +266,18 @@ pub(crate) fn parse_source( logical_path: impl Into, text: impl Into>, ) -> Result { - let recovered = parse_source_recovering(source, logical_path, text)?; + let logical_path = logical_path.into(); + let text = text.into(); + let input = InputStream::with_source_name(&text, logical_path.to_string_lossy()); + parse_input_stream(source, logical_path, input) +} + +pub(crate) fn parse_input_stream( + source: SourceId, + logical_path: impl Into, + input: InputStream, +) -> Result { + let recovered = parse_input_stream_recovering(source, logical_path, input)?; if recovered.diagnostics.is_empty() { Ok(recovered.file) } else { @@ -282,8 +294,20 @@ pub(crate) fn parse_source_recovering( ) -> Result { let logical_path = logical_path.into(); let text = text.into(); - let line_starts = line_starts(source, &text)?; let input = InputStream::with_source_name(&text, logical_path.to_string_lossy()); + parse_input_stream_recovering(source, logical_path, input) +} + +pub(crate) fn parse_input_stream_recovering( + source: SourceId, + logical_path: impl Into, + input: InputStream, +) -> Result { + let logical_path = logical_path.into(); + let text = input + .source_text() + .expect("InputStream always exposes its complete source text"); + let line_starts = line_starts(source, &text)?; let mut lexer = AntlRv4Lexer::with_hooks(input, LexerAdaptor::default()); lexer.remove_error_listeners(); let mut token_stream = CommonTokenStream::try_new(lexer).map_err(|error| FrontendError { diff --git a/src/bin_support/grammar/loader.rs b/src/bin_support/grammar/loader.rs index 09e78cb2..bd04cd28 100644 --- a/src/bin_support/grammar/loader.rs +++ b/src/bin_support/grammar/loader.rs @@ -1,9 +1,12 @@ use std::collections::{BTreeMap, BTreeSet}; use std::fs; use std::path::{Path, PathBuf}; +use std::rc::Rc; + +use antlr4_runtime::{CharStream as _, InputStream}; use super::diagnostic::{CompilationError, Diagnostic, Severity}; -use super::frontend::{SourceId, SourceSpan, parse_source, parse_source_recovering}; +use super::frontend::{SourceId, SourceSpan, parse_input_stream, parse_input_stream_recovering}; use super::model::{ GrammarId, ImportEdge, LoadedGrammarSet, LookupKind, LookupRecord, ParsedGrammarUnit, VocabularyEdge, VocabularySource, @@ -150,8 +153,11 @@ impl Loader { if let Some(source) = self.sources.id_for_canonical_path(&canonical) { return self.grammar_for_source.get(&source).copied(); } - let text = match fs::read_to_string(&canonical) { - Ok(text) => text, + let logical_path = user_spelling.unwrap_or(path).to_path_buf(); + let input = match fs::File::open(&canonical).and_then(|file| { + InputStream::from_reader_with_source_name(file, logical_path.to_string_lossy()) + }) { + Ok(input) => input, Err(error) => { self.diagnostics.push(Diagnostic::error( "G4L002", @@ -161,10 +167,12 @@ impl Loader { return None; } }; + let source_text = input + .source_text() + .expect("InputStream always exposes its complete source text"); let source = self.sources.next_id(); - let logical_path = user_spelling.unwrap_or(path).to_path_buf(); let file = if recover_syntax { - match parse_source_recovering(source, logical_path.clone(), text) { + match parse_input_stream_recovering(source, logical_path.clone(), input) { Ok(recovered) => { let has_diagnostics = !recovered.diagnostics.is_empty(); self.diagnostics.extend( @@ -182,16 +190,16 @@ impl Loader { } Err(error) => { self.record_frontend_error(&error); - self.retain_failed_source(canonical, source, logical_path); + self.retain_failed_source(canonical, source, logical_path, source_text); return None; } } } else { - match parse_source(source, logical_path.clone(), text) { + match parse_input_stream(source, logical_path.clone(), input) { Ok(file) => file, Err(error) => { self.record_frontend_error(&error); - self.retain_failed_source(canonical, source, logical_path); + self.retain_failed_source(canonical, source, logical_path, source_text); return None; } } @@ -215,8 +223,8 @@ impl Loader { canonical_path: PathBuf, source: SourceId, logical_path: PathBuf, + text: Rc, ) { - let text = fs::read_to_string(&canonical_path).unwrap_or_default(); self.sources .insert_failed(canonical_path, source, logical_path, text) .expect("canonical path checked before parsing"); diff --git a/src/bin_support/grammar/source.rs b/src/bin_support/grammar/source.rs index 0d3c2dfd..bcc9ba58 100644 --- a/src/bin_support/grammar/source.rs +++ b/src/bin_support/grammar/source.rs @@ -1,5 +1,6 @@ use std::collections::BTreeMap; use std::path::{Path, PathBuf}; +use std::rc::Rc; use super::frontend::{SourceFile, SourceId}; @@ -14,7 +15,7 @@ pub(crate) struct SourceSet { #[derive(Debug)] struct FailedSource { logical_path: PathBuf, - text: Box, + text: Rc, } impl SourceSet { @@ -39,7 +40,7 @@ impl SourceSet { canonical_path: PathBuf, source: SourceId, logical_path: PathBuf, - text: String, + text: Rc, ) -> Result<(), SourceId> { if let Some(id) = self.by_canonical_path.get(&canonical_path) { return Err(*id); @@ -49,13 +50,8 @@ impl SourceSet { .insert(canonical_path.clone(), source); self.canonical_paths.push(canonical_path); self.files.push(None); - self.failed_sources.insert( - source, - FailedSource { - logical_path, - text: text.into_boxed_str(), - }, - ); + self.failed_sources + .insert(source, FailedSource { logical_path, text }); Ok(()) } diff --git a/src/xpath/generated/x_path_lexer.rs b/src/xpath/generated/x_path_lexer.rs index c24dbfb6..ee398cdd 100644 --- a/src/xpath/generated/x_path_lexer.rs +++ b/src/xpath/generated/x_path_lexer.rs @@ -1,4 +1,4 @@ -// @generated by antlr-rust-runtime v0.19.1 - do not edit +// @generated by antlr-rust-runtime v0.21.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip]