From b093882d5179d427eb8c74234139b81630fae128 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Mon, 22 Jun 2026 14:24:20 +0200 Subject: [PATCH 1/2] feat: add generated parse_with_parser helper --- README.md | 20 +++++++++++++++++ docs/kotlin-build.md | 20 +++++++++++++++++ src/bin/antlr4-rust-gen.rs | 46 ++++++++++++++++++++++++++++++++++++-- 3 files changed, 84 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index e2afaf4a..b3796ce8 100644 --- a/README.md +++ b/README.md @@ -135,6 +135,26 @@ fn main() -> Result<(), antlr4_runtime::AntlrError> { } ``` +Use `parse_with_parser` when you want the compact setup path and also need the +parser afterward for diagnostics or the owned token stream: + +```rust +use antlr4_runtime::Parser; +use generated::json::{self, Json}; +use generated::json_lexer::JsonLexer; + +fn main() -> Result<(), antlr4_runtime::AntlrError> { + let output = json::parse_with_parser(r#"{"a":1}"#, JsonLexer::new, Json::json)?; + let syntax_errors = output.parser.number_of_syntax_errors(); + let tree = output.result; + let tokens = output.parser.into_token_stream(); + + println!("{} errors across {} tokens", syntax_errors, tokens.tokens().len()); + println!("{}", tree.text()); + Ok(()) +} +``` + Or construct each layer explicitly when you need to set source names, parser options, or custom error handling before invoking the entry rule: diff --git a/docs/kotlin-build.md b/docs/kotlin-build.md index 4a1a8e19..ccbefff8 100644 --- a/docs/kotlin-build.md +++ b/docs/kotlin-build.md @@ -74,6 +74,26 @@ let tree = kotlin_parser::parse("fun main() {}", KotlinLexer::new, KotlinParser: assert!(tree.text().contains("fun")); ``` +Use `parse_with_parser` when a caller also needs parser state after the entry +rule, such as syntax diagnostics or the token stream: + +```rust +use antlr4_runtime::Parser; +use generated::kotlin_lexer::KotlinLexer; +use generated::kotlin_parser::{self, KotlinParser}; + +let output = + kotlin_parser::parse_with_parser("fun main() {}", KotlinLexer::new, KotlinParser::kotlin_file) + .expect("entry rule parses"); +let syntax_errors = output.parser.number_of_syntax_errors(); +let tree = output.result; +let tokens = output.parser.into_token_stream(); + +assert_eq!(syntax_errors, 0); +assert!(tree.text().contains("fun")); +assert!(!tokens.tokens().is_empty()); +``` + The generated helper is additive. The explicit path is still available when the caller needs to name the input source, adjust parser options, or attach custom error handling before the entry rule: diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index 6aeb1a6b..ab9275f7 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -7903,23 +7903,56 @@ fn render_parser_base_initialization(members: &[IntMemberTemplate]) -> String { /// caller-selected lexer, token stream, parser, and entry rule in one call. fn render_parser_parse_convenience(type_name: &str) -> String { format!( - r#"/// Parses UTF-8 text by constructing the lexer, token stream, parser, and + r#"/// Result from [`parse_with_parser`]. +/// +/// Keeps the generated parser available after the entry rule runs so callers +/// can inspect diagnostics or recover the parser-owned token stream. +#[derive(Debug)] +pub struct ParseOutput +where + L: TokenSource, +{{ + pub result: R, + pub parser: {type_name}, +}} + +/// Parses UTF-8 text by constructing the lexer, token stream, parser, and /// caller-selected entry rule in one call. /// /// Pass the generated lexer constructor and a parser entry rule, for example /// `parse(src, MyGrammarLexer::new, {type_name}::file)`. +/// +/// Use [`parse_with_parser`] instead when the caller needs parser diagnostics +/// or the parser-owned token stream after the entry rule runs. pub fn parse( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, entry: impl FnOnce(&mut {type_name}) -> Result, ) -> Result +where + L: TokenSource, +{{ + parse_with_parser(input, lexer, entry).map(|output| output.result) +}} + +/// Parses UTF-8 text like [`parse`] while returning the parser after the entry +/// rule has run. +/// +/// This keeps the compact generated setup path available for callers that also +/// need `Parser::number_of_syntax_errors()` or `{type_name}::into_token_stream()`. +pub fn parse_with_parser( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, + entry: impl FnOnce(&mut {type_name}) -> Result, +) -> Result, antlr4_runtime::AntlrError> where L: TokenSource, {{ let lexer = lexer(antlr4_runtime::InputStream::new(input.as_ref())); let tokens = CommonTokenStream::new(lexer); let mut parser = {type_name}::new(tokens); - entry(&mut parser) + let result = entry(&mut parser)?; + Ok(ParseOutput {{ result, parser }}) }}"# ) } @@ -9250,10 +9283,19 @@ s : ; let rendered = render_parser("TParser", &minimal_parser_data(), None).expect("parser should render"); + assert!(rendered.contains("pub struct ParseOutput")); + assert!(rendered.contains("pub result: R,")); + assert!(rendered.contains("pub parser: TParser,")); assert!(rendered.contains("pub fn parse(")); + assert!(rendered.contains("pub fn parse_with_parser(")); assert!(rendered.contains("lexer: impl FnOnce(antlr4_runtime::InputStream) -> L")); assert!(rendered.contains("antlr4_runtime::InputStream::new(input.as_ref())")); assert!(rendered.contains("let tokens = CommonTokenStream::new(lexer);")); + assert!(rendered.contains("let result = entry(&mut parser)?;")); + assert!(rendered.contains("Ok(ParseOutput { result, parser })")); + assert!( + rendered.contains("parse_with_parser(input, lexer, entry).map(|output| output.result)") + ); assert!(rendered.contains("pub fn new(input: CommonTokenStream) -> Self")); } From 5fdf0c2268f8d7fe827c15d8801c82d98570d91c Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Mon, 22 Jun 2026 22:58:42 +0200 Subject: [PATCH 2/2] fix: address parse helper review feedback --- src/bin/antlr4-rust-gen.rs | 44 +++++++++++++++++++++++++++----------- 1 file changed, 31 insertions(+), 13 deletions(-) diff --git a/src/bin/antlr4-rust-gen.rs b/src/bin/antlr4-rust-gen.rs index ab9275f7..37359297 100644 --- a/src/bin/antlr4-rust-gen.rs +++ b/src/bin/antlr4-rust-gen.rs @@ -7902,13 +7902,14 @@ fn render_parser_base_initialization(members: &[IntMemberTemplate]) -> String { /// Renders the parser-module convenience that wires text input through the /// caller-selected lexer, token stream, parser, and entry rule in one call. fn render_parser_parse_convenience(type_name: &str) -> String { + let output_type_name = format!("{type_name}ParseOutput"); format!( r#"/// Result from [`parse_with_parser`]. /// /// Keeps the generated parser available after the entry rule runs so callers /// can inspect diagnostics or recover the parser-owned token stream. #[derive(Debug)] -pub struct ParseOutput +pub struct {output_type_name} where L: TokenSource, {{ @@ -7924,13 +7925,11 @@ where /// /// Use [`parse_with_parser`] instead when the caller needs parser diagnostics /// or the parser-owned token stream after the entry rule runs. -pub fn parse( +pub fn parse( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, entry: impl FnOnce(&mut {type_name}) -> Result, ) -> Result -where - L: TokenSource, {{ parse_with_parser(input, lexer, entry).map(|output| output.result) }} @@ -7940,19 +7939,17 @@ where /// /// This keeps the compact generated setup path available for callers that also /// need `Parser::number_of_syntax_errors()` or `{type_name}::into_token_stream()`. -pub fn parse_with_parser( +pub fn parse_with_parser( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, entry: impl FnOnce(&mut {type_name}) -> Result, -) -> Result, antlr4_runtime::AntlrError> -where - L: TokenSource, +) -> Result<{output_type_name}, antlr4_runtime::AntlrError> {{ let lexer = lexer(antlr4_runtime::InputStream::new(input.as_ref())); let tokens = CommonTokenStream::new(lexer); let mut parser = {type_name}::new(tokens); let result = entry(&mut parser)?; - Ok(ParseOutput {{ result, parser }}) + Ok({output_type_name} {{ result, parser }}) }}"# ) } @@ -9283,22 +9280,43 @@ s : ; let rendered = render_parser("TParser", &minimal_parser_data(), None).expect("parser should render"); - assert!(rendered.contains("pub struct ParseOutput")); + assert!(rendered.contains("pub struct TParserParseOutput")); assert!(rendered.contains("pub result: R,")); assert!(rendered.contains("pub parser: TParser,")); - assert!(rendered.contains("pub fn parse(")); - assert!(rendered.contains("pub fn parse_with_parser(")); + assert!(rendered.contains("pub fn parse(")); + assert!(rendered.contains("pub fn parse_with_parser(")); + assert!( + !rendered + .contains(") -> Result\nwhere\n L: TokenSource,") + ); + assert!(!rendered.contains( + ") -> Result, antlr4_runtime::AntlrError>\nwhere\n L: TokenSource," + )); assert!(rendered.contains("lexer: impl FnOnce(antlr4_runtime::InputStream) -> L")); assert!(rendered.contains("antlr4_runtime::InputStream::new(input.as_ref())")); assert!(rendered.contains("let tokens = CommonTokenStream::new(lexer);")); assert!(rendered.contains("let result = entry(&mut parser)?;")); - assert!(rendered.contains("Ok(ParseOutput { result, parser })")); + assert!(rendered.contains("Ok(TParserParseOutput { result, parser })")); assert!( rendered.contains("parse_with_parser(input, lexer, entry).map(|output| output.result)") ); assert!(rendered.contains("pub fn new(input: CommonTokenStream) -> Self")); } + #[test] + fn generated_parse_output_name_does_not_collide_with_parser_type() { + let rendered = render_parser("ParseOutput", &minimal_parser_data(), None) + .expect("parser should render"); + + assert!(rendered.contains("pub struct ParseOutputParseOutput")); + assert!(rendered.contains("pub parser: ParseOutput,")); + assert!( + rendered + .contains(") -> Result, antlr4_runtime::AntlrError>") + ); + assert!(rendered.contains("Ok(ParseOutputParseOutput { result, parser })")); + } + #[test] fn generated_parser_reports_lexer_errors_on_outer_success() { let rendered =