From a6f80b333163af1437c0962d8f42714d1187502d Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 7 Aug 2026 12:59:51 +0200 Subject: [PATCH 1/4] feat(codegen): add lex-only generated helpers Generate lex and lex_stream conveniences in every lexer module so callers can inspect token text, type, and channel without constructing a parser or wiring a TokenSink. Return the eagerly buffered CommonTokenStream to preserve token ownership, source diagnostics, hidden and custom channels, and EOF. The generated source uses existing revision-6 runtime APIs, so the compatibility revision remains unchanged. Document the token-dump workflow, cover custom streams and channels, and refresh all checked-in recognizers. --- README.md | 19 +++++ ...nerator__tests__lexer_lex_convenience.snap | 30 +++++++ .../antlr-rust-codegen/src/generator/tests.rs | 31 +++++++- crates/antlr-rust-codegen/src/lexer/render.rs | 2 + .../src/lexer/render_model.rs | 33 ++++++++ .../tests/antlr4_rust_gen_cli/lexer.rs | 78 +++++++++++++++++++ .../src/generated/antlr_v4_lexer.rs | 30 ++++++- .../src/generated/antlr_v4_parser.rs | 4 +- .../src/generated/rust_lexer.rs | 30 ++++++- .../src/generated/rust_parser.rs | 4 +- .../src/xpath/generated/x_path_lexer.rs | 30 ++++++- .../src/generated/toml_lexer.rs | 30 ++++++- .../src/generated/toml_parser.rs | 4 +- .../antlr-v4-grammar/self-hosted.sha256 | 4 +- 14 files changed, 312 insertions(+), 17 deletions(-) create mode 100644 crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap diff --git a/README.md b/README.md index ae3644b0..34107993 100644 --- a/README.md +++ b/README.md @@ -352,6 +352,25 @@ let parsed = json_parser::parse_stream(input, JsonLexer::new, JsonParser::json)? `parse_stream_with_parser` is the corresponding stream-based helper when the caller also needs the parser afterward. +### Inspecting tokens + +Generated lexer modules expose `lex` for token-only workflows. The returned +stream is already filled and retains EOF plus tokens on hidden or custom +channels: + +```rust +use generated::json_lexer::{self, JsonLexer}; + +let tokens = json_lexer::lex(r#"{"a":1}"#, JsonLexer::new); +for token in tokens.tokens() { + println!("{token}"); +} +``` + +`TokenView`'s display format includes token text, type, channel, line, and +column. Rules using `skip` do not emit tokens. Use `lex_stream` to supply a +named `InputStream`, `ByteStream`, or custom `CharStream`. + Construct each layer explicitly when you need parser options or custom error handling before invoking the entry rule: diff --git a/crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap b/crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap new file mode 100644 index 00000000..94e3e053 --- /dev/null +++ b/crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap @@ -0,0 +1,30 @@ +--- +source: crates/antlr-rust-codegen/src/generator/tests.rs +expression: "render_lexer_lex_convenience(\"TLexer\")" +--- +/// Lexes UTF-8 text into an eagerly filled token stream without constructing +/// a parser. +/// +/// The stream retains every emitted token, including EOF and tokens on hidden +/// or custom channels. Lexer rules using `skip` do not emit tokens. +/// +/// To print ANTLR-style records containing token text, type, and channel: +/// `for token in lex(src, TLexer::new).tokens() { println!("{token}"); }` +pub fn lex( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, +) -> antlr4_runtime::CommonTokenStream { + lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer) +} + +/// Lexes a caller-provided character stream without constructing a parser. +/// +/// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a +/// named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn lex_stream( + input: I, + lexer: impl FnOnce(I) -> L, +) -> antlr4_runtime::CommonTokenStream { + antlr4_runtime::CommonTokenStream::new(lexer(input)) +} diff --git a/crates/antlr-rust-codegen/src/generator/tests.rs b/crates/antlr-rust-codegen/src/generator/tests.rs index 56683de9..66155080 100644 --- a/crates/antlr-rust-codegen/src/generator/tests.rs +++ b/crates/antlr-rust-codegen/src/generator/tests.rs @@ -2895,6 +2895,35 @@ fn portable_local_semantics_reject_missing_generated_caller() { ); } +#[test] +fn renders_lex_convenience_without_a_parser() { + let rendered = render_lexer( + "TLexer", + &predicate_lexer_data(), + false, + SemUnknownPolicy::default(), + &SemPatternFile::default(), + false, + ) + .expect("lexer should render"); + + insta::assert_snapshot!( + "lexer_lex_convenience", + render_lexer_lex_convenience("TLexer") + ); + assert!(rendered.contains("pub fn lex(")); + assert!(rendered.contains( + "pub fn lex_stream(" + )); + assert!( + rendered.contains("lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer)") + ); + assert!(rendered.contains("antlr4_runtime::CommonTokenStream::new(lexer(input))")); + assert!( + rendered.contains("for token in lex(src, TLexer::new).tokens() { println!(\"{token}\"); }") + ); +} + #[test] fn renders_parse_convenience_without_replacing_manual_constructor() { let rendered = render_parser("TParser", &minimal_parser_data()).expect("parser should render"); @@ -6269,7 +6298,7 @@ fn lexer_default_policy_keeps_compiled_token_path() { "generated_lexer_lifecycle_facade", rendered_facade_declaration(&module, "__antlr4_rust_lexer_facade") ); - assert!(!module.contains("CommonToken")); + assert!(!module.contains("CommonToken::")); assert!(!module.contains("TokenFactory")); } diff --git a/crates/antlr-rust-codegen/src/lexer/render.rs b/crates/antlr-rust-codegen/src/lexer/render.rs index 29c9d5e7..292e488d 100644 --- a/crates/antlr-rust-codegen/src/lexer/render.rs +++ b/crates/antlr-rust-codegen/src/lexer/render.rs @@ -34,6 +34,7 @@ pub(crate) fn render_lexer_model(model: &LexerRenderModel<'_>) -> io::Result = OnceLock::new(); diff --git a/crates/antlr-rust-codegen/src/lexer/render_model.rs b/crates/antlr-rust-codegen/src/lexer/render_model.rs index e45d70d5..0a06c0d6 100644 --- a/crates/antlr-rust-codegen/src/lexer/render_model.rs +++ b/crates/antlr-rust-codegen/src/lexer/render_model.rs @@ -33,6 +33,39 @@ impl<'a> LexerRenderModel<'a> { } } +/// Renders lexer-module conveniences that buffer text or a caller-provided +/// character stream without constructing a parser. +pub(crate) fn render_lexer_lex_convenience(type_name: &str) -> String { + format!( + r#"/// Lexes UTF-8 text into an eagerly filled token stream without constructing +/// a parser. +/// +/// The stream retains every emitted token, including EOF and tokens on hidden +/// or custom channels. Lexer rules using `skip` do not emit tokens. +/// +/// To print ANTLR-style records containing token text, type, and channel: +/// `for token in lex(src, {type_name}::new).tokens() {{ println!("{{token}}"); }}` +pub fn lex( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, +) -> antlr4_runtime::CommonTokenStream {{ + lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer) +}} + +/// Lexes a caller-provided character stream without constructing a parser. +/// +/// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a +/// named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn lex_stream( + input: I, + lexer: impl FnOnce(I) -> L, +) -> antlr4_runtime::CommonTokenStream {{ + antlr4_runtime::CommonTokenStream::new(lexer(input)) +}}"# + ) +} + /// Renders the lexer-owned grammar metadata table. pub(crate) fn render_lexer_metadata(grammar_name: &str, data: &LexerCodegenData<'_>) -> String { format!( diff --git a/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs b/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs index 3c6014db..48d1742f 100644 --- a/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs +++ b/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs @@ -203,3 +203,81 @@ mod combined_literal_tests { "#, ); } + +#[test] +fn generated_lex_helpers_expose_hidden_and_custom_channels() { + let temp = temporary_directory("lex-helper-channels"); + let grammar = temp.path().join("Channels.g4"); + let out = temp.path().join("generated"); + fs::write( + &grammar, + "lexer grammar Channels;\n\ + channels { COMMENTS }\n\ + WORD: [a-z]+;\n\ + COMMENT: '#' ~[\\r\\n]* -> channel(COMMENTS);\n\ + WS: [ \\t]+ -> channel(HIDDEN);\n", + ) + .expect("grammar should be writable"); + + let output = run_antlr4_rust_gen(&[ + grammar.as_os_str(), + OsStr::new("--out-dir"), + out.as_os_str(), + ]); + assert!( + output.status.success(), + "stdout: {}\nstderr: {}", + utf8(&output.stdout), + utf8(&output.stderr) + ); + + assert_generated_project( + temp.path(), + &["channels.rs"], + r##" +#[cfg(test)] +mod lex_helper_tests { + use super::channels::{ + self, CHANNEL_COMMENTS, COMMENT, Channels, WORD, WS, + }; + use antlr4_runtime::{ + ByteStream, DEFAULT_CHANNEL, HIDDEN_CHANNEL, TOKEN_EOF, Token as _, + }; + + #[test] + fn buffers_every_emitted_channel_without_a_parser() { + let tokens = channels::lex("alpha # note", Channels::new); + let observed = tokens + .tokens() + .map(|token| ( + token.token_type(), + token.channel(), + token.text_or_empty(), + )) + .collect::>(); + + assert_eq!( + &observed[..3], + [ + (WORD, DEFAULT_CHANNEL, "alpha"), + (WS, HIDDEN_CHANNEL, " "), + (COMMENT, CHANNEL_COMMENTS, "# note"), + ] + ); + assert_eq!(observed[3].0, TOKEN_EOF); + assert_eq!(tokens.number_of_source_errors(), 0); + } + + #[test] + fn accepts_arbitrary_character_streams() { + let tokens = + channels::lex_stream(ByteStream::new(b"beta".to_vec()), Channels::new); + let first = tokens.get(0).expect("word token"); + + assert_eq!(first.token_type(), WORD); + assert_eq!(first.byte_span(), Some(0..4)); + } +} +"##, + ); +} diff --git a/crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs b/crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs index 6af107fe..54f84c89 100644 --- a/crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs +++ b/crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs @@ -1,6 +1,6 @@ -// @generated by antlr-rust-codegen v0.28.0 - do not edit +// @generated by antlr-rust-codegen v0.31.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.28.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.31.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { @@ -116,6 +116,32 @@ pub fn rule_names() -> &'static [&'static str] { METADATA.rule_names() } +/// Lexes UTF-8 text into an eagerly filled token stream without constructing +/// a parser. +/// +/// The stream retains every emitted token, including EOF and tokens on hidden +/// or custom channels. Lexer rules using `skip` do not emit tokens. +/// +/// To print ANTLR-style records containing token text, type, and channel: +/// `for token in lex(src, AntlRv4Lexer::new).tokens() { println!("{token}"); }` +pub fn lex( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, +) -> antlr4_runtime::CommonTokenStream { + lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer) +} + +/// Lexes a caller-provided character stream without constructing a parser. +/// +/// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a +/// named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn lex_stream( + input: I, + lexer: impl FnOnce(I) -> L, +) -> antlr4_runtime::CommonTokenStream { + antlr4_runtime::CommonTokenStream::new(lexer(input)) +} pub trait AntlRv4LexerHooks: Sized { fn handle_begin_argument(&mut self, ctx: &mut antlr4_runtime::LexerSemCtx<'_, I>) where diff --git a/crates/antlr-rust-g4-parser/src/generated/antlr_v4_parser.rs b/crates/antlr-rust-g4-parser/src/generated/antlr_v4_parser.rs index 29c797b0..79355bdd 100644 --- a/crates/antlr-rust-g4-parser/src/generated/antlr_v4_parser.rs +++ b/crates/antlr-rust-g4-parser/src/generated/antlr_v4_parser.rs @@ -1,6 +1,6 @@ -// @generated by antlr-rust-codegen v0.28.0 - do not edit +// @generated by antlr-rust-codegen v0.31.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.28.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.31.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { diff --git a/crates/antlr-rust-rs-parser/src/generated/rust_lexer.rs b/crates/antlr-rust-rs-parser/src/generated/rust_lexer.rs index faaeae32..8c63c819 100644 --- a/crates/antlr-rust-rs-parser/src/generated/rust_lexer.rs +++ b/crates/antlr-rust-rs-parser/src/generated/rust_lexer.rs @@ -1,6 +1,6 @@ -// @generated by antlr-rust-codegen v0.28.0 - do not edit +// @generated by antlr-rust-codegen v0.31.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.28.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.31.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { @@ -154,6 +154,32 @@ pub fn rule_names() -> &'static [&'static str] { METADATA.rule_names() } +/// Lexes UTF-8 text into an eagerly filled token stream without constructing +/// a parser. +/// +/// The stream retains every emitted token, including EOF and tokens on hidden +/// or custom channels. Lexer rules using `skip` do not emit tokens. +/// +/// To print ANTLR-style records containing token text, type, and channel: +/// `for token in lex(src, RustLexer::new).tokens() { println!("{token}"); }` +pub fn lex( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, +) -> antlr4_runtime::CommonTokenStream { + lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer) +} + +/// Lexes a caller-provided character stream without constructing a parser. +/// +/// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a +/// named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn lex_stream( + input: I, + lexer: impl FnOnce(I) -> L, +) -> antlr4_runtime::CommonTokenStream { + antlr4_runtime::CommonTokenStream::new(lexer(input)) +} static ATN_CELL: OnceLock = OnceLock::new(); diff --git a/crates/antlr-rust-rs-parser/src/generated/rust_parser.rs b/crates/antlr-rust-rs-parser/src/generated/rust_parser.rs index 3986c71c..8c602b9d 100644 --- a/crates/antlr-rust-rs-parser/src/generated/rust_parser.rs +++ b/crates/antlr-rust-rs-parser/src/generated/rust_parser.rs @@ -1,6 +1,6 @@ -// @generated by antlr-rust-codegen v0.28.0 - do not edit +// @generated by antlr-rust-codegen v0.31.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.28.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.31.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { diff --git a/crates/antlr-rust-runtime/src/xpath/generated/x_path_lexer.rs b/crates/antlr-rust-runtime/src/xpath/generated/x_path_lexer.rs index b82779f6..70073815 100644 --- a/crates/antlr-rust-runtime/src/xpath/generated/x_path_lexer.rs +++ b/crates/antlr-rust-runtime/src/xpath/generated/x_path_lexer.rs @@ -1,6 +1,6 @@ -// @generated by antlr-rust-codegen v0.28.0 - do not edit +// @generated by antlr-rust-codegen v0.31.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.28.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.31.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { @@ -43,6 +43,32 @@ pub fn rule_names() -> &'static [&'static str] { METADATA.rule_names() } +/// Lexes UTF-8 text into an eagerly filled token stream without constructing +/// a parser. +/// +/// The stream retains every emitted token, including EOF and tokens on hidden +/// or custom channels. Lexer rules using `skip` do not emit tokens. +/// +/// To print ANTLR-style records containing token text, type, and channel: +/// `for token in lex(src, XPathLexer::new).tokens() { println!("{token}"); }` +pub fn lex( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, +) -> antlr4_runtime::CommonTokenStream { + lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer) +} + +/// Lexes a caller-provided character stream without constructing a parser. +/// +/// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a +/// named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn lex_stream( + input: I, + lexer: impl FnOnce(I) -> L, +) -> antlr4_runtime::CommonTokenStream { + antlr4_runtime::CommonTokenStream::new(lexer(input)) +} static ATN_CELL: OnceLock = OnceLock::new(); diff --git a/crates/antlr-rust-toml-parser/src/generated/toml_lexer.rs b/crates/antlr-rust-toml-parser/src/generated/toml_lexer.rs index 241448b8..475ec971 100644 --- a/crates/antlr-rust-toml-parser/src/generated/toml_lexer.rs +++ b/crates/antlr-rust-toml-parser/src/generated/toml_lexer.rs @@ -1,6 +1,6 @@ -// @generated by antlr-rust-codegen v0.30.0 - do not edit +// @generated by antlr-rust-codegen v0.31.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.30.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.31.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { @@ -72,6 +72,32 @@ pub fn rule_names() -> &'static [&'static str] { METADATA.rule_names() } +/// Lexes UTF-8 text into an eagerly filled token stream without constructing +/// a parser. +/// +/// The stream retains every emitted token, including EOF and tokens on hidden +/// or custom channels. Lexer rules using `skip` do not emit tokens. +/// +/// To print ANTLR-style records containing token text, type, and channel: +/// `for token in lex(src, TomlLexer::new).tokens() { println!("{token}"); }` +pub fn lex( + input: impl AsRef, + lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, +) -> antlr4_runtime::CommonTokenStream { + lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer) +} + +/// Lexes a caller-provided character stream without constructing a parser. +/// +/// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a +/// named [`antlr4_runtime::InputStream`] or a byte-oriented +/// [`antlr4_runtime::ByteStream`]. +pub fn lex_stream( + input: I, + lexer: impl FnOnce(I) -> L, +) -> antlr4_runtime::CommonTokenStream { + antlr4_runtime::CommonTokenStream::new(lexer(input)) +} static ATN_CELL: OnceLock = OnceLock::new(); diff --git a/crates/antlr-rust-toml-parser/src/generated/toml_parser.rs b/crates/antlr-rust-toml-parser/src/generated/toml_parser.rs index d3115481..3052597c 100644 --- a/crates/antlr-rust-toml-parser/src/generated/toml_parser.rs +++ b/crates/antlr-rust-toml-parser/src/generated/toml_parser.rs @@ -1,6 +1,6 @@ -// @generated by antlr-rust-codegen v0.30.0 - do not edit +// @generated by antlr-rust-codegen v0.31.0 - do not edit // project: https://github.com/ophi-dev/antlr-rust-runtime -antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.30.0"); +antlr4_runtime::__antlr4_rust_require_codegen_api!(6, "0.31.0"); #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)] #[rustfmt::skip] mod __antlr4_rust_generated { diff --git a/third_party/antlr-v4-grammar/self-hosted.sha256 b/third_party/antlr-v4-grammar/self-hosted.sha256 index 8ade0a03..af6736db 100644 --- a/third_party/antlr-v4-grammar/self-hosted.sha256 +++ b/third_party/antlr-v4-grammar/self-hosted.sha256 @@ -2,5 +2,5 @@ 1286e542499e4480b3ab5ff60e4a4a7faf21134ca4c4f8f1f468f30095fa25cb third_party/antlr-v4-grammar/ANTLRv4Parser.g4 c7114545a75ab294215819962e92e570383dc830fd5768463dab04e6733bcb80 third_party/antlr-v4-grammar/predefined.tokens 5803594bd2c8dd2d5180f1ca08fc70dfc80308479d18a7c4a1b743fa523b55ec third_party/antlr-v4-grammar/antlr-v4.toml -2d58d0093ecc504b086b0ba3e6b42fcda9b58988d97759732a4851c4e64a876e crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs -2ec52bfe99c6529406c854b4b906510e11b9a31722a917f78bb0084a03ccb3e3 crates/antlr-rust-g4-parser/src/generated/antlr_v4_parser.rs +bf365fffcb3a71758299a62aa90ad9ad95c91ec34c47ccf07b80928693937c87 crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs +a2eeb7902e8330326be25a9dc8e3acdcb656525196f175925f3443f19187abef crates/antlr-rust-g4-parser/src/generated/antlr_v4_parser.rs From 65bfab9e16f230adebc29ba1c1504cca5c83c948 Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 7 Aug 2026 13:01:47 +0200 Subject: [PATCH 2/4] test(codegen): lock offline build fixture dependencies Seed the nested build-dependency workspace from the repository Cargo.lock before running it offline. This keeps its dependency graph identical to the tested workspace and allows Cargo to reuse locked packages whose versions are subsequently yanked, including wide 1.6.0. --- .../tests/antlr4_rust_gen_cli/multi_recognizer.rs | 2 ++ 1 file changed, 2 insertions(+) diff --git a/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/multi_recognizer.rs b/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/multi_recognizer.rs index 2366e2bb..f22e2246 100644 --- a/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/multi_recognizer.rs +++ b/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/multi_recognizer.rs @@ -340,6 +340,8 @@ fn three_build_dependency_consumers_share_codegen_and_track_only_resolved_inputs ), ) .expect("fixture workspace manifest should be writable"); + fs::copy(root.join("Cargo.lock"), workspace.join("Cargo.lock")) + .expect("fixture workspace should reuse the repository lockfile"); let first = run_fixture_workspace(workspace); assert_fixture_success(&first); From 3abf94d9da7b023d57250dbd4210678d504e19cd Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 7 Aug 2026 13:27:00 +0200 Subject: [PATCH 3/4] fix(codegen): make lex-only token dumps explicit Document vocabulary-resolved token names, always-visible numeric channels, buffered lexer diagnostics, and the CommonTokenStream construction panic. Move the README section outside the parser setup narrative. Exercise skipped rules and UTF-8 byte/column positions end to end, strengthen the legacy CommonToken guard, and let the lexer convenience snapshot own its rendered value. Regenerated recognizers retain generated-code API revision 6 while their stale generator banners now reflect the workspace's current 0.31.0 release. --- README.md | 51 ++++++++++++------- ...nerator__tests__lexer_lex_convenience.snap | 25 +++++++-- .../antlr-rust-codegen/src/generator/tests.rs | 24 +++------ .../src/lexer/render_model.rs | 23 ++++++++- .../tests/antlr4_rust_gen_cli/lexer.rs | 21 +++++--- .../src/generated/antlr_v4_lexer.rs | 23 ++++++++- .../src/generated/rust_lexer.rs | 23 ++++++++- .../src/xpath/generated/x_path_lexer.rs | 23 ++++++++- .../src/generated/toml_lexer.rs | 23 ++++++++- .../antlr-v4-grammar/self-hosted.sha256 | 2 +- 10 files changed, 181 insertions(+), 57 deletions(-) diff --git a/README.md b/README.md index 34107993..a21edc2a 100644 --- a/README.md +++ b/README.md @@ -352,25 +352,6 @@ let parsed = json_parser::parse_stream(input, JsonLexer::new, JsonParser::json)? `parse_stream_with_parser` is the corresponding stream-based helper when the caller also needs the parser afterward. -### Inspecting tokens - -Generated lexer modules expose `lex` for token-only workflows. The returned -stream is already filled and retains EOF plus tokens on hidden or custom -channels: - -```rust -use generated::json_lexer::{self, JsonLexer}; - -let tokens = json_lexer::lex(r#"{"a":1}"#, JsonLexer::new); -for token in tokens.tokens() { - println!("{token}"); -} -``` - -`TokenView`'s display format includes token text, type, channel, line, and -column. Rules using `skip` do not emit tokens. Use `lex_stream` to supply a -named `InputStream`, `ByteStream`, or custom `CharStream`. - Construct each layer explicitly when you need parser options or custom error handling before invoking the entry rule: @@ -399,6 +380,38 @@ from both the lexer and parser to suppress recovery output, as above, or call resolved half-open UTF-8 byte range for parser tokens and lexer failures, when the input stream can provide byte offsets. +### Inspecting tokens + +Generated lexer modules expose `lex` for token-only workflows. The returned +stream is already filled and retains EOF plus tokens on hidden or custom +channels: + +```rust +use antlr4_runtime::Token as _; +use generated::json_lexer::{self, JsonLexer}; + +let tokens = json_lexer::lex(r#"{"a":1}"#, JsonLexer::new); +let vocabulary = json_lexer::metadata().vocabulary(); +for token in tokens.tokens() { + println!( + "type={} channel={} text={:?}", + vocabulary.display_name(token.token_type()), + token.channel(), + token.text(), + ); +} +``` + +The vocabulary resolves symbolic or literal token names, and the explicit +channel value includes the default channel. Rules using `skip` do not emit +tokens. `number_of_source_errors()` reports buffered lexer diagnostics; after +iterating `tokens()`, call `drain_source_errors()` to retrieve them. + +Use `lex_stream` to supply a named `InputStream`, `ByteStream`, or custom +`CharStream`. Like `CommonTokenStream::new`, both helpers panic if token +buffering returns `TokenStoreError`; construct the lexer and use +`CommonTokenStream::try_new` when that error must be handled. + ### Reusing Recognizers Generated recognizers can be re-fed without reconstructing the lexer or parser. diff --git a/crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap b/crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap index 94e3e053..95cef103 100644 --- a/crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap +++ b/crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap @@ -1,6 +1,6 @@ --- source: crates/antlr-rust-codegen/src/generator/tests.rs -expression: "render_lexer_lex_convenience(\"TLexer\")" +expression: "&convenience" --- /// Lexes UTF-8 text into an eagerly filled token stream without constructing /// a parser. @@ -8,8 +8,18 @@ expression: "render_lexer_lex_convenience(\"TLexer\")" /// The stream retains every emitted token, including EOF and tokens on hidden /// or custom channels. Lexer rules using `skip` do not emit tokens. /// -/// To print ANTLR-style records containing token text, type, and channel: -/// `for token in lex(src, TLexer::new).tokens() { println!("{token}"); }` +/// With `use antlr4_runtime::Token as _;`, print each token's vocabulary name, +/// numeric channel, and text: +/// `let vocabulary = metadata().vocabulary(); for token in lex(src, TLexer::new).tokens() { println!("type={} channel={} text={:?}", vocabulary.display_name(token.token_type()), token.channel(), token.text()); }` +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Construct +/// the lexer and call [`antlr4_runtime::CommonTokenStream::try_new`] to handle +/// that error instead. pub fn lex( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, @@ -22,6 +32,15 @@ pub fn lex( /// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a /// named [`antlr4_runtime::InputStream`] or a byte-oriented /// [`antlr4_runtime::ByteStream`]. +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Call +/// [`antlr4_runtime::CommonTokenStream::try_new`] with the constructed lexer to +/// handle that error instead. pub fn lex_stream( input: I, lexer: impl FnOnce(I) -> L, diff --git a/crates/antlr-rust-codegen/src/generator/tests.rs b/crates/antlr-rust-codegen/src/generator/tests.rs index 66155080..1313a438 100644 --- a/crates/antlr-rust-codegen/src/generator/tests.rs +++ b/crates/antlr-rust-codegen/src/generator/tests.rs @@ -2907,21 +2907,9 @@ fn renders_lex_convenience_without_a_parser() { ) .expect("lexer should render"); - insta::assert_snapshot!( - "lexer_lex_convenience", - render_lexer_lex_convenience("TLexer") - ); - assert!(rendered.contains("pub fn lex(")); - assert!(rendered.contains( - "pub fn lex_stream(" - )); - assert!( - rendered.contains("lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer)") - ); - assert!(rendered.contains("antlr4_runtime::CommonTokenStream::new(lexer(input))")); - assert!( - rendered.contains("for token in lex(src, TLexer::new).tokens() { println!(\"{token}\"); }") - ); + let convenience = render_lexer_lex_convenience("TLexer"); + insta::assert_snapshot!("lexer_lex_convenience", &convenience); + assert!(rendered.contains(&convenience)); } #[test] @@ -6298,7 +6286,11 @@ fn lexer_default_policy_keeps_compiled_token_path() { "generated_lexer_lifecycle_facade", rendered_facade_declaration(&module, "__antlr4_rust_lexer_facade") ); - assert!(!module.contains("CommonToken::")); + assert!( + !module + .replace("CommonTokenStream", "") + .contains("CommonToken") + ); assert!(!module.contains("TokenFactory")); } diff --git a/crates/antlr-rust-codegen/src/lexer/render_model.rs b/crates/antlr-rust-codegen/src/lexer/render_model.rs index 0a06c0d6..03587d06 100644 --- a/crates/antlr-rust-codegen/src/lexer/render_model.rs +++ b/crates/antlr-rust-codegen/src/lexer/render_model.rs @@ -43,8 +43,18 @@ pub(crate) fn render_lexer_lex_convenience(type_name: &str) -> String { /// The stream retains every emitted token, including EOF and tokens on hidden /// or custom channels. Lexer rules using `skip` do not emit tokens. /// -/// To print ANTLR-style records containing token text, type, and channel: -/// `for token in lex(src, {type_name}::new).tokens() {{ println!("{{token}}"); }}` +/// With `use antlr4_runtime::Token as _;`, print each token's vocabulary name, +/// numeric channel, and text: +/// `let vocabulary = metadata().vocabulary(); for token in lex(src, {type_name}::new).tokens() {{ println!("type={{}} channel={{}} text={{:?}}", vocabulary.display_name(token.token_type()), token.channel(), token.text()); }}` +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Construct +/// the lexer and call [`antlr4_runtime::CommonTokenStream::try_new`] to handle +/// that error instead. pub fn lex( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, @@ -57,6 +67,15 @@ pub fn lex( /// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a /// named [`antlr4_runtime::InputStream`] or a byte-oriented /// [`antlr4_runtime::ByteStream`]. +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Call +/// [`antlr4_runtime::CommonTokenStream::try_new`] with the constructed lexer to +/// handle that error instead. pub fn lex_stream( input: I, lexer: impl FnOnce(I) -> L, diff --git a/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs b/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs index 48d1742f..b8dfb7f9 100644 --- a/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs +++ b/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs @@ -215,7 +215,9 @@ fn generated_lex_helpers_expose_hidden_and_custom_channels() { channels { COMMENTS }\n\ WORD: [a-z]+;\n\ COMMENT: '#' ~[\\r\\n]* -> channel(COMMENTS);\n\ - WS: [ \\t]+ -> channel(HIDDEN);\n", + WS: [ \\t]+ -> channel(HIDDEN);\n\ + UNICODE: '\\u00E9';\n\ + SKIPPED: '~' -> skip;\n", ) .expect("grammar should be writable"); @@ -238,7 +240,7 @@ fn generated_lex_helpers_expose_hidden_and_custom_channels() { #[cfg(test)] mod lex_helper_tests { use super::channels::{ - self, CHANNEL_COMMENTS, COMMENT, Channels, WORD, WS, + self, CHANNEL_COMMENTS, COMMENT, Channels, UNICODE, WORD, WS, }; use antlr4_runtime::{ ByteStream, DEFAULT_CHANNEL, HIDDEN_CHANNEL, TOKEN_EOF, Token as _, @@ -246,25 +248,28 @@ mod lex_helper_tests { #[test] fn buffers_every_emitted_channel_without_a_parser() { - let tokens = channels::lex("alpha # note", Channels::new); + let tokens = channels::lex("alpha ~\u{e9}#note", Channels::new); let observed = tokens .tokens() .map(|token| ( token.token_type(), token.channel(), token.text_or_empty(), + token.byte_span(), + token.column(), )) .collect::>(); assert_eq!( - &observed[..3], + &observed[..4], [ - (WORD, DEFAULT_CHANNEL, "alpha"), - (WS, HIDDEN_CHANNEL, " "), - (COMMENT, CHANNEL_COMMENTS, "# note"), + (WORD, DEFAULT_CHANNEL, "alpha", Some(0..5), 0), + (WS, HIDDEN_CHANNEL, " ", Some(5..6), 5), + (UNICODE, DEFAULT_CHANNEL, "\u{e9}", Some(7..9), 7), + (COMMENT, CHANNEL_COMMENTS, "#note", Some(9..14), 8), ] ); - assert_eq!(observed[3].0, TOKEN_EOF); + assert_eq!(observed[4].0, TOKEN_EOF); assert_eq!(tokens.number_of_source_errors(), 0); } diff --git a/crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs b/crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs index 54f84c89..825b3041 100644 --- a/crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs +++ b/crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs @@ -122,8 +122,18 @@ pub fn rule_names() -> &'static [&'static str] { /// The stream retains every emitted token, including EOF and tokens on hidden /// or custom channels. Lexer rules using `skip` do not emit tokens. /// -/// To print ANTLR-style records containing token text, type, and channel: -/// `for token in lex(src, AntlRv4Lexer::new).tokens() { println!("{token}"); }` +/// With `use antlr4_runtime::Token as _;`, print each token's vocabulary name, +/// numeric channel, and text: +/// `let vocabulary = metadata().vocabulary(); for token in lex(src, AntlRv4Lexer::new).tokens() { println!("type={} channel={} text={:?}", vocabulary.display_name(token.token_type()), token.channel(), token.text()); }` +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Construct +/// the lexer and call [`antlr4_runtime::CommonTokenStream::try_new`] to handle +/// that error instead. pub fn lex( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, @@ -136,6 +146,15 @@ pub fn lex( /// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a /// named [`antlr4_runtime::InputStream`] or a byte-oriented /// [`antlr4_runtime::ByteStream`]. +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Call +/// [`antlr4_runtime::CommonTokenStream::try_new`] with the constructed lexer to +/// handle that error instead. pub fn lex_stream( input: I, lexer: impl FnOnce(I) -> L, diff --git a/crates/antlr-rust-rs-parser/src/generated/rust_lexer.rs b/crates/antlr-rust-rs-parser/src/generated/rust_lexer.rs index 8c63c819..8c474d34 100644 --- a/crates/antlr-rust-rs-parser/src/generated/rust_lexer.rs +++ b/crates/antlr-rust-rs-parser/src/generated/rust_lexer.rs @@ -160,8 +160,18 @@ pub fn rule_names() -> &'static [&'static str] { /// The stream retains every emitted token, including EOF and tokens on hidden /// or custom channels. Lexer rules using `skip` do not emit tokens. /// -/// To print ANTLR-style records containing token text, type, and channel: -/// `for token in lex(src, RustLexer::new).tokens() { println!("{token}"); }` +/// With `use antlr4_runtime::Token as _;`, print each token's vocabulary name, +/// numeric channel, and text: +/// `let vocabulary = metadata().vocabulary(); for token in lex(src, RustLexer::new).tokens() { println!("type={} channel={} text={:?}", vocabulary.display_name(token.token_type()), token.channel(), token.text()); }` +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Construct +/// the lexer and call [`antlr4_runtime::CommonTokenStream::try_new`] to handle +/// that error instead. pub fn lex( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, @@ -174,6 +184,15 @@ pub fn lex( /// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a /// named [`antlr4_runtime::InputStream`] or a byte-oriented /// [`antlr4_runtime::ByteStream`]. +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Call +/// [`antlr4_runtime::CommonTokenStream::try_new`] with the constructed lexer to +/// handle that error instead. pub fn lex_stream( input: I, lexer: impl FnOnce(I) -> L, diff --git a/crates/antlr-rust-runtime/src/xpath/generated/x_path_lexer.rs b/crates/antlr-rust-runtime/src/xpath/generated/x_path_lexer.rs index 70073815..0e50d50a 100644 --- a/crates/antlr-rust-runtime/src/xpath/generated/x_path_lexer.rs +++ b/crates/antlr-rust-runtime/src/xpath/generated/x_path_lexer.rs @@ -49,8 +49,18 @@ pub fn rule_names() -> &'static [&'static str] { /// The stream retains every emitted token, including EOF and tokens on hidden /// or custom channels. Lexer rules using `skip` do not emit tokens. /// -/// To print ANTLR-style records containing token text, type, and channel: -/// `for token in lex(src, XPathLexer::new).tokens() { println!("{token}"); }` +/// With `use antlr4_runtime::Token as _;`, print each token's vocabulary name, +/// numeric channel, and text: +/// `let vocabulary = metadata().vocabulary(); for token in lex(src, XPathLexer::new).tokens() { println!("type={} channel={} text={:?}", vocabulary.display_name(token.token_type()), token.channel(), token.text()); }` +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Construct +/// the lexer and call [`antlr4_runtime::CommonTokenStream::try_new`] to handle +/// that error instead. pub fn lex( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, @@ -63,6 +73,15 @@ pub fn lex( /// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a /// named [`antlr4_runtime::InputStream`] or a byte-oriented /// [`antlr4_runtime::ByteStream`]. +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Call +/// [`antlr4_runtime::CommonTokenStream::try_new`] with the constructed lexer to +/// handle that error instead. pub fn lex_stream( input: I, lexer: impl FnOnce(I) -> L, diff --git a/crates/antlr-rust-toml-parser/src/generated/toml_lexer.rs b/crates/antlr-rust-toml-parser/src/generated/toml_lexer.rs index 475ec971..4821c89e 100644 --- a/crates/antlr-rust-toml-parser/src/generated/toml_lexer.rs +++ b/crates/antlr-rust-toml-parser/src/generated/toml_lexer.rs @@ -78,8 +78,18 @@ pub fn rule_names() -> &'static [&'static str] { /// The stream retains every emitted token, including EOF and tokens on hidden /// or custom channels. Lexer rules using `skip` do not emit tokens. /// -/// To print ANTLR-style records containing token text, type, and channel: -/// `for token in lex(src, TomlLexer::new).tokens() { println!("{token}"); }` +/// With `use antlr4_runtime::Token as _;`, print each token's vocabulary name, +/// numeric channel, and text: +/// `let vocabulary = metadata().vocabulary(); for token in lex(src, TomlLexer::new).tokens() { println!("type={} channel={} text={:?}", vocabulary.display_name(token.token_type()), token.channel(), token.text()); }` +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Construct +/// the lexer and call [`antlr4_runtime::CommonTokenStream::try_new`] to handle +/// that error instead. pub fn lex( input: impl AsRef, lexer: impl FnOnce(antlr4_runtime::InputStream) -> L, @@ -92,6 +102,15 @@ pub fn lex( /// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a /// named [`antlr4_runtime::InputStream`] or a byte-oriented /// [`antlr4_runtime::ByteStream`]. +/// +/// `number_of_source_errors()` reports buffered lexer diagnostics. After +/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics. +/// +/// # Panics +/// +/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Call +/// [`antlr4_runtime::CommonTokenStream::try_new`] with the constructed lexer to +/// handle that error instead. pub fn lex_stream( input: I, lexer: impl FnOnce(I) -> L, diff --git a/third_party/antlr-v4-grammar/self-hosted.sha256 b/third_party/antlr-v4-grammar/self-hosted.sha256 index af6736db..642763b7 100644 --- a/third_party/antlr-v4-grammar/self-hosted.sha256 +++ b/third_party/antlr-v4-grammar/self-hosted.sha256 @@ -2,5 +2,5 @@ 1286e542499e4480b3ab5ff60e4a4a7faf21134ca4c4f8f1f468f30095fa25cb third_party/antlr-v4-grammar/ANTLRv4Parser.g4 c7114545a75ab294215819962e92e570383dc830fd5768463dab04e6733bcb80 third_party/antlr-v4-grammar/predefined.tokens 5803594bd2c8dd2d5180f1ca08fc70dfc80308479d18a7c4a1b743fa523b55ec third_party/antlr-v4-grammar/antlr-v4.toml -bf365fffcb3a71758299a62aa90ad9ad95c91ec34c47ccf07b80928693937c87 crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs +9d4d36b3187171f034515462bd45c73af8d25a984199c856a8073bb7e5dba6f1 crates/antlr-rust-g4-parser/src/generated/antlr_v4_lexer.rs a2eeb7902e8330326be25a9dc8e3acdcb656525196f175925f3443f19187abef crates/antlr-rust-g4-parser/src/generated/antlr_v4_parser.rs From 1947c6ef666624eaf80d580e4bd3a604bf24235f Mon Sep 17 00:00:00 2001 From: Konstantin Vyatkin Date: Fri, 7 Aug 2026 13:34:00 +0200 Subject: [PATCH 4/4] test(codegen): snapshot lex-only token sequence Use the existing inline-Insta convention for disposable generated-project crates to pin the complete vocabulary name, channel, text, byte span, and column sequence. Keep lexer diagnostics as a separate invariant. --- .../tests/antlr4_rust_gen_cli/lexer.rs | 78 ++++++++++++++----- 1 file changed, 60 insertions(+), 18 deletions(-) diff --git a/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs b/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs index b8dfb7f9..abd2b930 100644 --- a/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs +++ b/crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs @@ -236,23 +236,23 @@ fn generated_lex_helpers_expose_hidden_and_custom_channels() { assert_generated_project( temp.path(), &["channels.rs"], - r##" + r####" +// Inline snapshot is intentional: the temporary generated crate is deleted +// after this test, so an external snapshot cannot be retained in the repository. #[cfg(test)] +#[allow(clippy::disallowed_methods)] // `insta` assertion macros unwrap internal I/O. mod lex_helper_tests { - use super::channels::{ - self, CHANNEL_COMMENTS, COMMENT, Channels, UNICODE, WORD, WS, - }; - use antlr4_runtime::{ - ByteStream, DEFAULT_CHANNEL, HIDDEN_CHANNEL, TOKEN_EOF, Token as _, - }; + use super::channels::{self, Channels, WORD}; + use antlr4_runtime::{ByteStream, Token as _}; #[test] fn buffers_every_emitted_channel_without_a_parser() { let tokens = channels::lex("alpha ~\u{e9}#note", Channels::new); + let vocabulary = channels::metadata().vocabulary(); let observed = tokens .tokens() .map(|token| ( - token.token_type(), + vocabulary.display_name(token.token_type()), token.channel(), token.text_or_empty(), token.byte_span(), @@ -260,16 +260,58 @@ mod lex_helper_tests { )) .collect::>(); - assert_eq!( - &observed[..4], - [ - (WORD, DEFAULT_CHANNEL, "alpha", Some(0..5), 0), - (WS, HIDDEN_CHANNEL, " ", Some(5..6), 5), - (UNICODE, DEFAULT_CHANNEL, "\u{e9}", Some(7..9), 7), - (COMMENT, CHANNEL_COMMENTS, "#note", Some(9..14), 8), - ] + insta::assert_debug_snapshot!( + observed, + @r###" + [ + ( + "WORD", + 0, + "alpha", + Some( + 0..5, + ), + 0, + ), + ( + "WS", + 1, + " ", + Some( + 5..6, + ), + 5, + ), + ( + "'\\u00E9'", + 0, + "é", + Some( + 7..9, + ), + 7, + ), + ( + "COMMENT", + 2, + "#note", + Some( + 9..14, + ), + 8, + ), + ( + "EOF", + 0, + "", + Some( + 14..14, + ), + 13, + ), + ] + "### ); - assert_eq!(observed[4].0, TOKEN_EOF); assert_eq!(tokens.number_of_source_errors(), 0); } @@ -283,6 +325,6 @@ mod lex_helper_tests { assert_eq!(first.byte_span(), Some(0..4)); } } -"##, +"####, ); }