feat(codegen): add lex-only generated helpers

README.md

@@ -380,6 +380,38 @@ from both the lexer and parser to suppress recovery output, as above, or call
 resolved half-open UTF-8 byte range for parser tokens and lexer failures, when
 the input stream can provide byte offsets.
 
+### Inspecting tokens
+
+Generated lexer modules expose `lex` for token-only workflows. The returned
+stream is already filled and retains EOF plus tokens on hidden or custom
+channels:
+
+```rust
+use antlr4_runtime::Token as _;
+use generated::json_lexer::{self, JsonLexer};
+
+let tokens = json_lexer::lex(r#"{"a":1}"#, JsonLexer::new);
+let vocabulary = json_lexer::metadata().vocabulary();
+for token in tokens.tokens() {
+    println!(
+        "type={} channel={} text={:?}",
+        vocabulary.display_name(token.token_type()),
+        token.channel(),
+        token.text(),
+    );
+}
+```
+
+The vocabulary resolves symbolic or literal token names, and the explicit
+channel value includes the default channel. Rules using `skip` do not emit
+tokens. `number_of_source_errors()` reports buffered lexer diagnostics; after
+iterating `tokens()`, call `drain_source_errors()` to retrieve them.
+
+Use `lex_stream` to supply a named `InputStream`, `ByteStream`, or custom
+`CharStream`. Like `CommonTokenStream::new`, both helpers panic if token
+buffering returns `TokenStoreError`; construct the lexer and use
+`CommonTokenStream::try_new` when that error must be handled.
+
 ### Reusing Recognizers
 
 Generated recognizers can be re-fed without reconstructing the lexer or parser.

crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__lexer_lex_convenience.snap

@@ -0,0 +1,49 @@
+---
+source: crates/antlr-rust-codegen/src/generator/tests.rs
+expression: "&convenience"
+---
+/// Lexes UTF-8 text into an eagerly filled token stream without constructing
+/// a parser.
+///
+/// The stream retains every emitted token, including EOF and tokens on hidden
+/// or custom channels. Lexer rules using `skip` do not emit tokens.
+///
+/// With `use antlr4_runtime::Token as _;`, print each token's vocabulary name,
+/// numeric channel, and text:
+/// `let vocabulary = metadata().vocabulary(); for token in lex(src, TLexer::new).tokens() { println!("type={} channel={} text={:?}", vocabulary.display_name(token.token_type()), token.channel(), token.text()); }`
+///
+/// `number_of_source_errors()` reports buffered lexer diagnostics. After
+/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics.
+///
+/// # Panics
+///
+/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Construct
+/// the lexer and call [`antlr4_runtime::CommonTokenStream::try_new`] to handle
+/// that error instead.
+pub fn lex<L: antlr4_runtime::TokenSource>(
+    input: impl AsRef<str>,
+    lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
+) -> antlr4_runtime::CommonTokenStream<L> {
+    lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer)
+}
+
+/// Lexes a caller-provided character stream without constructing a parser.
+///
+/// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a
+/// named [`antlr4_runtime::InputStream`] or a byte-oriented
+/// [`antlr4_runtime::ByteStream`].
+///
+/// `number_of_source_errors()` reports buffered lexer diagnostics. After
+/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics.
+///
+/// # Panics
+///
+/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Call
+/// [`antlr4_runtime::CommonTokenStream::try_new`] with the constructed lexer to
+/// handle that error instead.
+pub fn lex_stream<I: antlr4_runtime::CharStream, L: antlr4_runtime::TokenSource>(
+    input: I,
+    lexer: impl FnOnce(I) -> L,
+) -> antlr4_runtime::CommonTokenStream<L> {
+    antlr4_runtime::CommonTokenStream::new(lexer(input))
+}

crates/antlr-rust-codegen/src/generator/tests.rs

@@ -2895,6 +2895,23 @@ fn portable_local_semantics_reject_missing_generated_caller() {
     );
 }
 
+#[test]
+fn renders_lex_convenience_without_a_parser() {
+    let rendered = render_lexer(
+        "TLexer",
+        &predicate_lexer_data(),
+        false,
+        SemUnknownPolicy::default(),
+        &SemPatternFile::default(),
+        false,
+    )
+    .expect("lexer should render");
+
+    let convenience = render_lexer_lex_convenience("TLexer");
+    insta::assert_snapshot!("lexer_lex_convenience", &convenience);
+    assert!(rendered.contains(&convenience));
+}
+
 #[test]
 fn renders_parse_convenience_without_replacing_manual_constructor() {
     let rendered = render_parser("TParser", &minimal_parser_data()).expect("parser should render");
@@ -6269,7 +6286,11 @@ fn lexer_default_policy_keeps_compiled_token_path() {
         "generated_lexer_lifecycle_facade",
         rendered_facade_declaration(&module, "__antlr4_rust_lexer_facade")
     );
-    assert!(!module.contains("CommonToken"));
+    assert!(
+        !module
+            .replace("CommonTokenStream", "")
+            .contains("CommonToken")
+    );
     assert!(!module.contains("TokenFactory"));
 }
 

crates/antlr-rust-codegen/src/lexer/render.rs

@@ -34,6 +34,7 @@ pub(crate) fn render_lexer_model(model: &LexerRenderModel<'_>) -> io::Result<Str
     let embedded = model.embedded;
     let type_name = rust_type_name(grammar_name);
     let metadata = render_lexer_metadata(grammar_name, data);
+    let lex_convenience = render_lexer_lex_convenience(&type_name);
     let token_constants = render_lexer_token_constants(data);
     let lexer_state_constants = render_lexer_state_constants(data);
     // Embedded mode: lexer action/predicate bodies are verbatim Rust from the
@@ -263,6 +264,7 @@ use std::sync::OnceLock;
 {token_constants}
 {lexer_state_constants}
 {metadata}
+{lex_convenience}
 {typed_hook_adapter}
 
 static ATN_CELL: OnceLock<LexerAtn> = OnceLock::new();

crates/antlr-rust-codegen/src/lexer/render_model.rs

@@ -33,6 +33,58 @@ impl<'a> LexerRenderModel<'a> {
     }
 }
 
+/// Renders lexer-module conveniences that buffer text or a caller-provided
+/// character stream without constructing a parser.
+pub(crate) fn render_lexer_lex_convenience(type_name: &str) -> String {
+    format!(
+        r#"/// Lexes UTF-8 text into an eagerly filled token stream without constructing
+/// a parser.
+///
+/// The stream retains every emitted token, including EOF and tokens on hidden
+/// or custom channels. Lexer rules using `skip` do not emit tokens.
+///
+/// With `use antlr4_runtime::Token as _;`, print each token's vocabulary name,
+/// numeric channel, and text:
+/// `let vocabulary = metadata().vocabulary(); for token in lex(src, {type_name}::new).tokens() {{ println!("type={{}} channel={{}} text={{:?}}", vocabulary.display_name(token.token_type()), token.channel(), token.text()); }}`
+///
+/// `number_of_source_errors()` reports buffered lexer diagnostics. After
+/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics.
+///
+/// # Panics
+///
+/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Construct
+/// the lexer and call [`antlr4_runtime::CommonTokenStream::try_new`] to handle
+/// that error instead.
+pub fn lex<L: antlr4_runtime::TokenSource>(
+    input: impl AsRef<str>,
+    lexer: impl FnOnce(antlr4_runtime::InputStream) -> L,
+) -> antlr4_runtime::CommonTokenStream<L> {{
+    lex_stream(antlr4_runtime::InputStream::new(input.as_ref()), lexer)
+}}
+
+/// Lexes a caller-provided character stream without constructing a parser.
+///
+/// Unlike [`lex`], this accepts any [`antlr4_runtime::CharStream`], including a
+/// named [`antlr4_runtime::InputStream`] or a byte-oriented
+/// [`antlr4_runtime::ByteStream`].
+///
+/// `number_of_source_errors()` reports buffered lexer diagnostics. After
+/// iterating `tokens()`, `drain_source_errors()` retrieves those diagnostics.
+///
+/// # Panics
+///
+/// Panics if buffering returns an [`antlr4_runtime::TokenStoreError`]. Call
+/// [`antlr4_runtime::CommonTokenStream::try_new`] with the constructed lexer to
+/// handle that error instead.
+pub fn lex_stream<I: antlr4_runtime::CharStream, L: antlr4_runtime::TokenSource>(
+    input: I,
+    lexer: impl FnOnce(I) -> L,
+) -> antlr4_runtime::CommonTokenStream<L> {{
+    antlr4_runtime::CommonTokenStream::new(lexer(input))
+}}"#
+    )
+}
+
 /// Renders the lexer-owned grammar metadata table.
 pub(crate) fn render_lexer_metadata(grammar_name: &str, data: &LexerCodegenData<'_>) -> String {
     format!(

crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/lexer.rs

@@ -203,3 +203,86 @@ mod combined_literal_tests {
 "#,
     );
 }
+
+#[test]
+fn generated_lex_helpers_expose_hidden_and_custom_channels() {
+    let temp = temporary_directory("lex-helper-channels");
+    let grammar = temp.path().join("Channels.g4");
+    let out = temp.path().join("generated");
+    fs::write(
+        &grammar,
+        "lexer grammar Channels;\n\
+         channels { COMMENTS }\n\
+         WORD: [a-z]+;\n\
+         COMMENT: '#' ~[\\r\\n]* -> channel(COMMENTS);\n\
+         WS: [ \\t]+ -> channel(HIDDEN);\n\
+         UNICODE: '\\u00E9';\n\
+         SKIPPED: '~' -> skip;\n",
+    )
+    .expect("grammar should be writable");
+
+    let output = run_antlr4_rust_gen(&[
+        grammar.as_os_str(),
+        OsStr::new("--out-dir"),
+        out.as_os_str(),
+    ]);
+    assert!(
+        output.status.success(),
+        "stdout: {}\nstderr: {}",
+        utf8(&output.stdout),
+        utf8(&output.stderr)
+    );
+
+    assert_generated_project(
+        temp.path(),
+        &["channels.rs"],
+        r##"
+#[cfg(test)]
+mod lex_helper_tests {
+    use super::channels::{
+        self, CHANNEL_COMMENTS, COMMENT, Channels, UNICODE, WORD, WS,
+    };
+    use antlr4_runtime::{
+        ByteStream, DEFAULT_CHANNEL, HIDDEN_CHANNEL, TOKEN_EOF, Token as _,
+    };
+
+    #[test]
+    fn buffers_every_emitted_channel_without_a_parser() {
+        let tokens = channels::lex("alpha ~\u{e9}#note", Channels::new);
+        let observed = tokens
+            .tokens()
+            .map(|token| (
+                token.token_type(),
+                token.channel(),
+                token.text_or_empty(),
+                token.byte_span(),
+                token.column(),
+            ))
+            .collect::<Vec<_>>();
+
+        assert_eq!(
+            &observed[..4],
+            [
+                (WORD, DEFAULT_CHANNEL, "alpha", Some(0..5), 0),
+                (WS, HIDDEN_CHANNEL, " ", Some(5..6), 5),
+                (UNICODE, DEFAULT_CHANNEL, "\u{e9}", Some(7..9), 7),
+                (COMMENT, CHANNEL_COMMENTS, "#note", Some(9..14), 8),
+            ]
+        );
+        assert_eq!(observed[4].0, TOKEN_EOF);
+        assert_eq!(tokens.number_of_source_errors(), 0);
+    }
+
+    #[test]
+    fn accepts_arbitrary_character_streams() {
+        let tokens =
+            channels::lex_stream(ByteStream::new(b"beta".to_vec()), Channels::new);
+        let first = tokens.get(0).expect("word token");
+
+        assert_eq!(first.token_type(), WORD);
+        assert_eq!(first.byte_span(), Some(0..4));
+    }
+}
+"##,
+    );
+}

crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/multi_recognizer.rs

@@ -340,6 +340,8 @@ fn three_build_dependency_consumers_share_codegen_and_track_only_resolved_inputs
         ),
     )
     .expect("fixture workspace manifest should be writable");
+    fs::copy(root.join("Cargo.lock"), workspace.join("Cargo.lock"))
+        .expect("fixture workspace should reuse the repository lockfile");
 
     let first = run_fixture_workspace(workspace);
     assert_fixture_success(&first);