perf(prediction): eliminate provable tail-call contexts

README.md

@@ -170,8 +170,20 @@ Using the same package release for `antlr4-rust-gen` and
 `antlr-rust-runtime` remains the recommended workflow, but matching the
 generated-code API is the compile-time requirement.
 
-The bundled generator currently emits revision 13, which moves the iterative
-generated listener tree-walk engine into
+The bundled generator currently emits revision 14. Generated parsers now embed
+packed parser ATN format 3, whose rule-transition tags carry validated
+grammar-agnostic tail-call markers. Parser and lexer prediction reuse the
+existing caller context when every continuation from a rule call's follow state
+is a plain epsilon path to the enclosing rule stop; full-context construction
+also omits the same redundant frames. SLL accuracy is preserved by default, and
+the reduced-accuracy parser mode is available only through an explicit
+simulator constructor.
+
+Revision 12 and 13 generated recognizers remain accepted because the runtime
+still provides their source API and reads packed parser ATN formats 1 and 2.
+Regenerate them with revision 14 to emit tail-call metadata.
+
+Revision 13 moved the iterative generated listener tree-walk engine into
 `antlr4_runtime::generated::walk_generated`. Generated parsers retain their
 public plain and validated walker surfaces and emit only grammar-specific
 listener callback adapters; traversal order, invocation-state threading, and
@@ -209,8 +221,8 @@ error name must collapse into one; see `docs/migration.md`). Revision 9
 similarly imports the grammar-independent generated support surface (the typed
 terminal/error-node wrappers, context child-iteration helpers, and
 embedded-action input facade) from the runtime's `generated` module. Revisions
-12 and 13 are accepted generated-code contracts; recognizers from revision 11
-or earlier must be regenerated when upgrading to this release.
+12, 13, and 14 are accepted generated-code contracts; recognizers from revision
+11 or earlier must be regenerated when upgrading to this release.
 
 Generated modules created before this check was introduced cannot be detected
 retroactively. When first upgrading to a release that includes the check,

crates/antlr-rust-codegen/src/generator/snapshots/antlr_rust_codegen__generator__tests__generated_module_file_header.snap

@@ -4,7 +4,7 @@ expression: "generated_module_header.replace(env!(\"CARGO_PKG_VERSION\"),\n\"<ge
 ---
 // @generated by antlr-rust-codegen v<generator-version> - do not edit
 // project: https://github.com/ophi-dev/antlr-rust-runtime
-antlr4_runtime::__antlr4_rust_require_codegen_api!(13, "<generator-version>");
+antlr4_runtime::__antlr4_rust_require_codegen_api!(14, "<generator-version>");
 #[allow(warnings, missing_docs, clippy::all, clippy::pedantic, clippy::nursery)]
 #[rustfmt::skip]
 mod __antlr4_rust_generated {

crates/antlr-rust-codegen/src/grammar/atn/lexer.rs

@@ -1180,6 +1180,7 @@ fn lower(
         }
     }
     add_rule_return_edges(&mut atn);
+    atn.identify_tail_calls();
     atn
 }
 
@@ -1218,6 +1219,7 @@ fn lower_transition(transition: &FinalizedTransition) -> LexerTransition {
             rule_index: *rule_index,
             follow_state: *follow,
             precedence: *precedence,
+            tail_call: false,
         },
         FinalizedTransitionKind::Predicate {
             rule_index,
@@ -1814,6 +1816,7 @@ fn runtime_edge(
             rule_index,
             follow_state,
             precedence,
+            ..
         } => (
             *follow_state,
             3,

crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/cli.rs

@@ -99,7 +99,8 @@ fn generated_modules_enforce_codegen_api_compatibility() {
         "__antlr4_rust_require_codegen_api!({},",
         antlr4_runtime::__ANTLR4_RUST_CODEGEN_API
     );
-    let previous = "__antlr4_rust_require_codegen_api!(12,";
+    let previous = "__antlr4_rust_require_codegen_api!(13,";
+    let oldest_supported = "__antlr4_rust_require_codegen_api!(12,";
     let unsupported = "__antlr4_rust_require_codegen_api!(11,";
     let mut previous_parser = parser;
     let check_start = previous_parser
@@ -109,11 +110,22 @@ fn generated_modules_enforce_codegen_api_compatibility() {
     fs::write(&parser_path, &previous_parser).expect("parser should be writable");
     assert_generated_modules_compile(temp.path(), &modules);
 
-    let mut incompatible_parser = previous_parser;
-    let check_start = incompatible_parser
+    let mut oldest_parser = previous_parser;
+    let check_start = oldest_parser
         .find(previous)
         .expect("parser check should contain the previous API revision");
-    incompatible_parser.replace_range(check_start..check_start + previous.len(), unsupported);
+    oldest_parser.replace_range(check_start..check_start + previous.len(), oldest_supported);
+    fs::write(&parser_path, &oldest_parser).expect("parser should be writable");
+    assert_generated_modules_compile(temp.path(), &modules);
+
+    let mut incompatible_parser = oldest_parser;
+    let check_start = incompatible_parser
+        .find(oldest_supported)
+        .expect("parser check should contain the oldest supported API revision");
+    incompatible_parser.replace_range(
+        check_start..check_start + oldest_supported.len(),
+        unsupported,
+    );
     fs::write(parser_path, incompatible_parser).expect("parser should be writable");
 
     let output = run_generated_project(temp.path(), &modules, "");
@@ -129,7 +141,7 @@ fn generated_modules_enforce_codegen_api_compatibility() {
         .collect::<Vec<_>>()
         .join("\n");
     assert!(
-        diagnostic.contains("supports revisions 12 and 13"),
+        diagnostic.contains("supports revisions 12, 13, and 14"),
         "diagnostic should name the supported revisions: {diagnostic}"
     );
     insta::assert_snapshot!(

crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/snapshots/antlr4_rust_gen_cli__cli__generated_codegen_api_checks.snap

@@ -2,5 +2,5 @@
 source: crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/cli.rs
 expression: normalize_current_package_version(&checks)
 ---
-lexer: antlr4_runtime::__antlr4_rust_require_codegen_api!(13, "<generator-version>");
-parser: antlr4_runtime::__antlr4_rust_require_codegen_api!(13, "<generator-version>");
+lexer: antlr4_runtime::__antlr4_rust_require_codegen_api!(14, "<generator-version>");
+parser: antlr4_runtime::__antlr4_rust_require_codegen_api!(14, "<generator-version>");

crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/snapshots/antlr4_rust_gen_cli__cli__generated_codegen_api_mismatch_diagnostic.snap

@@ -2,5 +2,5 @@
 source: crates/antlr-rust-codegen/tests/antlr4_rust_gen_cli/cli.rs
 expression: normalize_current_package_version(&diagnostic)
 ---
-error: antlr4-rust generated-code API mismatch: antlr4-rust-gen v<generator-version> emitted generated-code API revision 11, but the selected antlr-rust-runtime supports revisions 12 and 13; regenerate this recognizer with a compatible antlr4-rust-gen or select a compatible antlr-rust-runtime dependency
+error: antlr4-rust generated-code API mismatch: antlr4-rust-gen v<generator-version> emitted generated-code API revision 11, but the selected antlr-rust-runtime supports revisions 12, 13, and 14; regenerate this recognizer with a compatible antlr4-rust-gen or select a compatible antlr-rust-runtime dependency
  --> src/codegen_api_parser.rs:3:1

crates/antlr-rust-runtime/src/atn/lexer.rs

@@ -68,6 +68,7 @@ trait LexerContextOps {
     fn singleton(&mut self, parent: LexerContextId, return_state: usize) -> LexerContextId;
     fn merge(&mut self, left: LexerContextId, right: LexerContextId) -> LexerContextId;
     fn node(&self, context: LexerContextId) -> LexerContextNode;
+    fn has_empty_path(&self, context: LexerContextId) -> bool;
 }
 
 struct BorrowedLexerContexts<'a> {
@@ -87,6 +88,10 @@ impl LexerContextOps for BorrowedLexerContexts<'_> {
     fn node(&self, context: LexerContextId) -> LexerContextNode {
         self.contexts.node(context)
     }
+
+    fn has_empty_path(&self, context: LexerContextId) -> bool {
+        self.contexts.has_empty_path(context)
+    }
 }
 
 struct SharedLexerContexts<'a, I> {
@@ -115,6 +120,13 @@ where
     fn node(&self, context: LexerContextId) -> LexerContextNode {
         self.lexer.lexer_prediction_store().contexts.node(context)
     }
+
+    fn has_empty_path(&self, context: LexerContextId) -> bool {
+        self.lexer
+            .lexer_prediction_store()
+            .contexts
+            .has_empty_path(context)
+    }
 }
 
 /// Ordered lexer configurations with graph-structured caller contexts.
@@ -1948,12 +1960,17 @@ fn close_config<C, P>(
             LexerTransition::Rule {
                 target,
                 follow_state,
+                tail_call,
                 ..
             } => {
                 let mut next = config.clone();
                 set_config_state(atn, &mut next, *target);
                 next.passed_non_greedy |= state.non_greedy;
-                next.context = contexts.singleton(config.context, *follow_state);
+                next.context = if *tail_call && !contexts.has_empty_path(config.context) {
+                    config.context
+                } else {
+                    contexts.singleton(config.context, *follow_state)
+                };
                 close_config(atn, next, contexts, closure, semantic_predicate);
             }
             LexerTransition::Predicate {
@@ -2344,6 +2361,7 @@ mod tests {
             rule_index: 1,
             follow_state: 3,
             precedence: 0,
+            tail_call: false,
         });
         atn.add_state(call_fragment);
         atn.add_state(LexerAtnState::new(3, AtnStateKind::RuleStop).with_rule_index(0));
@@ -2365,6 +2383,76 @@ mod tests {
         atn
     }
 
+    fn tail_call_atn() -> LexerAtn {
+        let mut atn = LexerAtn::new(1);
+        for (state_number, kind, rule_index) in [
+            (0, AtnStateKind::RuleStart, 0),
+            (1, AtnStateKind::RuleStop, 0),
+            (2, AtnStateKind::RuleStart, 1),
+            (3, AtnStateKind::RuleStop, 1),
+        ] {
+            atn.add_state(LexerAtnState::new(state_number, kind).with_rule_index(rule_index));
+        }
+        atn.state_mut(0)
+            .expect("caller start")
+            .add_transition(LexerTransition::Rule {
+                target: 2,
+                rule_index: 1,
+                follow_state: 1,
+                precedence: 0,
+                tail_call: false,
+            });
+        atn.state_mut(2)
+            .expect("callee start")
+            .add_transition(LexerTransition::Epsilon { target: 3 });
+        atn.set_rule_to_start_state(vec![0, 2]);
+        atn.set_rule_to_stop_state(vec![1, 3]);
+        atn.set_rule_to_token_type(vec![1, INVALID_TOKEN_TYPE]);
+        atn.identify_tail_calls();
+        atn
+    }
+
+    #[test]
+    fn marked_lexer_tail_call_does_not_allocate_a_redundant_context() {
+        let atn = tail_call_atn();
+        assert!(atn.state(0).expect("caller start").transitions[0].is_tail_call());
+        let mut contexts = LexerContextArena::new();
+        let mut workspace = PredictionWorkspace::default();
+        let parent = contexts.singleton(EMPTY_LEXER_CONTEXT, 99);
+        let before = contexts.len();
+
+        let config = LexerConfig {
+            state: 0,
+            position: 0,
+            consumed_eof: false,
+            alt_rule_index: Some(0),
+            passed_non_greedy: false,
+            context: parent,
+            actions: Vec::new(),
+        };
+        let _ = epsilon_closure(&atn, [config], &mut contexts, &mut workspace, &mut |_| true);
+        assert_eq!(
+            contexts.len(),
+            before,
+            "tail call with no empty lexer path must reuse its caller context"
+        );
+
+        let local = LexerConfig {
+            state: 0,
+            position: 0,
+            consumed_eof: false,
+            alt_rule_index: Some(0),
+            passed_non_greedy: false,
+            context: EMPTY_LEXER_CONTEXT,
+            actions: Vec::new(),
+        };
+        let _ = epsilon_closure(&atn, [local], &mut contexts, &mut workspace, &mut |_| true);
+        assert!(
+            contexts.len() > before,
+            "an empty lexer path must retain the conservative return frame"
+        );
+    }
+
     // `BLOCK_COMMENT: ('/**/' | '/*' ~[!] .*? '*/'); OTHER: .;`
     //
     // `#[rustfmt::skip]`: this serialized ATN is a positional stream emitted by