fix(codegen): union token sets for a group label shared across alternatives

src/bin/antlr4-rust-gen.rs

@@ -8214,19 +8214,42 @@ fn context_label_accessor(
         .filter(|element| element.label.as_deref() == Some(label.as_str()))
         .collect::<Vec<_>>();
     let first = declarations.first()?;
+    // Token-backed declarations may carry different token sets per alternative
+    // (`r x=(A|B) r | r x=(C|D) r`); ANTLR still declares a single token field
+    // for the label, so union the sets and let the per-alternative selector
+    // checks below reject layouts where one occurrence lookup cannot serve
+    // every alternative. Rule references keep requiring one shared target.
+    let union_token_sets = declarations
+        .iter()
+        .all(|element| !element.token_types.is_empty());
     if (first.target.is_empty() && first.token_types.is_empty())
         || declarations.iter().any(|element| {
             !element.stable_accessor
-                || !same_context_ref_target(element, first)
                 || element.is_list != first.is_list
+                || !(union_token_sets || same_context_ref_target(element, first))
         })
     {
         return None;
     }
 
-    let target = first.target.clone();
-    let token_types = first.token_types.clone();
     let is_list = first.is_list;
+    let reference = embedded::ElementRef {
+        label: None,
+        // Only read for rule-backed labels (rendering gates on empty
+        // `token_types`), where the guard above already forced every
+        // declaration onto one shared target.
+        target: first.target.clone(),
+        token_types: declarations
+            .iter()
+            .flat_map(|element| element.token_types.iter().copied())
+            .collect::<BTreeSet<_>>()
+            .into_iter()
+            .collect(),
+        is_block: first.is_block,
+        is_list,
+        cardinality: embedded::ChildCardinality::ONE,
+        stable_accessor: true,
+    };
     let mut selector = None;
     let mut cardinalities = Vec::with_capacity(alternatives.len());
     for alternative in alternatives {
@@ -8241,7 +8264,7 @@ fn context_label_accessor(
                 alternative
                     .refs
                     .iter()
-                    .filter(|element| context_ref_can_match_target(element, first))
+                    .filter(|element| context_ref_can_match_target(element, &reference))
                     .map(|element| element.cardinality),
             );
             if target_cardinality.max != Some(0) {
@@ -8252,7 +8275,7 @@ fn context_label_accessor(
         }
 
         let alternative_selector =
-            context_label_selector(alternative, &matching, &label, first, is_list)?;
+            context_label_selector(alternative, &matching, &label, &reference, is_list)?;
         if selector.is_some_and(|existing| existing != alternative_selector) {
             return None;
         }
@@ -8273,8 +8296,8 @@ fn context_label_accessor(
     }
     Some(ContextLabelAccessor {
         source_name: label,
-        target,
-        token_types,
+        target: reference.target,
+        token_types: reference.token_types,
         cardinality,
         selector: selector?,
     })
@@ -8302,7 +8325,17 @@ fn context_label_selector(
     }
     let element = matching[0].1;
     if !element.cardinality.is_repeated() {
-        return Some(ContextLabelSelector::Nth(start));
+        // An optional labeled occurrence (`x=A? C` with `C` in the accessor's
+        // token set) must not be shadowed by a following match sliding into
+        // its position when it is absent.
+        let shadowed_when_absent = element.cardinality.min == 0
+            && alternative.refs[first_position + 1..]
+                .iter()
+                .any(|following| {
+                    context_ref_can_match_target(following, target)
+                        && following.cardinality.max != Some(0)
+                });
+        return (!shadowed_when_absent).then_some(ContextLabelSelector::Nth(start));
     }
     let has_following_target = alternative.refs[first_position + 1..]
         .iter()
@@ -13501,6 +13534,38 @@ mod tests {
         insta::assert_snapshot!("typed_context_accessors_latest_context", latest_context);
     }
 
+    #[test]
+    fn token_group_label_across_alternatives_unions_sets_and_guards_shadowing() {
+        let data = parser_fixture_data("multi-alternative-label/T.g4");
+        let rendered = render_parser("TParser", &data).expect("parser should render");
+
+        // The `op` label spans two alternatives with different token groups;
+        // the accessor must match the union of both sets.
+        let calc_context = rendered
+            .split_once("impl<'a, State> CalcContext<'a, State> {")
+            .expect("calc context impl")
+            .1
+            .split_once("impl<State> std::fmt::Display for CalcContext")
+            .expect("calc context display impl")
+            .0;
+        insta::assert_snapshot!("multi_alternative_label_calc_context", calc_context);
+
+        // `lead = PLUS? PLUS unary`: with `lead` absent the unlabeled PLUS
+        // slides into `.nth(0)`, so no accessor may be emitted at all.
+        let shadowed_context = rendered
+            .split_once("impl<'a, State> ShadowedContext<'a, State> {")
+            .expect("shadowed context impl")
+            .1
+            .split_once("impl<State> std::fmt::Display for ShadowedContext")
+            .expect("shadowed context display impl")
+            .0;
+        assert!(
+            !shadowed_context.contains("pub fn lead("),
+            "optional labeled token shadowed by a following union match must drop its accessor\n{shadowed_context}"
+        );
+        insta::assert_snapshot!("multi_alternative_label_shadowed_context", shadowed_context);
+    }
+
     #[test]
     fn non_embedded_parser_action_disables_generated_rule() {
         let rendered =

src/bin/snapshots/antlr4_rust_gen__tests__multi_alternative_label_calc_context.snap

@@ -0,0 +1,66 @@
+---
+source: src/bin/antlr4-rust-gen.rs
+expression: calc_context
+---
+
+    pub fn child_count(&self) -> usize {
+        match &self.__node {
+            __GeneratedRuleContext::Stored(node) => node.child_count(),
+            __GeneratedRuleContext::Active { context, .. } => context.child_count(),
+        }
+    }
+
+    pub fn start(&self) -> __GeneratedTokenView {
+        let token = match &self.__node {
+            __GeneratedRuleContext::Stored(node) => node.start(),
+            __GeneratedRuleContext::Active { context, tokens, .. } => context.start(tokens),
+        };
+        __GeneratedTokenView { text: token.map(|token| token.text_or_empty().to_owned()).unwrap_or_default() }
+    }
+
+    pub fn text(&self) -> String {
+        match &self.__node {
+            __GeneratedRuleContext::Stored(node) => node.text(),
+            __GeneratedRuleContext::Active { context, storage, tokens } => context.text(storage, tokens),
+        }
+    }
+    pub fn calc_children(&self) -> impl Iterator<Item = CalcContext<'a>> + '_ {
+        __rule_children(self.__node, 2)
+            .map(move |node| CalcContext::__from_child_node(node, &self.__invocation_states))
+    }
+    pub fn unary(&self) -> Option<UnaryContext<'a>> {
+        __rule_children(self.__node, 4)
+            .next()
+            .map(|node| UnaryContext::__from_child_node(node, &self.__invocation_states))
+    }
+    pub fn star_token(&self) -> Option<TerminalNode<'a>> {
+        __token_children(self.__node, 8)
+            .next()
+            .map(TerminalNode::new)
+    }
+    pub fn slash_token(&self) -> Option<TerminalNode<'a>> {
+        __token_children(self.__node, 9)
+            .next()
+            .map(TerminalNode::new)
+    }
+    pub fn percent_token(&self) -> Option<TerminalNode<'a>> {
+        __token_children(self.__node, 10)
+            .next()
+            .map(TerminalNode::new)
+    }
+    pub fn plus_token(&self) -> Option<TerminalNode<'a>> {
+        __token_children(self.__node, 11)
+            .next()
+            .map(TerminalNode::new)
+    }
+    pub fn minus_token(&self) -> Option<TerminalNode<'a>> {
+        __token_children(self.__node, 12)
+            .next()
+            .map(TerminalNode::new)
+    }
+    pub fn op(&self) -> Option<TerminalNode<'a>> {
+        __token_children_matching(self.__node, &[8, 9, 10, 11, 12])
+            .nth(0)
+            .map(TerminalNode::new)
+    }
+}

src/bin/snapshots/antlr4_rust_gen__tests__multi_alternative_label_shadowed_context.snap

@@ -0,0 +1,36 @@
+---
+source: src/bin/antlr4-rust-gen.rs
+expression: shadowed_context
+---
+
+    pub fn child_count(&self) -> usize {
+        match &self.__node {
+            __GeneratedRuleContext::Stored(node) => node.child_count(),
+            __GeneratedRuleContext::Active { context, .. } => context.child_count(),
+        }
+    }
+
+    pub fn start(&self) -> __GeneratedTokenView {
+        let token = match &self.__node {
+            __GeneratedRuleContext::Stored(node) => node.start(),
+            __GeneratedRuleContext::Active { context, tokens, .. } => context.start(tokens),
+        };
+        __GeneratedTokenView { text: token.map(|token| token.text_or_empty().to_owned()).unwrap_or_default() }
+    }
+
+    pub fn text(&self) -> String {
+        match &self.__node {
+            __GeneratedRuleContext::Stored(node) => node.text(),
+            __GeneratedRuleContext::Active { context, storage, tokens } => context.text(storage, tokens),
+        }
+    }
+    pub fn unary(&self) -> Result<UnaryContext<'a>, MissingChildError> {
+        __rule_children(self.__node, 4)
+            .next()
+            .map(|node| UnaryContext::__from_child_node(node, &self.__invocation_states))
+            .ok_or_else(|| MissingChildError::new("ShadowedContext", "unary"))
+    }
+    pub fn plus_tokens(&self) -> impl Iterator<Item = TerminalNode<'a>> + '_ {
+        __token_children(self.__node, 11).map(TerminalNode::new)
+    }
+}

tests/antlr4_rust_gen_cli.rs

@@ -458,6 +458,77 @@ mod grouped_token_tests {
     );
 }
 
+#[test]
+fn token_group_label_shared_across_alternatives_unions_the_sets() {
+    let temp = temporary_directory("multi-alternative-label");
+    let grammar = Path::new(env!("CARGO_MANIFEST_DIR"))
+        .join("tests/fixtures/antlr4-rust-gen/multi-alternative-label/T.g4");
+    let out = temp.path().join("generated");
+
+    let output = run_antlr4_rust_gen(&[
+        grammar.as_os_str(),
+        OsStr::new("--out-dir"),
+        out.as_os_str(),
+    ]);
+    assert!(
+        output.status.success(),
+        "stdout: {}\nstderr: {}",
+        utf8(&output.stdout),
+        utf8(&output.stderr)
+    );
+    assert_generated_project(
+        temp.path(),
+        &["t_lexer.rs", "t_parser.rs"],
+        r#"
+#[cfg(test)]
+mod multi_alternative_label_tests {
+    use super::t_lexer::TLexer;
+    use super::t_parser::*;
+    use antlr4_runtime::{CommonTokenStream, InputStream, Parser as _};
+
+    fn parse(input: &str) -> antlr4_runtime::ParsedFile {
+        let lexer = TLexer::new(InputStream::new(input));
+        let tokens = CommonTokenStream::new(lexer);
+        let mut parser = TParser::new(tokens);
+        let root = parser.expr().expect("input should parse");
+        assert_eq!(parser.number_of_syntax_errors(), 0);
+        parser.into_parsed_file(root)
+    }
+
+    #[test]
+    fn op_label_is_available_on_every_alternative_group() {
+        let parsed = parse("a * b + c < d");
+        let expr = parsed
+            .tree()
+            .as_rule()
+            .expect("expr rule")
+            .downcast_ref::<ExprContext>()
+            .expect("typed expr context");
+        let relation = expr.relation().expect("relation child");
+        assert_eq!(relation.op().expect("relation operator").to_string(), "<");
+
+        let calc = relation
+            .relation_children()
+            .next()
+            .expect("left relation")
+            .calc()
+            .expect("calc child");
+        assert_eq!(calc.op().expect("additive operator").to_string(), "+");
+
+        let product = calc.calc_children().next().expect("left calc");
+        assert_eq!(
+            product.op().expect("multiplicative operator").to_string(),
+            "*"
+        );
+
+        let leaf = product.calc_children().next().expect("leaf calc");
+        assert!(leaf.op().is_none(), "primary alternative carries no operator");
+    }
+}
+"#,
+    );
+}
+
 #[test]
 fn compile_parse_tree_pattern_matches_and_binds_against_generated_parser() {
     let temp = temporary_directory("tree-pattern-compile");