From c7c1fb4f079484fadfb47f209cd06132c18e3d91 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 6 Jan 2026 17:03:56 -0800 Subject: [PATCH 1/7] Add UNION RECURSIVE grammar and parser tests --- docs/dev/union_recursive_task1_context.md | 22 +++++++++++ docs/dev/union_recursive_tasks.md | 37 +++++++++++++++++++ ppl/src/main/antlr/OpenSearchPPLLexer.g4 | 2 + ppl/src/main/antlr/OpenSearchPPLParser.g4 | 20 +++++++++- .../sql/ppl/antlr/PPLSyntaxParserTest.java | 24 ++++++++++++ 5 files changed, 104 insertions(+), 1 deletion(-) create mode 100644 docs/dev/union_recursive_task1_context.md create mode 100644 docs/dev/union_recursive_tasks.md diff --git a/docs/dev/union_recursive_task1_context.md b/docs/dev/union_recursive_task1_context.md new file mode 100644 index 00000000000..e3619dd4ccf --- /dev/null +++ b/docs/dev/union_recursive_task1_context.md @@ -0,0 +1,22 @@ +# Task 1 Context - UNION RECURSIVE Syntax/Grammar + +## What changed +- Added `UNION RECURSIVE` command grammar and recursive subpipeline rule in `ppl/src/main/antlr/OpenSearchPPLParser.g4`. +- Added lexer tokens `UNION` and `RECURSIVE` in `ppl/src/main/antlr/OpenSearchPPLLexer.g4`. +- Added parser tests (pass + fail) in `ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java`. + +## Notes/decisions +- The recursive block is parsed as `recursiveSubPipeline : PIPE? subSearch` so it can include an optional leading pipe. +- The name/options use generic `ident EQUAL ...` pairs in grammar for now; AST work should interpret these as `name`, `max_depth`, `max_rows` with validation in Task 2+. +- Changes in `language-grammar/` were reverted per request; only `ppl/` grammar is updated. + +## Tests run +- `./gradlew :ppl:test --tests org.opensearch.sql.ppl.antlr.PPLSyntaxParserTest` + +## Files touched +- `ppl/src/main/antlr/OpenSearchPPLParser.g4` +- `ppl/src/main/antlr/OpenSearchPPLLexer.g4` +- `ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java` + +## Next task pointer +- Task 2: Add AST and logical plan nodes for UNION RECURSIVE, including parsing of `name`, `max_depth`, and `max_rows` options from the grammar output. diff --git a/docs/dev/union_recursive_tasks.md b/docs/dev/union_recursive_tasks.md new file mode 100644 index 00000000000..799df11a2de --- /dev/null +++ b/docs/dev/union_recursive_tasks.md @@ -0,0 +1,37 @@ +# UNION RECURSIVE in PPL - High-Level Task Breakdown + +This document captures the implementation plan for supporting `UNION RECURSIVE` in OpenSearch PPL. + +## 1) Add syntax/grammar support +- Parse `UNION RECURSIVE name= [max_depth=] [max_rows=] [ ]`. +- Treat the recursive subpipeline as a bracketed pipeline node in the AST. +- Extend existing UNION grammar/entrypoints to support the recursive variant. +- Add parser tests for valid/invalid forms (missing name, missing subpipeline, invalid bounds). + +## 2) Extend AST and logical plan nodes +- Add/extend AST nodes to represent anchor pipeline, recursive subpipeline, and options. +- Carry the recursive relation name so it can be resolved inside the subpipeline. +- Add logical plan nodes that map cleanly to Calcite recursive operators (e.g., `RepeatUnion`). +- Validate schema alignment between anchor and recursive outputs. + +## 3) Implement analyzer/resolver rules +- Bind the recursive relation name inside the recursive subpipeline scope. +- Enforce scoping rules (recursive name visible only inside the bracketed block). +- Validate naming conflicts and prevent shadowing by other relations. + +## 4) Add Calcite translation +- Translate anchor + recursive block into Calcite recursive relational algebra. +- Ensure anchor is the base and recursive block references the recursive relation. +- Preserve bag semantics (`UNION ALL`) and unioned output. +- Wire through optional limits (max_depth, max_rows) as planner hints or runtime controls. + +## 5) Enforce runtime safety controls +- Add execution-time guards for max_depth and max_rows. +- Decide defaults and behavior when limits are hit (truncate vs error). +- Add tests for fixpoint termination and limits. + +## 6) Tests and documentation +- Add parser/planner tests for syntax and plan generation. +- Add execution tests for common patterns (BoM, tree traversal, reachability). +- Update PPL user docs with syntax, examples, and limitations. +- Add negative tests for schema mismatch between anchor and recursive block. diff --git a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 index 71162e81bd8..d7f24efa3ff 100644 --- a/ppl/src/main/antlr/OpenSearchPPLLexer.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLLexer.g4 @@ -51,6 +51,8 @@ TIMECHART: 'TIMECHART'; APPENDCOL: 'APPENDCOL'; ADDTOTALS: 'ADDTOTALS'; ADDCOLTOTALS: 'ADDCOLTOTALS'; +UNION: 'UNION'; +RECURSIVE: 'RECURSIVE'; ROW: 'ROW'; COL: 'COL'; EXPAND: 'EXPAND'; diff --git a/ppl/src/main/antlr/OpenSearchPPLParser.g4 b/ppl/src/main/antlr/OpenSearchPPLParser.g4 index 7045796a03c..09a482dba0f 100644 --- a/ppl/src/main/antlr/OpenSearchPPLParser.g4 +++ b/ppl/src/main/antlr/OpenSearchPPLParser.g4 @@ -42,6 +42,10 @@ subSearch : searchCommand (PIPE commands)* ; +recursiveSubPipeline + : PIPE? subSearch + ; + // commands pplCommands : describeCommand @@ -79,6 +83,7 @@ commands | addtotalsCommand | addcoltotalsCommand | appendCommand + | unionRecursiveCommand | expandCommand | flattenCommand | reverseCommand @@ -131,6 +136,7 @@ commandName | REX | APPENDPIPE | REPLACE + | UNION ; searchCommand @@ -543,6 +549,18 @@ appendCommand : APPEND LT_SQR_PRTHS searchCommand? (PIPE commands)* RT_SQR_PRTHS ; +unionRecursiveCommand + : UNION RECURSIVE unionRecursiveNameArg unionRecursiveOption* LT_SQR_PRTHS recursiveSubPipeline RT_SQR_PRTHS + ; + +unionRecursiveNameArg + : ident EQUAL ident + ; + +unionRecursiveOption + : ident EQUAL integerLiteral + ; + multisearchCommand : MULTISEARCH (LT_SQR_PRTHS subSearch RT_SQR_PRTHS)+ ; @@ -1523,6 +1541,7 @@ searchableKeyWord | singleFieldRelevanceFunctionName | multiFieldRelevanceFunctionName | commandName + | RECURSIVE | collectionFunctionName | REGEX | explainMode @@ -1658,4 +1677,3 @@ searchableKeyWord | ROW | COL ; - diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java index dfe79c71edd..903fda525a0 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/antlr/PPLSyntaxParserTest.java @@ -95,6 +95,30 @@ public void testSearchFieldsCommandCrossClusterShouldPass() { assertNotEquals(null, tree); } + @Test + public void testUnionRecursiveCommandShouldPass() { + ParseTree tree = + new PPLSyntaxParser() + .parse( + "source=bill_of_materials | fields component, quantity" + + " | union recursive name=bom_qty max_depth=10 max_rows=100" + + " [ | search source=bill_of_materials | fields component, quantity ]" + + " | fields component, quantity"); + assertNotEquals(null, tree); + } + + @Test + public void testUnionRecursiveCommandWithoutNameShouldFail() { + exceptionRule.expect(SyntaxCheckException.class); + new PPLSyntaxParser().parse("source=t | union recursive max_depth=10 [ | search source=t ]"); + } + + @Test + public void testUnionRecursiveCommandWithoutSubpipelineShouldFail() { + exceptionRule.expect(SyntaxCheckException.class); + new PPLSyntaxParser().parse("source=t | union recursive name=foo"); + } + @Test public void testPerSecondFunctionInTimechartShouldPass() { ParseTree tree = new PPLSyntaxParser().parse("source=t | timechart per_second(a)"); From b262cf79b9ea3dba617bbeb46697b2be6421edeb Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 6 Jan 2026 17:26:57 -0800 Subject: [PATCH 2/7] Add UNION RECURSIVE AST and analyzer hooks --- .../org/opensearch/sql/analysis/Analyzer.java | 6 +++ .../sql/ast/AbstractNodeVisitor.java | 5 ++ .../sql/ast/EmptySourcePropagateVisitor.java | 10 ++++ .../org/opensearch/sql/ast/dsl/AstDSL.java | 10 ++++ .../sql/ast/tree/UnionRecursive.java | 48 +++++++++++++++++++ docs/dev/union_recursive_task2_context.md | 38 +++++++++++++++ .../opensearch/sql/ppl/parser/AstBuilder.java | 38 +++++++++++++++ .../sql/ppl/utils/PPLQueryDataAnonymizer.java | 16 +++++++ .../sql/ppl/parser/AstBuilderTest.java | 14 ++++++ .../ppl/utils/PPLQueryDataAnonymizerTest.java | 10 ++++ 10 files changed, 195 insertions(+) create mode 100644 core/src/main/java/org/opensearch/sql/ast/tree/UnionRecursive.java create mode 100644 docs/dev/union_recursive_task2_context.md diff --git a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java index 24cef144c97..c2692bde9de 100644 --- a/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java +++ b/core/src/main/java/org/opensearch/sql/analysis/Analyzer.java @@ -100,6 +100,7 @@ import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.ast.tree.UnionRecursive; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; @@ -855,6 +856,11 @@ public LogicalPlan visitMultisearch(Multisearch node, AnalysisContext context) { throw getOnlyForCalciteException("Multisearch"); } + @Override + public LogicalPlan visitUnionRecursive(UnionRecursive node, AnalysisContext context) { + throw getOnlyForCalciteException("UnionRecursive"); + } + private LogicalSort buildSort( LogicalPlan child, AnalysisContext context, Integer count, List sortFields) { ExpressionReferenceOptimizer optimizer = diff --git a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java index a6ef5e7547a..97881837272 100644 --- a/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java @@ -87,6 +87,7 @@ import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.ast.tree.UnionRecursive; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; @@ -146,6 +147,10 @@ public T visitAppendPipe(AppendPipe node, C context) { return visitChildren(node, context); } + public T visitUnionRecursive(UnionRecursive node, C context) { + return visitChildren(node, context); + } + public T visitFilter(Filter node, C context) { return visitChildren(node, context); } diff --git a/core/src/main/java/org/opensearch/sql/ast/EmptySourcePropagateVisitor.java b/core/src/main/java/org/opensearch/sql/ast/EmptySourcePropagateVisitor.java index 40c6d2e7acb..e7289796c3f 100644 --- a/core/src/main/java/org/opensearch/sql/ast/EmptySourcePropagateVisitor.java +++ b/core/src/main/java/org/opensearch/sql/ast/EmptySourcePropagateVisitor.java @@ -13,6 +13,7 @@ import org.opensearch.sql.ast.tree.Lookup; import org.opensearch.sql.ast.tree.Relation; import org.opensearch.sql.ast.tree.TableFunction; +import org.opensearch.sql.ast.tree.UnionRecursive; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; @@ -65,6 +66,15 @@ public UnresolvedPlan visitAppendCol(AppendCol node, Void context) { return new AppendCol(node.isOverride(), subSearch).attach(child); } + @Override + public UnresolvedPlan visitUnionRecursive(UnionRecursive node, Void context) { + UnresolvedPlan recursiveSubsearch = node.getRecursiveSubsearch().accept(this, context); + UnresolvedPlan child = node.getChild().get(0).accept(this, context); + return new UnionRecursive( + node.getRelationName(), node.getMaxDepth(), node.getMaxRows(), recursiveSubsearch) + .attach(child); + } + // TODO: Revisit lookup logic here but for now we don't see use case yet @Override public UnresolvedPlan visitLookup(Lookup node, Void context) { diff --git a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java index bf54d2ffd89..12598c309e5 100644 --- a/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java +++ b/core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java @@ -79,6 +79,7 @@ import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.ast.tree.UnionRecursive; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -569,6 +570,15 @@ public static AppendPipe appendPipe(UnresolvedPlan input, UnresolvedPlan subquer return new AppendPipe(subquery).attach(input); } + public static UnionRecursive unionRecursive( + UnresolvedPlan input, + String relationName, + Integer maxDepth, + Integer maxRows, + UnresolvedPlan recursiveSubsearch) { + return new UnionRecursive(relationName, maxDepth, maxRows, recursiveSubsearch).attach(input); + } + public static Trendline.TrendlineComputation computation( Integer numDataPoints, Field dataField, String alias, Trendline.TrendlineType type) { return new Trendline.TrendlineComputation(numDataPoints, dataField, alias, type); diff --git a/core/src/main/java/org/opensearch/sql/ast/tree/UnionRecursive.java b/core/src/main/java/org/opensearch/sql/ast/tree/UnionRecursive.java new file mode 100644 index 00000000000..56cb45da261 --- /dev/null +++ b/core/src/main/java/org/opensearch/sql/ast/tree/UnionRecursive.java @@ -0,0 +1,48 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ast.tree; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import lombok.EqualsAndHashCode; +import lombok.Getter; +import lombok.RequiredArgsConstructor; +import lombok.Setter; +import lombok.ToString; +import org.opensearch.sql.ast.AbstractNodeVisitor; +import org.opensearch.sql.ast.Node; + +/** Logical plan node of UNION RECURSIVE, the interface for recursive union queries. */ +@Getter +@Setter +@ToString +@EqualsAndHashCode(callSuper = false) +@RequiredArgsConstructor +public class UnionRecursive extends UnresolvedPlan { + + private final String relationName; + private final Integer maxDepth; + private final Integer maxRows; + private final UnresolvedPlan recursiveSubsearch; + + private UnresolvedPlan child; + + @Override + public UnionRecursive attach(UnresolvedPlan child) { + this.child = child; + return this; + } + + @Override + public List getChild() { + return this.child == null ? ImmutableList.of() : ImmutableList.of(this.child); + } + + @Override + public T accept(AbstractNodeVisitor visitor, C context) { + return visitor.visitUnionRecursive(this, context); + } +} diff --git a/docs/dev/union_recursive_task2_context.md b/docs/dev/union_recursive_task2_context.md new file mode 100644 index 00000000000..58134d9d278 --- /dev/null +++ b/docs/dev/union_recursive_task2_context.md @@ -0,0 +1,38 @@ +# Task 2 Context - UNION RECURSIVE AST + Logical Plan Hooks + +## What changed +- Added AST node `UnionRecursive` in `core/src/main/java/org/opensearch/sql/ast/tree/UnionRecursive.java`. +- Added AST DSL helper `unionRecursive(...)` in `core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java`. +- Updated visitors and analyzer hooks: + - `core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java` + - `core/src/main/java/org/opensearch/sql/ast/EmptySourcePropagateVisitor.java` + - `core/src/main/java/org/opensearch/sql/analysis/Analyzer.java` +- Added AST builder support to parse name/max_depth/max_rows and build `UnionRecursive` in `ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java`. +- Added anonymizer support in `ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java`. +- Added tests: + - `ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java` + - `ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java` + +## Notes/decisions +- `UnionRecursive` stores `relationName`, `maxDepth`, `maxRows`, and `recursiveSubsearch`, with the pipeline child attached separately (same pattern as `Append`). +- Name/options parsing uses `StringUtils.unquoteIdentifier` and validates: + - name argument must be `name=` + - only `max_depth` and `max_rows` are accepted; duplicates are rejected +- Analyzer throws `getOnlyForCalciteException("UnionRecursive")` to keep this Calcite-only for now. + +## Tests run +- `./gradlew :ppl:test --tests org.opensearch.sql.ppl.parser.AstBuilderTest --tests org.opensearch.sql.ppl.utils.PPLQueryDataAnonymizerTest` + +## Files touched +- `core/src/main/java/org/opensearch/sql/ast/tree/UnionRecursive.java` +- `core/src/main/java/org/opensearch/sql/ast/dsl/AstDSL.java` +- `core/src/main/java/org/opensearch/sql/ast/AbstractNodeVisitor.java` +- `core/src/main/java/org/opensearch/sql/ast/EmptySourcePropagateVisitor.java` +- `core/src/main/java/org/opensearch/sql/analysis/Analyzer.java` +- `ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java` +- `ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java` +- `ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java` +- `ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java` + +## Next task pointer +- Task 3: Analyzer/resolver rules for recursive relation scoping and name resolution inside the recursive block. diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 3f4f3049365..5a45609fe2c 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -112,6 +112,7 @@ import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.ast.tree.UnionRecursive; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Window; import org.opensearch.sql.calcite.plan.OpenSearchConstants; @@ -1208,6 +1209,43 @@ public UnresolvedPlan visitAppendCommand(OpenSearchPPLParser.AppendCommandContex return new Append(subsearch); } + @Override + public UnresolvedPlan visitUnionRecursiveCommand( + OpenSearchPPLParser.UnionRecursiveCommandContext ctx) { + String nameKey = StringUtils.unquoteIdentifier(ctx.unionRecursiveNameArg().ident(0).getText()); + String relationName = + StringUtils.unquoteIdentifier(ctx.unionRecursiveNameArg().ident(1).getText()); + if (!"name".equalsIgnoreCase(nameKey)) { + throw new SemanticCheckException("UNION RECURSIVE requires name="); + } + + Integer maxDepth = null; + Integer maxRows = null; + for (OpenSearchPPLParser.UnionRecursiveOptionContext optionCtx : ctx.unionRecursiveOption()) { + String optionName = StringUtils.unquoteIdentifier(optionCtx.ident().getText()); + int value = Integer.parseInt(optionCtx.integerLiteral().getText()); + switch (optionName.toLowerCase(Locale.ROOT)) { + case "max_depth": + if (maxDepth != null) { + throw new SemanticCheckException("max_depth specified more than once"); + } + maxDepth = value; + break; + case "max_rows": + if (maxRows != null) { + throw new SemanticCheckException("max_rows specified more than once"); + } + maxRows = value; + break; + default: + throw new SemanticCheckException("invalid UNION RECURSIVE option: " + optionName); + } + } + + UnresolvedPlan recursiveSubsearch = visitSubSearch(ctx.recursiveSubPipeline().subSearch()); + return new UnionRecursive(relationName, maxDepth, maxRows, recursiveSubsearch); + } + @Override public UnresolvedPlan visitMultisearchCommand(OpenSearchPPLParser.MultisearchCommandContext ctx) { List subsearches = new ArrayList<>(); diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java index 7d04ad8e6ad..ac8e4b9bc41 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizer.java @@ -101,6 +101,7 @@ import org.opensearch.sql.ast.tree.SubqueryAlias; import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; +import org.opensearch.sql.ast.tree.UnionRecursive; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; @@ -654,6 +655,21 @@ public String visitAppend(Append node, String context) { return StringUtils.format("%s | append [%s ]", child, subsearch); } + @Override + public String visitUnionRecursive(UnionRecursive node, String context) { + String child = node.getChild().get(0).accept(this, context); + String subsearch = anonymizeData(node.getRecursiveSubsearch()); + StringBuilder options = new StringBuilder(); + options.append("name=").append(maskField(node.getRelationName())); + if (node.getMaxDepth() != null) { + options.append(" max_depth=").append(MASK_LITERAL); + } + if (node.getMaxRows() != null) { + options.append(" max_rows=").append(MASK_LITERAL); + } + return StringUtils.format("%s | union recursive %s [%s ]", child, options, subsearch); + } + @Override public String visitMultisearch(Multisearch node, String context) { List anonymizedSubsearches = new ArrayList<>(); diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index 9e1cfe05a4b..d5ed02ce313 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -50,6 +50,7 @@ import static org.opensearch.sql.ast.dsl.AstDSL.stringLiteral; import static org.opensearch.sql.ast.dsl.AstDSL.tableFunction; import static org.opensearch.sql.ast.dsl.AstDSL.trendline; +import static org.opensearch.sql.ast.dsl.AstDSL.unionRecursive; import static org.opensearch.sql.ast.dsl.AstDSL.unresolvedArg; import static org.opensearch.sql.ast.tree.Trendline.TrendlineType.SMA; import static org.opensearch.sql.lang.PPLLangSpec.PPL_SPEC; @@ -1017,6 +1018,19 @@ public void testAppendPipe() { defaultStatsArgs()))); } + @Test + public void testUnionRecursive() { + assertEqual( + "source=t | fields a | union recursive name=rel max_depth=2 max_rows=5 [ source=t |" + + " fields a ]", + unionRecursive( + projectWithArg(relation("t"), defaultFieldsArgs(), field("a")), + "rel", + 2, + 5, + projectWithArg(relation("t"), defaultFieldsArgs(), field("a")))); + } + public void testTrendline() { assertEqual( "source=t | trendline sma(5, test_field) as test_field_alias sma(1, test_field_2) as" diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index 2fd08988f6b..ce673ce206c 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -547,6 +547,16 @@ public void testAppend() { anonymize("source=t | stats count() by b | append [ ]")); } + @Test + public void testUnionRecursive() { + assertEquals( + "source=table | fields + identifier | union recursive name=identifier max_depth=*** " + + "max_rows=*** [source=table | where identifier = *** | fields + identifier ]", + anonymize( + "source=t | fields a | union recursive name=bom max_depth=3 max_rows=100" + + " [ source=b | where a = 1 | fields a ]")); + } + @Test // Same as SQL, select * from a as b -> SELECT * FROM table AS identifier public void testSubqueryAlias() { From cb22b3d6df00f19fa42b2737f80aedb866172417 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 6 Jan 2026 18:06:45 -0800 Subject: [PATCH 3/7] Add UNION RECURSIVE subsearch validation --- docs/dev/union_recursive_task3_context.md | 31 ++++ .../opensearch/sql/ppl/parser/AstBuilder.java | 2 + .../ppl/utils/UnionRecursiveValidator.java | 149 ++++++++++++++++++ .../sql/ppl/parser/AstBuilderTest.java | 38 ++++- .../ppl/utils/PPLQueryDataAnonymizerTest.java | 2 +- 5 files changed, 219 insertions(+), 3 deletions(-) create mode 100644 docs/dev/union_recursive_task3_context.md create mode 100644 ppl/src/main/java/org/opensearch/sql/ppl/utils/UnionRecursiveValidator.java diff --git a/docs/dev/union_recursive_task3_context.md b/docs/dev/union_recursive_task3_context.md new file mode 100644 index 00000000000..a35e5cbfdfd --- /dev/null +++ b/docs/dev/union_recursive_task3_context.md @@ -0,0 +1,31 @@ +# Task 3 Context - UNION RECURSIVE Analyzer/Resolver Rules + +## What changed +- Added `UnionRecursiveValidator` to enforce recursive relation usage and scoping in + `ppl/src/main/java/org/opensearch/sql/ppl/utils/UnionRecursiveValidator.java`. +- Wired validation into `ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java` for + `UNION RECURSIVE` subsearches. + +## Validation rules +- Recursive subsearch must reference the recursive relation name at least once. +- The recursive relation name cannot be used as an alias inside the recursive block. +- The recursive relation name cannot be combined with other sources in a multi-source relation. + +## Tests added/updated +- Updated `testUnionRecursive` to reference the recursive relation. +- Added negative tests for missing recursive reference, alias conflicts, and mixed sources in + `ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java`. +- Updated `ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java` to + reference the recursive relation in the subsearch. + +## Tests run +- `./gradlew :ppl:test --tests org.opensearch.sql.ppl.parser.AstBuilderTest --tests org.opensearch.sql.ppl.utils.PPLQueryDataAnonymizerTest --tests org.opensearch.sql.ppl.calcite.CalcitePPLUnionRecursiveTest --tests org.opensearch.sql.ppl.antlr.PPLSyntaxParserTest` + +## Files touched +- `ppl/src/main/java/org/opensearch/sql/ppl/utils/UnionRecursiveValidator.java` +- `ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java` +- `ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java` +- `ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java` + +## Next task pointer +- Task 4: Calcite translation for `UNION RECURSIVE` (RepeatUnion + recursion scoping). diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java index 5a45609fe2c..51d81a4f556 100644 --- a/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java +++ b/ppl/src/main/java/org/opensearch/sql/ppl/parser/AstBuilder.java @@ -131,6 +131,7 @@ import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParser.StatsByClauseContext; import org.opensearch.sql.ppl.antlr.parser.OpenSearchPPLParserBaseVisitor; import org.opensearch.sql.ppl.utils.ArgumentFactory; +import org.opensearch.sql.ppl.utils.UnionRecursiveValidator; /** Class of building the AST. Refines the visit path and build the AST nodes */ public class AstBuilder extends OpenSearchPPLParserBaseVisitor { @@ -1243,6 +1244,7 @@ public UnresolvedPlan visitUnionRecursiveCommand( } UnresolvedPlan recursiveSubsearch = visitSubSearch(ctx.recursiveSubPipeline().subSearch()); + UnionRecursiveValidator.validate(recursiveSubsearch, relationName); return new UnionRecursive(relationName, maxDepth, maxRows, recursiveSubsearch); } diff --git a/ppl/src/main/java/org/opensearch/sql/ppl/utils/UnionRecursiveValidator.java b/ppl/src/main/java/org/opensearch/sql/ppl/utils/UnionRecursiveValidator.java new file mode 100644 index 00000000000..7fa5ea33b14 --- /dev/null +++ b/ppl/src/main/java/org/opensearch/sql/ppl/utils/UnionRecursiveValidator.java @@ -0,0 +1,149 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.utils; + +import java.util.List; +import java.util.Locale; +import org.opensearch.sql.ast.Node; +import org.opensearch.sql.ast.expression.QualifiedName; +import org.opensearch.sql.ast.tree.Append; +import org.opensearch.sql.ast.tree.AppendPipe; +import org.opensearch.sql.ast.tree.Join; +import org.opensearch.sql.ast.tree.Lookup; +import org.opensearch.sql.ast.tree.Relation; +import org.opensearch.sql.ast.tree.RelationSubquery; +import org.opensearch.sql.ast.tree.SubqueryAlias; +import org.opensearch.sql.ast.tree.UnionRecursive; +import org.opensearch.sql.ast.tree.UnresolvedPlan; +import org.opensearch.sql.exception.SemanticCheckException; + +public final class UnionRecursiveValidator { + + private UnionRecursiveValidator() {} + + public static void validate(UnresolvedPlan recursiveSubsearch, String relationName) { + ValidationState state = new ValidationState(relationName); + state.visit(recursiveSubsearch); + if (!state.isRecursiveRelationReferenced()) { + throw new SemanticCheckException( + "UNION RECURSIVE subsearch must reference recursive relation: " + relationName); + } + } + + private static final class ValidationState { + private final String relationName; + private final String relationNameLower; + private boolean recursiveRelationReferenced; + + private ValidationState(String relationName) { + this.relationName = relationName; + this.relationNameLower = relationName.toLowerCase(Locale.ROOT); + } + + private boolean isRecursiveRelationReferenced() { + return recursiveRelationReferenced; + } + + private void visit(UnresolvedPlan plan) { + if (plan == null) { + return; + } + if (plan instanceof Relation relation) { + visitRelation(relation); + return; + } + if (plan instanceof SubqueryAlias alias) { + visitSubqueryAlias(alias); + return; + } + if (plan instanceof RelationSubquery relationSubquery) { + visitRelationSubquery(relationSubquery); + return; + } + if (plan instanceof Join join) { + for (UnresolvedPlan child : join.getChildren()) { + visit(child); + } + return; + } + if (plan instanceof Lookup lookup) { + visitChildren(lookup.getChild()); + visit(lookup.getLookupRelation()); + return; + } + if (plan instanceof Append append) { + visitChildren(append.getChild()); + visit(append.getSubSearch()); + return; + } + if (plan instanceof AppendPipe appendPipe) { + visitChildren(appendPipe.getChild()); + visit(appendPipe.getSubQuery()); + return; + } + if (plan instanceof UnionRecursive unionRecursive) { + visitChildren(unionRecursive.getChild()); + visit(unionRecursive.getRecursiveSubsearch()); + return; + } + visitChildren(plan.getChild()); + } + + private void visitChildren(List children) { + if (children == null) { + return; + } + for (Node child : children) { + if (child instanceof UnresolvedPlan planChild) { + visit(planChild); + } + } + } + + private void visitRelation(Relation relation) { + List names = relation.getQualifiedNames(); + boolean containsRecursive = false; + for (QualifiedName name : names) { + if (isRecursiveQualifiedName(name)) { + containsRecursive = true; + break; + } + } + if (containsRecursive) { + if (names.size() > 1) { + throw new SemanticCheckException( + "UNION RECURSIVE relation name cannot be combined with other sources: " + + relationName); + } + recursiveRelationReferenced = true; + } + } + + private void visitSubqueryAlias(SubqueryAlias alias) { + if (isRecursiveName(alias.getAlias())) { + throw new SemanticCheckException( + "UNION RECURSIVE relation name conflicts with alias: " + relationName); + } + visitChildren(alias.getChild()); + } + + private void visitRelationSubquery(RelationSubquery relationSubquery) { + if (isRecursiveName(relationSubquery.getAliasAsTableName())) { + throw new SemanticCheckException( + "UNION RECURSIVE relation name conflicts with alias: " + relationName); + } + visitChildren(relationSubquery.getChild()); + } + + private boolean isRecursiveQualifiedName(QualifiedName name) { + return name.getParts().size() == 1 && isRecursiveName(name.getParts().get(0)); + } + + private boolean isRecursiveName(String name) { + return name != null && name.toLowerCase(Locale.ROOT).equals(relationNameLower); + } + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java index d5ed02ce313..a1a1a17d920 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/parser/AstBuilderTest.java @@ -1021,14 +1021,48 @@ public void testAppendPipe() { @Test public void testUnionRecursive() { assertEqual( - "source=t | fields a | union recursive name=rel max_depth=2 max_rows=5 [ source=t |" + "source=t | fields a | union recursive name=rel max_depth=2 max_rows=5 [ source=rel |" + " fields a ]", unionRecursive( projectWithArg(relation("t"), defaultFieldsArgs(), field("a")), "rel", 2, 5, - projectWithArg(relation("t"), defaultFieldsArgs(), field("a")))); + projectWithArg(relation("rel"), defaultFieldsArgs(), field("a")))); + } + + @Test + public void testUnionRecursiveRequiresRecursiveReference() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> plan("source=t | union recursive name=rel [ source=t | fields a ]")); + assertTrue( + exception + .getMessage() + .contains("UNION RECURSIVE subsearch must reference recursive relation")); + } + + @Test + public void testUnionRecursiveAliasConflict() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> plan("source=t | union recursive name=rel [ source=s as rel | fields a ]")); + assertTrue( + exception.getMessage().contains("UNION RECURSIVE relation name conflicts with alias")); + } + + @Test + public void testUnionRecursiveMixedRelationReference() { + SemanticCheckException exception = + assertThrows( + SemanticCheckException.class, + () -> plan("source=t | union recursive name=rel [ source=rel,other | fields a ]")); + assertTrue( + exception + .getMessage() + .contains("UNION RECURSIVE relation name cannot be combined with other sources")); } public void testTrendline() { diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java index ce673ce206c..4b09304ec65 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/utils/PPLQueryDataAnonymizerTest.java @@ -554,7 +554,7 @@ public void testUnionRecursive() { + "max_rows=*** [source=table | where identifier = *** | fields + identifier ]", anonymize( "source=t | fields a | union recursive name=bom max_depth=3 max_rows=100" - + " [ source=b | where a = 1 | fields a ]")); + + " [ source=bom | where a = 1 | fields a ]")); } @Test From 452c3d889b3a6fdc133f3a98dbd8f372aa617c88 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Tue, 6 Jan 2026 18:07:03 -0800 Subject: [PATCH 4/7] Add Calcite translation for UNION RECURSIVE --- .../sql/calcite/CalcitePlanContext.java | 41 ++++++++ .../sql/calcite/CalciteRelNodeVisitor.java | 94 +++++++++++++++++++ docs/dev/union_recursive_task4_context.md | 32 +++++++ .../calcite/CalcitePPLUnionRecursiveTest.java | 44 +++++++++ 4 files changed, 211 insertions(+) create mode 100644 docs/dev/union_recursive_task4_context.md create mode 100644 ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java index d9cc7251e14..ffbf4cfd082 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java @@ -8,7 +8,9 @@ import static org.opensearch.sql.calcite.utils.OpenSearchTypeFactory.TYPE_FACTORY; import java.sql.Connection; +import java.util.ArrayDeque; import java.util.ArrayList; +import java.util.Deque; import java.util.HashMap; import java.util.List; import java.util.Map; @@ -17,6 +19,7 @@ import java.util.function.BiFunction; import lombok.Getter; import lombok.Setter; +import org.apache.calcite.rel.type.RelDataType; import org.apache.calcite.rex.RexCorrelVariable; import org.apache.calcite.rex.RexLambdaRef; import org.apache.calcite.rex.RexNode; @@ -59,6 +62,7 @@ public class CalcitePlanContext { private final Stack correlVar = new Stack<>(); private final Stack> windowPartitions = new Stack<>(); + private final Deque recursiveRelations = new ArrayDeque<>(); @Getter public Map rexLambdaRefMap; @@ -99,6 +103,7 @@ private CalcitePlanContext(CalcitePlanContext parent) { this.rexLambdaRefMap = new HashMap<>(); // New map for lambda variables this.capturedVariables = new ArrayList<>(); // New list for captured variables this.inLambdaContext = true; // Mark that we're inside a lambda + this.recursiveRelations.addAll(parent.recursiveRelations); } public RexNode resolveJoinCondition( @@ -130,6 +135,29 @@ public Optional peekCorrelVar() { } } + public void pushRecursiveRelation(String relationName, RelDataType rowType) { + recursiveRelations.push(new RecursiveRelationInfo(relationName, rowType)); + } + + public Optional findRecursiveRelation(String relationName) { + if (relationName == null) { + return Optional.empty(); + } + String relationNameLower = relationName.toLowerCase(java.util.Locale.ROOT); + for (RecursiveRelationInfo info : recursiveRelations) { + if (info.nameLower.equals(relationNameLower)) { + return Optional.of(info); + } + } + return Optional.empty(); + } + + public void popRecursiveRelation() { + if (!recursiveRelations.isEmpty()) { + recursiveRelations.pop(); + } + } + /** * Creates a clone of this context that shares the relBuilder with the parent. This allows lambda * expressions to reference fields from the current row while having their own lambda variable @@ -206,4 +234,17 @@ public RexLambdaRef captureVariable(RexNode fieldRef, String fieldName) { return lambdaRef; } + + @Getter + public static final class RecursiveRelationInfo { + private final String name; + private final String nameLower; + private final RelDataType rowType; + + private RecursiveRelationInfo(String name, RelDataType rowType) { + this.name = name; + this.nameLower = name.toLowerCase(java.util.Locale.ROOT); + this.rowType = rowType; + } + } } diff --git a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java index 19dce3e3609..72d3b6b2bfe 100644 --- a/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java +++ b/core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java @@ -91,6 +91,7 @@ import org.opensearch.sql.ast.expression.ParseMethod; import org.opensearch.sql.ast.expression.PatternMethod; import org.opensearch.sql.ast.expression.PatternMode; +import org.opensearch.sql.ast.expression.QualifiedName; import org.opensearch.sql.ast.expression.Span; import org.opensearch.sql.ast.expression.SpanUnit; import org.opensearch.sql.ast.expression.UnresolvedExpression; @@ -142,6 +143,7 @@ import org.opensearch.sql.ast.tree.TableFunction; import org.opensearch.sql.ast.tree.Trendline; import org.opensearch.sql.ast.tree.Trendline.TrendlineType; +import org.opensearch.sql.ast.tree.UnionRecursive; import org.opensearch.sql.ast.tree.UnresolvedPlan; import org.opensearch.sql.ast.tree.Values; import org.opensearch.sql.ast.tree.Window; @@ -184,6 +186,13 @@ public RelNode analyze(UnresolvedPlan unresolved, CalcitePlanContext context) { @Override public RelNode visitRelation(Relation node, CalcitePlanContext context) { + Optional recursiveRelation = + findRecursiveRelation(node, context); + if (recursiveRelation.isPresent()) { + CalcitePlanContext.RecursiveRelationInfo relationInfo = recursiveRelation.get(); + context.relBuilder.transientScan(relationInfo.getName(), relationInfo.getRowType()); + return context.relBuilder.peek(); + } DataSourceSchemaIdentifierNameResolver nameResolver = new DataSourceSchemaIdentifierNameResolver( dataSourceService, node.getTableQualifiedName().getParts()); @@ -207,6 +216,19 @@ public RelNode visitRelation(Relation node, CalcitePlanContext context) { return scan; } + private Optional findRecursiveRelation( + Relation node, CalcitePlanContext context) { + List qualifiedNames = node.getQualifiedNames(); + if (qualifiedNames.size() != 1) { + return Optional.empty(); + } + QualifiedName qualifiedName = qualifiedNames.get(0); + if (qualifiedName.getParts().size() != 1) { + return Optional.empty(); + } + return context.findRecursiveRelation(qualifiedName.getParts().get(0)); + } + // This is a tool method to add an existed RelOptTable to builder stack, not used for now private RelBuilder scan(RelOptTable tableSchema, CalcitePlanContext context) { final RelNode scan = @@ -2229,6 +2251,78 @@ public RelNode visitAppend(Append node, CalcitePlanContext context) { return mergeTableAndResolveColumnConflict(mainNode, subsearchNode, context); } + @Override + public RelNode visitUnionRecursive(UnionRecursive node, CalcitePlanContext context) { + visitChildren(node, context); + RelNode anchorNode = context.relBuilder.build(); + RelDataType anchorRowType = anchorNode.getRowType(); + + context.pushRecursiveRelation(node.getRelationName(), anchorRowType); + try { + UnresolvedPlan prunedSubSearch = + node.getRecursiveSubsearch().accept(new EmptySourcePropagateVisitor(), null); + prunedSubSearch.accept(this, context); + } finally { + context.popRecursiveRelation(); + } + + RelNode recursiveNode = context.relBuilder.build(); + validateUnionRecursiveSchema(anchorRowType, recursiveNode.getRowType(), node.getRelationName()); + + context.relBuilder.push(anchorNode); + context.relBuilder.push(recursiveNode); + int iterationLimit = node.getMaxDepth() == null ? -1 : node.getMaxDepth(); + context.relBuilder.repeatUnion(node.getRelationName(), true, iterationLimit); + + if (node.getMaxRows() != null) { + PlanUtils.replaceTop( + context.relBuilder, + LogicalSystemLimit.create( + SystemLimitType.QUERY_SIZE_LIMIT, + context.relBuilder.peek(), + context.relBuilder.literal(node.getMaxRows()))); + } + + return context.relBuilder.peek(); + } + + private void validateUnionRecursiveSchema( + RelDataType anchorRowType, RelDataType recursiveRowType, String relationName) { + List anchorFields = anchorRowType.getFieldList(); + List recursiveFields = recursiveRowType.getFieldList(); + + if (anchorFields.size() != recursiveFields.size()) { + throw new SemanticCheckException( + "UNION RECURSIVE schema mismatch for relation " + + relationName + + ": anchor field count " + + anchorFields.size() + + " does not match recursive field count " + + recursiveFields.size()); + } + + for (int i = 0; i < anchorFields.size(); i++) { + RelDataTypeField anchorField = anchorFields.get(i); + RelDataTypeField recursiveField = recursiveFields.get(i); + if (!anchorField.getName().equalsIgnoreCase(recursiveField.getName())) { + throw new SemanticCheckException( + "UNION RECURSIVE schema mismatch for relation " + + relationName + + ": anchor field name " + + anchorField.getName() + + " does not match recursive field name " + + recursiveField.getName()); + } + if (!SqlTypeUtil.equalSansNullability(anchorField.getType(), recursiveField.getType())) { + throw new SemanticCheckException( + "UNION RECURSIVE schema mismatch for relation " + + relationName + + ": field type mismatch for " + + anchorField.getName()); + } + } + } + private RelNode mergeTableAndResolveColumnConflict( RelNode mainNode, RelNode subqueryNode, CalcitePlanContext context) { // Use shared schema merging logic that handles type conflicts via field renaming diff --git a/docs/dev/union_recursive_task4_context.md b/docs/dev/union_recursive_task4_context.md new file mode 100644 index 00000000000..93d223c9117 --- /dev/null +++ b/docs/dev/union_recursive_task4_context.md @@ -0,0 +1,32 @@ +# Task 4 Context - UNION RECURSIVE Calcite Translation + +## What changed +- Added recursive relation scoping to `core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java`: + - Maintains a stack of recursive relation info (name + row type). + - Provides push/pop and lookup for recursive relations. +- Updated `core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java`: + - Resolves recursive relation references to `transientScan` when in recursive scope. + - Added `visitUnionRecursive` to build `RepeatUnion` with `max_depth` and `max_rows`. + - Validates anchor/recursive schema alignment (field count, names, types). + - Applies `LogicalSystemLimit` when `max_rows` is set. +- Added Calcite planner tests: + - `ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java`. + +## Notes/decisions +- `RepeatUnion` is built by pushing anchor then recursive nodes and calling + `relBuilder.repeatUnion(relationName, true, iterationLimit)`. +- Recursive relation references are only rewritten inside the recursive block; + outside the block, relation names resolve normally. +- Schema checks use case-insensitive name comparison and + `SqlTypeUtil.equalSansNullability` for types. + +## Tests run +- `./gradlew :ppl:test --tests org.opensearch.sql.ppl.parser.AstBuilderTest --tests org.opensearch.sql.ppl.utils.PPLQueryDataAnonymizerTest --tests org.opensearch.sql.ppl.calcite.CalcitePPLUnionRecursiveTest --tests org.opensearch.sql.ppl.antlr.PPLSyntaxParserTest` + +## Files touched +- `core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java` +- `core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java` +- `ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java` + +## Next task pointer +- Task 5: Runtime safety controls for max depth/rows enforcement and termination behavior. diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java new file mode 100644 index 00000000000..05a956fd053 --- /dev/null +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java @@ -0,0 +1,44 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.sql.ppl.calcite; + +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.test.CalciteAssert; +import org.junit.Assert; +import org.junit.Test; +import org.opensearch.sql.exception.SemanticCheckException; + +public class CalcitePPLUnionRecursiveTest extends CalcitePPLAbstractTest { + + public CalcitePPLUnionRecursiveTest() { + super(CalciteAssert.SchemaSpec.SCOTT_WITH_TEMPORAL); + } + + @Test + public void testUnionRecursivePlan() { + String ppl = + "source=EMP | fields EMPNO, MGR | union recursive name=emp_hierarchy max_depth=1" + + " [ source=emp_hierarchy | fields EMPNO, MGR ] | fields EMPNO, MGR"; + RelNode root = getRelNode(ppl); + String expectedLogical = + "LogicalRepeatUnion(iterationLimit=[1], all=[true])\n" + + " LogicalTableSpool(readType=[LAZY], writeType=[LAZY], table=[[emp_hierarchy]])\n" + + " LogicalProject(EMPNO=[$0], MGR=[$3])\n" + + " LogicalTableScan(table=[[scott, EMP]])\n" + + " LogicalTableSpool(readType=[LAZY], writeType=[LAZY], table=[[emp_hierarchy]])\n" + + " LogicalTableScan(table=[[emp_hierarchy]])\n"; + verifyLogical(root, expectedLogical); + } + + @Test + public void testUnionRecursiveSchemaMismatch() { + String ppl = + "source=EMP | fields EMPNO, MGR | union recursive name=emp_hierarchy" + + " [ source=emp_hierarchy | fields EMPNO ]"; + Throwable t = Assert.assertThrows(SemanticCheckException.class, () -> getRelNode(ppl)); + Assert.assertTrue(t.getMessage().contains("UNION RECURSIVE schema mismatch")); + } +} From dc043822a95e01653da908734b372caa380d94cc Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 7 Jan 2026 08:43:44 -0800 Subject: [PATCH 5/7] Add SQL generation test for UNION RECURSIVE --- docs/dev/union_recursive_task4_context.md | 5 + .../rel2sql/OpenSearchRelToSqlConverter.java | 91 +++++++++++++++++++ .../calcite/CalcitePPLUnionRecursiveTest.java | 17 ++++ 3 files changed, 113 insertions(+) create mode 100644 ppl/src/main/java/org/apache/calcite/rel/rel2sql/OpenSearchRelToSqlConverter.java diff --git a/docs/dev/union_recursive_task4_context.md b/docs/dev/union_recursive_task4_context.md index 93d223c9117..150868d5ff6 100644 --- a/docs/dev/union_recursive_task4_context.md +++ b/docs/dev/union_recursive_task4_context.md @@ -11,6 +11,9 @@ - Applies `LogicalSystemLimit` when `max_rows` is set. - Added Calcite planner tests: - `ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java`. +- Added SQL generation support for recursive CTEs: + - `ppl/src/main/java/org/apache/calcite/rel/rel2sql/OpenSearchRelToSqlConverter.java`. + - SQL test in `ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java`. ## Notes/decisions - `RepeatUnion` is built by pushing anchor then recursive nodes and calling @@ -22,10 +25,12 @@ ## Tests run - `./gradlew :ppl:test --tests org.opensearch.sql.ppl.parser.AstBuilderTest --tests org.opensearch.sql.ppl.utils.PPLQueryDataAnonymizerTest --tests org.opensearch.sql.ppl.calcite.CalcitePPLUnionRecursiveTest --tests org.opensearch.sql.ppl.antlr.PPLSyntaxParserTest` +- `./gradlew :ppl:test --tests org.opensearch.sql.ppl.calcite.CalcitePPLUnionRecursiveTest` ## Files touched - `core/src/main/java/org/opensearch/sql/calcite/CalcitePlanContext.java` - `core/src/main/java/org/opensearch/sql/calcite/CalciteRelNodeVisitor.java` +- `ppl/src/main/java/org/apache/calcite/rel/rel2sql/OpenSearchRelToSqlConverter.java` - `ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java` ## Next task pointer diff --git a/ppl/src/main/java/org/apache/calcite/rel/rel2sql/OpenSearchRelToSqlConverter.java b/ppl/src/main/java/org/apache/calcite/rel/rel2sql/OpenSearchRelToSqlConverter.java new file mode 100644 index 00000000000..3fd97d1f211 --- /dev/null +++ b/ppl/src/main/java/org/apache/calcite/rel/rel2sql/OpenSearchRelToSqlConverter.java @@ -0,0 +1,91 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.apache.calcite.rel.rel2sql; + +import com.google.common.collect.ImmutableList; +import java.util.List; +import java.util.Objects; +import org.apache.calcite.plan.RelOptTable; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.RepeatUnion; +import org.apache.calcite.rel.core.TableSpool; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rel.type.RelDataTypeField; +import org.apache.calcite.sql.SqlIdentifier; +import org.apache.calcite.sql.SqlLiteral; +import org.apache.calcite.sql.SqlNode; +import org.apache.calcite.sql.SqlNodeList; +import org.apache.calcite.sql.SqlOperator; +import org.apache.calcite.sql.SqlSelect; +import org.apache.calcite.sql.SqlWith; +import org.apache.calcite.sql.SqlWithItem; +import org.apache.calcite.sql.fun.SqlStdOperatorTable; +import org.apache.calcite.sql.parser.SqlParserPos; + +/** OpenSearch-specific RelToSqlConverter extension for recursive CTE support. */ +public class OpenSearchRelToSqlConverter extends RelToSqlConverter { + + public OpenSearchRelToSqlConverter(org.apache.calcite.sql.SqlDialect dialect) { + super(dialect); + } + + /** Convert a RelNode to a SqlNode while preserving top-level WITH RECURSIVE clauses. */ + public SqlNode toSqlNode(RelNode rel) { + Result result = visitRoot(rel); + SqlNode node = result.node; + if (node.getKind() == org.apache.calcite.sql.SqlKind.WITH) { + return node; + } + return result.asStatement(); + } + + /** Visits a RepeatUnion; called by {@link #dispatch} via reflection. */ + public Result visit(RepeatUnion e) { + Result seedResult = visitInput(e, 0); + Result recursiveResult = visitInput(e, 1); + + SqlNode seedNode = seedResult.asSelect(); + SqlNode recursiveNode = recursiveResult.asSelect(); + SqlOperator operator = e.all ? SqlStdOperatorTable.UNION_ALL : SqlStdOperatorTable.UNION; + SqlNode unionNode = operator.createCall(SqlImplementor.POS, seedNode, recursiveNode); + + SqlIdentifier relationName = buildRelationName(e.getTransientTable()); + SqlNodeList columnList = buildColumnList(e.getRowType()); + SqlWithItem withItem = + new SqlWithItem( + SqlImplementor.POS, + relationName, + columnList, + unionNode, + SqlLiteral.createBoolean(true, SqlImplementor.POS)); + SqlNodeList withList = new SqlNodeList(ImmutableList.of(withItem), SqlImplementor.POS); + + SqlSelect body = wrapSelect(relationName); + SqlWith withNode = new SqlWith(SqlImplementor.POS, withList, body); + return result(withNode, ImmutableList.of(Clause.FROM), e, null); + } + + /** Visits a TableSpool; called by {@link #dispatch} via reflection. */ + public Result visit(TableSpool e) { + return visitInput(e, 0); + } + + private static SqlIdentifier buildRelationName(RelOptTable transientTable) { + RelOptTable table = Objects.requireNonNull(transientTable, "transientTable"); + List qualifiedName = table.getQualifiedName(); + return new SqlIdentifier(qualifiedName, SqlParserPos.QUOTED_ZERO); + } + + private static SqlNodeList buildColumnList(RelDataType rowType) { + List columns = + rowType.getFieldList().stream() + .map(RelDataTypeField::getName) + .map(name -> new SqlIdentifier(name, SqlParserPos.QUOTED_ZERO)) + .map(SqlNode.class::cast) + .toList(); + return new SqlNodeList(columns, SqlParserPos.QUOTED_ZERO); + } +} diff --git a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java index 05a956fd053..a6588fa97ef 100644 --- a/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java +++ b/ppl/src/test/java/org/opensearch/sql/ppl/calcite/CalcitePPLUnionRecursiveTest.java @@ -5,7 +5,11 @@ package org.opensearch.sql.ppl.calcite; +import static org.junit.Assert.assertTrue; + import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.rel2sql.OpenSearchRelToSqlConverter; +import org.apache.calcite.sql.SqlNode; import org.apache.calcite.test.CalciteAssert; import org.junit.Assert; import org.junit.Test; @@ -41,4 +45,17 @@ public void testUnionRecursiveSchemaMismatch() { Throwable t = Assert.assertThrows(SemanticCheckException.class, () -> getRelNode(ppl)); Assert.assertTrue(t.getMessage().contains("UNION RECURSIVE schema mismatch")); } + + @Test + public void testUnionRecursiveSparkSqlIncludesCte() { + String ppl = + "source=EMP | fields EMPNO, MGR | union recursive name=emp_hierarchy max_depth=1" + + " [ source=emp_hierarchy | fields EMPNO, MGR ]"; + RelNode root = getRelNode(ppl); + OpenSearchRelToSqlConverter converter = + new OpenSearchRelToSqlConverter(OpenSearchSparkSqlDialect.DEFAULT); + SqlNode sqlNode = converter.toSqlNode(root); + String sql = sqlNode.toSqlString(OpenSearchSparkSqlDialect.DEFAULT).getSql(); + assertTrue(sql.contains("WITH RECURSIVE")); + } } From 40215a22bc20278db122546aff722f86e3c8de99 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 7 Jan 2026 08:59:51 -0800 Subject: [PATCH 6/7] Add UNION RECURSIVE docs and yaml rest test --- docs/dev/union_recursive_task6_context.md | 25 ++++++++ docs/user/ppl/cmd/union_recursive.md | 46 ++++++++++++++ docs/user/ppl/index.md | 3 +- .../rest-api-spec/test/issues/5000.yml | 61 +++++++++++++++++++ 4 files changed, 134 insertions(+), 1 deletion(-) create mode 100644 docs/dev/union_recursive_task6_context.md create mode 100644 docs/user/ppl/cmd/union_recursive.md create mode 100644 integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml diff --git a/docs/dev/union_recursive_task6_context.md b/docs/dev/union_recursive_task6_context.md new file mode 100644 index 00000000000..8083df49c86 --- /dev/null +++ b/docs/dev/union_recursive_task6_context.md @@ -0,0 +1,25 @@ +# Task 6 Context - UNION RECURSIVE Tests and Docs + +## What changed +- Added user-facing command documentation: + - `docs/user/ppl/cmd/union_recursive.md` + - Added command entry in `docs/user/ppl/index.md` +- Added YAML REST integration test for recursive queries: + - `integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml` + +## Notes/decisions +- The REST test enables Calcite, creates an `edges` index, and validates a + `union recursive` query with `max_depth=1` for deterministic output. +- Documentation calls out Calcite requirement, schema alignment, and UNION ALL + semantics. + +## Tests run +- Not run (YAML REST tests can be executed via `./gradlew :integ-test:yamlRestTest`). + +## Files touched +- `docs/user/ppl/cmd/union_recursive.md` +- `docs/user/ppl/index.md` +- `integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml` + +## Next task pointer +- Task 7: None (Task 6 completes the planned work). diff --git a/docs/user/ppl/cmd/union_recursive.md b/docs/user/ppl/cmd/union_recursive.md new file mode 100644 index 00000000000..2fa06065905 --- /dev/null +++ b/docs/user/ppl/cmd/union_recursive.md @@ -0,0 +1,46 @@ +# union recursive + +## Description + +The `union recursive` command evaluates a recursive subsearch and unions its output with the +anchor pipeline until a fixpoint or configured limits are reached. This is the PPL equivalent of +SQL `WITH RECURSIVE`. + +## Syntax + + +| union recursive name= [max_depth=] [max_rows=] + [ ] +| + +- name: required. Logical name of the recursive relation. +- max_depth: optional. Maximum number of recursive iterations. +- max_rows: optional. Maximum total rows produced by the recursive relation. +- recursive_subsearch: required. Bracketed subsearch that must reference the recursive relation and + return the same schema as the anchor. + +## Usage notes + +- The anchor pipeline defines the base rows (iteration 0). +- The recursive subsearch is evaluated repeatedly and unioned with the anchor output. +- `union recursive` uses UNION ALL semantics; duplicates are not removed. +- The recursive relation name is only visible inside the bracketed recursive block. + +## Example: Graph traversal + +```ppl +source=edges | where parent = "A" | fields parent, child +| union recursive name=rel max_depth=3 [ + source=edges as e + | join right = r on e.parent = r.child rel as r + | fields e.parent as parent, e.child as child + ] +| sort parent, child +``` + +## Limitations + +- Requires the Calcite engine. Set `plugins.calcite.enabled` to true. +- The anchor and recursive subsearch must return the same field names and types. +- Recursive relation names cannot be used as aliases inside the recursive block. +- Duplicates are preserved unless you remove them in the pipeline. diff --git a/docs/user/ppl/index.md b/docs/user/ppl/index.md index 30ad7159182..030da3f6402 100644 --- a/docs/user/ppl/index.md +++ b/docs/user/ppl/index.md @@ -68,6 +68,7 @@ source=accounts | [spath command](cmd/spath.md) | 3.3 | experimental (since 3.3) | Extract fields from structured text data. | | [patterns command](cmd/patterns.md) | 2.4 | stable (since 2.4) | Extract log patterns from a text field and append the results to the search result. | | [join command](cmd/join.md) | 3.0 | stable (since 3.0) | Combine two datasets together. | +| [union recursive command](cmd/union_recursive.md) | 3.5 | experimental (since 3.5) | Recursively union results from an anchor pipeline and a recursive subsearch. | | [append command](cmd/append.md) | 3.3 | experimental (since 3.3) | Append the result of a sub-search to the bottom of the input search results. | | [appendcol command](cmd/appendcol.md) | 3.1 | experimental (since 3.1) | Append the result of a sub-search and attach it alongside the input search results. | | [lookup command](cmd/lookup.md) | 3.0 | experimental (since 3.0) | Add or replace data from a lookup index. | @@ -99,4 +100,4 @@ source=accounts * **Optimization** - [Optimization](../../user/optimization/optimization.rst) * **Limitations** - - [Limitations](limitations/limitations.md) \ No newline at end of file + - [Limitations](limitations/limitations.md) diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml new file mode 100644 index 00000000000..12f3e8483d4 --- /dev/null +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml @@ -0,0 +1,61 @@ +setup: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled: true + - do: + indices.create: + index: edges + body: + mappings: + properties: + parent: + type: keyword + child: + type: keyword + - do: + bulk: + index: edges + refresh: true + body: + - '{"index": {"_id": "1"}}' + - '{"parent": "A", "child": "B"}' + - '{"index": {"_id": "2"}}' + - '{"parent": "B", "child": "C"}' + - '{"index": {"_id": "3"}}' + - '{"parent": "B", "child": "D"}' + +--- +teardown: + - do: + query.settings: + body: + transient: + plugins.calcite.enabled : false + +--- +"union recursive basic": + - skip: + features: + - headers + - allowed_warnings + - do: + headers: + Content-Type: 'application/json' + ppl: + body: + query: | + source=edges as e + | where parent = "A" + | fields parent, child + | union recursive name=rel max_depth=1 [ + source=edges as e + | join right = r on e.parent = r.child rel as r + | fields e.parent as parent, e.child as child + ] + | sort parent, child + + - match: { total: 3 } + - match: { "schema": [ { "name": "parent", "type": "string" }, { "name": "child", "type": "string" } ] } + - match: { "datarows": [["A", "B"], ["B", "C"], ["B", "D"]] } From 4ddf3925f168772084b2e42e943b274e4d3650c8 Mon Sep 17 00:00:00 2001 From: Peng Huo Date: Wed, 7 Jan 2026 13:10:07 -0800 Subject: [PATCH 7/7] Refactor UNION RECURSIVE yaml rest test data --- docs/dev/union_recursive_task7_context.md | 19 ++++++ .../rest-api-spec/test/issues/5000.yml | 62 ++++++++++++------- 2 files changed, 60 insertions(+), 21 deletions(-) create mode 100644 docs/dev/union_recursive_task7_context.md diff --git a/docs/dev/union_recursive_task7_context.md b/docs/dev/union_recursive_task7_context.md new file mode 100644 index 00000000000..d9873ad291f --- /dev/null +++ b/docs/dev/union_recursive_task7_context.md @@ -0,0 +1,19 @@ +# Task 7 Context - Refactor UNION RECURSIVE YAML REST Test + +## What changed +- Updated YAML REST test data, mapping, and query to the BoM example: + - `integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml` + +## Notes/decisions +- Uses the provided BoM mapping and bulk data. +- Uses the provided `union recursive` query and expected results. + +## Tests run +- Not run (YAML REST tests can be executed via `./gradlew :integ-test:yamlRestTest`). + +## Files touched +- `integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml` +- `docs/dev/union_recursive_task7_context.md` + +## Next task pointer +- None. diff --git a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml index 12f3e8483d4..ee5e405dd9f 100644 --- a/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml +++ b/integ-test/src/yamlRestTest/resources/rest-api-spec/test/issues/5000.yml @@ -6,25 +6,37 @@ setup: plugins.calcite.enabled: true - do: indices.create: - index: edges + index: bom body: mappings: properties: parent: type: keyword - child: + component: type: keyword + quantity: + type: integer - do: bulk: - index: edges + index: bom refresh: true body: - - '{"index": {"_id": "1"}}' - - '{"parent": "A", "child": "B"}' - - '{"index": {"_id": "2"}}' - - '{"parent": "B", "child": "C"}' - - '{"index": {"_id": "3"}}' - - '{"parent": "B", "child": "D"}' + - '{"index": {}}' + - '{"parent": "Bike", "component": "Frame", "quantity": 1}' + - '{"index": {}}' + - '{"parent": "Bike", "component": "Wheels", "quantity": 2}' + - '{"index": {}}' + - '{"parent": "Bike", "component": "Drivetrain", "quantity": 1}' + - '{"index": {}}' + - '{"parent": "Wheels", "component": "Spokes", "quantity": 32}' + - '{"index": {}}' + - '{"parent": "Wheels", "component": "Tire", "quantity": 1}' + - '{"index": {}}' + - '{"parent": "Drivetrain", "component": "Crankset", "quantity": 1}' + - '{"index": {}}' + - '{"parent": "Drivetrain", "component": "Chain", "quantity": 1}' + - '{"index": {}}' + - '{"parent": "Crankset", "component": "Pedal", "quantity": 2}' --- teardown: @@ -35,7 +47,7 @@ teardown: plugins.calcite.enabled : false --- -"union recursive basic": +"union recursive bom quantities": - skip: features: - headers @@ -46,16 +58,24 @@ teardown: ppl: body: query: | - source=edges as e - | where parent = "A" - | fields parent, child - | union recursive name=rel max_depth=1 [ - source=edges as e - | join right = r on e.parent = r.child rel as r - | fields e.parent as parent, e.child as child + source=bom + | where parent='Bike' + | fields component, quantity + | union recursive name=bom_qty [ + source=bom + | join left=b right=t on b.parent = t.component bom_qty + | eval quantity=b.quantity * t.quantity + | fields component, quantity ] - | sort parent, child + | where component not in [source=bom | fields parent] - - match: { total: 3 } - - match: { "schema": [ { "name": "parent", "type": "string" }, { "name": "child", "type": "string" } ] } - - match: { "datarows": [["A", "B"], ["B", "C"], ["B", "D"]] } + - match: { total: 5 } + - match: { size: 5 } + - match: { "schema": [ { "name": "component", "type": "string" }, { "name": "quantity", "type": "int" } ] } + - match: + "datarows": + - ["Chain", 1] + - ["Frame", 1] + - ["Pedal", 2] + - ["Spokes", 64] + - ["Tire", 2]