diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java index fa73b8ec745c6..8b4c5035c9744 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java @@ -36,16 +36,22 @@ import org.elasticsearch.xpack.esql.plan.QuerySettings; import org.elasticsearch.xpack.esql.plan.logical.Enrich; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.NamedSubquery; +import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.plan.logical.ViewUnionAll; import org.elasticsearch.xpack.esql.session.Configuration; import org.hamcrest.Matcher; import org.hamcrest.Matchers; import java.util.ArrayList; +import java.util.Arrays; import java.util.HashMap; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; import java.util.Set; import java.util.function.Supplier; +import java.util.stream.Collectors; import static junit.framework.Assert.assertTrue; import static org.elasticsearch.test.ESTestCase.expectThrows; @@ -77,6 +83,7 @@ public class TestAnalyzer { private Supplier minimumTransportVersion = TransportVersionUtils::randomCompatibleVersion; private ExternalSourceResolution externalSourceResolution = ExternalSourceResolution.EMPTY; private boolean stripErrorPrefix; + private Map views = new LinkedHashMap<>(); TestAnalyzer() {} @@ -290,6 +297,14 @@ public TestAnalyzer addEnrichError(String policyName, Enrich.Mode mode, String r return this; } + /** + * Add a view definition. + */ + public TestAnalyzer addView(String name, String query) { + views.put(name, query); + return this; + } + /** * Adds the standard set of enrich policy resolutions used by many analyzer tests. */ @@ -455,7 +470,83 @@ public TestAnalyzer minimumTransportVersion(TransportVersion minimumTransportVer * Build the analyzer, parse the query, and analyze it. */ public LogicalPlan query(String query, QueryParams params) { - return buildAnalyzer().analyze(EsqlTestUtils.TEST_PARSER.parseQuery(query, params)); + return buildAnalyzer().analyze(parseQuery(query, params)); + } + + private LogicalPlan parseQuery(String query, QueryParams params) { + var parsed = EsqlTestUtils.TEST_PARSER.parseQuery(query, params); + if (views.isEmpty()) { + return parsed; + } + return resolveViews(parsed); + } + + // This most primitive view resolution only works for the simple cases being tested + private LogicalPlan resolveViews(LogicalPlan parsed) { + var viewDefinitions = resolveViews(views); + return parsed.transformDown(UnresolvedRelation.class, ur -> { + List resolved = Arrays.stream(ur.indexPattern().indexPattern().split("\\s*\\,\\s*")).map(indexPattern -> { + var view = viewDefinitions.get(indexPattern); + return view == null + ? (LogicalPlan) (makeUnresolvedRelation(ur, indexPattern)) + : new NamedSubquery(view.source(), view, indexPattern); + }).toList(); + if (resolved.size() == 1) { + var subplan = resolved.get(0); + if (subplan instanceof NamedSubquery n) { + return n.child(); + } + return subplan; + } + List unresolvedRelations = new ArrayList<>(); + List namedSubqueries = new ArrayList<>(); + for (LogicalPlan l : resolved) { + if (l instanceof UnresolvedRelation u) { + unresolvedRelations.add(u); + } else if (l instanceof NamedSubquery n) { + namedSubqueries.add(n); + } else { + throw new IllegalArgumentException("Only support UnresolvedRelation and NamedSubquery in Views Analyzer Tests"); + } + } + LinkedHashMap subplans = new LinkedHashMap<>(); + if (unresolvedRelations.size() == 1) { + subplans.put(null, unresolvedRelations.get(0)); + } else if (unresolvedRelations.size() > 1) { + String indexPattern = unresolvedRelations.stream() + .map(u -> u.indexPattern().indexPattern()) + .collect(Collectors.joining(",")); + subplans.put(null, makeUnresolvedRelation(unresolvedRelations.get(0), indexPattern)); + } + for (NamedSubquery namedSubquery : namedSubqueries) { + subplans.put(namedSubquery.name(), namedSubquery.child()); + } + if (subplans.size() == 1) { + return namedSubqueries.get(0).child(); + } else { + return new ViewUnionAll(ur.source(), subplans, List.of()); + } + }); + } + + private static UnresolvedRelation makeUnresolvedRelation(UnresolvedRelation plan, String indexPattern) { + return new UnresolvedRelation( + plan.source(), + new IndexPattern(plan.source(), indexPattern), + plan.frozen(), + plan.metadataFields(), + plan.indexMode(), + plan.unresolvedMessage(), + plan.telemetryLabel() + ); + } + + private static Map resolveViews(Map views) { + var parsedViews = new HashMap(); + for (Map.Entry entry : views.entrySet()) { + parsedViews.put(entry.getKey(), TEST_PARSER.parseQuery(entry.getValue())); + } + return parsedViews; } /** diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec index 0b790fddf95d2..94e0fd0c8ee42 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec @@ -420,3 +420,40 @@ rehired_count:long | is_rehired:boolean 11 | true 12 | false ; + +// Testing views used inside subqueries +// These tests verify that view references inside user-written subqueries are resolved correctly. +// The parser produces a plain UnionAll for "FROM index, (FROM ...)" syntax, and ViewResolver +// must recurse into it to resolve any view references in the subquery children. + +viewInSubquery +required_capability: subquery_in_from_command +required_capability: views_with_no_branching +required_capability: views_crud_as_index_actions + +FROM sample_data, (FROM country_airports) +| STATS sd = COUNT(client_ip), ca = COUNT(count), total = COUNT() +; + +sd:long | ca:long | total:long +7 | 15 | 22 +; + +viewInSubqueryMultipleRows +required_capability: subquery_in_from_command +required_capability: views_with_no_branching +required_capability: views_crud_as_index_actions + +FROM sample_data, (FROM country_airports) +| STATS total = COUNT() BY country +| SORT total DESC, country ASC +| LIMIT 5 +; + +total:long | country:keyword +7 | null +1 | Argentina +1 | Australia +1 | Brazil +1 | Canada +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index aa6cf68324755..a18019c62b3ca 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -3064,7 +3064,7 @@ private static LogicalPlan rebuildUnionAll( } } } - return new UnionAll(unionAll.source(), newChildren, newOutput); + return unionAll.replaceSubPlansAndOutput(newChildren, newOutput); } /** @@ -3310,7 +3310,7 @@ private static UnionAll rebuildUnionAllOutput( newOutput.add(oldAttr); } } - return new UnionAll(unionAll.source(), newChildren, newOutput); + return unionAll.replaceSubPlansAndOutput(newChildren, newOutput); } /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/NamedSubquery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/NamedSubquery.java new file mode 100644 index 0000000000000..a1779f9e8443a --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/NamedSubquery.java @@ -0,0 +1,68 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.plan.logical; + +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.Objects; + +/** + * A {@link Subquery} that carries the view name it was resolved from. + *

+ * Unlike plain {@link Subquery}, the name participates in {@link #equals} and {@link #hashCode}, + * which allows {@code Node.transformDown} to distinguish a newly-tagged subquery from its + * untagged predecessor. After view resolution, a post-processing pass converts + * {@link UnionAll} nodes containing {@code NamedSubquery} children into {@link ViewUnionAll}. + *

+ * This class should only be used during query re-writing and not survive in the final query plan. + * If we decide to keep named subqueries as a feature later, we should add serialization support. + */ +public class NamedSubquery extends Subquery { + private final String name; + + public NamedSubquery(Source source, LogicalPlan subqueryPlan, String name) { + super(source, subqueryPlan); + this.name = Objects.requireNonNull(name); + } + + public String name() { + return name; + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, NamedSubquery::new, child(), name); + } + + @Override + public UnaryPlan replaceChild(LogicalPlan newChild) { + return new NamedSubquery(source(), newChild, name); + } + + @Override + public int hashCode() { + return Objects.hash(name, child()); + } + + @Override + public boolean equals(Object obj) { + if (this == obj) { + return true; + } + if (obj == null || getClass() != obj.getClass()) { + return false; + } + NamedSubquery other = (NamedSubquery) obj; + return Objects.equals(name, other.name) && Objects.equals(child(), other.child()); + } + + @Override + public String nodeString(NodeStringFormat format) { + return nodeName() + "[" + name + "]"; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Subquery.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Subquery.java index de2cfc1a17afb..15d19eca6c30e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Subquery.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Subquery.java @@ -21,28 +21,19 @@ public class Subquery extends UnaryPlan implements TelemetryAware, SortAgnostic { public static final NamedWriteableRegistry.Entry ENTRY = new NamedWriteableRegistry.Entry(LogicalPlan.class, "Subquery", Subquery::new); - private final String name; // named subqueries are views (information useful for debugging planning) - - // subquery alias/qualifier could be added in the future if needed public Subquery(Source source, LogicalPlan subqueryPlan) { - this(source, subqueryPlan, null); - } - - public Subquery(Source source, LogicalPlan subqueryPlan, String name) { super(source, subqueryPlan); - this.name = name; } private Subquery(StreamInput in) throws IOException { - this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(LogicalPlan.class), null); + this(Source.readFrom((PlanStreamInput) in), in.readNamedWriteable(LogicalPlan.class)); } @Override public void writeTo(StreamOutput out) throws IOException { Source.EMPTY.writeTo(out); out.writeNamedWriteable(child()); - // View names are not needed on the data nodes, only on the coordinating node for debugging purposes } @Override @@ -51,13 +42,13 @@ public String getWriteableName() { } @Override - protected NodeInfo info() { - return NodeInfo.create(this, Subquery::new, child(), name); + protected NodeInfo info() { + return NodeInfo.create(this, Subquery::new, child()); } @Override public UnaryPlan replaceChild(LogicalPlan newChild) { - return new Subquery(source(), newChild, name); + return new Subquery(source(), newChild); } @Override @@ -91,7 +82,7 @@ public boolean equals(Object obj) { @Override public String nodeString(NodeStringFormat format) { - return nodeName() + "[" + (name == null ? "" : name) + "]"; + return nodeName() + "[]"; } public LogicalPlan plan() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/ViewUnionAll.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/ViewUnionAll.java new file mode 100644 index 0000000000000..e30d97c86ed03 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/ViewUnionAll.java @@ -0,0 +1,100 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.plan.logical; + +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.tree.NodeInfo; +import org.elasticsearch.xpack.esql.core.tree.Source; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.Objects; +import java.util.SequencedSet; + +/** + * A {@link UnionAll} produced by view resolution, as opposed to user-written subqueries. + * This type marker allows {@link org.elasticsearch.xpack.esql.view.ViewResolver} to distinguish + * between unions it has already processed (view-produced) and unions from the parser (subqueries) + * that may still contain unresolved view references. + */ +public class ViewUnionAll extends UnionAll { + private final LinkedHashMap namedSubqueries = new LinkedHashMap<>(); + + public ViewUnionAll(Source source, LinkedHashMap children, List output) { + super(source, children.values().stream().toList(), output); + namedSubqueries.putAll(children); + } + + @Override + public LogicalPlan replaceChildren(List newChildren) { + return new ViewUnionAll(source(), asSubqueryMap(newChildren), output()); + } + + @Override + protected NodeInfo info() { + return NodeInfo.create(this, ViewUnionAll::new, namedSubqueries, output()); + } + + @Override + public ViewUnionAll replaceSubPlans(List subPlans) { + return new ViewUnionAll(source(), asSubqueryMap(subPlans), output()); + } + + @Override + public ViewUnionAll replaceSubPlansAndOutput(List subPlans, List output) { + return new ViewUnionAll(source(), asSubqueryMap(subPlans), output); + } + + // Currently for testing only, could also be useful for EXPLAIN and PROFILE + public Map namedSubqueries() { + return namedSubqueries; + } + + private LinkedHashMap asSubqueryMap(List children) { + SequencedSet names = namedSubqueries.sequencedKeySet(); + assert children.size() == names.size(); + LinkedHashMap newSubqueries = new LinkedHashMap<>(); + for (LogicalPlan child : children) { + newSubqueries.put(names.removeFirst(), child); + } + return newSubqueries; + } + + @Override + public String nodeString(NodeStringFormat format) { + return nodeName() + "[" + namedSubqueries.keySet() + "]"; + } + + @Override + public int hashCode() { + // Standard Map.hashCode() uses sum of (key ^ value) per entry, which is separable: + // swapping values between keys can produce the same sum. Instead, we use multiplication + // (non-separable) so that each key is bound to its value in the hash. + int h = 0; + for (Map.Entry entry : namedSubqueries.entrySet()) { + int k = entry.getKey().hashCode(); + int v = Objects.hashCode(entry.getValue()); + h += k * (v + 1); + } + return Objects.hash(ViewUnionAll.class, h); + } + + @Override + public boolean equals(Object o) { + // Map equality is order independent, but does require the same keys map to the same sub-plans + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + ViewUnionAll other = (ViewUnionAll) o; + + return Objects.equals(namedSubqueries, other.namedSubqueries()); + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewResolver.java index 6f2040560eb88..4c0628e7d17b5 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/view/ViewResolver.java @@ -26,14 +26,18 @@ import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Fork; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.NamedSubquery; import org.elasticsearch.xpack.esql.plan.logical.Subquery; +import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.UnionAll; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.plan.logical.ViewUnionAll; import java.util.ArrayList; import java.util.Arrays; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -118,14 +122,12 @@ public void replaceViews( listener.onResponse(new ViewResolutionResult(plan, viewQueries)); return; } - replaceViews( - plan, - parser, - new LinkedHashSet<>(), - viewQueries, - 0, - listener.delegateFailureAndWrap((l, rewritten) -> listener.onResponse(new ViewResolutionResult(rewritten, viewQueries))) - ); + replaceViews(plan, parser, new LinkedHashSet<>(), viewQueries, 0, listener.delegateFailureAndWrap((l, rewritten) -> { + LogicalPlan postProcessed = rewriteUnionAllsWithNamedSubqueries(rewritten); + postProcessed = compactNestedViewUnionAlls(postProcessed); + postProcessed = postProcessed.transformDown(NamedSubquery.class, UnaryPlan::child); + listener.onResponse(new ViewResolutionResult(postProcessed, viewQueries)); + })); } private void replaceViews( @@ -142,10 +144,11 @@ private void replaceViews( plan.transformDown((p, planListener) -> { switch (p) { - case UnionAll union -> { - // UnionAll is the result of this re-writing, so we assume rewriting is completed - // TODO: This could conflicts with subquery feature, perhaps we need a new plan node type? - planListener.onResponse(union); + case ViewUnionAll viewUnion -> { + // ViewUnionAll is the result of view resolution, so we skip it. + // Plain UnionAll (from user-written subqueries) matches the Fork case below + // and its children are recursed into with proper seen-set scoping. + planListener.onResponse(viewUnion); return; } case Fork fork -> { @@ -184,6 +187,9 @@ private void replaceViewsFork( viewQueries, depth + 1, l.delegateFailureAndWrap((subListener, newPlan) -> { + if (newPlan instanceof Subquery sq && sq.child() instanceof NamedSubquery named) { + newPlan = named; + } if (newPlan.equals(subplan) == false) { var updatedSubplansInner = updatedSubplans; if (updatedSubplansInner == null) { @@ -200,7 +206,7 @@ private void replaceViewsFork( } chain.andThenApply(updatedSubplans -> { if (updatedSubplans != null) { - return new Fork(fork.source(), updatedSubplans, fork.output()); + return fork.replaceSubPlans(updatedSubplans); } return (LogicalPlan) fork; }).addListener(listener); @@ -259,7 +265,7 @@ private void replaceViewsUnresolvedRelation( chain.andThenApply(ignored -> { var unresolvedPatterns = buildUnresolvedPatterns(response, seenViews, patterns); if (unresolvedPatterns.isEmpty() && subqueries.size() == 1) { - // only one view, no need for UnionAll, return view plan directly + // Only one view resolved with no remaining index patterns - return its plan directly. return subqueries.getFirst().plan(); } if (unresolvedPatterns.isEmpty() == false) { @@ -384,12 +390,17 @@ record ViewPlan(String name, LogicalPlan plan) {} private LogicalPlan buildPlanFromBranches(UnresolvedRelation ur, List subqueries, int depth) { List unresolvedRelations = new ArrayList<>(); - List otherPlans = new ArrayList<>(); + LinkedHashMap otherPlans = new LinkedHashMap<>(); for (ViewPlan lp : subqueries) { if (lp.plan instanceof UnresolvedRelation urp && urp.indexMode() == IndexMode.STANDARD) { unresolvedRelations.add(urp); + } else if (lp.plan instanceof NamedSubquery namedSubquery) { + assert namedSubquery.name().equals(lp.name); + assert otherPlans.containsKey(lp.name) == false; + otherPlans.put(lp.name, lp.plan); } else { - otherPlans.add(new Subquery(ur.source(), lp.plan, lp.name)); + assert otherPlans.containsKey(lp.name) == false; + otherPlans.put(lp.name, new NamedSubquery(ur.source(), lp.plan, lp.name)); } } if (unresolvedRelations.isEmpty() == false) { @@ -405,24 +416,161 @@ private LogicalPlan buildPlanFromBranches(UnresolvedRelation ur, List ur.indexMode(), ur.unresolvedMessage() ); - otherPlans.addFirst(mergedUnresolved); + assert otherPlans.containsKey(null) == false; + otherPlans.putFirst(null, mergedUnresolved); } if (otherPlans.size() == 1) { - return otherPlans.getFirst(); + return otherPlans.values().stream().findFirst().get(); } traceUnionAllBranches(depth, otherPlans); - return new UnionAll(ur.source(), otherPlans, List.of()); + return new ViewUnionAll(ur.source(), otherPlans, List.of()); + } + + /** + * Top-down rewrite that: + *

    + *
  1. Unwraps {@code Subquery[NamedSubquery[X]]} → {@code NamedSubquery[X]}
    2. + *
  2. Converts plain {@link UnionAll} nodes containing at least one {@link NamedSubquery} + * child into {@link ViewUnionAll} nodes
    3. + *
+ * This handles user-written {@code UNION ALL (FROM my_view)} where the parser creates a + * {@link Subquery} wrapper and view resolution replaces its child with a {@link NamedSubquery}. + */ + static LogicalPlan rewriteUnionAllsWithNamedSubqueries(LogicalPlan plan) { + // Replace Subquery/NamedSubquery with just NamedSubquery + plan = plan.transformDown(Subquery.class, sq -> sq.child() instanceof NamedSubquery n ? n : sq); + + // Any UnionAll containing at least one NamedSubquery should be replaced by ViewUnionAll + plan = plan.transformDown(UnionAll.class, unionAll -> { + if (unionAll instanceof ViewUnionAll) { + return unionAll; + } + LinkedHashMap subPlans = new LinkedHashMap<>(); + boolean hasNamedSubqueries = false; + for (LogicalPlan child : unionAll.children()) { + if (child instanceof NamedSubquery named) { + assert subPlans.containsKey(named.name()) == false; + subPlans.put(named.name(), named.child()); + hasNamedSubqueries = true; + } else if (child instanceof Subquery unnamed) { + // This named subquery is only maintained if it exists together with a named subquery + String name = "unnamed_view_" + Integer.toHexString(unnamed.toString().hashCode()); + assert subPlans.containsKey(name) == false; + subPlans.put(name, unnamed.child()); + } else { + assert subPlans.containsKey(null) == false; + subPlans.put(null, child); + } + } + if (hasNamedSubqueries) { + unionAll = new ViewUnionAll(unionAll.source(), subPlans, unionAll.output()); + } + return unionAll; + }); + return plan; } - private void traceUnionAllBranches(int depth, List plans) { + /** + * Bottom-up rewrite that flattens nested {@link ViewUnionAll} structures. When a + * {@link NamedSubquery} entry in a {@link ViewUnionAll} wraps another {@link ViewUnionAll}, + * its entries are merged into the parent: index patterns are combined and named entries are + * lifted. Flattening is skipped when it would create duplicate named entries (e.g. when + * sibling views share a common subview). + */ + static LogicalPlan compactNestedViewUnionAlls(LogicalPlan plan) { + List children = plan.children(); + List newChildren = null; + for (int i = 0; i < children.size(); i++) { + LogicalPlan child = children.get(i); + LogicalPlan newChild = compactNestedViewUnionAlls(child); + if (newChild != child) { + if (newChildren == null) { + newChildren = new ArrayList<>(children); + } + newChildren.set(i, newChild); + } + } + LogicalPlan current = (newChildren != null) ? plan.replaceChildren(newChildren) : plan; + + if (current instanceof ViewUnionAll vua) { + return tryFlattenViewUnionAll(vua); + } + return current; + } + + private static LogicalPlan tryFlattenViewUnionAll(ViewUnionAll vua) { + // Trial pass: collect all entries from full flattening and check for conflicts + LinkedHashMap flat = new LinkedHashMap<>(); + List indexPatternURs = new ArrayList<>(); + boolean hasInnerVua = false; + + for (Map.Entry entry : vua.namedSubqueries().entrySet()) { + String key = entry.getKey(); + LogicalPlan value = entry.getValue(); + + LogicalPlan inner = (value instanceof NamedSubquery ns) ? ns.child() : value; + if (key != null && inner instanceof ViewUnionAll innerVua) { + hasInnerVua = true; + for (Map.Entry innerEntry : innerVua.namedSubqueries().entrySet()) { + if (innerEntry.getKey() == null && innerEntry.getValue() instanceof UnresolvedRelation innerUr) { + indexPatternURs.add(innerUr); + } else if (innerEntry.getKey() != null && flat.containsKey(innerEntry.getKey())) { + return vua; // conflict: duplicate named entry across siblings, abort + } else { + flat.put(innerEntry.getKey(), innerEntry.getValue()); + } + } + } else if (key == null && value instanceof UnresolvedRelation ur) { + indexPatternURs.add(ur); + } else { + if (key != null && flat.containsKey(key)) { + return vua; // conflict + } + flat.put(key, value); + } + } + + if (hasInnerVua == false) { + return vua; + } + + if (indexPatternURs.isEmpty() == false) { + flat.putFirst(null, mergeUnresolvedRelations(indexPatternURs)); + } + + if (flat.size() == 1) { + return flat.values().iterator().next(); + } + return new ViewUnionAll(vua.source(), flat, vua.output()); + } + + private static UnresolvedRelation mergeUnresolvedRelations(List unresolvedRelations) { + UnresolvedRelation template = unresolvedRelations.getFirst(); + List patterns = new ArrayList<>(); + for (UnresolvedRelation ur : unresolvedRelations) { + patterns.add(ur.indexPattern().indexPattern()); + } + return new UnresolvedRelation( + template.source(), + new IndexPattern(template.indexPattern().source(), String.join(",", patterns)), + template.frozen(), + template.metadataFields(), + template.indexMode(), + template.unresolvedMessage() + ); + } + + private void traceUnionAllBranches(int depth, Map plans) { if (log.isTraceEnabled() == false) { return; } String tab = " ".repeat(depth); log.trace("{} creating UnionAll with {} branches:", tab, plans.size()); String branchPrefix = " " + tab; - for (LogicalPlan p : plans) { - log.trace("{} branch plan=\n{}{}", tab, branchPrefix, p.toString().replace("\n", "\n" + branchPrefix)); + for (Map.Entry entry : plans.entrySet()) { + String name = entry.getKey(); + LogicalPlan p = entry.getValue(); + log.trace("{} branch plan[{}]=\n{}{}", tab, branchPrefix, name, p.toString().replace("\n", "\n" + branchPrefix)); } } @@ -433,6 +581,13 @@ private LogicalPlan resolve(View view, BiFunction p // Parse the view query with the view name, which causes all Source objects // to be tagged with the view name during parsing - return parser.apply(view.query(), view.name()); + LogicalPlan subquery = parser.apply(view.query(), view.name()); + if (subquery instanceof UnresolvedRelation ur) { + // Simple UnresolvedRelation subqueries are not kept as views, so we can compact them together and avoid branched plans + return ur; + } else { + // More complex subqueries are maintained with the view name for branch identification + return new NamedSubquery(subquery.source(), subquery, view.name()); + } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index d6910ed1250b4..cbf076f1c6336 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -113,6 +113,7 @@ import org.elasticsearch.xpack.esql.plan.logical.UnionAll; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.UriParts; +import org.elasticsearch.xpack.esql.plan.logical.ViewUnionAll; import org.elasticsearch.xpack.esql.plan.logical.fuse.FuseScoreEval; import org.elasticsearch.xpack.esql.plan.logical.inference.Completion; import org.elasticsearch.xpack.esql.plan.logical.inference.Rerank; @@ -5111,6 +5112,59 @@ public void testSubqueryInFrom() { assertEquals("languages", subqueryIndex.indexPattern()); } + public void testViewInFrom() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.VIEWS_WITH_NO_BRANCHING.isEnabled()); + LogicalPlan plan = basic().addLanguages().addView("view", "FROM languages | WHERE language_code > 1").query(""" + FROM test, view + | WHERE emp_no > 10000 + | SORT emp_no, language_code + """); + + Limit limit = as(plan, Limit.class); + OrderBy orderBy = as(limit.child(), OrderBy.class); + List order = orderBy.order(); + assertEquals(2, order.size()); + ReferenceAttribute empNo = as(order.get(0).child(), ReferenceAttribute.class); + assertEquals("emp_no", empNo.name()); + ReferenceAttribute languageCode = as(order.get(1).child(), ReferenceAttribute.class); + assertEquals("language_code", languageCode.name()); + Filter filter = as(orderBy.child(), Filter.class); + GreaterThan greaterThan = as(filter.condition(), GreaterThan.class); + empNo = as(greaterThan.left(), ReferenceAttribute.class); + assertEquals("emp_no", empNo.name()); + Literal literal = as(greaterThan.right(), Literal.class); + assertEquals(10000, literal.value()); + ViewUnionAll viewUnionAll = as(filter.child(), ViewUnionAll.class); + assertEquals(2, viewUnionAll.children().size()); + + Project viewProject = as(viewUnionAll.children().get(0), Project.class); + List projections = viewProject.projections(); + assertEquals(13, projections.size()); // all fields from the two indices + Eval viewEval = as(viewProject.child(), Eval.class); + List aliases = viewEval.fields(); // nullEvals from languages index + assertEquals(2, aliases.size()); + assertEquals("language_code", aliases.get(0).name()); + Literal nullLiteral = as(aliases.get(0).child(), Literal.class); + assertNull(nullLiteral.value()); + assertEquals(INTEGER, nullLiteral.dataType()); + assertEquals("language_name", aliases.get(1).name()); + nullLiteral = as(aliases.get(1).child(), Literal.class); + assertNull(nullLiteral.value()); + assertEquals(KEYWORD, nullLiteral.dataType()); + EsRelation subqueryIndex = as(viewEval.child(), EsRelation.class); + assertEquals("test", subqueryIndex.indexPattern()); + + viewProject = as(viewUnionAll.children().get(1), Project.class); + projections = viewProject.projections(); + assertEquals(13, projections.size()); // all fields from the two indices + viewEval = as(viewProject.child(), Eval.class); + aliases = viewEval.fields(); // nullEvals from test index + assertEquals(11, aliases.size()); + Filter subqueryFilter = as(viewEval.child(), Filter.class); + subqueryIndex = as(subqueryFilter.child(), EsRelation.class); + assertEquals("languages", subqueryIndex.indexPattern()); + } + /** * If there is only one subquery in the main from command, the subquery is merged into the main index pattern */ @@ -5136,6 +5190,31 @@ public void testSubqueryInFromWithoutMainIndexPattern() { assertEquals("languages", relation.indexPattern()); } + /** + * If there is only one view in the main from command, the view is merged into the main index pattern + */ + public void testViewInFromWithoutMainIndexPattern() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.VIEWS_WITH_NO_BRANCHING.isEnabled()); + LogicalPlan plan = basic().addLanguages().addView("view", "FROM languages | WHERE language_code > 1").query(""" + FROM view + | WHERE language_name is not null + """); + + Limit limit = as(plan, Limit.class); + Filter filter = as(limit.child(), Filter.class); + IsNotNull isNotNull = as(filter.condition(), IsNotNull.class); + FieldAttribute language_name = as(isNotNull.field(), FieldAttribute.class); + assertEquals("language_name", language_name.name()); + filter = as(filter.child(), Filter.class); + GreaterThan greaterThan = as(filter.condition(), GreaterThan.class); + FieldAttribute language_code = as(greaterThan.left(), FieldAttribute.class); + assertEquals("language_code", language_code.name()); + Literal literal = as(greaterThan.right(), Literal.class); + assertEquals(1, literal.value()); + EsRelation relation = as(filter.child(), EsRelation.class); + assertEquals("languages", relation.indexPattern()); + } + public void testMultipleSubqueriesInFrom() { assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); LogicalPlan plan = basic().addLanguages().addSampleData().addLanguagesLookup().query(""" @@ -5246,6 +5325,116 @@ public void testMultipleSubqueriesInFrom() { assertEquals("test", subqueryIndex.indexPattern()); } + public void testMultipleViewsInFrom() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING.isEnabled()); + LogicalPlan plan = basic().addLanguages() + .addSampleData() + .addLanguagesLookup() + .addView("view1", "FROM languages | WHERE language_code > 10 | RENAME language_name as languageName") + .addView("view2", "FROM sample_data | STATS max(@timestamp)") + .addView("view3", "FROM test | EVAL language_code = languages | LOOKUP JOIN languages_lookup ON language_code") + .query(""" + FROM test, view1, view2, view3 + | WHERE emp_no > 10000 + | STATS count(*) by emp_no, language_code + | RENAME emp_no AS empNo, language_code AS languageCode + | MV_EXPAND languageCode + """); + + Limit limit = as(plan, Limit.class); + MvExpand mvExpand = as(limit.child(), MvExpand.class); + NamedExpression mvExpandTarget = as(mvExpand.target(), NamedExpression.class); + assertEquals("languageCode", mvExpandTarget.name()); + ReferenceAttribute mvExpandExpanded = as(mvExpand.expanded(), ReferenceAttribute.class); + assertEquals("languageCode", mvExpandExpanded.name()); + Project rename = as(mvExpand.child(), Project.class); + List projections = rename.projections(); + assertEquals(3, projections.size()); + Alias a = as(projections.get(1), Alias.class); + assertEquals("empNo", a.name()); + ReferenceAttribute ra = as(a.child(), ReferenceAttribute.class); + assertEquals("emp_no", ra.name()); + a = as(projections.get(2), Alias.class); + assertEquals("languageCode", a.name()); + ra = as(a.child(), ReferenceAttribute.class); + assertEquals("language_code", ra.name()); + Aggregate aggregate = as(rename.child(), Aggregate.class); + List aggregates = aggregate.aggregates(); + assertEquals(3, aggregates.size()); + a = as(aggregates.get(0), Alias.class); + assertEquals("count(*)", a.name()); + List groupings = aggregate.groupings(); + assertEquals(2, groupings.size()); + ra = as(groupings.get(0), ReferenceAttribute.class); + assertEquals("emp_no", ra.name()); + ra = as(groupings.get(1), ReferenceAttribute.class); + assertEquals("language_code", ra.name()); + Filter filter = as(aggregate.child(), Filter.class); + GreaterThan greaterThan = as(filter.condition(), GreaterThan.class); + ReferenceAttribute empNo = as(greaterThan.left(), ReferenceAttribute.class); + assertEquals("emp_no", empNo.name()); + Literal literal = as(greaterThan.right(), Literal.class); + assertEquals(10000, literal.value()); + ViewUnionAll viewUninAll = as(filter.child(), ViewUnionAll.class); + assertEquals(4, viewUninAll.children().size()); + + Project viewProject = as(viewUninAll.children().get(0), Project.class); + projections = viewProject.projections(); + assertEquals(15, projections.size()); // all fields from the other legs + Eval viewEval = as(viewProject.child(), Eval.class); + List aliases = viewEval.fields(); // nullEvals from the other legs + assertEquals(4, aliases.size()); + EsRelation subqueryIndex = as(viewEval.child(), EsRelation.class); + assertEquals("test", subqueryIndex.indexPattern()); + + viewProject = as(viewUninAll.children().get(1), Project.class); + projections = viewProject.projections(); + assertEquals(15, projections.size()); // all fields from the other legs + viewEval = as(viewProject.child(), Eval.class); + aliases = viewEval.fields(); // nullEvals from the other legs + assertEquals(13, aliases.size()); + rename = as(viewEval.child(), Project.class); + List renameProjections = rename.projections(); + assertEquals(2, renameProjections.size()); + FieldAttribute language_code = as(renameProjections.get(0), FieldAttribute.class); + assertEquals("language_code", language_code.name()); + a = as(renameProjections.get(1), Alias.class); + assertEquals("languageName", a.name()); + FieldAttribute language_name = as(a.child(), FieldAttribute.class); + assertEquals("language_name", language_name.name()); + Filter subqueryFilter = as(rename.child(), Filter.class); + greaterThan = as(subqueryFilter.condition(), GreaterThan.class); + language_code = as(greaterThan.left(), FieldAttribute.class); + assertEquals("language_code", language_code.name()); + literal = as(greaterThan.right(), Literal.class); + assertEquals(10, literal.value()); + subqueryIndex = as(subqueryFilter.child(), EsRelation.class); + assertEquals("languages", subqueryIndex.indexPattern()); + + viewProject = as(viewUninAll.children().get(2), Project.class); + projections = viewProject.projections(); + assertEquals(15, projections.size()); // all fields from the other legs + viewEval = as(viewProject.child(), Eval.class); + aliases = viewEval.fields(); // nullEvals from the other legs + assertEquals(14, aliases.size()); + Aggregate subqueryAggregate = as(viewEval.child(), Aggregate.class); + subqueryIndex = as(subqueryAggregate.child(), EsRelation.class); + assertEquals("sample_data", subqueryIndex.indexPattern()); + + viewProject = as(viewUninAll.children().get(3), Project.class); + projections = viewProject.projections(); + assertEquals(15, projections.size()); // all fields from the other legs + viewEval = as(viewProject.child(), Eval.class); + aliases = viewEval.fields(); // nullEvals from the other legs + assertEquals(2, aliases.size()); + LookupJoin lookupJoin = as(viewEval.child(), LookupJoin.class); + subqueryIndex = as(lookupJoin.right(), EsRelation.class); + assertEquals("languages_lookup", subqueryIndex.indexPattern()); + viewEval = as(lookupJoin.left(), Eval.class); + subqueryIndex = as(viewEval.child(), EsRelation.class); + assertEquals("test", subqueryIndex.indexPattern()); + } + public void testMultipleSubqueryInFromWithoutMainIndexPattern() { assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); LogicalPlan plan = basic().addLanguages().addSampleData().addLanguagesLookup().query(""" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/approximation/ApproximationSupportTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/approximation/ApproximationSupportTests.java index 520cba4ec8d7a..4ec6350dbc464 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/approximation/ApproximationSupportTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/approximation/ApproximationSupportTests.java @@ -66,6 +66,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.MMR; import org.elasticsearch.xpack.esql.plan.logical.MetricsInfo; +import org.elasticsearch.xpack.esql.plan.logical.NamedSubquery; import org.elasticsearch.xpack.esql.plan.logical.ParameterizedQuery; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.Subquery; @@ -75,6 +76,7 @@ import org.elasticsearch.xpack.esql.plan.logical.UnionAll; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedExternalRelation; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.plan.logical.ViewUnionAll; import org.elasticsearch.xpack.esql.plan.logical.fuse.Fuse; import org.elasticsearch.xpack.esql.plan.logical.fuse.FuseScoreEval; import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan; @@ -134,11 +136,13 @@ public class ApproximationSupportTests extends ESTestCase { Lookup.class, MMR.class, Subquery.class, + NamedSubquery.class, // Non-unary plans are not supported yet. // These require more complicated expression tree traversal. Fork.class, UnionAll.class, + ViewUnionAll.class, Join.class, InlineJoin.class, LookupJoin.class, diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/ViewUnionAllTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/ViewUnionAllTests.java new file mode 100644 index 0000000000000..87e470f2f75e6 --- /dev/null +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/logical/ViewUnionAllTests.java @@ -0,0 +1,116 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ +package org.elasticsearch.xpack.esql.plan.logical; + +import org.elasticsearch.index.IndexMode; +import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; +import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.plan.IndexPattern; + +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import static org.hamcrest.Matchers.contains; +import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.instanceOf; + +public class ViewUnionAllTests extends ESTestCase { + + public void testIsInstanceOfUnionAll() { + ViewUnionAll viewUnion = new ViewUnionAll(Source.EMPTY, viewMap(), List.of()); + assertThat(viewUnion, instanceOf(UnionAll.class)); + assertThat(viewUnion, instanceOf(Fork.class)); + } + + public void testReplaceChildrenPreservesType() { + LogicalPlan child1 = relation("index1"); + LogicalPlan child2 = relation("index2"); + ViewUnionAll original = new ViewUnionAll(Source.EMPTY, viewMap(child1), List.of()); + + LogicalPlan replaced = original.replaceChildren(List.of(child2)); + assertThat(replaced, instanceOf(ViewUnionAll.class)); + assertEquals(List.of(child2), replaced.children()); + } + + public void testReplaceSubPlansPreservesType() { + LogicalPlan child1 = relation("index1"); + LogicalPlan child2 = relation("index2"); + ViewUnionAll original = new ViewUnionAll(Source.EMPTY, viewMap(child1), List.of()); + assertThat(original.namedSubqueries(), equalTo(Map.of("view_0", child1))); + + ViewUnionAll replaced = original.replaceSubPlans(List.of(child2)); + assertEquals(List.of(child2), replaced.children()); + assertThat(replaced.namedSubqueries(), equalTo(Map.of("view_0", child2))); + } + + public void testReplaceSubPlansAndOutputPreservesType() { + LogicalPlan child1 = relation("index1"); + LogicalPlan child2 = relation("index2"); + ViewUnionAll original = new ViewUnionAll(Source.EMPTY, viewMap(child1), List.of()); + assertThat(original.namedSubqueries(), equalTo(Map.of("view_0", child1))); + + Attribute col1 = new ReferenceAttribute(Source.EMPTY, null, "col", DataType.KEYWORD); + ViewUnionAll replaced = original.replaceSubPlansAndOutput(List.of(child2), List.of(col1)); + assertEquals(List.of(child2), replaced.children()); + assertThat(replaced.namedSubqueries(), equalTo(Map.of("view_0", child2))); + assertThat(replaced.output(), contains(col1)); + } + + public void testEqualsAndHashCode() { + LogicalPlan child1 = relation("index1"); + LogicalPlan child2 = relation("index2"); + + ViewUnionAll a = new ViewUnionAll(Source.EMPTY, viewMap(child1, child2), List.of()); + ViewUnionAll b = new ViewUnionAll(Source.EMPTY, viewMap(child1, child2), List.of()); + ViewUnionAll c = new ViewUnionAll(Source.EMPTY, viewMap(child1), List.of()); + ViewUnionAll d = new ViewUnionAll(Source.EMPTY, viewMap(child2, child1), List.of()); + + // a and b are identical + assertEquals(a, b); + assertEquals(a.hashCode(), b.hashCode()); + + // a and c are different + assertNotEquals(a, c); + assertNotEquals(a.hashCode(), c.hashCode()); + + // a and d are different + assertNotEquals(a, d); + assertNotEquals(a.hashCode(), d.hashCode()); + + // If we replace subplans we can make d match a + d = d.replaceSubPlans(List.of(child1, child2)); + assertEquals(a, d); + assertEquals(a.hashCode(), d.hashCode()); + } + + public void testNotEqualToPlainUnionAll() { + LogicalPlan child = relation("index1"); + + ViewUnionAll viewUnion = new ViewUnionAll(Source.EMPTY, viewMap(child), List.of()); + UnionAll plainUnion = new UnionAll(Source.EMPTY, List.of(child), List.of()); + + // ViewUnionAll and UnionAll with same children should NOT be equal (different getClass()) + assertNotEquals(viewUnion, plainUnion); + assertNotEquals(plainUnion, viewUnion); + } + + private LinkedHashMap viewMap(LogicalPlan... children) { + LinkedHashMap namedChildren = LinkedHashMap.newLinkedHashMap(children.length); + for (int i = 0; i < children.length; i++) { + namedChildren.put("view_" + i, children[i]); + } + return namedChildren; + } + + private static UnresolvedRelation relation(String name) { + return new UnresolvedRelation(Source.EMPTY, new IndexPattern(Source.EMPTY, name), false, List.of(), IndexMode.STANDARD, null); + } +} diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java index a642bc52ec5ef..592c5a9c4915d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/tree/EsqlNodeSubclassTests.java @@ -54,6 +54,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Grok; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.UnionAll; +import org.elasticsearch.xpack.esql.plan.logical.ViewUnionAll; import org.elasticsearch.xpack.esql.plan.logical.join.JoinConfig; import org.elasticsearch.xpack.esql.plan.logical.join.JoinType; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; @@ -89,6 +90,7 @@ import java.util.EnumSet; import java.util.HashMap; import java.util.HashSet; +import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; @@ -158,7 +160,13 @@ public static List nodeSubclasses() throws IOException { .toList(); } - private static final List> CLASSES_WITH_MIN_TWO_CHILDREN = List.of(Concat.class, CIDRMatch.class, Fork.class, UnionAll.class); + private static final List> CLASSES_WITH_MIN_TWO_CHILDREN = List.of( + Concat.class, + CIDRMatch.class, + Fork.class, + UnionAll.class, + ViewUnionAll.class + ); // List of classes that are "unresolved" NamedExpression subclasses, therefore not suitable for use with logical/physical plan nodes. private static final List> UNRESOLVED_CLASSES = List.of( @@ -363,6 +371,9 @@ protected Object makeArg(Type argType) { private static Object makeArg(Class> toBuildClass, Type argType) throws Exception { if (argType instanceof ParameterizedType pt) { + if (pt.getRawType() == LinkedHashMap.class) { + return makeOrderedMap(toBuildClass, pt); + } if (pt.getRawType() == Map.class) { return makeMap(toBuildClass, pt); } @@ -466,10 +477,11 @@ public void accept(Page page) { return randomInt(); } else if (argClass == JoinType.class) { return JoinTypes.LEFT; - } else if (List.of(Fork.class, MergeExec.class, UnionAll.class).contains(toBuildClass) && argType == LogicalPlan.class) { - // limit recursion of plans, in order to prevent stackoverflow errors - return randomEsRelation(); - } + } else if (List.of(Fork.class, MergeExec.class, UnionAll.class, ViewUnionAll.class).contains(toBuildClass) + && argType == LogicalPlan.class) { + // limit recursion of plans, in order to prevent stackoverflow errors + return randomEsRelation(); + } if (Expression.class == argClass) { /* @@ -582,6 +594,17 @@ private static Object makeMap(Class> toBuildClass, Parameteriz return map; } + private static Object makeOrderedMap(Class> toBuildClass, ParameterizedType pt) throws Exception { + LinkedHashMap map = new LinkedHashMap<>(); + int size = randomSizeForCollection(toBuildClass); + while (map.size() < size) { + Object key = makeArg(toBuildClass, pt.getActualTypeArguments()[0]); + Object value = makeArg(toBuildClass, pt.getActualTypeArguments()[1]); + map.put(key, value); + } + return map; + } + private static int randomSizeForCollection(Class> toBuildClass) { if (CompoundOutputEval.class.isAssignableFrom(toBuildClass) || CompoundOutputEvalExec.class.isAssignableFrom(toBuildClass)) { // subclasses of CompoundOutputEval/Exec must have map and list that match in size diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java index 68f61692ea038..f41184e870c82 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/view/InMemoryViewServiceTests.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.ConfigurationTestUtils; import org.elasticsearch.xpack.esql.SerializationTestUtils; import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -33,6 +34,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Subquery; import org.elasticsearch.xpack.esql.plan.logical.UnionAll; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.plan.logical.ViewUnionAll; import org.elasticsearch.xpack.esql.session.Configuration; import org.elasticsearch.xpack.esql.telemetry.PlanTelemetry; import org.hamcrest.BaseMatcher; @@ -66,6 +68,7 @@ import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.not; import static org.hamcrest.Matchers.startsWith; public class InMemoryViewServiceTests extends AbstractStatementParserTests { @@ -316,7 +319,7 @@ public void testReplaceViewsPlans() { LogicalPlan plan = query("FROM view1, view2, view3"); LogicalPlan rewritten = replaceViews(plan); // We cannot express the expected plan easily, so we check its structure instead - assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(rewritten, instanceOf(ViewUnionAll.class)); List subqueries = rewritten.children(); assertThat(subqueries.size(), equalTo(3)); assertThat( @@ -397,7 +400,7 @@ public void testReplaceViewsPlanWildcard() { LogicalPlan plan = query("FROM view*"); LogicalPlan rewritten = replaceViews(plan); // We cannot express the expected plan easily, so we check its structure instead - assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(rewritten, instanceOf(ViewUnionAll.class)); List subqueries = rewritten.children(); assertThat(subqueries.size(), equalTo(3)); assertThat( @@ -418,7 +421,7 @@ public void testReplaceViewsPlanWildcardWithIndex() { LogicalPlan plan = query("FROM view*"); LogicalPlan rewritten = replaceViews(plan); // We cannot express the expected plan easily, so we check its structure instead - assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(rewritten, instanceOf(ViewUnionAll.class)); List subqueries = rewritten.children(); assertThat(subqueries.size(), equalTo(4)); assertThat( @@ -512,12 +515,12 @@ public void testReplaceViewsNestedPlansWildcard() { LogicalPlan plan = query("FROM view_1_*"); LogicalPlan rewritten = replaceViews(plan); // We cannot express the expected plan easily, so we check its structure instead - assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(rewritten, instanceOf(ViewUnionAll.class)); List subqueries = rewritten.children(); assertThat(subqueries.size(), equalTo(2)); for (LogicalPlan child : subqueries) { child = (child instanceof Subquery subquery) ? subquery.child() : child; - assertThat(child, instanceOf(UnionAll.class)); + assertThat(child, instanceOf(ViewUnionAll.class)); List subchildren = child.children(); assertThat(subchildren.size(), equalTo(2)); assertThat( @@ -542,13 +545,13 @@ public void testReplaceViewsNestedPlansWildcardWithIndex() { LogicalPlan plan = query("FROM view_1_*"); LogicalPlan rewritten = replaceViews(plan); // We cannot express the expected plan easily, so we check its structure instead - assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(rewritten, instanceOf(ViewUnionAll.class)); List subqueries = rewritten.children(); assertThat(subqueries.size(), equalTo(3)); assertThat(subqueries.getFirst(), matchesPlan(query("FROM view_1_*"))); for (LogicalPlan child : subqueries.subList(1, 3)) { child = (child instanceof Subquery subquery) ? subquery.child() : child; - assertThat(child, instanceOf(UnionAll.class)); + assertThat(child, instanceOf(ViewUnionAll.class)); List subchildren = child.children(); assertThat(subchildren.size(), equalTo(2)); assertThat( @@ -576,12 +579,12 @@ public void testReplaceViewsNestedPlansWildcards() { LogicalPlan plan = query("FROM view_1_*, view_2_*, view_3_*"); LogicalPlan rewritten = replaceViews(plan); // We cannot express the expected plan easily, so we check its structure instead - assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat(rewritten, instanceOf(ViewUnionAll.class)); List subqueries = rewritten.children(); assertThat(subqueries.size(), equalTo(6)); for (LogicalPlan child : subqueries) { child = (child instanceof Subquery subquery) ? subquery.child() : child; - assertThat(child, instanceOf(UnionAll.class)); + assertThat(child, instanceOf(ViewUnionAll.class)); List subchildren = child.children(); assertThat(subchildren.size(), equalTo(2)); assertThat( @@ -951,6 +954,115 @@ public void testSerializationSubqueryWithSourceFromViewQuery() { ); } + // --- Behavioral test: views combined with subqueries --- + + public void testViewInsideSubqueryIsResolved() { + assumeTrue("Requires views with branching support", EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING.isEnabled()); + addView("my_view", "FROM emp | WHERE emp.age > 30"); + // Parser produces a plain UnionAll for "FROM index, (FROM subquery)" syntax + LogicalPlan plan = query("FROM emp2, (FROM my_view)"); + assertThat(plan, instanceOf(UnionAll.class)); + assertThat("Parser should produce plain UnionAll, not ViewUnionAll", plan, not(instanceOf(ViewUnionAll.class))); + + // ViewResolver should recurse into the plain UnionAll and resolve my_view + LogicalPlan rewritten = replaceViews(plan); + // The top-level UnionAll should be replaced by ViewUnionAll because it contains a named subquery from the resolved view + assertThat("Top-level UnionAll should be re-written to ViewUnionAll with view name", rewritten, instanceOf(ViewUnionAll.class)); + // After resolution, the subquery's UnresolvedRelation[my_view] should become + // the view definition: FROM emp | WHERE emp.age > 30 + List children = rewritten.children(); + assertThat(children.size(), equalTo(2)); + // One child should match the resolved view definition, the other should be emp2 + assertThat(children, containsInAnyOrder(matchesPlan(query("FROM emp | WHERE emp.age > 30")), matchesPlan(query("FROM emp2")))); + } + + public void testSubqueryInsideViewIsResolved() { + assumeTrue("Requires views with branching support", EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING.isEnabled()); + addView("my_view", "FROM emp1, (FROM emp3 | WHERE emp.age > 35) | WHERE emp.age > 30"); + // Parser produces a plain UnionAll for "FROM index, (FROM subquery)" syntax + LogicalPlan plan = query("FROM emp2, (FROM my_view)"); + assertThat(plan, instanceOf(UnionAll.class)); + assertThat("Parser should produce plain UnionAll, not ViewUnionAll", plan, not(instanceOf(ViewUnionAll.class))); + + // ViewResolver should recurse into the plain UnionAll and resolve my_view + LogicalPlan rewritten = replaceViews(plan); + // The top-level UnionAll is replaced by a ViewUnionAll as a result of compacting the nested subqueries + assertThat(rewritten, instanceOf(UnionAll.class)); + assertThat("Top-level UnionAll should be re-written to ViewUnionAll with view name", rewritten, instanceOf(ViewUnionAll.class)); + // After resolution, the subquery's UnresolvedRelation[my_view] should become + // the view definition: FROM emp1 | WHERE emp.age > 30 + List children = rewritten.children(); + assertThat(children.size(), equalTo(2)); + // One child should match the resolved view definition, the other should be emp2 + assertThat( + children, + containsInAnyOrder( + matchesPlan(query("FROM emp1, (FROM emp3 | WHERE emp.age > 35) | WHERE emp.age > 30")), + matchesPlan(query("FROM emp2")) + ) + ); + } + + public void testViewNamedInUnionAll() { + assumeTrue("Requires views with branching support", EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING.isEnabled()); + addView("my_view1", "FROM emp1 | WHERE emp.age > 30"); + addView("my_view2", "FROM emp2 | WHERE emp.age > 30"); + // Parser produces a plain UnionAll for "FROM index, (FROM subquery)" syntax + LogicalPlan plan = query("FROM (FROM my_view1), (FROM my_view2)"); + assertThat(plan, instanceOf(UnionAll.class)); + assertThat("Parser should produce plain UnionAll, not ViewUnionAll", plan, not(instanceOf(ViewUnionAll.class))); + + // ViewResolver should recurse into the plain UnionAll and resolve my_view + LogicalPlan rewritten = replaceViews(plan); + // The top-level UnionAll is replaced by a ViewUnionAll as a result of compacting the nested subqueries + assertThat( + "Top-level UnionAll should changed to include view names after view resolution", + rewritten, + instanceOf(ViewUnionAll.class) + ); + // The names should match the correct sub-plans + ViewUnionAll namedUnionAll = (ViewUnionAll) rewritten; + for (Map.Entry entry : namedUnionAll.namedSubqueries().entrySet()) { + if (entry.getKey().equals("my_view1")) { + assertThat(entry.getValue(), matchesPlan(query("FROM emp1 | WHERE emp.age > 30"))); + } else if (entry.getKey().equals("my_view2")) { + assertThat(entry.getValue(), matchesPlan(query("FROM emp2 | WHERE emp.age > 30"))); + } else { + fail("Unexpected named sub-plan: " + entry); + } + } + } + + public void testIndexCompactionWithNestedNamedSubqueries() { + assumeTrue("Requires views with branching support", EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING.isEnabled()); + addView("my_view1", "FROM emp1 | WHERE emp.age > 30"); + addView("my_view2", "FROM emp2, my_view1"); + addView("my_view3", "FROM emp3, my_view2"); + addView("my_view4", "FROM emp4, my_view3"); + LogicalPlan plan = query("FROM emp5, my_view4"); + assertThat(plan, instanceOf(UnresolvedRelation.class)); + + // ViewResolver should recurse into the plain UnionAll and resolve my_view + LogicalPlan rewritten = replaceViews(plan); + // The top-level UnionAll is replaced by a ViewUnionAll as a result of compacting the nested subqueries + assertThat( + "Top-level UnionAll should changed to include view names after view resolution", + rewritten, + instanceOf(ViewUnionAll.class) + ); + // The names should match the correct sub-plans + ViewUnionAll namedUnionAll = (ViewUnionAll) rewritten; + for (Map.Entry entry : namedUnionAll.namedSubqueries().entrySet()) { + if (entry.getKey() == null) { + assertThat(entry.getValue(), matchesPlan(query("FROM emp5, emp4, emp3, emp2"))); + } else if (entry.getKey().equals("my_view1")) { + assertThat(entry.getValue(), matchesPlan(query("FROM emp1 | WHERE emp.age > 30"))); + } else { + fail("Unexpected named sub-plan: " + entry); + } + } + } + private LogicalPlan replaceViews(LogicalPlan plan) { PlainActionFuture future = new PlainActionFuture<>(); viewResolver.replaceViews(plan, this::parse, future);