diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java index 4d1687656c62f..76a365bc129a2 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java @@ -18,6 +18,7 @@ import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.APPROXIMATION_V2; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METRICS_GROUP_BY_ALL; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING; @@ -62,6 +63,12 @@ protected void shouldSkipTest(String testName) throws IOException { testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName()) ); + // FORK is not supported with unmapped_fields="load", see https://github.com/elastic/elasticsearch/issues/142033 + assumeFalse( + "FORK is not supported with unmapped_fields=\"load\"", + testCase.requiredCapabilities.contains(OPTIONAL_FIELDS_V2.capabilityName()) + ); + assumeFalse( "Tests using subqueries are skipped since we don't support nested subqueries", testCase.requiredCapabilities.contains(SUBQUERY_IN_FROM_COMMAND.capabilityName()) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index f1a81d23d4e21..02e9cf9f1679d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -78,7 +78,8 @@ FROM partial_mapping_sample_data 2024-10-23T13:52:55.015Z | 173.21.3.15 | 8268152 | Connection error? | 1 | English ; -doesNotLoadUnmappedFieldsLookupJoin +doesNotLoadUnmappedFieldsLookupJoin-Ignore +// temporarily forbidding "load" with LOOKUP JOIN required_capability: optional_fields_v2 required_capability: join_lookup_v12 FROM partial_mapping_sample_data diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 007c2f545aba8..69a77fc4d206d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -278,7 +278,7 @@ public LogicalPlan analyze(LogicalPlan plan) { } public LogicalPlan verify(LogicalPlan plan, BitSet partialMetrics) { - Collection failures = verifier.verify(plan, partialMetrics); + Collection failures = verifier.verify(plan, partialMetrics, context().unmappedResolution()); if (failures.isEmpty() == false) { throw new VerificationException(failures); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index fa3684e23176e..ebe6e5587201e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.index.IndexMode; import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.xpack.esql.LicenseAware; import org.elasticsearch.xpack.esql.capabilities.ConfigurationAware; @@ -31,6 +32,7 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.Fork; import org.elasticsearch.xpack.esql.plan.logical.InlineStats; import org.elasticsearch.xpack.esql.plan.logical.Insist; import org.elasticsearch.xpack.esql.plan.logical.Limit; @@ -38,6 +40,8 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.Project; +import org.elasticsearch.xpack.esql.plan.logical.Subquery; +import org.elasticsearch.xpack.esql.plan.logical.UnionAll; import org.elasticsearch.xpack.esql.plan.logical.promql.PromqlCommand; import org.elasticsearch.xpack.esql.telemetry.FeatureMetric; import org.elasticsearch.xpack.esql.telemetry.Metrics; @@ -86,9 +90,10 @@ public Verifier(Metrics metrics, XPackLicenseState licenseState, List verify(LogicalPlan plan, BitSet partialMetrics) { + Collection verify(LogicalPlan plan, BitSet partialMetrics, UnmappedResolution unmappedResolution) { assert partialMetrics != null; Failures failures = new Failures(); @@ -102,6 +107,10 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics) { return failures.failures(); } + if (unmappedResolution == UnmappedResolution.LOAD) { + checkLoadModeDisallowedCommands(plan, failures); + } + // collect plan checkers var planCheckers = planCheckers(plan); planCheckers.addAll(extraCheckers); @@ -340,6 +349,24 @@ private static void checkLimitBy(LogicalPlan plan, Failures failures) { } } + /** + * {@code unmapped_fields="load"} does not yet support branching commands (FORK, LOOKUP JOIN, subqueries/views). + * See https://github.com/elastic/elasticsearch/issues/142033 + */ + private static void checkLoadModeDisallowedCommands(LogicalPlan plan, Failures failures) { + plan.forEachDown(p -> { + if (p instanceof Fork && p instanceof UnionAll == false) { + failures.add(fail(p, "FORK is not supported with unmapped_fields=\"load\"")); + } + if (p instanceof Subquery) { + failures.add(fail(p, "Subqueries and views are not supported with unmapped_fields=\"load\"")); + } + if (p instanceof EsRelation esRelation && esRelation.indexMode() == IndexMode.LOOKUP) { + failures.add(fail(p, "LOOKUP JOIN is not supported with unmapped_fields=\"load\"")); + } + }); + } + private void licenseCheck(LogicalPlan plan, Failures failures) { Consumer> licenseCheck = n -> { if (n instanceof LicenseAware la && la.licenseCheck(licenseState) == false) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index fa3543219e426..0baf1902eb8af 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -7,7 +7,6 @@ package org.elasticsearch.xpack.esql.analysis; -import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -2471,11 +2470,11 @@ public void testSubquerysMixAndLookupJoinNullify() { assertThat(Expressions.names(plan.output()), is(List.of("COUNT(*)", "empNo", "languageCode", "does_not_exist2"))); } - // same tree as above, except for the source nodes + // unmapped_fields="load" disallows subqueries and LOOKUP JOIN (see #142033) public void testSubquerysMixAndLookupJoinLoad() { assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); - var plan = analyzeStatement(setUnmappedLoad(""" + var e = expectThrows(VerificationException.class, () -> analyzeStatement(setUnmappedLoad(""" FROM test, (FROM languages | WHERE language_code > 10 @@ -2489,36 +2488,12 @@ public void testSubquerysMixAndLookupJoinLoad() { | STATS COUNT(*) BY emp_no, language_code, does_not_exist2 | RENAME emp_no AS empNo, language_code AS languageCode | MV_EXPAND languageCode - """)); - - // TODO: golden testing - assertThat(Expressions.names(plan.output()), is(List.of("COUNT(*)", "empNo", "languageCode", "does_not_exist2"))); - - List esRelations = plan.collect(EsRelation.class); - assertThat( - esRelations.stream().map(EsRelation::indexPattern).toList(), - is( - List.of( - "test", // FROM - "languages", - "sample_data", - "test", // LOOKUP JOIN - "languages_lookup" - ) - ) - ); - for (var esr : esRelations) { - if (esr.indexMode() != IndexMode.LOOKUP) { - var dne = esr.output().stream().filter(a -> a.name().startsWith("does_not_exist")).toList(); - assertThat(dne.size(), is(2)); - var dne1 = as(dne.getFirst(), FieldAttribute.class); - var dne2 = as(dne.getLast(), FieldAttribute.class); - var pukesf1 = as(dne1.field(), PotentiallyUnmappedKeywordEsField.class); - var pukesf2 = as(dne2.field(), PotentiallyUnmappedKeywordEsField.class); - assertThat(pukesf1.getName(), is("does_not_exist1")); - assertThat(pukesf2.getName(), is("does_not_exist2")); - } - } + """))); + String msg = e.getMessage(); + assertThat(msg, containsString("Found 4 problems")); + assertThat(msg, containsString("Subqueries and views are not supported with unmapped_fields=\"load\"")); + assertThat(msg, containsString("LOOKUP JOIN is not supported with unmapped_fields=\"load\"")); + assertThat(msg, not(containsString("FORK is not supported"))); } public void testFailSubquerysWithNoMainAndStatsOnlyNullify() { @@ -3795,6 +3770,163 @@ public void testStatsFilteredAggAfterEvalWithDottedUnmappedFieldFromIndex() { } } + public void testLoadModeDisallowsFork() { + verificationFailure( + setUnmappedLoad("FROM test | FORK (WHERE emp_no > 1) (WHERE emp_no < 100)"), + "FORK is not supported with unmapped_fields=\"load\"" + ); + } + + public void testLoadModeDisallowsForkWithStats() { + verificationFailure( + setUnmappedLoad("FROM test | FORK (STATS c = COUNT(*)) (STATS d = AVG(salary))"), + "FORK is not supported with unmapped_fields=\"load\"" + ); + } + + public void testLoadModeDisallowsForkWithMultipleBranches() { + verificationFailure(setUnmappedLoad(""" + FROM test + | FORK (WHERE emp_no > 1) + (WHERE emp_no < 100) + (WHERE salary > 50000) + """), "FORK is not supported with unmapped_fields=\"load\""); + } + + public void testLoadModeDisallowsLookupJoin() { + verificationFailure( + setUnmappedLoad("FROM test | EVAL language_code = languages | LOOKUP JOIN languages_lookup ON language_code"), + "LOOKUP JOIN is not supported with unmapped_fields=\"load\"" + ); + } + + public void testLoadModeDisallowsLookupJoinAfterFilter() { + verificationFailure(setUnmappedLoad(""" + FROM test + | WHERE emp_no > 1 + | EVAL language_code = languages + | LOOKUP JOIN languages_lookup ON language_code + | KEEP emp_no, language_name + """), "LOOKUP JOIN is not supported with unmapped_fields=\"load\""); + } + + public void testLoadModeDisallowsSubquery() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + + verificationFailure( + setUnmappedLoad("FROM test, (FROM languages | WHERE language_code > 1)"), + "Subqueries and views are not supported with unmapped_fields=\"load\"" + ); + } + + public void testLoadModeAllowsSingleSubquery() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + + // A single subquery without a main index is merged into the main query during analysis, + // so there is no Subquery node in the plan and no branching — this is allowed. + var plan = analyzeStatement(setUnmappedLoad("FROM (FROM languages | WHERE language_code > 1)")); + + // TODO: golden testing + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var relation = as(filter.child(), EsRelation.class); + assertThat(relation.indexPattern(), is("languages")); + } + + public void testLoadModeDisallowsMultipleSubqueries() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + + verificationFailure(setUnmappedLoad(""" + FROM test, + (FROM languages | WHERE language_code > 1), + (FROM sample_data | STATS max(@timestamp)) + """), "Subqueries and views are not supported with unmapped_fields=\"load\""); + } + + public void testLoadModeDisallowsNestedSubqueries() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + + verificationFailure( + setUnmappedLoad("FROM test, (FROM languages, (FROM sample_data | STATS count(*)) | WHERE language_code > 10)"), + "Subqueries and views are not supported with unmapped_fields=\"load\"" + ); + } + + public void testLoadModeDisallowsSubqueryWithLookupJoin() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + + verificationFailure(setUnmappedLoad(""" + FROM test, + (FROM test + | EVAL language_code = languages + | LOOKUP JOIN languages_lookup ON language_code) + """), "Subqueries and views are not supported with unmapped_fields=\"load\""); + } + + public void testLoadModeDisallowsForkAndLookupJoin() { + var query = setUnmappedLoad(""" + FROM test + | EVAL language_code = languages + | LOOKUP JOIN languages_lookup ON language_code + | FORK (WHERE emp_no > 1) (WHERE emp_no < 100) + """); + verificationFailure(query, "FORK is not supported with unmapped_fields=\"load\""); + verificationFailure(query, "LOOKUP JOIN is not supported with unmapped_fields=\"load\""); + } + + public void testLoadModeDisallowsSubqueryAndFork() { + assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); + + var query = setUnmappedLoad(""" + FROM test, (FROM languages | WHERE language_code > 1) + | FORK (WHERE emp_no > 1) (WHERE emp_no < 100) + """); + verificationFailure(query, "Subqueries and views are not supported with unmapped_fields=\"load\""); + verificationFailure(query, "FORK is not supported with unmapped_fields=\"load\""); + } + + public void testLoadModeAllowsInlineStats() { + var plan = analyzeStatement(setUnmappedLoad(""" + FROM test + | INLINE STATS c = COUNT(*) BY emp_no + """)); + + // TODO: golden testing + var limit = as(plan, Limit.class); + var inlineStats = as(limit.child(), InlineStats.class); + var agg = as(inlineStats.child(), Aggregate.class); + assertThat(Expressions.names(agg.groupings()), is(List.of("emp_no"))); + var relation = as(agg.child(), EsRelation.class); + assertThat(relation.indexPattern(), is("test")); + } + + public void testLoadModeAllowsInlineStatsWithUnmappedFields() { + var plan = analyzeStatement(setUnmappedLoad(""" + FROM test + | INLINE STATS s = COUNT(does_not_exist1), c = COUNT(*) BY does_not_exist2, emp_no + """)); + + // TODO: golden testing + var limit = as(plan, Limit.class); + var inlineStats = as(limit.child(), InlineStats.class); + var agg = as(inlineStats.child(), Aggregate.class); + assertThat(Expressions.names(agg.groupings()), is(List.of("does_not_exist2", "emp_no"))); + assertThat(Expressions.names(agg.aggregates()), is(List.of("s", "c", "does_not_exist2", "emp_no"))); + var relation = as(agg.child(), EsRelation.class); + assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> a.name().startsWith("does_not_exist")).toList(); + assertThat(dneAttrs, hasSize(2)); + } + + public void testNullifyModeAllowsFork() { + var plan = analyzeStatement(setUnmappedNullify("FROM test | FORK (WHERE emp_no > 1) (WHERE emp_no < 100)")); + + // TODO: golden testing + var limit = as(plan, Limit.class); + var fork = as(limit.child(), Fork.class); + assertThat(fork.children(), hasSize(2)); + } + private void verificationFailure(String statement, String expectedFailure) { var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); assertThat(e.getMessage(), containsString(expectedFailure));