From aaff42c21622134d09cf339db6e94d4df233640a Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 10 Mar 2026 15:57:24 +0100 Subject: [PATCH 1/8] Randomly nullify also in multi-node spec tests --- .../xpack/esql/qa/multi_node/EsqlSpecIT.java | 5 +++++ .../xpack/esql/qa/single_node/EsqlSpecIT.java | 8 +------- .../xpack/esql/qa/rest/EsqlSpecTestCase.java | 13 +++++++++++++ 3 files changed, 19 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java index ba24668a5b29d..82d29b4c203b1 100644 --- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java @@ -36,4 +36,9 @@ protected boolean enableRoundingDoubleValuesOnAsserting() { protected boolean supportsSourceFieldMapping() { return false; } + + @Override + protected String maybeRandomizeQuery(String query) { + randomlyNullify(query); + } } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java index 7793b39b02e57..bbc38c1dae555 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java @@ -55,13 +55,7 @@ protected boolean supportsSourceFieldMapping() { @Override protected String maybeRandomizeQuery(String query) { - // TODO we should implement more generic randomization for SET parameters - return randomBoolean() - && testCase.expectedWarnings().isEmpty() // avoid shifting warnings positions in source query - && testCase.expectedWarningsRegex().isEmpty() // regexp might also contain line/position - && query.startsWith("SET") == false // avoid conflicts with provided settings - ? "SET unmapped_fields=\"nullify\"; " + query - : query; + randomlyNullify(query); } @Before diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java index 7a2c583ce2ef6..a87e49c332aa6 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java @@ -346,6 +346,19 @@ protected String maybeRandomizeQuery(String query) { return query; } + /** + * Intended to be used in {@link #maybeRandomizeQuery(String)} except in test cases that do not support {@code nullify} + * (e.g. old test cases in bwc tests) + */ + public String randomlyNullify(String query) { + return randomBoolean() + && testCase.expectedWarnings().isEmpty() // avoid shifting warnings positions in source query + && testCase.expectedWarningsRegex().isEmpty() // regexp might also contain line/position + && query.startsWith("SET") == false // avoid conflicts with provided settings + ? "SET unmapped_fields=\"nullify\"; " + query + : query; + } + /** * Returns true if the cluster under test supports the given ESQL capability. * Subclasses may override this to check additional clusters (e.g. remote clusters in CCS). From 5f9fb6b64fd8f500ae354a30eaa97cc5aa82bf5d Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 10 Mar 2026 15:58:57 +0100 Subject: [PATCH 2/8] Remove redundant capability checks --- .../src/main/resources/unmapped-nullify.csv-spec | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec index 4c7522bada0ba..13723d98a3124 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec @@ -13,11 +13,15 @@ null ; keepMapped -required_capability: date_nanos_type -required_capability: optional_fields_nullify_tech_preview required_capability: optional_fields_fix_unmapped_field_detection -SET unmapped_fields="nullify"; FROM date_nanos | SORT millis ASC | WHERE millis < "2000-01-01" | EVAL nanos = MV_MIN(nanos) | KEEP nanos; +SET unmapped_fields="nullify"; +FROM date_nanos +| SORT millis ASC +| WHERE millis < "2000-01-01" +| EVAL nanos = MV_MIN(nanos) +| KEEP nanos +; nanos:date_nanos 2023-03-23T12:15:03.360103847Z From b8fd1957e96e73beefa43ec35a5a1cabeace12e8 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 10 Mar 2026 16:07:13 +0100 Subject: [PATCH 3/8] Move child output removal into collectUnresolved --- .../esql/analysis/rules/ResolveUnmapped.java | 30 +++++++++---------- 1 file changed, 14 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java index 1e2fca2e20c98..6e242050671f0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java @@ -87,21 +87,8 @@ private static LogicalPlan resolve(LogicalPlan plan, boolean load) { if (plan.childrenResolved() == false) { return plan; } - var unresolved = collectUnresolved(plan); - if (unresolved.isEmpty()) { - return plan; - } - // Filter out unresolved attributes that exist in the children's output. These attributes are not truly unmapped; - // they just haven't been resolved yet by ResolveRefs (e.g. because the children only became resolved after ImplicitCasting). - // ResolveRefs will wire them up in the next iteration of the resolution batch. - Set childOutputNames = new java.util.HashSet<>(); - for (LogicalPlan child : plan.children()) { - for (Attribute attr : child.output()) { - childOutputNames.add(attr.name()); - } - } - unresolved.removeIf(ua -> childOutputNames.contains(ua.name())); + var unresolved = collectUnresolved(plan); if (unresolved.isEmpty()) { return plan; } @@ -348,11 +335,22 @@ private static LinkedHashSet unresolvedLinkedSet(List collectUnresolved(LogicalPlan plan) { + Set childOutputNames = new java.util.HashSet<>(); + for (LogicalPlan child : plan.children()) { + for (Attribute attr : child.output()) { + childOutputNames.add(attr.name()); + } + } + List unresolved = new ArrayList<>(); Consumer collectUnresolved = ua -> { - // Exclude metadata fields so they fail with a proper verification error instead of being silently nullified/loaded. if ((ua instanceof UnresolvedPattern || ua instanceof UnresolvedTimestamp) == false - && MetadataAttribute.isSupported(ua.name()) == false) { + // Exclude metadata fields so they fail with a proper verification error instead of being silently nullified/loaded. + && MetadataAttribute.isSupported(ua.name()) == false + // Filter out unresolved attributes that exist in the children's output. These attributes are not truly unmapped; + // they just haven't been resolved yet by ResolveRefs (e.g. because the children only became resolved after ImplicitCasting). + // ResolveRefs will wire them up in the next iteration of the resolution batch. + && childOutputNames.contains(ua.name()) == false) { unresolved.add(ua); } }; From fe1ce6de96ade39c5d31331eff740b49b2a5f80c Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 10 Mar 2026 16:14:49 +0100 Subject: [PATCH 4/8] Fix tests D'oh. --- .../org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java | 2 +- .../org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java | 2 +- .../testFixtures/src/main/resources/unmapped-nullify.csv-spec | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java index 82d29b4c203b1..577158bb79269 100644 --- a/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/multi-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/multi_node/EsqlSpecIT.java @@ -39,6 +39,6 @@ protected boolean supportsSourceFieldMapping() { @Override protected String maybeRandomizeQuery(String query) { - randomlyNullify(query); + return randomlyNullify(query); } } diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java index bbc38c1dae555..e130c981c6373 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/EsqlSpecIT.java @@ -55,7 +55,7 @@ protected boolean supportsSourceFieldMapping() { @Override protected String maybeRandomizeQuery(String query) { - randomlyNullify(query); + return randomlyNullify(query); } @Before diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec index 13723d98a3124..b4137fe6e3b6f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec @@ -15,7 +15,7 @@ null keepMapped required_capability: optional_fields_fix_unmapped_field_detection -SET unmapped_fields="nullify"; +SET unmapped_fields="nullify"\; FROM date_nanos | SORT millis ASC | WHERE millis < "2000-01-01" From 35962e0925d8c871be5fd6caae95604655755c95 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 11 Mar 2026 11:11:09 +0100 Subject: [PATCH 5/8] Add reproducer --- .../main/resources/unmapped-nullify.csv-spec | 1 + .../esql/analysis/AnalyzerUnmappedTests.java | 38 +++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec index b4137fe6e3b6f..f0b7dafcae9b4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec @@ -12,6 +12,7 @@ null null ; +// Reproducer for https://github.com/elastic/elasticsearch/issues/141870 keepMapped required_capability: optional_fields_fix_unmapped_field_detection diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 9c580624a8d8d..378b3fd115581 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -56,6 +56,7 @@ import java.util.List; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; +import static org.elasticsearch.xpack.esql.EsqlTestUtils.asLimit; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzeStatement; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTests.withInlinestatsWarning; @@ -3123,6 +3124,43 @@ public void testQstrAfterSortWithUnmappedField() { assertThat(orderBy, not(nullValue())); } + /* + * Reproducer for https://github.com/elastic/elasticsearch/issues/141870 + * ResolveRefs processes the EVAL only after ImplicitCasting processes the implicit cast in the WHERE. + * This means that ResolveUnmapped will see the EVAL with a yet-to-be-resolved reference to nanos. + * It should not treat it as unmapped, because there is clearly a nanos attribute in the EVAL's input. + * + * Limit[1000[INTEGER],false,false] + * \_Eval[[MVMIN(nanos{r}#6) AS nanos#11]] + * \_Filter[millis{r}#4 < 946684800000[DATETIME]] + * \_OrderBy[[Order[millis{r}#4,ASC,LAST]]] + * \_Row[[TODATETIME(1970-01-01T00:00:00Z[KEYWORD]) AS millis#4, TODATENANOS(1970-01-01T00:00:00Z[KEYWORD]) AS nanos#6]] + */ + public void testDoNotResolveUnmappedFieldPresentInChildren() { + var plan = analyzeStatement(setUnmappedNullify(""" + ROW millis = "1970-01-01T00:00:00Z"::date, nanos = "1970-01-01T00:00:00Z"::date_nanos + | SORT millis ASC + | WHERE millis < "2000-01-01" + | EVAL nanos = MV_MIN(nanos) + """)); + + var limit = asLimit(plan, 1000); + var eval = as(limit.child(), Eval.class); + var filter = as(eval.child(), Filter.class); + var orderBy = as(filter.child(), OrderBy.class); + // There should be no EVAL injected with NULL for nanos + var row = as(orderBy.child(), Row.class); + + var output = plan.output(); + assertThat(output, hasSize(2)); + var millisAttr = output.get(0); + var nanosAttr = output.get(1); + assertThat(millisAttr.name(), is("millis")); + assertThat(millisAttr.dataType(), is(DataType.DATETIME)); + assertThat(nanosAttr.name(), is("nanos")); + assertThat(nanosAttr.dataType(), is(DataType.DATE_NANOS)); + } + private void verificationFailure(String statement, String expectedFailure) { var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); assertThat(e.getMessage(), containsString(expectedFailure)); From 1e70cba19303ec6241105789effb6aaaa1593cee Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 11 Mar 2026 11:19:33 +0100 Subject: [PATCH 6/8] Randomly also test load --- .../xpack/esql/analysis/AnalyzerUnmappedTests.java | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 378b3fd115581..5ffa6f304667e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -3137,12 +3137,14 @@ public void testQstrAfterSortWithUnmappedField() { * \_Row[[TODATETIME(1970-01-01T00:00:00Z[KEYWORD]) AS millis#4, TODATENANOS(1970-01-01T00:00:00Z[KEYWORD]) AS nanos#6]] */ public void testDoNotResolveUnmappedFieldPresentInChildren() { - var plan = analyzeStatement(setUnmappedNullify(""" + String query = """ ROW millis = "1970-01-01T00:00:00Z"::date, nanos = "1970-01-01T00:00:00Z"::date_nanos | SORT millis ASC | WHERE millis < "2000-01-01" | EVAL nanos = MV_MIN(nanos) - """)); + """; + boolean useNullify = randomBoolean(); + var plan = analyzeStatement(useNullify? setUnmappedNullify(query) : setUnmappedLoad(query)); var limit = asLimit(plan, 1000); var eval = as(limit.child(), Eval.class); From cd235dc2b48db61ee9415ac142c1927341d8f3e8 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 11 Mar 2026 10:28:05 +0000 Subject: [PATCH 7/8] [CI] Auto commit changes from spotless --- .../xpack/esql/analysis/AnalyzerUnmappedTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 5ffa6f304667e..6176f2b2fc9e3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -3144,7 +3144,7 @@ public void testDoNotResolveUnmappedFieldPresentInChildren() { | EVAL nanos = MV_MIN(nanos) """; boolean useNullify = randomBoolean(); - var plan = analyzeStatement(useNullify? setUnmappedNullify(query) : setUnmappedLoad(query)); + var plan = analyzeStatement(useNullify ? setUnmappedNullify(query) : setUnmappedLoad(query)); var limit = asLimit(plan, 1000); var eval = as(limit.child(), Eval.class); From 000e342f415aa11a6fb1c525c242b5f2b9f48de0 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 11 Mar 2026 15:03:44 +0000 Subject: [PATCH 8/8] [CI] Auto commit changes from spotless --- .../xpack/esql/analysis/rules/ResolveUnmapped.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java index 65f2dbb7297ca..78292ab80a82d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java @@ -363,9 +363,9 @@ private static LinkedHashSet collectUnresolved(LogicalPlan // The aggs will "export" the aliases as UnresolvedAttributes part of their .aggregates(); we don't need to consider those // as they'll be resolved as refs once the aliased expression is resolved. && aliasedGroupings.contains(ua.name()) == false - // Filter out unresolved attributes that exist in the children's output. These attributes are not truly unmapped; - // they just haven't been resolved yet by ResolveRefs (e.g. because the children only became resolved after ImplicitCasting). - // ResolveRefs will wire them up in the next iteration of the resolution batch. + // Filter out unresolved attributes that exist in the children's output. These attributes are not truly unmapped; + // they just haven't been resolved yet by ResolveRefs (e.g. because the children only became resolved after ImplicitCasting). + // ResolveRefs will wire them up in the next iteration of the resolution batch. && childOutputNames.contains(ua.name()) == false) { unresolved.putIfAbsent(ua.name(), ua); }