From 54c80ca46d475e757b6aff1a78bc90628427aed7 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 13 Mar 2026 16:00:49 +0100 Subject: [PATCH 01/69] Add reproducer tests --- .../src/main/resources/unmapped-load.csv-spec | 31 ++++++++++++ .../esql/analysis/AnalyzerUnmappedTests.java | 49 +++++++++++++++++++ 2 files changed, 80 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index f1a81d23d4e21..5a7a463323c3f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -936,6 +936,37 @@ FROM partial_mapping_sample_data 2024-10-23T13:51:54.732Z | Connection error? | 725449 ; +// https://github.com/elastic/elasticsearch/issues/141994 +// A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded +// from _source even when not explicitly referenced in a downstream expression (no KEEP, no SORT on message, etc.). +fieldIsPartiallyUnmappedMultiIndexWithoutKeep +required_capability: optional_fields_v2 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +; +ignoreOrder:true + +// no_mapping_sample_data also has values for @timestamp, client_ip, and event_duration, +// but we cannot load them because they're not partially mapped as KEYWORD, but as other types. +// TODO: This is inconsistent, how do we resolve this? +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | sample_data +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | sample_data +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | sample_data +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | sample_data +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | sample_data +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | sample_data +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | 42 | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +; + // https://github.com/elastic/elasticsearch/issues/143991 statsFilteredAggAfterEvalWithDottedUnmappedField required_capability: optional_fields_v2 diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index fa3543219e426..ed94e1534ed3d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -9,6 +9,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; +import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Alias; @@ -19,6 +20,7 @@ import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; +import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.expression.function.aggregate.Count; @@ -52,10 +54,15 @@ import org.elasticsearch.xpack.esql.plan.logical.Sample; import org.elasticsearch.xpack.esql.plan.logical.Subquery; import org.elasticsearch.xpack.esql.plan.logical.UnionAll; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; import java.util.List; +import java.util.Map; +import java.util.Set; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.asLimit; @@ -3795,6 +3802,48 @@ public void testStatsFilteredAggAfterEvalWithDottedUnmappedFieldFromIndex() { } } + /** + * Reproducer for https://github.com/elastic/elasticsearch/issues/141994 + * When unmapped_fields="load" and a keyword field is partially unmapped (mapped in some concrete indices but not in others), + * the field should be resolved as {@link PotentiallyUnmappedKeywordEsField} in the EsRelation output, even if the field is + * not explicitly referenced in any downstream expression (e.g. KEEP, SORT, WHERE). + */ + public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() { + String query = setUnmappedLoad(""" + FROM test* + | SORT emp_no + """); + + var statement = EsqlTestUtils.TEST_PARSER.createStatement(query); + var mapping = EsqlTestUtils.loadMapping("mapping-basic.json"); + var indexResolution = IndexResolution.valid( + new EsIndex( + "test*", + mapping, + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of("first_name") + ) + ); + var indexResolutions = Map.of(new IndexPattern(Source.EMPTY, "test*"), indexResolution); + var analyzer = AnalyzerTestUtils.analyzer( + indexResolutions, + EsqlTestUtils.TEST_VERIFIER, + EsqlTestUtils.configuration(query), + statement + ); + var plan = analyzer.analyze(statement.plan()); + + var limit = as(plan, Limit.class); + var order = as(limit.child(), OrderBy.class); + var relation = as(order.child(), EsRelation.class); + + var firstNameAttr = relation.output().stream().filter(a -> a.name().equals("first_name")).findFirst().orElseThrow(); + var fieldAttr = as(firstNameAttr, FieldAttribute.class); + as(fieldAttr.field(), PotentiallyUnmappedKeywordEsField.class); + } + private void verificationFailure(String statement, String expectedFailure) { var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); assertThat(e.getMessage(), containsString(expectedFailure)); From 80773aeb32531d90b0b084330afbdb0e9f050c67 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 13 Mar 2026 16:31:14 +0000 Subject: [PATCH 02/69] [CI] Auto commit changes from spotless --- .../xpack/esql/analysis/AnalyzerUnmappedTests.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index ed94e1534ed3d..f281f18c98e12 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -38,6 +38,9 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Add; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.LessThan; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.Dissect; import org.elasticsearch.xpack.esql.plan.logical.Enrich; @@ -54,9 +57,6 @@ import org.elasticsearch.xpack.esql.plan.logical.Sample; import org.elasticsearch.xpack.esql.plan.logical.Subquery; import org.elasticsearch.xpack.esql.plan.logical.UnionAll; -import org.elasticsearch.xpack.esql.index.EsIndex; -import org.elasticsearch.xpack.esql.index.IndexResolution; -import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.join.JoinTypes; import org.elasticsearch.xpack.esql.plan.logical.join.LookupJoin; From 4d3759285bfb810271c4241bbc6355b140a6e39b Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 13 Mar 2026 18:17:00 +0100 Subject: [PATCH 03/69] WIP: check what cursor did there --- .../src/main/resources/unmapped-load.csv-spec | 41 +++++++++---------- .../xpack/esql/analysis/Analyzer.java | 9 ++++ 2 files changed, 29 insertions(+), 21 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 5a7a463323c3f..457cffffa9f1b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -938,33 +938,32 @@ FROM partial_mapping_sample_data // https://github.com/elastic/elasticsearch/issues/141994 // A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded -// from _source even when not explicitly referenced in a downstream expression (no KEEP, no SORT on message, etc.). +// from _source even when not explicitly referenced in a downstream expression like KEEP. fieldIsPartiallyUnmappedMultiIndexWithoutKeep +required_capability: index_metadata_field required_capability: optional_fields_v2 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index +| DROP @timestamp, client_ip, event_duration +| SORT _index, message DESC ; -ignoreOrder:true - -// no_mapping_sample_data also has values for @timestamp, client_ip, and event_duration, -// but we cannot load them because they're not partially mapped as KEYWORD, but as other types. -// TODO: This is inconsistent, how do we resolve this? -@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword -2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | sample_data -2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | sample_data -2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | sample_data -2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | sample_data -2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | sample_data -2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | sample_data -2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | sample_data -null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data -null | null | null | Connection error? | no_mapping_sample_data -null | null | null | Connection error? | no_mapping_sample_data -null | null | null | Connection error? | no_mapping_sample_data -null | null | null | 42 | no_mapping_sample_data -null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data -null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data + +message:keyword | _index:keyword +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connected to 10.1.0.3! | no_mapping_sample_data +Connected to 10.1.0.2! | no_mapping_sample_data +Connected to 10.1.0.1! | no_mapping_sample_data +42 | no_mapping_sample_data +Disconnected | sample_data +Connection error | sample_data +Connection error | sample_data +Connection error | sample_data +Connected to 10.1.0.3 | sample_data +Connected to 10.1.0.2 | sample_data +Connected to 10.1.0.1 | sample_data ; // https://github.com/elastic/elasticsearch/issues/143991 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 007c2f545aba8..cc3031c5f19a0 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -347,6 +347,15 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR var attributes = mappingAsAttributes(plan.source(), esIndex.mapping()); attributes.addAll(metadata.stream().map(NamedExpression::toAttribute).toList()); + if (context.unmappedResolution() == UnmappedResolution.LOAD && plan.indexMode() != IndexMode.LOOKUP) { + for (int i = 0; i < attributes.size(); i++) { + if (attributes.get(i) instanceof FieldAttribute fa + && fa.dataType() == KEYWORD + && esIndex.isPartiallyUnmappedField(fa.name())) { + attributes.set(i, ResolveRefs.insistKeyword(fa)); + } + } + } return new EsRelation( plan.source(), esIndex.name(), From db022987d7d36e2718fab5aa8358989bb16300eb Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 24 Mar 2026 13:50:04 +0100 Subject: [PATCH 04/69] Update test --- .../src/main/resources/unmapped-load.csv-spec | 2 +- .../esql/analysis/AnalyzerUnmappedTests.java | 43 +++++++------------ 2 files changed, 16 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index fec8dc1c8bead..60f8006fefab1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -884,8 +884,8 @@ required_capability: optional_fields_v2 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index | DROP @timestamp, client_ip, event_duration -| SORT _index, message DESC ; +ignoreOrder:true message:keyword | _index:keyword Connection error? | no_mapping_sample_data diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 3846d3a4052e9..5cc86da1b71a8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -14,17 +14,14 @@ import org.elasticsearch.xpack.esql.TestAnalyzer; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; -import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; -import org.elasticsearch.xpack.esql.expression.FieldAttribute; import org.elasticsearch.xpack.esql.index.EsIndex; -import org.elasticsearch.xpack.esql.index.IndexResolution; -import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Filter; @@ -568,31 +565,21 @@ public void testTbucketWithUnmappedTimestampWithFork() { * not explicitly referenced in any downstream expression (e.g. KEEP, SORT, WHERE). */ public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() { - String query = setUnmappedLoad(""" - FROM test* - | SORT emp_no - """); - - var statement = EsqlTestUtils.TEST_PARSER.createStatement(query); - var mapping = EsqlTestUtils.loadMapping("mapping-basic.json"); - var indexResolution = IndexResolution.valid( - new EsIndex( - "test*", - mapping, - Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), - Map.of(), - Map.of(), - Set.of("first_name") + var plan = analyzer() + .addIndex( + new EsIndex( + "test*", + EsqlTestUtils.loadMapping("mapping-basic.json"), + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of("first_name") + ) ) - ); - var indexResolutions = Map.of(new IndexPattern(Source.EMPTY, "test*"), indexResolution); - var analyzer = AnalyzerTestUtils.analyzer( - indexResolutions, - EsqlTestUtils.TEST_VERIFIER, - EsqlTestUtils.configuration(query), - statement - ); - var plan = analyzer.analyze(statement.plan()); + .statement(setUnmappedLoad(""" + FROM test* + | SORT emp_no + """)); var limit = as(plan, Limit.class); var order = as(limit.child(), OrderBy.class); From c5c502f7eb77326dfdb2daa553ae77a9348c0580 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 24 Mar 2026 13:50:18 +0100 Subject: [PATCH 05/69] Spotless --- .../esql/analysis/AnalyzerUnmappedTests.java | 26 +++++++++---------- 1 file changed, 12 insertions(+), 14 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 5cc86da1b71a8..0b5db4b4389ac 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -565,21 +565,19 @@ public void testTbucketWithUnmappedTimestampWithFork() { * not explicitly referenced in any downstream expression (e.g. KEEP, SORT, WHERE). */ public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() { - var plan = analyzer() - .addIndex( - new EsIndex( - "test*", - EsqlTestUtils.loadMapping("mapping-basic.json"), - Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), - Map.of(), - Map.of(), - Set.of("first_name") - ) + var plan = analyzer().addIndex( + new EsIndex( + "test*", + EsqlTestUtils.loadMapping("mapping-basic.json"), + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of("first_name") ) - .statement(setUnmappedLoad(""" - FROM test* - | SORT emp_no - """)); + ).statement(setUnmappedLoad(""" + FROM test* + | SORT emp_no + """)); var limit = as(plan, Limit.class); var order = as(limit.child(), OrderBy.class); From 464c088b0acabff1a99fcd56e42c0c232dd0a818 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 12:22:02 +0100 Subject: [PATCH 06/69] WIP --- .../java/org/elasticsearch/xpack/esql/analysis/Analyzer.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 4e23e25c87cbd..8915f83e9ac46 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -348,6 +348,7 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR var attributes = mappingAsAttributes(plan.source(), esIndex.mapping()); attributes.addAll(metadata.stream().map(NamedExpression::toAttribute).toList()); + // TODO: at least factor out into a helper method. if (context.unmappedResolution() == UnmappedResolution.LOAD && plan.indexMode() != IndexMode.LOOKUP) { for (int i = 0; i < attributes.size(); i++) { if (attributes.get(i) instanceof FieldAttribute fa @@ -671,6 +672,8 @@ protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) { default -> plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> maybeResolveAttribute(ua, childrenOutput)); }; + // TODO: this looks at used partially mapped field attributes. It should probably get (mostly?) obsolete with the change to + // ResolveTables. Also has overlap with Shmuel's work. return context.unmappedResolution() == UnmappedResolution.LOAD ? resolvePartiallyMapped(resolved, context) : resolved; } From c9899267d027f88d5a9ff3dd8a1774dd66046c76 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 14:07:32 +0100 Subject: [PATCH 07/69] WIP: more tests --- .../src/main/resources/unmapped-load.csv-spec | 109 ++++++++++++++++++ .../esql/analysis/AnalyzerUnmappedTests.java | 102 ++++++++++++++++ 2 files changed, 211 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 60f8006fefab1..8391adfe12e0f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -904,6 +904,115 @@ Connected to 10.1.0.2 | sample_data Connected to 10.1.0.1 | sample_data ; +// EVAL on a different field; the partially-mapped keyword field (message) is not explicitly referenced by any command, +// so it must be resolved as PotentiallyUnmappedKeywordEsField at the EsRelation level (the new fix). +fieldIsPartiallyUnmappedMultiIndexWithEvalOnOtherField +required_capability: index_metadata_field +required_capability: optional_fields_v2 + +SET unmapped_fields="load"\; +//TODO: Will not work because event_duration is partially mapped too; need to use a field that is fully mapped in both indices +FROM sample_data, no_mapping_sample_data METADATA _index +| EVAL duration_ms = event_duration / 1000 +| DROP @timestamp, client_ip, event_duration +; +ignoreOrder:true + +message:keyword | _index:keyword | duration_ms:long +Connected to 10.1.0.1 | sample_data | 1756 +Connected to 10.1.0.2 | sample_data | 2764 +Connected to 10.1.0.3 | sample_data | 3450 +Connection error | sample_data | 5033 +Connection error | sample_data | 8268 +Connection error | sample_data | 725 +Disconnected | sample_data | 1232 +Connected to 10.1.0.1!| no_mapping_sample_data | null +Connected to 10.1.0.2!| no_mapping_sample_data | null +Connected to 10.1.0.3!| no_mapping_sample_data | null +Connection error? | no_mapping_sample_data | null +Connection error? | no_mapping_sample_data | null +Connection error? | no_mapping_sample_data | null +42 | no_mapping_sample_data | null +; + +// RENAME on an unrelated field; the partially-mapped keyword field (message) is not explicitly referenced. +fieldIsPartiallyUnmappedMultiIndexWithRenameOnOtherField +required_capability: index_metadata_field +required_capability: optional_fields_v2 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +// TODO: likely won't work because event_duration is partially mapped too; need to use a field that is fully mapped in both indices +| RENAME event_duration AS duration +| DROP @timestamp, client_ip +; +ignoreOrder:true + +duration:long | message:keyword | _index:keyword +1756467 | Connected to 10.1.0.1 | sample_data +5033755 | Connection error | sample_data +8268153 | Connection error | sample_data +725448 | Connection error | sample_data +1232382 | Disconnected | sample_data +2764889 | Connected to 10.1.0.2 | sample_data +3450233 | Connected to 10.1.0.3 | sample_data +null | Connected to 10.1.0.1! | no_mapping_sample_data +null | Connection error? | no_mapping_sample_data +null | Connection error? | no_mapping_sample_data +null | Connection error? | no_mapping_sample_data +null | 42 | no_mapping_sample_data +null | Connected to 10.1.0.2! | no_mapping_sample_data +null | Connected to 10.1.0.3! | no_mapping_sample_data +; + +// Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. +fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop +required_capability: index_metadata_field +required_capability: optional_fields_v2 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| DROP *_ip, *_duration, @timestamp +; +ignoreOrder:true + +message:keyword | _index:keyword +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connected to 10.1.0.3! | no_mapping_sample_data +Connected to 10.1.0.2! | no_mapping_sample_data +Connected to 10.1.0.1! | no_mapping_sample_data +42 | no_mapping_sample_data +Disconnected | sample_data +Connection error | sample_data +Connection error | sample_data +Connection error | sample_data +Connected to 10.1.0.3 | sample_data +Connected to 10.1.0.2 | sample_data +Connected to 10.1.0.1 | sample_data +; + +// DROP the partially-mapped field itself — verify no errors from converting then pruning. +fieldIsPartiallyUnmappedMultiIndexDropPartiallyMappedField +required_capability: index_metadata_field +required_capability: optional_fields_v2 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| DROP @timestamp, client_ip, event_duration, message +| SORT _index +| LIMIT 5 +; + +_index:keyword +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +; + maxOverTimeOfKeyword required_capability: optional_fields_v2 required_capability: ts_command_v0 diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 0b5db4b4389ac..e70ea91263bb2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -18,7 +18,9 @@ import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.KeywordEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -588,6 +590,106 @@ public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() { as(fieldAttr.field(), PotentiallyUnmappedKeywordEsField.class); } + /** + * Verify that a non-keyword partially-mapped field (e.g. integer) is NOT converted to + * {@link PotentiallyUnmappedKeywordEsField}, since the fix only applies to keyword fields. + */ + public void testPartiallyMappedNonKeywordFieldNotLoadedWithoutExplicitReference() { + var plan = analyzer().addIndex( + new EsIndex( + "test*", + EsqlTestUtils.loadMapping("mapping-basic.json"), + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of("salary") // salary is integer, not keyword + ) + ).statement(setUnmappedLoad(""" + FROM test* + | SORT emp_no + """)); + + var limit = as(plan, Limit.class); + var order = as(limit.child(), OrderBy.class); + var relation = as(order.child(), EsRelation.class); + + var salaryAttr = relation.output().stream().filter(a -> a.name().equals("salary")).findFirst().orElseThrow(); + var fieldAttr = as(salaryAttr, FieldAttribute.class); + assertThat(fieldAttr.field(), not(instanceOf(PotentiallyUnmappedKeywordEsField.class))); + assertThat(fieldAttr.field().getClass(), is(EsField.class)); + assertThat(fieldAttr.field().getDataType(), is(DataType.INTEGER)); + } + + /** + * When both a keyword and non-keyword field are partially mapped, only the keyword field + * should be converted to {@link PotentiallyUnmappedKeywordEsField}. + */ + public void testOnlyPartiallyMappedKeywordFieldLoaded() { + var plan = analyzer().addIndex( + new EsIndex( + "test*", + EsqlTestUtils.loadMapping("mapping-basic.json"), + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of("first_name", "salary") // keyword + integer + ) + ).statement(setUnmappedLoad(""" + FROM test* + | SORT emp_no + """)); + + var limit = as(plan, Limit.class); + var order = as(limit.child(), OrderBy.class); + var relation = as(order.child(), EsRelation.class); + + // keyword field should be converted + var firstName = relation.output().stream().filter(a -> a.name().equals("first_name")).findFirst().orElseThrow(); + as(as(firstName, FieldAttribute.class).field(), PotentiallyUnmappedKeywordEsField.class); + + // integer field should NOT be loaded + var salary = relation.output().stream().filter(a -> a.name().equals("salary")).findFirst().orElseThrow(); + var salaryField = as(salary, FieldAttribute.class).field(); + assertThat(salaryField, not(instanceOf(PotentiallyUnmappedKeywordEsField.class))); + assertThat(salaryField.getClass(), is(EsField.class)); + assertThat(salaryField.getDataType(), is(DataType.INTEGER)); + } + + /** + * Verify that a keyword field that is NOT partially unmapped is not converted to + * {@link PotentiallyUnmappedKeywordEsField}, even when another keyword field is. + */ + public void testNonPartiallyMappedKeywordFieldNotLoadedFromSource() { + var plan = analyzer().addIndex( + new EsIndex( + "test*", + EsqlTestUtils.loadMapping("mapping-basic.json"), + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of("first_name") // only first_name is partially unmapped + ) + ).statement(setUnmappedLoad(""" + FROM test* + | SORT emp_no + """)); + + var limit = as(plan, Limit.class); + var order = as(limit.child(), OrderBy.class); + var relation = as(order.child(), EsRelation.class); + + // first_name (partially unmapped keyword) should be converted + var firstName = relation.output().stream().filter(a -> a.name().equals("first_name")).findFirst().orElseThrow(); + as(as(firstName, FieldAttribute.class).field(), PotentiallyUnmappedKeywordEsField.class); + + // last_name (keyword but NOT partially unmapped) should NOT be converted + var lastName = relation.output().stream().filter(a -> a.name().equals("last_name")).findFirst().orElseThrow(); + var lastNameField = as(lastName, FieldAttribute.class).field(); + assertThat(lastNameField, not(instanceOf(PotentiallyUnmappedKeywordEsField.class))); + assertThat(lastNameField, instanceOf(KeywordEsField.class)); + assertThat(lastNameField.getDataType(), is(DataType.KEYWORD)); + } + public void testTbucketWithUnmappedTimestampWithLookupJoin() { var query = """ FROM test From 65c765aec286610f4a691aa01851b8b59d95c9dc Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 14:33:12 +0100 Subject: [PATCH 08/69] Add ability to unmap fields in CsvTestsDataLoader --- .../xpack/esql/CsvTestsDataLoader.java | 86 ++++++++++++++++--- .../elasticsearch/xpack/esql/CsvTests.java | 5 +- 2 files changed, 76 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 522611eb829db..5bc69b15b50e7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -737,23 +737,38 @@ private static void load(RestClient client, TestDataset dataset, IndexCreator in public static String readMappingFile(TestDataset dataset) throws IOException { String mappingJsonText = dataset.loadMappings(); - if (dataset.typeMapping == null || dataset.typeMapping.isEmpty()) { + boolean hasTypeMappingOverrides = dataset.typeMapping != null && dataset.typeMapping.isEmpty() == false; + if (hasTypeMappingOverrides == false && dataset.dynamic == null) { return mappingJsonText; } boolean modified = false; ObjectMapper mapper = new ObjectMapper(); JsonNode mappingNode = mapper.readTree(mappingJsonText); - JsonNode propertiesNode = mappingNode.path("properties"); - for (Map.Entry entry : dataset.typeMapping.entrySet()) { - String key = entry.getKey(); - String newType = entry.getValue(); - - if (propertiesNode.has(key)) { - modified = true; - ((ObjectNode) propertiesNode.get(key)).put("type", newType); + if (hasTypeMappingOverrides) { + ObjectNode propertiesNode = (ObjectNode) mappingNode.path("properties"); + for (Map.Entry entry : dataset.typeMapping.entrySet()) { + String key = entry.getKey(); + String newType = entry.getValue(); + + if (newType == null) { + // null value means remove the field from the mapping + if (propertiesNode.has(key)) { + propertiesNode.remove(key); + modified = true; + } + } else if (propertiesNode.has(key)) { + ((ObjectNode) propertiesNode.get(key)).put("type", newType); + modified = true; + } } } + + if (dataset.dynamic != null) { + ((ObjectNode) mappingNode).put("dynamic", dataset.dynamic); + modified = true; + } + if (modified) { return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(mappingNode); } @@ -1038,22 +1053,23 @@ public record TestDataset( String dataFileName, String settingFileName, boolean allowSubFields, - @Nullable Map typeMapping, // Override mappings read from mappings file - @Nullable Map dynamicTypeMapping, // Define mappings not in the mapping files, but available from field-caps + @Nullable Map typeMapping, + @Nullable Map dynamicTypeMapping, + @Nullable String dynamic, List inferenceEndpoints, List requiredCapabilities ) { public TestDataset(String indexName) { - this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, null, List.of(), List.of()); + this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, null, null, List.of(), List.of()); } public TestDataset(String indexName, String mappingFileName, String dataFileName) { - this(indexName, mappingFileName, dataFileName, null, true, null, null, List.of(), List.of()); + this(indexName, mappingFileName, dataFileName, null, true, null, null, null, List.of(), List.of()); } public TestDataset(String indexName, String mappingFileName, String dataFileName, String settingFileName) { - this(indexName, mappingFileName, dataFileName, settingFileName, true, null, null, List.of(), List.of()); + this(indexName, mappingFileName, dataFileName, settingFileName, true, null, null, null, List.of(), List.of()); } public TestDataset withIndex(String indexName) { @@ -1065,6 +1081,7 @@ public TestDataset withIndex(String indexName) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1079,6 +1096,7 @@ public TestDataset withData(String dataFileName) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1093,6 +1111,7 @@ public TestDataset noData() { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1107,6 +1126,7 @@ public TestDataset withSetting(String settingFileName) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1121,11 +1141,20 @@ public TestDataset noSubfields() { false, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); } + /** + * Overrides the types of fields in the mapping file. Each entry maps a field name to its new type + * (e.g. {@code Map.of("client_ip", "keyword")} changes client_ip from ip to keyword). + * A {@code null} value removes the field from the mapping entirely, making it unmapped for this index. + *

+ * This affects both the Elasticsearch index mapping (via {@link CsvTestsDataLoader#readMappingFile}) and + * the in-memory mapping used by CSV unit tests. + */ public TestDataset withTypeMapping(Map typeMapping) { return new TestDataset( indexName, @@ -1135,11 +1164,18 @@ public TestDataset withTypeMapping(Map typeMapping) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); } + /** + * Adds field mappings that are not present in the mapping file, simulating fields discovered via field-caps + * at runtime (e.g. runtime fields). These fields are only added to the in-memory mapping used by CSV unit + * tests and are not included in the Elasticsearch index mapping. Fields already present in the mapping file + * are not overridden. + */ public TestDataset withDynamicTypeMapping(Map dynamicTypeMapping) { return new TestDataset( indexName, @@ -1149,6 +1185,26 @@ public TestDataset withDynamicTypeMapping(Map dynamicTypeMapping allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, + inferenceEndpoints, + requiredCapabilities + ); + } + + /** + * Sets the "dynamic" mapping parameter (e.g. "false", "strict", "runtime"). + * This prevents unmapped fields in the data from being automatically indexed. + */ + public TestDataset withDynamic(String dynamic) { + return new TestDataset( + indexName, + mappingFileName, + dataFileName, + settingFileName, + allowSubFields, + typeMapping, + dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1163,6 +1219,7 @@ public TestDataset withInferenceEndpoints(String... inferenceEndpoints) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, List.of(inferenceEndpoints), requiredCapabilities ); @@ -1177,6 +1234,7 @@ public TestDataset withRequiredCapabilities(EsqlCapabilities.Cap... requiredCapa allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, List.of(requiredCapabilities) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 9da68dcda6799..6d7dbd1b32b13 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -505,7 +505,10 @@ private static Map createMappingForIndex(CsvTestsDataLoader.Tes var mapping = new TreeMap<>(LoadMapping.loadMapping(dataset.streamMapping())); if (dataset.typeMapping() != null) { for (var entry : dataset.typeMapping().entrySet()) { - if (mapping.containsKey(entry.getKey())) { + if (entry.getValue() == null) { + // null value means remove the field from the mapping + mapping.remove(entry.getKey()); + } else if (mapping.containsKey(entry.getKey())) { DataType dataType = DataType.fromTypeName(entry.getValue()); EsField field = mapping.get(entry.getKey()); EsField editedField = new EsField( From 8d058cae95e2cc30cbe02d7e58e9cfe6679daf0c Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 14:57:17 +0100 Subject: [PATCH 09/69] Add employees_no_names index --- .../xpack/esql/CsvTestsDataLoader.java | 17 ++++++++ .../src/main/resources/unmapped-load.csv-spec | 41 +++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 5bc69b15b50e7..7359be5d5bcde 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -43,6 +43,7 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -88,6 +89,10 @@ public class CsvTestsDataLoader { new TestDataset("employees", "mapping-default.json", "employees.csv").noSubfields(), new TestDataset("voyager", "mapping-voyager.json", "voyager.csv").noSubfields(), new TestDataset("employees_incompatible", "mapping-default-incompatible.json", "employees_incompatible.csv").noSubfields(), + new TestDataset("employees", "mapping-default.json", "employees.csv").withIndex("employees_no_names") + .withTypeMapping(removeFields("first_name", "last_name")) + .withDynamic("false") + .noSubfields(), new TestDataset("all_types", "mapping-all-types.json", "all-types.csv"), new TestDataset("hosts"), new TestDataset("apps"), @@ -735,6 +740,18 @@ private static void load(RestClient client, TestDataset dataset, IndexCreator in } } + /** + * Creates a type mapping that removes the given fields from the mapping. + * For use with {@link TestDataset#withTypeMapping}. + */ + private static Map removeFields(String... fields) { + var map = new HashMap(); + for (String field : fields) { + map.put(field, null); + } + return map; + } + public static String readMappingFile(TestDataset dataset) throws IOException { String mappingJsonText = dataset.loadMappings(); boolean hasTypeMappingOverrides = dataset.typeMapping != null && dataset.typeMapping.isEmpty() == false; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 8391adfe12e0f..0078a9efd3b11 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -874,6 +874,47 @@ FROM partial_mapping_sample_data 2024-10-23T13:51:54.732Z | Connection error? | 725449 ; +// Confirm that the employees_no_names index does not have first_name and last_name mapped. +employeesNoNamesFieldsRemoved +required_capability: index_metadata_field +required_capability: optional_fields_v2 + +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword +10001 | Georgi | Facello | employees +10001 | null | null | employees_no_names +10002 | Bezalel | Simmel | employees +10002 | null | null | employees_no_names +10003 | Parto | Bamford | employees +10003 | null | null | employees_no_names +; + +employeesNoNamesLoad +required_capability: index_metadata_field +required_capability: optional_fields_v2 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword +10001 | Georgi | Facello | employees +10001 | Georgi | Facello | employees_no_names +10002 | Bezalel | Simmel | employees +10002 | Bezalel | Simmel | employees_no_names +10003 | Parto | Bamford | employees +10003 | Parto | Bamford | employees_no_names +; + + // https://github.com/elastic/elasticsearch/issues/141994 // A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded // from _source even when not explicitly referenced in a downstream expression like KEEP. From 9a2e72c00662b970469c4466c29e291ace3f1b6f Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 17:06:05 +0100 Subject: [PATCH 10/69] Fix, add tests --- .../src/main/resources/unmapped-load.csv-spec | 121 +++++++++++------- 1 file changed, 74 insertions(+), 47 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 0078a9efd3b11..9fa044b9cb0a5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -945,65 +945,92 @@ Connected to 10.1.0.2 | sample_data Connected to 10.1.0.1 | sample_data ; -// EVAL on a different field; the partially-mapped keyword field (message) is not explicitly referenced by any command, -// so it must be resolved as PotentiallyUnmappedKeywordEsField at the EsRelation level (the new fix). -fieldIsPartiallyUnmappedMultiIndexWithEvalOnOtherField +// EVAL on a fully mapped field (salary); the partially-mapped keyword fields (first_name, last_name) are not explicitly +// referenced by any command — they remain in the output implicitly and get loaded. +fieldIsPartiallyUnmappedWithEvalOnOtherMappedField required_capability: index_metadata_field required_capability: optional_fields_v2 SET unmapped_fields="load"\; -//TODO: Will not work because event_duration is partially mapped too; need to use a field that is fully mapped in both indices -FROM sample_data, no_mapping_sample_data METADATA _index -| EVAL duration_ms = event_duration / 1000 -| DROP @timestamp, client_ip, event_duration +FROM employees, employees_no_names METADATA _index +| EVAL salary_k = salary / 1000 +| DROP gender, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | salary_k:integer +10001 | Georgi | Facello | employees | 57 +10001 | Georgi | Facello | employees_no_names | 57 +10002 | Bezalel | Simmel | employees | 56 +10002 | Bezalel | Simmel | employees_no_names | 56 +10003 | Parto | Bamford | employees | 61 +10003 | Parto | Bamford | employees_no_names | 61 ; -ignoreOrder:true -message:keyword | _index:keyword | duration_ms:long -Connected to 10.1.0.1 | sample_data | 1756 -Connected to 10.1.0.2 | sample_data | 2764 -Connected to 10.1.0.3 | sample_data | 3450 -Connection error | sample_data | 5033 -Connection error | sample_data | 8268 -Connection error | sample_data | 725 -Disconnected | sample_data | 1232 -Connected to 10.1.0.1!| no_mapping_sample_data | null -Connected to 10.1.0.2!| no_mapping_sample_data | null -Connected to 10.1.0.3!| no_mapping_sample_data | null -Connection error? | no_mapping_sample_data | null -Connection error? | no_mapping_sample_data | null -Connection error? | no_mapping_sample_data | null -42 | no_mapping_sample_data | null -; - -// RENAME on an unrelated field; the partially-mapped keyword field (message) is not explicitly referenced. -fieldIsPartiallyUnmappedMultiIndexWithRenameOnOtherField +fieldIsPartiallyUnmappedWithRenameOnOtherMappedField required_capability: index_metadata_field required_capability: optional_fields_v2 SET unmapped_fields="load"\; -FROM sample_data, no_mapping_sample_data METADATA _index -// TODO: likely won't work because event_duration is partially mapped too; need to use a field that is fully mapped in both indices -| RENAME event_duration AS duration -| DROP @timestamp, client_ip +FROM employees, employees_no_names METADATA _index +| RENAME salary AS pay +| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:keyword +10001 | Georgi | Facello | 57305 | employees +10001 | Georgi | Facello | 57305 | employees_no_names +10002 | Bezalel | Simmel | 56371 | employees +10002 | Bezalel | Simmel | 56371 | employees_no_names +10003 | Parto | Bamford | 61805 | employees +10003 | Parto | Bamford | 61805 | employees_no_names +; + +// EVAL on a partially mapped field (last_name). +// first_name is not referenced by any command but must still be loaded. +fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField +required_capability: index_metadata_field +required_capability: optional_fields_v2 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| EVAL last_name = TO_UPPER(last_name) +| DROP gender, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword +10001 | Georgi | employees | FACELLO +10001 | Georgi | employees_no_names | FACELLO +10002 | Bezalel | employees | SIMMEL +10002 | Bezalel | employees_no_names | SIMMEL +10003 | Parto | employees | BAMFORD +10003 | Parto | employees_no_names | BAMFORD +; + +fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField +required_capability: index_metadata_field +required_capability: optional_fields_v2 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| RENAME last_name AS surname +| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 ; -ignoreOrder:true -duration:long | message:keyword | _index:keyword -1756467 | Connected to 10.1.0.1 | sample_data -5033755 | Connection error | sample_data -8268153 | Connection error | sample_data -725448 | Connection error | sample_data -1232382 | Disconnected | sample_data -2764889 | Connected to 10.1.0.2 | sample_data -3450233 | Connected to 10.1.0.3 | sample_data -null | Connected to 10.1.0.1! | no_mapping_sample_data -null | Connection error? | no_mapping_sample_data -null | Connection error? | no_mapping_sample_data -null | Connection error? | no_mapping_sample_data -null | 42 | no_mapping_sample_data -null | Connected to 10.1.0.2! | no_mapping_sample_data -null | Connected to 10.1.0.3! | no_mapping_sample_data +emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index:keyword +10001 | Georgi | Facello | 57305 | employees +10001 | Georgi | Facello | 57305 | employees_no_names +10002 | Bezalel | Simmel | 56371 | employees +10002 | Bezalel | Simmel | 56371 | employees_no_names +10003 | Parto | Bamford | 61805 | employees +10003 | Parto | Bamford | 61805 | employees_no_names ; // Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. From a123267075492a30c3f19b9321a71acc27f1dcc7 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 17:23:13 +0100 Subject: [PATCH 11/69] Fix capabilities --- .../src/main/resources/unmapped-load.csv-spec | 15 +++++++-------- .../xpack/esql/action/EsqlCapabilities.java | 7 +++++++ 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 9fa044b9cb0a5..5980450fc073c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -877,7 +877,6 @@ FROM partial_mapping_sample_data // Confirm that the employees_no_names index does not have first_name and last_name mapped. employeesNoNamesFieldsRemoved required_capability: index_metadata_field -required_capability: optional_fields_v2 FROM employees, employees_no_names METADATA _index | KEEP emp_no, first_name, last_name, _index @@ -896,7 +895,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword employeesNoNamesLoad required_capability: index_metadata_field -required_capability: optional_fields_v2 +required_capability: optional_fields_fix_load_partially_mapped_keyword SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -920,7 +919,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword // from _source even when not explicitly referenced in a downstream expression like KEEP. fieldIsPartiallyUnmappedMultiIndexWithoutKeep required_capability: index_metadata_field -required_capability: optional_fields_v2 +required_capability: optional_fields_fix_load_partially_mapped_keyword SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -949,7 +948,7 @@ Connected to 10.1.0.1 | sample_data // referenced by any command — they remain in the output implicitly and get loaded. fieldIsPartiallyUnmappedWithEvalOnOtherMappedField required_capability: index_metadata_field -required_capability: optional_fields_v2 +required_capability: optional_fields_fix_load_partially_mapped_keyword SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -970,7 +969,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | s fieldIsPartiallyUnmappedWithRenameOnOtherMappedField required_capability: index_metadata_field -required_capability: optional_fields_v2 +required_capability: optional_fields_fix_load_partially_mapped_keyword SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -993,7 +992,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:k // first_name is not referenced by any command but must still be loaded. fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField required_capability: index_metadata_field -required_capability: optional_fields_v2 +required_capability: optional_fields_fix_load_partially_mapped_keyword SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1014,7 +1013,7 @@ emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField required_capability: index_metadata_field -required_capability: optional_fields_v2 +required_capability: optional_fields_fix_load_partially_mapped_keyword SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1036,7 +1035,7 @@ emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index: // Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop required_capability: index_metadata_field -required_capability: optional_fields_v2 +required_capability: optional_fields_fix_load_partially_mapped_keyword SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 1e3077aa674a6..3b77045346c04 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -253,6 +253,13 @@ public enum Cap { */ OPTIONAL_FIELDS_V2(Build.current().isSnapshot()), + /** + * Fix for partially mapped keyword fields not being loaded from _source even when not explicitly referenced + * by a downstream command (e.g. KEEP, WHERE, SORT). + * See https://github.com/elastic/elasticsearch/issues/141994 + */ + OPTIONAL_FIELDS_FIX_LOAD_PARTIALLY_MAPPED_KEYWORD(OPTIONAL_FIELDS_V2.isEnabled()), + /** * Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently * supported. From 773fad4f19a8f08e85c5ccc10b8a4a459d735615 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 18:05:18 +0100 Subject: [PATCH 12/69] Don't mess with partially mapped non-KEYWORD fields at all --- .../xpack/esql/CsvTestsDataLoader.java | 7 +++ .../src/main/resources/unmapped-load.csv-spec | 22 ++++++++ .../xpack/esql/analysis/Analyzer.java | 52 +------------------ .../esql/analysis/AnalyzerUnmappedTests.java | 31 +++++++++++ 4 files changed, 61 insertions(+), 51 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 7359be5d5bcde..2f9231a54a1c1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -93,6 +93,13 @@ public class CsvTestsDataLoader { .withTypeMapping(removeFields("first_name", "last_name")) .withDynamic("false") .noSubfields(), + new TestDataset("employees", "mapping-default.json", "employees.csv").withIndex("employees_gender_text") + .withTypeMapping(Map.of("gender", "text")) + .noSubfields(), + new TestDataset("employees", "mapping-default.json", "employees.csv").withIndex("employees_no_gender") + .withTypeMapping(removeFields("gender")) + .withDynamic("false") + .noSubfields(), new TestDataset("all_types", "mapping-all-types.json", "all-types.csv"), new TestDataset("hosts"), new TestDataset("apps"), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 5980450fc073c..d1247f7baeaed 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -914,6 +914,28 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword ; +// Verify that a partially-mapped TEXT field is not automatically loaded from _source. +// gender is mapped as text in employees_gender_text and unmapped in employees_no_gender. +partiallyMappedTextFieldNotLoaded +required_capability: index_metadata_field +required_capability: optional_fields_fix_load_partially_mapped_keyword + +SET unmapped_fields="load"\; +FROM employees_gender_text, employees_no_gender METADATA _index +| DROP first_name, last_name, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | gender:text | _index:keyword +10001 | M | employees_gender_text +10001 | null | employees_no_gender +10002 | F | employees_gender_text +10002 | null | employees_no_gender +10003 | M | employees_gender_text +10003 | null | employees_no_gender +; + // https://github.com/elastic/elasticsearch/issues/141994 // A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded // from _source even when not explicitly referenced in a downstream expression like KEEP. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 8915f83e9ac46..cc2d917863cec 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -672,9 +672,7 @@ protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) { default -> plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> maybeResolveAttribute(ua, childrenOutput)); }; - // TODO: this looks at used partially mapped field attributes. It should probably get (mostly?) obsolete with the change to - // ResolveTables. Also has overlap with Shmuel's work. - return context.unmappedResolution() == UnmappedResolution.LOAD ? resolvePartiallyMapped(resolved, context) : resolved; + return resolved; } private LogicalPlan resolveAggregate(Aggregate aggregate, List childrenOutput) { @@ -1250,54 +1248,6 @@ public static FieldAttribute insistKeyword(Attribute attribute) { ); } - /** - * This will inspect current node/{@code plan}'s expressions and check if any of the {@code FieldAttribute}s refer to fields that - * are partially unmapped across the indices involved in the plan fragment. If so, replace their field with an "insisted" EsField. - */ - private static LogicalPlan resolvePartiallyMapped(LogicalPlan plan, AnalyzerContext context) { - var indexResolutions = collectIndexResolutions(plan, context); - Map insistedMap = new HashMap<>(); - var transformed = plan.transformExpressionsOnly(FieldAttribute.class, fa -> { - var esField = fa.field(); - var isInsisted = esField instanceof PotentiallyUnmappedKeywordEsField || esField instanceof InvalidMappedField; - if (isInsisted == false) { - var existing = insistedMap.get(fa); - if (existing != null) { // field shows up multiple times in the node; return first processing - return existing; - } - // Field is partially unmapped. - if (indexResolutions.stream().anyMatch(r -> r.get().isPartiallyUnmappedField(fa.name()))) { - FieldAttribute newFA = fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa); - insistedMap.put(fa, newFA); - return newFA; - } - } - return fa; - }); - return insistedMap.isEmpty() ? transformed : propagateInsistedFields(transformed, insistedMap); - } - - /** - * Push only those fields from the {@code insistedMap} into {@code EsRelation}s in the {@code plan} that wrap a - * {@code PotentiallyUnmappedKeywordEsField}. - */ - private static LogicalPlan propagateInsistedFields(LogicalPlan plan, Map insistedMap) { - return plan.transformUp(EsRelation.class, esr -> { - var newOutput = new ArrayList(); - boolean updated = false; - for (Attribute attr : esr.output()) { - var newFA = insistedMap.get(attr); - if (newFA != null && newFA.field() instanceof PotentiallyUnmappedKeywordEsField) { - newOutput.add(newFA); - updated = true; - } else { - newOutput.add(attr); - } - } - return updated ? esr.withAttributes(newOutput) : esr; - }); - } - private LogicalPlan resolveFuse(Fuse fuse, List childrenOutput) { Source source = fuse.source(); Attribute score = fuse.score(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index e70ea91263bb2..bb97f502f1c8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -22,6 +22,7 @@ import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.KeywordEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; +import org.elasticsearch.xpack.esql.core.type.TextEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; @@ -620,6 +621,36 @@ public void testPartiallyMappedNonKeywordFieldNotLoadedWithoutExplicitReference( assertThat(fieldAttr.field().getDataType(), is(DataType.INTEGER)); } + /** + * Verify that a partially-mapped text field is NOT converted to {@link PotentiallyUnmappedKeywordEsField}, + * since the fix only applies to keyword fields. Text fields should not be loaded from _source. + */ + public void testPartiallyMappedTextFieldNotLoaded() { + var plan = analyzer().addIndex( + new EsIndex( + "test*", + EsqlTestUtils.loadMapping("mapping-basic.json"), + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of("gender") // gender is text, not keyword + ) + ).statement(setUnmappedLoad(""" + FROM test* + | SORT emp_no + """)); + + var limit = as(plan, Limit.class); + var order = as(limit.child(), OrderBy.class); + var relation = as(order.child(), EsRelation.class); + + var genderAttr = relation.output().stream().filter(a -> a.name().equals("gender")).findFirst().orElseThrow(); + var fieldAttr = as(genderAttr, FieldAttribute.class); + assertThat(fieldAttr.field(), not(instanceOf(PotentiallyUnmappedKeywordEsField.class))); + assertThat(fieldAttr.field(), instanceOf(TextEsField.class)); + assertThat(fieldAttr.field().getDataType(), is(DataType.TEXT)); + } + /** * When both a keyword and non-keyword field are partially mapped, only the keyword field * should be converted to {@link PotentiallyUnmappedKeywordEsField}. From fc294518a8e70cf77edd8e88387e895b025e8d0e Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 18:31:21 +0100 Subject: [PATCH 13/69] Add tests --- .../src/main/resources/unmapped-load.csv-spec | 25 +++++++ .../esql/analysis/AnalyzerUnmappedTests.java | 67 +++++++++++++++++++ 2 files changed, 92 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index d1247f7baeaed..9ffd11ce05daa 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -936,6 +936,31 @@ emp_no:integer | gender:text | _index:keyword 10003 | null | employees_no_gender ; +// Verify that partially-mapped non-keyword fields (date, ip, long) are NOT loaded from _source when not mentioned +// exlicitly, at least until we solve https://github.com/elastic/elasticsearch/issues/141995. +// All fields in sample_data are partially mapped (no_mapping_sample_data has no mapped fields). +// message (keyword) should be loaded from _source; @timestamp (date), client_ip (ip), event_duration (long) should be +// null. +partiallyMappedNonKeywordFieldsNotLoaded +required_capability: index_metadata_field +required_capability: optional_fields_fix_load_partially_mapped_keyword + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| SORT _index, message DESC +| LIMIT 6 +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +; + // https://github.com/elastic/elasticsearch/issues/141994 // A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded // from _source even when not explicitly referenced in a downstream expression like KEEP. diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index bb97f502f1c8d..ff555263634c2 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -651,6 +651,73 @@ public void testPartiallyMappedTextFieldNotLoaded() { assertThat(fieldAttr.field().getDataType(), is(DataType.TEXT)); } + /** + * Verify that partially-mapped fields of ALL non-keyword types are NOT converted to + * {@link PotentiallyUnmappedKeywordEsField}. Only keyword fields should be loaded from _source. + * This iterates over all {@link DataType} values that can appear as ES mapped field types. + */ + public void testPartiallyMappedNonKeywordFieldsNotLoaded() { + // Types that cannot appear as regular ES mapped fields in an EsIndex mapping + Set excludedTypes = Set.of( + DataType.KEYWORD, // this is the type we DO convert — not a negative test case + DataType.NULL, // not a real mapped field type + DataType.UNSUPPORTED, // not a real mapped field type + DataType.DOC_DATA_TYPE, // internal _doc type + DataType.TSID_DATA_TYPE, // internal _tsid type + DataType.SOURCE, // internal _source type + DataType.DATE_PERIOD, // ESQL-internal, not an ES mapping type + DataType.TIME_DURATION, // ESQL-internal, not an ES mapping type + DataType.OBJECT, // not a leaf field type + DataType.GEOHASH, // ESQL-internal grid type, not a real ES mapped field type + DataType.GEOTILE, // ESQL-internal grid type, not a real ES mapped field type + DataType.GEOHEX // ESQL-internal grid type, not a real ES mapped field type + ); + + for (DataType dataType : DataType.values()) { + if (excludedTypes.contains(dataType)) { + continue; + } + // Build a minimal mapping: one keyword field (emp_no stand-in for SORT) and one field of the type under test + Map mapping = Map.of( + "sort_field", + new EsField("sort_field", DataType.INTEGER, Map.of(), true, EsField.TimeSeriesFieldType.NONE), + "test_field", + new EsField("test_field", dataType, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + + var plan = analyzer().addIndex( + new EsIndex( + "test*", + mapping, + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Set.of("test_field") // partially unmapped + ) + ).statement(setUnmappedLoad(""" + FROM test* + | SORT sort_field + """)); + + var limit = as(plan, Limit.class); + var order = as(limit.child(), OrderBy.class); + var relation = as(order.child(), EsRelation.class); + + var testFieldAttr = relation.output().stream().filter(a -> a.name().equals("test_field")).findFirst().orElseThrow(); + var fieldAttr = as(testFieldAttr, FieldAttribute.class); + assertThat( + "Partially-mapped " + dataType + " field should not be converted to PotentiallyUnmappedKeywordEsField", + fieldAttr.field(), + not(instanceOf(PotentiallyUnmappedKeywordEsField.class)) + ); + assertThat( + "Partially-mapped " + dataType + " field should retain its original data type", + fieldAttr.dataType(), + is(dataType.widenSmallNumeric()) + ); + } + } + /** * When both a keyword and non-keyword field are partially mapped, only the keyword field * should be converted to {@link PotentiallyUnmappedKeywordEsField}. From 631d8bc2cb14507c892babb4649631abab24c2f7 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 25 Mar 2026 18:43:04 +0100 Subject: [PATCH 14/69] Refactor a little --- .../xpack/esql/analysis/Analyzer.java | 30 +++++++++++++------ 1 file changed, 21 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index cc2d917863cec..31bd3621236d4 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -348,16 +348,11 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR var attributes = mappingAsAttributes(plan.source(), esIndex.mapping()); attributes.addAll(metadata.stream().map(NamedExpression::toAttribute).toList()); - // TODO: at least factor out into a helper method. - if (context.unmappedResolution() == UnmappedResolution.LOAD && plan.indexMode() != IndexMode.LOOKUP) { - for (int i = 0; i < attributes.size(); i++) { - if (attributes.get(i) instanceof FieldAttribute fa - && fa.dataType() == KEYWORD - && esIndex.isPartiallyUnmappedField(fa.name())) { - attributes.set(i, ResolveRefs.insistKeyword(fa)); - } - } + + if (context.unmappedResolution() == UnmappedResolution.LOAD) { + loadPartiallyMappedKeywordFields(attributes, esIndex, plan.indexMode()); } + return new EsRelation( plan.source(), esIndex.name(), @@ -369,6 +364,23 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR ); } + /** + * When {@code SET unmapped_fields="load"}, convert partially-mapped keyword fields to use + * {@link PotentiallyUnmappedKeywordEsField} so they are loaded from {@code _source} on shards where they are unmapped. + * Non-keyword fields are left unchanged (they will return null on unmapped shards, consistent with their default behavior). + */ + private static void loadPartiallyMappedKeywordFields(List attributes, EsIndex esIndex, IndexMode indexMode) { + if (indexMode != IndexMode.LOOKUP) { + for (int i = 0; i < attributes.size(); i++) { + if (attributes.get(i) instanceof FieldAttribute fa + && fa.dataType() == KEYWORD + && esIndex.isPartiallyUnmappedField(fa.name())) { + attributes.set(i, ResolveRefs.insistKeyword(fa)); + } + } + } + } + private List resolveMetadata(List metadata, AnalyzerContext context) { LinkedHashMap resolved = new LinkedHashMap<>(); Set allTags = null; From 821f3c428888ceea90f309c202ae958243233488 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 11:00:42 +0100 Subject: [PATCH 15/69] Fix some tests --- .../load/analysis.expected | 4 ++-- .../rest-api-spec/test/esql/260_flattened_subfield.yml | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected index 812e7c610bd25..337900ed01ba1 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] -\_Project[[!event_duration]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3] \ No newline at end of file +\_Project[[event_duration{f}#0]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#0, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml index dc1a5e13abad2..d2bc1e885f821 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml @@ -121,7 +121,7 @@ setup: - match: { error.type: verification_exception } - contains: { error.reason: 'Found 1 problem' } - - contains: { error.reason: 'line 1:56: Cannot use field [foo.bar] due to ambiguities being mapped as [2] incompatible types: [keyword] enforced by INSIST command, and [unsupported] in index mappings' } + - contains: { error.reason: 'line 1:56: Cannot use field [foo.bar] with unsupported type [flattened]' } --- 'loading from two indices with different mappings 3': From b9ce17d2aa6193202dad9f3b081ea3e793aea095 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 12:29:10 +0100 Subject: [PATCH 16/69] Fix CsvTests --- .../xpack/esql/CsvTestUtils.java | 3 ++- .../xpack/esql/CsvTestsDataLoader.java | 1 + .../elasticsearch/xpack/esql/CsvTests.java | 22 +++++++++++++++- .../TestPhysicalOperationProviders.java | 25 +++++++++++++------ 4 files changed, 41 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index 69655970c9abd..bfc56daecae21 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -417,7 +417,8 @@ public void close() { for (int i = 0; i < entries.length; i++) { String[] header = entries[i].split(":"); String name = header[0].trim(); - String typeName = (typeMapping != null && typeMapping.containsKey(name)) ? typeMapping.get(name) + String typeName = (typeMapping != null && typeMapping.containsKey(name) && typeMapping.get(name) != null) + ? typeMapping.get(name) : header.length > 1 ? header[1].trim() : null; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 8702bcf78ef9c..1f6882baf1489 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -120,6 +120,7 @@ public class CsvTestsDataLoader { new TestDataset("ul_logs"), new TestDataset("sample_data"), new TestDataset("partial_mapping_sample_data"), + // TODO: withTypeMapping should be redundant here, mapping-no_mapping_sample_data.json doesn't provide any mapping where types could be overridden new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv").withTypeMapping( Stream.of("timestamp", "client_ip", "event_duration").collect(toMap(k -> k, k -> "keyword")) ), diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 34357c9c54739..4275cbfa19528 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -535,6 +535,26 @@ private static Map createMappingForIndex(CsvTestsDataLoader.Tes return mapping; } + /** + * Recursively collects all field names from a mapping, including subfields with dot-separated names. + */ + private static Set collectAllFieldNames(Map mapping) { + Set names = new HashSet<>(); + for (var entry : mapping.entrySet()) { + names.add(entry.getKey()); + collectSubFieldNames(entry.getKey(), entry.getValue(), names); + } + return names; + } + + private static void collectSubFieldNames(String prefix, EsField field, Set names) { + for (var entry : field.getProperties().entrySet()) { + String fullName = prefix + "." + entry.getKey(); + names.add(fullName); + collectSubFieldNames(fullName, entry.getValue(), names); + } + } + record MappingPerIndex(String index, Map mapping) {} record MergedResult(Map mapping, Set partiallyUnmappedFields) {} @@ -735,7 +755,7 @@ private static TestPhysicalOperationProviders testOperationProviders( for (CsvTestsDataLoader.MultiIndexTestDataset datasets : allDatasets.values()) { for (CsvTestsDataLoader.TestDataset dataset : datasets.datasets()) { var testData = loadPageFromCsv(dataset.streamData(), dataset.typeMapping()); - Set mappedFields = LoadMapping.loadMapping(dataset.streamMapping()).keySet(); + Set mappedFields = collectAllFieldNames(createMappingForIndex(dataset)); indexPages.add( new TestPhysicalOperationProviders.IndexPage(dataset.indexName(), testData.v1(), testData.v2(), mappedFields) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 297c5f042c69c..5ec2a480730c7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -287,7 +287,6 @@ private BiFunction getBlockExtraction(DriverCo if (attribute instanceof FieldAttribute fa) { if (fa.field() instanceof MultiTypeEsField m) { return (doc, copier) -> getBlockForMultiType(context, doc, m, copier); - } if (fa.field() instanceof PotentiallyUnmappedKeywordEsField k) { return (doc, copier) -> switch (extractBlockForSingleDoc(doc, k.getName(), copier)) { @@ -300,13 +299,23 @@ private BiFunction getBlockExtraction(DriverCo return (doc, copier) -> getNullsBlock(doc); } } - return (indexDoc, blockCopier) -> switch (extractBlockForSingleDoc(indexDoc, attribute.name(), blockCopier)) { - case BlockResultMissing missing -> throw new EsqlIllegalArgumentException( - "Cannot find column named [{}] in {}", - missing.columnName, - missing.columnNames - ); - case BlockResultSuccess success -> success.block; + return (indexDoc, blockCopier) -> { + // For regular FieldAttributes, check if the field is mapped in this index. + // If not, return nulls (simulates missing doc values for unmapped fields). + if (attribute instanceof FieldAttribute fa) { + var indexPage = getIndexPage(indexDoc); + if (indexPage.mappedFields().contains(fa.fieldName().string()) == false) { + return getNullsBlock(indexDoc); + } + } + return switch (extractBlockForSingleDoc(indexDoc, attribute.name(), blockCopier)) { + case BlockResultMissing missing -> throw new EsqlIllegalArgumentException( + "Cannot find column named [{}] in {}", + missing.columnName, + missing.columnNames + ); + case BlockResultSuccess success -> success.block; + }; }; } From e96a9528689a7c73a7980c321bdf428f08a79cbc Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 12:58:45 +0100 Subject: [PATCH 17/69] Update capabilities --- .../generative/GenerativeForkRestTest.java | 4 +- .../xpack/esql/CsvTestsDataLoader.java | 3 +- .../src/main/resources/flattened.csv-spec | 30 ++--- .../src/main/resources/unmapped-load.csv-spec | 116 +++++++++--------- .../xpack/esql/action/EsqlCapabilities.java | 27 ++-- .../xpack/esql/plan/QuerySettings.java | 4 +- .../esql/analysis/AnalyzerUnmappedTests.java | 2 +- .../optimizer/UnmappedGoldenTestCase.java | 2 +- .../xpack/esql/parser/SetParserTests.java | 4 +- .../xpack/esql/plan/QuerySettingsTests.java | 2 +- .../test/esql/260_flattened_subfield.yml | 2 +- 11 files changed, 97 insertions(+), 99 deletions(-) diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java index 55c342d9ed85d..b9e3f6809e5c0 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java @@ -18,7 +18,7 @@ import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.APPROXIMATION_V5; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METRICS_GROUP_BY_ALL; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING; @@ -66,7 +66,7 @@ protected void shouldSkipTest(String testName) throws IOException { // FORK is not supported with unmapped_fields="load", see https://github.com/elastic/elasticsearch/issues/142033 assumeFalse( "FORK is not supported with unmapped_fields=\"load\"", - testCase.requiredCapabilities.contains(OPTIONAL_FIELDS_V2.capabilityName()) + testCase.requiredCapabilities.contains(OPTIONAL_FIELDS_V4.capabilityName()) ); assumeFalse( diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 1f6882baf1489..f2afa5017a42b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -120,7 +120,8 @@ public class CsvTestsDataLoader { new TestDataset("ul_logs"), new TestDataset("sample_data"), new TestDataset("partial_mapping_sample_data"), - // TODO: withTypeMapping should be redundant here, mapping-no_mapping_sample_data.json doesn't provide any mapping where types could be overridden + // TODO: withTypeMapping should be redundant here, mapping-no_mapping_sample_data.json doesn't provide any mapping where types could + // be overridden new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv").withTypeMapping( Stream.of("timestamp", "client_ip", "event_duration").collect(toMap(k -> k, k -> "keyword")) ), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec index 419044aeb717f..71dc0d8b156de 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec @@ -1,6 +1,6 @@ flattened root fields required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -17,7 +17,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened KEEP subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -42,7 +42,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened DROP subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -68,7 +68,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened DROP subfield after KEEP-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -94,7 +94,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -111,7 +111,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE and KEEP *-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -129,7 +129,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE and KEEP specific subfields-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -147,7 +147,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with EVAL TO_UPPER-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -173,7 +173,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with EVAL LENGTH-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -199,7 +199,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed SORT on flattened subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -224,7 +224,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed RENAME flattened subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -250,7 +250,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with STATS count-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -265,7 +265,7 @@ count(*):long | resource.attributes.agent.type:keyword # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with STATS count_distinct-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -280,7 +280,7 @@ count_distinct(resource.attributes.host.name):long # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with INLINE STATS-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -302,7 +302,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with MV_EXPAND-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 9ffd11ce05daa..b56ff7bc46565 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -3,7 +3,7 @@ #################### // These ones verify we don't do anything extra when unmapped_fields is not set to "load" doesNotLoadUnmappedFieldsSort -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 FROM partial_mapping_sample_data | SORT @timestamp DESC @@ -20,7 +20,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsStats -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 FROM partial_mapping_sample_data | STATS count(*) BY message @@ -36,7 +36,7 @@ count(*):long | message:keyword ; doesNotLoadUnmappedFieldsInlineStats -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 FROM partial_mapping_sample_data | INLINE STATS count = COUNT(*) BY message @@ -51,7 +51,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsInlineStatsNoGrouping -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 FROM partial_mapping_sample_data | INLINE STATS max_duration = MAX(event_duration) @@ -66,7 +66,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsEnrich -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: enrich_load FROM partial_mapping_sample_data @@ -84,7 +84,7 @@ FROM partial_mapping_sample_data doesNotLoadUnmappedFieldsLookupJoin-Ignore // temporarily forbidding "load" with LOOKUP JOIN -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: join_lookup_v12 FROM partial_mapping_sample_data @@ -105,7 +105,7 @@ FROM partial_mapping_sample_data ###################### unmappedFieldDoesNotAppearLast -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -118,7 +118,7 @@ FROM partial_mapping_sample_data ; fieldDoesNotExistSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -137,7 +137,7 @@ FROM partial_mapping_sample_data ; fieldIsUnmappedSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -156,7 +156,7 @@ FROM partial_mapping_sample_data ; fieldUnmappedInSourceButExcludedSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -176,7 +176,7 @@ FROM partial_mapping_excluded_source_sample_data ; fieldUnmappedInSourceButSourceDisabledSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -196,7 +196,7 @@ FROM partial_mapping_no_source_sample_data ; statsByUnmappedFieldExistsInSource -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -213,7 +213,7 @@ s:long | c:long | unmapped_message:keyword ; inlineStatsByUnmappedFieldExistsInSource -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -231,7 +231,7 @@ FROM partial_mapping_sample_data ; unmappedFieldNonExistentTsIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -250,7 +250,7 @@ FROM k8s_unmapped ; statsImplicitLastOverTimeUnmappedTsIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -268,7 +268,7 @@ time_bucket:datetime | x:keyword ; statsByUnmappedFieldTsIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -293,7 +293,7 @@ s:long | c:long | event_log:keyword ; explicitLastOverTimeUnmappedTsIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -313,7 +313,7 @@ qa | 2024-05-10T00:10:00.000Z | vulputate qu ; fieldIsNestedAndUnmapped -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -332,7 +332,7 @@ FROM partial_mapping_sample_data ; fieldIsNestedAndNonExistent -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -355,7 +355,7 @@ FROM partial_mapping_sample_data ############################### noFieldExistsMultiParametersSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -374,7 +374,7 @@ FROM partial_mapping_sample_data ; mixedFieldsMultiParametersSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -397,7 +397,7 @@ FROM partial_mapping_sample_data ##################### mixedFieldsMultiParametersMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -423,7 +423,7 @@ sample_data | 2023-10-23T12:15:03.360Z | null | Connected ; fieldDoesNotExistMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -449,7 +449,7 @@ sample_data | 2023-10-23T12:15:03.360Z | null ; fieldIsUnmappedMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -475,7 +475,7 @@ FROM partial_mapping_sample_data, sample_data METADATA _index ; fieldIsPartiallyUnmappedMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -501,7 +501,7 @@ sample_data | Connected to 10.1.0.1 ; fieldIsPartiallyUnmappedAndRenamedMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data @@ -528,7 +528,7 @@ Connected to 10.1.0.1 ; fieldIsPartiallyUnmappedPartiallySourceIsDisabledMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -555,7 +555,7 @@ partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to ; partialMappingStats -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -574,7 +574,7 @@ max(@timestamp):date | count(*):long | message:keyword ; partialMappingCoalesce -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -603,7 +603,7 @@ FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data MET ; partialMappingKeywordCast -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -631,7 +631,7 @@ FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data MET ; partialMappingStatsAfterCast -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -647,7 +647,7 @@ count(*):long | message::INT:integer ; filterOnPartiallyUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -666,7 +666,7 @@ sample_data | Connection error ; sortOnPartiallyUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -684,7 +684,7 @@ no_mapping_sample_data | Connected to 10.1.0.2! ; filterOnFullyUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -700,7 +700,7 @@ FROM partial_mapping_sample_data ; filterOnConjunctionMappedAndUnmapped -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -716,7 +716,7 @@ FROM partial_mapping_sample_data ; filterOnConjunctionMappedAndNonExistent -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -728,7 +728,7 @@ _index:keyword | message:keyword | does_not_exist:keyword ; filterOnDisjunctionMappedAndUnmapped -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -745,7 +745,7 @@ FROM partial_mapping_sample_data ; filterOnDisjunctionMappedAndNonExistent -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -761,7 +761,7 @@ sample_data | Connection error | null ; sortOnNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM no_mapping_sample_data METADATA _index @@ -779,7 +779,7 @@ no_mapping_sample_data | null ; sortOnUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -796,7 +796,7 @@ FROM partial_mapping_sample_data ; sortOnExpressionOnUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -814,7 +814,7 @@ FROM partial_mapping_sample_data ; sortOnMultipleFieldsWithNonExistent -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -832,7 +832,7 @@ no_mapping_sample_data | Connected to 10.1.0.2! | null ; sortOnMultipleFieldsWithUnmapped -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -850,7 +850,7 @@ FROM partial_mapping_sample_data ; filterOnNonExistentFieldCastToLong -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -862,7 +862,7 @@ _index:keyword | message:keyword | does_not_exist:keyword ; filterOnUnmappedFieldCastToLong -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -895,7 +895,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword employeesNoNamesLoad required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -918,7 +918,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword // gender is mapped as text in employees_gender_text and unmapped in employees_no_gender. partiallyMappedTextFieldNotLoaded required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM employees_gender_text, employees_no_gender METADATA _index @@ -943,7 +943,7 @@ emp_no:integer | gender:text | _index:keyword // null. partiallyMappedNonKeywordFieldsNotLoaded required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -966,7 +966,7 @@ null | null | null | Connected to 10. // from _source even when not explicitly referenced in a downstream expression like KEEP. fieldIsPartiallyUnmappedMultiIndexWithoutKeep required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -995,7 +995,7 @@ Connected to 10.1.0.1 | sample_data // referenced by any command — they remain in the output implicitly and get loaded. fieldIsPartiallyUnmappedWithEvalOnOtherMappedField required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1016,7 +1016,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | s fieldIsPartiallyUnmappedWithRenameOnOtherMappedField required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1039,7 +1039,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:k // first_name is not referenced by any command but must still be loaded. fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1060,7 +1060,7 @@ emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1082,7 +1082,7 @@ emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index: // Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop required_capability: index_metadata_field -required_capability: optional_fields_fix_load_partially_mapped_keyword +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -1110,7 +1110,7 @@ Connected to 10.1.0.1 | sample_data // DROP the partially-mapped field itself — verify no errors from converting then pruning. fieldIsPartiallyUnmappedMultiIndexDropPartiallyMappedField required_capability: index_metadata_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -1128,7 +1128,7 @@ no_mapping_sample_data ; maxOverTimeOfKeyword -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -1152,7 +1152,7 @@ us | 2024-05-10T00:04:00.000Z | k8s_unmapped // https://github.com/elastic/elasticsearch/issues/143991 statsFilteredAggAfterEvalWithDottedUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: optional_fields_detect_unmapped_fields_in_agg_filters SET unmapped_fields="load"\; @@ -1167,7 +1167,7 @@ null | foo // https://github.com/elastic/elasticsearch/issues/143991 statsFilteredAggAfterEvalWithDottedUnmappedFieldLoadsFromSource -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: optional_fields_detect_unmapped_fields_in_agg_filters SET unmapped_fields="load"\; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 0655bae7154aa..b4fe0b7eb9ead 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -226,7 +226,7 @@ public enum Cap { /** * Support for optional fields (might or might not be present in the mappings) using FAIL/NULLIFY only. - * Compared to {@link #OPTIONAL_FIELDS_V2}, this does not enable support for LOAD. + * Compared to {@link #OPTIONAL_FIELDS_V4}, this does not enable support for LOAD. */ OPTIONAL_FIELDS_NULLIFY_TECH_PREVIEW, @@ -249,16 +249,19 @@ public enum Cap { /** * Support for optional fields (might or might not be present in the mappings) using FAIL/NULLIFY/LOAD. - * V2: prevent pushing down filters and sorts to Lucene of potentially unmapped fields. - */ - OPTIONAL_FIELDS_V2(Build.current().isSnapshot()), - - /** - * Fix for partially mapped keyword fields not being loaded from _source even when not explicitly referenced + *

    + *
  • V2: prevent pushing down filters and sorts to Lucene of potentially unmapped fields; + *
  • V3:
      + *
    • Reject loading sub-fields of flattened fields when {@code unmapped_fields="load"} + * See https://github.com/elastic/elasticsearch/issues/143494 + *
    • Support for union type like resolution for load + *
    + *
  • V4: fix for partially mapped keyword fields not being loaded from _source even when not explicitly referenced * by a downstream command (e.g. KEEP, WHERE, SORT). * See https://github.com/elastic/elasticsearch/issues/141994 + *
*/ - OPTIONAL_FIELDS_FIX_LOAD_PARTIALLY_MAPPED_KEYWORD(OPTIONAL_FIELDS_V2.isEnabled()), + OPTIONAL_FIELDS_V4(Build.current().isSnapshot()), /** * Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently @@ -2366,18 +2369,12 @@ public enum Cap { * */ CHANGE_POINT_SUPPORT_NULL_COLUMN, - + /** * Support CHANGE_POINT arguments in any order */ CHANGE_POINT_ARGS_ANY_ORDER, - /** - * Reject loading sub-fields of flattened fields when {@code unmapped_fields="load"} - * See https://github.com/elastic/elasticsearch/issues/143494 - */ - REJECT_LOADING_FLATTENED_SUBFIELDS(OPTIONAL_FIELDS_V2.isEnabled()), - FIX_DIV_ERROR_MESSAGE, /** diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java index e2fede89460dd..0f83a7ba08673 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java @@ -95,12 +95,12 @@ public class QuerySettings { String resolution = Foldables.stringLiteralValueOf(value, "Unexpected value"); try { UnmappedResolution res = UnmappedResolution.valueOf(resolution.toUpperCase(Locale.ROOT)); - if (res == UnmappedResolution.LOAD && EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled() == false) { + if (res == UnmappedResolution.LOAD && EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled() == false) { throw new IllegalArgumentException("'LOAD' is only supported in snapshot builds"); } return res; } catch (Exception exc) { - var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled() + var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled() ? UnmappedResolution.values() : Arrays.stream(UnmappedResolution.values()).filter(e -> e != UnmappedResolution.LOAD).toArray(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 02aa28b9c8f2f..5df84fa3e6d97 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -1144,7 +1144,7 @@ private static String setUnmappedNullify(String query) { } private static String setUnmappedLoad(String query) { - assumeTrue("Requires OPTIONAL_FIELDS_V2", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled()); + assumeTrue("Requires OPTIONAL_FIELDS_V4", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled()); return "SET unmapped_fields=\"load\"; " + query; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java index cf3dd01f9d741..4641abdb2c1fe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java @@ -41,7 +41,7 @@ private Optional tryRunTestsNullifyOnly(String query, EnumSet } private Optional tryRunTestsLoadOnly(String query, EnumSet stages, String... nestedPaths) { - return EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled() + return EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled() ? builder(setUnmappedLoad(query)).nestedPath(ArrayUtils.prepend("load", nestedPaths)).stages(stages).tryRun() : Optional.empty(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java index 598713ed116b3..85218c8d2c328 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java @@ -199,7 +199,7 @@ private Object settingValue(EsqlStatement query, int position) { } public void testSetUnmappedFields_snapshot() { - assumeTrue("OPTIONAL_FIELDS_V2 option required", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled()); + assumeTrue("OPTIONAL_FIELDS_V4 option required", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled()); var modes = List.of("FAIL", "NULLIFY", "LOAD"); verifySetUnmappedFields(modes); @@ -236,7 +236,7 @@ public void testSetUnmappedFieldsWrongValue() { v -> Arrays.stream(UnmappedResolution.values()).anyMatch(x -> x.name().equalsIgnoreCase(v)), () -> randomAlphaOfLengthBetween(0, 10) ); - var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled() + var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled() ? UnmappedResolution.values() : Arrays.stream(UnmappedResolution.values()).filter(e -> e != UnmappedResolution.LOAD).toArray(); expectValidationError( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java index 6454ed88cbf28..20cd2b40bba39 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java @@ -114,7 +114,7 @@ public void testValidate_UnmappedFields_techPreview() { } public void testValidate_UnmappedFields_allValues() { - assumeTrue("Requires unmapped fields", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled()); + assumeTrue("Requires unmapped fields", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled()); validateUnmappedFields("FAIL", "NULLIFY", "LOAD"); } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml index d2bc1e885f821..6bfc375732d23 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml @@ -6,7 +6,7 @@ setup: - method: POST path: /_query parameters: [] - capabilities: [ reject_loading_flattened_subfields ] + capabilities: [ optional_fields_v4 ] reason: "Loading a subfield a flattened field must fail when unmapped_fields=load" - do: indices.create: From fafe2921fc285b77680f6a289c81a64973607408 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 13:07:01 +0100 Subject: [PATCH 18/69] Spotless --- .../org/elasticsearch/xpack/esql/action/EsqlCapabilities.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index b4fe0b7eb9ead..de4296205404f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -2369,7 +2369,7 @@ public enum Cap { * */ CHANGE_POINT_SUPPORT_NULL_COLUMN, - + /** * Support CHANGE_POINT arguments in any order */ From 59cd9fa8aa6a5e34c108c3bcaf2cba6adf561420 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 13:09:02 +0100 Subject: [PATCH 19/69] Improve comment --- .../xpack/esql/planner/TestPhysicalOperationProviders.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 5ec2a480730c7..02d1b630384ef 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -301,7 +301,7 @@ private BiFunction getBlockExtraction(DriverCo } return (indexDoc, blockCopier) -> { // For regular FieldAttributes, check if the field is mapped in this index. - // If not, return nulls (simulates missing doc values for unmapped fields). + // If not, return nulls (simulates unmapped fields that happen to still be in _source/the csv file). if (attribute instanceof FieldAttribute fa) { var indexPage = getIndexPage(indexDoc); if (indexPage.mappedFields().contains(fa.fieldName().string()) == false) { From 5f07932d31535ddb4d3b2e635f043eb19a777ce9 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 13:16:52 +0100 Subject: [PATCH 20/69] Improve another comment --- .../org/elasticsearch/xpack/esql/CsvTestsDataLoader.java | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index f2afa5017a42b..819006628bcec 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -1229,10 +1229,8 @@ public TestDataset withTypeMapping(Map typeMapping) { } /** - * Adds field mappings that are not present in the mapping file, simulating fields discovered via field-caps - * at runtime (e.g. runtime fields). These fields are only added to the in-memory mapping used by CSV unit - * tests and are not included in the Elasticsearch index mapping. Fields already present in the mapping file - * are not overridden. + * Adds field mappings that are not present in the mapping file, but will be in the field caps response, e.g. because during + * ingestion more fields are added dynamically. Required for csv tests which do not ingest the csvs into real indices. */ public TestDataset withDynamicTypeMapping(Map dynamicTypeMapping) { return new TestDataset( From 81f02057b40407e6821d678b6ce3afcab13cf4c4 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 13:44:32 +0100 Subject: [PATCH 21/69] More capability fixes --- .../testFixtures/src/main/resources/views.csv-spec | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec index 2f512bff2c3f1..f6e0df5f3d350 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec @@ -459,7 +459,7 @@ total:long | country:keyword ; unmappedFields_Load_with_Count -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -475,7 +475,7 @@ rehired_count:long ; unmappedFields_Load_with_WithNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -491,7 +491,7 @@ rehired_count:long | null_count:long ; unmappedFields_Load_with_EvalOnNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -509,7 +509,7 @@ c:long ; unmappedFields_Load_with_WhereOnNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -526,7 +526,7 @@ c:long ; unmappedFields_Load_with_StatsByNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -542,7 +542,7 @@ c:long | nonexistent_field:keyword ; unmappedFields_Load_with_SortByNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions From 3484535667e710e135d824f1eed0577db4789830 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 13:44:42 +0100 Subject: [PATCH 22/69] Remove redundant type override That index has no mapping, and thus type overrides also make no sense. --- .../org/elasticsearch/xpack/esql/CsvTestsDataLoader.java | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 819006628bcec..948118ed69f10 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -120,11 +120,7 @@ public class CsvTestsDataLoader { new TestDataset("ul_logs"), new TestDataset("sample_data"), new TestDataset("partial_mapping_sample_data"), - // TODO: withTypeMapping should be redundant here, mapping-no_mapping_sample_data.json doesn't provide any mapping where types could - // be overridden - new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv").withTypeMapping( - Stream.of("timestamp", "client_ip", "event_duration").collect(toMap(k -> k, k -> "keyword")) - ), + new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv"), new TestDataset( "partial_mapping_no_source_sample_data", "mapping-partial_mapping_no_source_sample_data.json", From 3a6d552bc7bd185ecf863ad666d1ae411751eee6 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 13:57:44 +0100 Subject: [PATCH 23/69] Improve comment --- .../testFixtures/src/main/resources/unmapped-load.csv-spec | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index b56ff7bc46565..be02989b489a1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -914,8 +914,9 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword ; -// Verify that a partially-mapped TEXT field is not automatically loaded from _source. -// gender is mapped as text in employees_gender_text and unmapped in employees_no_gender. +// Verify that a partially-mapped TEXT field is not automatically loaded from _source for the unmapped field index. +// employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while +// employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. partiallyMappedTextFieldNotLoaded required_capability: index_metadata_field required_capability: optional_fields_v4 From d67b21052a68ecc83b5581b0f1f5e46af347ab23 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 13:58:36 +0100 Subject: [PATCH 24/69] Simplify loadPartiallyMappedKeywordFields We don't have to consider lookup indices. They are forbidden and cannot have partially mapped fields, anyway. --- .../elasticsearch/xpack/esql/analysis/Analyzer.java | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index b523856857231..68a98bebffddd 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -371,13 +371,11 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR * Non-keyword fields are left unchanged (they will return null on unmapped shards, consistent with their default behavior). */ private static void loadPartiallyMappedKeywordFields(List attributes, EsIndex esIndex, IndexMode indexMode) { - if (indexMode != IndexMode.LOOKUP) { - for (int i = 0; i < attributes.size(); i++) { - if (attributes.get(i) instanceof FieldAttribute fa - && fa.dataType() == KEYWORD - && esIndex.isPartiallyUnmappedField(fa.name())) { - attributes.set(i, ResolveRefs.insistKeyword(fa)); - } + for (int i = 0; i < attributes.size(); i++) { + if (attributes.get(i) instanceof FieldAttribute fa + && fa.dataType() == KEYWORD + && esIndex.isPartiallyUnmappedField(fa.name())) { + attributes.set(i, ResolveRefs.insistKeyword(fa)); } } } From 4daa8cd28e0fa1ea8c61295e7e9600838d83bcbe Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 26 Mar 2026 16:57:52 +0100 Subject: [PATCH 25/69] More Andrei tests :) --- .../src/main/resources/unmapped-load.csv-spec | 122 ++++++++++++++++++ 1 file changed, 122 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index be02989b489a1..5e87ee687a92b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -914,6 +914,128 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword ; +employeesNoNamesLoadStats +required_capability: index_metadata_field +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +| STATS count(*) BY first_name +; +ignoreOrder:true + +count(*):long | first_name:keyword +2 | Bezalel +2 | Georgi +2 | Parto +; + +employeesNoNamesLoadStatsByFirstName +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names +| KEEP emp_no, first_name, last_name +| STATS c=count(*) BY first_name +| SORT first_name ASC NULLS FIRST +| LIMIT 5 +; + +c:long | first_name:keyword +20 | null +2 | Alejandro +2 | Amabile +2 | Anneke +2 | Anoosh +; + +employeesNoNamesLoadInlineStats +required_capability: index_metadata_field +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| INLINE STATS c=count(*) BY first_name +| SORT first_name NULLS FIRST, emp_no, _index +| LIMIT 6 +; + +emp_no:integer | last_name:keyword | _index:keyword | c:long | first_name:keyword +10030 | Demeyer | employees | 20 | null +10030 | Demeyer | employees_no_names | 20 | null +10031 | Joslin | employees | 20 | null +10031 | Joslin | employees_no_names | 20 | null +10032 | Reistad | employees | 20 | null +10032 | Reistad | employees_no_names | 20 | null +; + +employeesNoNamesLoadInlineStatsCount +required_capability: index_metadata_field +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| INLINE STATS c=count(first_name) BY last_name +| SORT first_name NULLS FIRST, emp_no DESC, _index +| LIMIT 22 +; + +emp_no:integer | first_name:keyword | _index:keyword | c:long | last_name:keyword +10039 | null | employees | 0 | Brender +10039 | null | employees_no_names | 0 | Brender +10038 | null | employees | 2 | Lortz +10038 | null | employees_no_names | 2 | Lortz +10037 | null | employees | 0 | Makrucki +10037 | null | employees_no_names | 0 | Makrucki +10036 | null | employees | 0 | Portugali +10036 | null | employees_no_names | 0 | Portugali +10035 | null | employees | 0 | Chappelet +10035 | null | employees_no_names | 0 | Chappelet +10034 | null | employees | 0 | Swan +10034 | null | employees_no_names | 0 | Swan +10033 | null | employees | 0 | Merlo +10033 | null | employees_no_names | 0 | Merlo +10032 | null | employees | 2 | Reistad +10032 | null | employees_no_names | 2 | Reistad +10031 | null | employees | 0 | Joslin +10031 | null | employees_no_names | 0 | Joslin +10030 | null | employees | 0 | Demeyer +10030 | null | employees_no_names | 0 | Demeyer +10059 | Alejandro | employees | 2 | McAlpine +10059 | Alejandro | employees_no_names | 2 | McAlpine +; + +employeesNoNamesLoadStatsWhereStartsWith +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names +| KEEP emp_no, first_name, last_name +| STATS c=count(first_name) WHERE STARTS_WITH(last_name, "M") BY last_name +| SORT c DESC, last_name +| LIMIT 12 +; + +c:long | last_name:keyword +2 | Malabarba +2 | Maliniak +2 | Mandell +2 | McAlpine +2 | McClurg +2 | McFarlin +2 | Meriste +2 | Mondadori +2 | Montemayor +2 | Morton +0 | Awdeh +0 | Azuma +; + // Verify that a partially-mapped TEXT field is not automatically loaded from _source for the unmapped field index. // employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while // employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. From 7f81ee47cdf6695fa2403c32983b07c0fcefc313 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 31 Mar 2026 19:38:34 +0200 Subject: [PATCH 26/69] Small cleanup --- .../org/elasticsearch/xpack/esql/analysis/Analyzer.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 68a98bebffddd..1562519274abb 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -351,7 +351,7 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR attributes.addAll(metadata.stream().map(NamedExpression::toAttribute).toList()); if (context.unmappedResolution() == UnmappedResolution.LOAD) { - loadPartiallyMappedKeywordFields(attributes, esIndex, plan.indexMode()); + loadPartiallyMappedKeywordFields(attributes, esIndex); } return new EsRelation( @@ -370,11 +370,11 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR * {@link PotentiallyUnmappedKeywordEsField} so they are loaded from {@code _source} on shards where they are unmapped. * Non-keyword fields are left unchanged (they will return null on unmapped shards, consistent with their default behavior). */ - private static void loadPartiallyMappedKeywordFields(List attributes, EsIndex esIndex, IndexMode indexMode) { + private static void loadPartiallyMappedKeywordFields(List attributes, EsIndex esIndex) { for (int i = 0; i < attributes.size(); i++) { if (attributes.get(i) instanceof FieldAttribute fa && fa.dataType() == KEYWORD - && esIndex.isPartiallyUnmappedField(fa.name())) { + && esIndex.isPartiallyUnmappedField(fa.fieldName().string())) { attributes.set(i, ResolveRefs.insistKeyword(fa)); } } From 0fe03fd9b95e98c059a0cd01f30696f0af76ed16 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Tue, 31 Mar 2026 20:25:59 +0000 Subject: [PATCH 27/69] [CI] Auto commit changes from spotless --- .../xpack/esql/CsvTestsDataLoader.java | 2 +- .../xpack/esql/analysis/Analyzer.java | 140 +++++++++--------- .../esql/analysis/AnalyzerUnmappedTests.java | 4 +- 3 files changed, 73 insertions(+), 73 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 2d10c123e3a36..f4d55b591027b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -43,8 +43,8 @@ import java.net.URL; import java.nio.charset.StandardCharsets; import java.util.ArrayList; -import java.util.HashMap; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 03c5e092f45c1..149dd1e7bbf2c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -312,14 +312,14 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR return plan.unresolvedMessage().equals(indexResolutionMessage) ? plan : new UnresolvedRelation( - plan.source(), - plan.indexPattern(), - plan.frozen(), - metadata, - plan.indexMode(), - indexResolutionMessage, - plan.telemetryLabel() - ); + plan.source(), + plan.indexPattern(), + plan.frozen(), + metadata, + plan.indexMode(), + indexResolutionMessage, + plan.telemetryLabel() + ); } // assert indexResolution.matches(plan.indexPattern().indexPattern()) : "Expected index resolution to match the index pattern"; IndexPattern table = plan.indexPattern(); @@ -821,14 +821,14 @@ private LogicalPlan resolveMvExpand(MvExpand p, List childrenOutput) resolved, resolved.resolved() ? new ReferenceAttribute( - resolved.source(), - resolved.qualifier(), - resolved.name(), - resolved.dataType(), - resolved.nullable(), - null, - false - ) + resolved.source(), + resolved.qualifier(), + resolved.name(), + resolved.dataType(), + resolved.nullable(), + null, + false + ) : resolved ); } @@ -1102,7 +1102,7 @@ private LogicalPlan resolveFork(Fork fork) { // forkColumns do not contain NO_FIELD because Fork.outputUnion removes it. if (logicalPlan instanceof Project == false || (subPlanColumns.equals(forkColumns) == false - && subqueryReferencingIndexWithEmptyMapping(fork, logicalPlan, forkColumns) == false)) { + && subqueryReferencingIndexWithEmptyMapping(fork, logicalPlan, forkColumns) == false)) { changed = true; List newOutput = new ArrayList<>(); for (String attrName : forkColumns) { @@ -1621,10 +1621,10 @@ private LogicalPlan resolveRename(Rename rename, UnmappedResolution unmappedReso return unmappedResolution == UnmappedResolution.DEFAULT ? new Project(rename.source(), rename.child(), projectionsForRename(rename, rename.child().output(), log)) : new ResolvingProject( - rename.source(), - rename.child(), - inputAttributes -> projectionsForRename(rename, inputAttributes, log) - ); + rename.source(), + rename.child(), + inputAttributes -> projectionsForRename(rename, inputAttributes, log) + ); } /** @@ -2353,10 +2353,10 @@ private static Expression castStringLiteral(Expression from, DataType target, Co return isTemporalAmount(target) ? castStringLiteralToTemporalAmount(from) : new Literal( - from.source(), - EsqlDataTypeConverter.convert(from.fold(FoldContext.small() /* TODO remove me */), target, configuration), - target - ); + from.source(), + EsqlDataTypeConverter.convert(from.fold(FoldContext.small() /* TODO remove me */), target, configuration), + target + ); } catch (Exception e) { return unresolvedAttribute(from, target.toString(), e); } @@ -2515,50 +2515,50 @@ private Expression resolveConvertFunction(ConvertFunction convert, List supportedTypes = convert.supportedTypes(); - if (supportedTypes.contains(fa.dataType()) && canConvertOriginalTypes(mtf, supportedTypes)) { - // Build the mapping between index name and conversion expressions - Map indexToConversionExpressions = new HashMap<>(); - for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { - String indexName = entry.getKey(); - AbstractConvertFunction originalConversionFunction = (AbstractConvertFunction) entry.getValue(); - Expression originalField = originalConversionFunction.field(); - Expression newConvertFunction = convertExpression.replaceChildren(Collections.singletonList(originalField)); - indexToConversionExpressions.put(indexName, newConvertFunction); + // This is an explicit casting of a union typed field that has been converted to MultiTypeEsField in EsRelation by + // DateMillisToNanosInEsRelation, it is not necessary to cast it again to the same type, replace the implicit casting + // with explicit casting. However, it is useful to differentiate implicit and explicit casting in some cases, for + // example, an expression like multiTypeEsField(synthetic=false, date_nanos)::date_nanos::datetime is rewritten to + // multiTypeEsField(synthetic=true, date_nanos)::datetime, the implicit casting is overwritten by explicit casting and + // the multiTypeEsField is not casted to datetime directly. + if (((Expression) convert).dataType() == mtf.getDataType()) { + return createIfDoesNotAlreadyExist(fa, mtf, unionFieldAttributes); } - // The only code that creates MultiTypeEsField with synthetic=false (reaching this branch) is - // DateMillisToNanosInEsRelation, which runs in the "Initialize" batch before ResolveUnmapped. At that point, - // unmapped fields haven't been detected yet, so potentiallyUnmappedExpression is always null. - if (mtf.getPotentiallyUnmappedExpression() != null) { - throw new IllegalStateException("Unexpected potentially unmapped expression for [" + fa.fieldName() + "]"); + + // Data type is different between implicit(date_nanos) and explicit casting, if the conversion is supported, create a + // new MultiTypeEsField with explicit casting type, and add it to unionFieldAttributes. + Set supportedTypes = convert.supportedTypes(); + if (supportedTypes.contains(fa.dataType()) && canConvertOriginalTypes(mtf, supportedTypes)) { + // Build the mapping between index name and conversion expressions + Map indexToConversionExpressions = new HashMap<>(); + for (Map.Entry entry : mtf.getIndexToConversionExpressions().entrySet()) { + String indexName = entry.getKey(); + AbstractConvertFunction originalConversionFunction = (AbstractConvertFunction) entry.getValue(); + Expression originalField = originalConversionFunction.field(); + Expression newConvertFunction = convertExpression.replaceChildren(Collections.singletonList(originalField)); + indexToConversionExpressions.put(indexName, newConvertFunction); + } + // The only code that creates MultiTypeEsField with synthetic=false (reaching this branch) is + // DateMillisToNanosInEsRelation, which runs in the "Initialize" batch before ResolveUnmapped. At that point, + // unmapped fields haven't been detected yet, so potentiallyUnmappedExpression is always null. + if (mtf.getPotentiallyUnmappedExpression() != null) { + throw new IllegalStateException("Unexpected potentially unmapped expression for [" + fa.fieldName() + "]"); + } + MultiTypeEsField multiTypeEsField = new MultiTypeEsField( + fa.fieldName().string(), + convertExpression.dataType(), + false, + indexToConversionExpressions, + fa.field().getTimeSeriesFieldType(), + null + ); + return createIfDoesNotAlreadyExist(fa, multiTypeEsField, unionFieldAttributes); } - MultiTypeEsField multiTypeEsField = new MultiTypeEsField( - fa.fieldName().string(), - convertExpression.dataType(), - false, - indexToConversionExpressions, - fa.field().getTimeSeriesFieldType(), - null + } else if (convert.field() instanceof AbstractConvertFunction subConvert) { + return convertExpression.replaceChildren( + Collections.singletonList(resolveConvertFunction(subConvert, unionFieldAttributes)) ); - return createIfDoesNotAlreadyExist(fa, multiTypeEsField, unionFieldAttributes); } - } else if (convert.field() instanceof AbstractConvertFunction subConvert) { - return convertExpression.replaceChildren( - Collections.singletonList(resolveConvertFunction(subConvert, unionFieldAttributes)) - ); - } return convertExpression; } @@ -3004,11 +3004,11 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { UnionAll.class, unionAll -> unionAll.resolved() ? implicitCastingUnionAllOutput( - unionAll, - planWithConvertFunctionsReplaced, - updatedUnionAllOutput, - context.configuration() - ) + unionAll, + planWithConvertFunctionsReplaced, + updatedUnionAllOutput, + context.configuration() + ) : unionAll ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 66329d90c4e73..ac5d226dd23f8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -22,10 +22,10 @@ import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.KeywordEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.TextEsField; -import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; @@ -34,8 +34,8 @@ import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Limit; -import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.hamcrest.Matcher; import org.hamcrest.Matchers; From 0caffa7e21708266ddc326ac4d567833d2128ab5 Mon Sep 17 00:00:00 2001 From: Mouhcine Aitounejjar <5037925+mouhc1ne@users.noreply.github.com> Date: Tue, 31 Mar 2026 16:38:53 -0400 Subject: [PATCH 28/69] Fix compile errors due to EsIndex signature change, missed during conflict resolution. --- .../xpack/esql/analysis/AnalyzerUnmappedTests.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 66329d90c4e73..20fba7052359a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -849,7 +849,7 @@ public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() { Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of("first_name") + Map.of("first_name", Set.of("test2")) ) ).statement(setUnmappedLoad(""" FROM test* @@ -877,7 +877,7 @@ public void testPartiallyMappedNonKeywordFieldNotLoadedWithoutExplicitReference( Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of("salary") // salary is integer, not keyword + Map.of("salary", Set.of("test2")) // salary is integer, not keyword ) ).statement(setUnmappedLoad(""" FROM test* @@ -907,7 +907,7 @@ public void testPartiallyMappedTextFieldNotLoaded() { Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of("gender") // gender is text, not keyword + Map.of("gender", Set.of("test2")) // gender is text, not keyword ) ).statement(setUnmappedLoad(""" FROM test* @@ -966,7 +966,7 @@ public void testPartiallyMappedNonKeywordFieldsNotLoaded() { Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of("test_field") // partially unmapped + Map.of("test_field", Set.of("test2")) // partially unmapped ) ).statement(setUnmappedLoad(""" FROM test* @@ -1004,7 +1004,7 @@ public void testOnlyPartiallyMappedKeywordFieldLoaded() { Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of("first_name", "salary") // keyword + integer + Map.of("first_name", Set.of("test2"), "salary", Set.of("test2")) // keyword + integer ) ).statement(setUnmappedLoad(""" FROM test* @@ -1039,7 +1039,7 @@ public void testNonPartiallyMappedKeywordFieldNotLoadedFromSource() { Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of("first_name") // only first_name is partially unmapped + Map.of("first_name", Set.of("test2")) // only first_name is partially unmapped ) ).statement(setUnmappedLoad(""" FROM test* From edb3953f913dc45ad59882d568ebb545bf95d333 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 1 Apr 2026 18:56:40 +0200 Subject: [PATCH 29/69] Remove duplicate no_mapping_sample_data --- .../java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index f4d55b591027b..0910f0a1897ef 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -123,9 +123,6 @@ public class CsvTestsDataLoader { new TestDataset("sample_data").withIndex("cloned_sample_data"), new TestDataset("partial_mapping_sample_data"), new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv"), - new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv").withTypeMapping( - Stream.of("timestamp", "client_ip", "event_duration").collect(toMap(k -> k, k -> "keyword")) - ), new TestDataset( "partial_mapping_no_source_sample_data", "mapping-partial_mapping_no_source_sample_data.json", From 24d608d9ecf37b099df7681b486705aeef71459e Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 2 Apr 2026 15:10:01 +0200 Subject: [PATCH 30/69] Fix capability --- .../src/main/resources/unmapped-load.csv-spec | 30 +++++++++---------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index e5bae8215a137..628be476bca91 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1052,7 +1052,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword employeesNoNamesLoad required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1073,7 +1073,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword employeesNoNamesLoadStats required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1091,7 +1091,7 @@ count(*):long | first_name:keyword ; employeesNoNamesLoadStatsByFirstName -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names @@ -1111,7 +1111,7 @@ c:long | first_name:keyword employeesNoNamesLoadInlineStats required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1132,7 +1132,7 @@ emp_no:integer | last_name:keyword | _index:keyword | c:long | first_name:ke employeesNoNamesLoadInlineStatsCount required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1168,7 +1168,7 @@ emp_no:integer | first_name:keyword | _index:keyword | c:long | last_name:ke ; employeesNoNamesLoadStatsWhereStartsWith -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names @@ -1198,7 +1198,7 @@ c:long | last_name:keyword // employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. partiallyMappedTextFieldNotLoaded required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees_gender_text, employees_no_gender METADATA _index @@ -1223,7 +1223,7 @@ emp_no:integer | gender:text | _index:keyword // null. partiallyMappedNonKeywordFieldsNotLoaded required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -1246,7 +1246,7 @@ null | null | null | Connected to 10. // from _source even when not explicitly referenced in a downstream expression like KEEP. fieldIsPartiallyUnmappedMultiIndexWithoutKeep required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -1275,7 +1275,7 @@ Connected to 10.1.0.1 | sample_data // referenced by any command — they remain in the output implicitly and get loaded. fieldIsPartiallyUnmappedWithEvalOnOtherMappedField required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1296,7 +1296,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | s fieldIsPartiallyUnmappedWithRenameOnOtherMappedField required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1319,7 +1319,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:k // first_name is not referenced by any command but must still be loaded. fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1340,7 +1340,7 @@ emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1362,7 +1362,7 @@ emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index: // Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -1390,7 +1390,7 @@ Connected to 10.1.0.1 | sample_data // DROP the partially-mapped field itself — verify no errors from converting then pruning. fieldIsPartiallyUnmappedMultiIndexDropPartiallyMappedField required_capability: index_metadata_field -required_capability: optional_fields_v4 +required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index From eee93311f01689d854c2da2dd1441e6c8e104377 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 2 Apr 2026 15:18:31 +0200 Subject: [PATCH 31/69] Update tests --- .../src/main/resources/unmapped-load.csv-spec | 22 ++++++++++++++++++- .../load/analysis.expected | 2 +- .../load/analysis.expected | 2 +- 3 files changed, 23 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 628be476bca91..f536e092156df 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1196,7 +1196,27 @@ c:long | last_name:keyword // Verify that a partially-mapped TEXT field is not automatically loaded from _source for the unmapped field index. // employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while // employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. -partiallyMappedTextFieldNotLoaded +partiallyMappedTextFieldNotLoadedKeep +required_capability: index_metadata_field +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees_gender_text, employees_no_gender METADATA _index +| KEEP emp_no, gender, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | gender:text | _index:keyword +10001 | M | employees_gender_text +10001 | null | employees_no_gender +10002 | F | employees_gender_text +10002 | null | employees_no_gender +10003 | M | employees_gender_text +10003 | null | employees_no_gender +; + +partiallyMappedTextFieldNotLoadedDrop required_capability: index_metadata_field required_capability: optional_fields_v5 diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected index fba946502a112..f2d64fa28372c 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[event_duration{r}#0]] \_Eval[[$$event_duration$converted_to$long{f(MultiTypeEsField)$}#1 AS event_duration#0]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f}#5, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f(PotentiallyUnmappedKeywordEsField)}#5, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected index 4160e1709b996..68726db5f431b 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[ts{r}#0]] \_Eval[[$$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1 AS ts#0]] - \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, client_ip{f}#2, event_duration{f}#3, message{f}#4, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file + \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#4, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file From fa1c1853ae38ee66d52bb44377c81cad7111e3a1 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 2 Apr 2026 16:51:22 +0200 Subject: [PATCH 32/69] Move detection of PUNKs/PUKs to ResolveTable --- .../xpack/esql/analysis/Analyzer.java | 113 +++++------------- 1 file changed, 29 insertions(+), 84 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 1dce433a43ba4..b7f78175b3c70 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -354,7 +354,7 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR attributes.addAll(metadata.stream().map(NamedExpression::toAttribute).toList()); if (context.unmappedResolution() == UnmappedResolution.LOAD) { - loadPartiallyMappedKeywordFields(attributes, esIndex); + loadPartiallyMappedFields(attributes, esIndex); } return new EsRelation( @@ -369,16 +369,24 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR } /** - * When {@code SET unmapped_fields="load"}, convert partially-mapped keyword fields to use - * {@link PotentiallyUnmappedKeywordEsField} so they are loaded from {@code _source} on shards where they are unmapped. - * Non-keyword fields are left unchanged (they will return null on unmapped shards, consistent with their default behavior). + * When {@code SET unmapped_fields="load"}, convert partially-mapped fields so they can be used across indices where they + * may not exist: + *
    + *
  • Keyword fields are converted to use {@link PotentiallyUnmappedKeywordEsField} so they are loaded from + * {@code _source} on shards where they are unmapped.
    • + *
  • Non-keyword fields are marked as {@link InvalidMappedField#potentiallyUnmapped potentially unmapped} so that + * explicit casts (e.g. {@code field::long}) can resolve them via the union types / {@code MultiTypeEsField} + * mechanism.
    • + *
*/ - private static void loadPartiallyMappedKeywordFields(List attributes, EsIndex esIndex) { + private static void loadPartiallyMappedFields(List attributes, EsIndex esIndex) { for (int i = 0; i < attributes.size(); i++) { - if (attributes.get(i) instanceof FieldAttribute fa - && fa.dataType() == KEYWORD - && esIndex.isPartiallyUnmappedField(fa.fieldName().string())) { - attributes.set(i, ResolveRefs.insistKeyword(fa)); + if (attributes.get(i) instanceof FieldAttribute fa && esIndex.isPartiallyUnmappedField(fa.fieldName().string())) { + if (fa.dataType() == KEYWORD) { + attributes.set(i, ResolveRefs.insistKeyword(fa)); + } else { + attributes.set(i, ResolveRefs.invalidInsistAttribute(fa, esIndex)); + } } } } @@ -1247,7 +1255,7 @@ private Attribute resolveInsistAttribute(Attribute attribute, List ch return resolvedCol; } - private static FieldAttribute invalidInsistAttribute(FieldAttribute fa, EsIndex esIndex) { + static FieldAttribute invalidInsistAttribute(FieldAttribute fa, EsIndex esIndex) { InvalidMappedField field = InvalidMappedField.potentiallyUnmapped(fa.name(), getTypesToIndices(fa, esIndex)); return new FieldAttribute(fa.source(), null, fa.qualifier(), fa.name(), field); } @@ -1272,64 +1280,6 @@ public static FieldAttribute insistKeyword(Attribute attribute) { ); } - /** - * This will inspect current node/{@code plan}'s expressions and check if any of the {@code FieldAttribute}s refer to fields that - * are partially unmapped across the indices involved in the plan fragment. If so, replace their field with an "insisted" EsField. - */ - private static LogicalPlan resolvePartiallyMapped(LogicalPlan plan, AnalyzerContext context) { - var indexResolutions = collectIndexResolutions(plan, context); - Map insistedMap = new HashMap<>(); - var transformed = plan.transformExpressionsOnly(FieldAttribute.class, fa -> { - var esField = fa.field(); - if (esField instanceof PotentiallyUnmappedKeywordEsField - || esField instanceof InvalidMappedField imf && imf.isPotentiallyUnmapped()) { - return fa; - } - var existing = insistedMap.get(fa); - if (existing != null) { // field shows up multiple times in the node; return first processing - return existing; - } - - if (indexResolutions.isEmpty()) { - throw new IllegalStateException("Unmapped fields with empty index resolutions."); - } - if (indexResolutions.size() > 1) { - throw new IllegalStateException( - Strings.format("Multiple index patterns should be disabled with unmapped fields", indexResolutions) - ); - } - EsIndex esIndex = indexResolutions.getFirst().get(); - if (esIndex.isPartiallyUnmappedField(fa.name())) { - FieldAttribute newFA = fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa, esIndex); - insistedMap.put(fa, newFA); - return newFA; - } - return fa; - }); - return insistedMap.isEmpty() ? transformed : propagateInsistedFields(transformed, insistedMap); - } - - /** - * Push only those fields from the {@code insistedMap} into {@code EsRelation}s in the {@code plan} that wrap a - * {@code PotentiallyUnmappedKeywordEsField}. - */ - private static LogicalPlan propagateInsistedFields(LogicalPlan plan, Map insistedMap) { - return plan.transformUp(EsRelation.class, esr -> { - var newOutput = new ArrayList(); - boolean updated = false; - for (Attribute attr : esr.output()) { - var newFA = insistedMap.get(attr); - if (newFA != null && newFA.field() instanceof PotentiallyUnmappedKeywordEsField) { - newOutput.add(newFA); - updated = true; - } else { - newOutput.add(attr); - } - } - return updated ? esr.withAttributes(newOutput) : esr; - }); - } - private LogicalPlan resolveFuse(Fuse fuse, List childrenOutput) { Source source = fuse.source(); Attribute score = fuse.score(); @@ -2704,17 +2654,14 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { return relation; } return relation.transformExpressionsUp(FieldAttribute.class, f -> { - if (f.field() instanceof InvalidMappedField imf && allDates(context, relation, imf)) { + if (f.field() instanceof InvalidMappedField imf && allDates(context, imf)) { HashMap typeResolutions = new HashMap<>(); var convert = new ToDateNanos(f.source(), f, context.configuration()); imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions)); - // This rule runs in the "Initialize" batch, before ResolveUnmapped. The isFieldMappedInAllIndices - // check above should prevent reaching here for fields that are unmapped in any index when in LOAD mode. - if (imf.isPotentiallyUnmapped()) { - throw new IllegalStateException( - "Unexpected potentially unmapped field [" + imf.getName() + "] in DateMillisToNanosInEsRelation" - ); - } + // The allDates check filters out fields that are not mapped in all indices, which includes + // potentiallyUnmapped fields. This assertion guards against future changes breaking that invariant. + assert imf.isPotentiallyUnmapped() == false + : "Unexpected potentially unmapped field [" + imf.getName() + "] in DateMillisToNanosInEsRelation"; var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions, null); return new FieldAttribute( f.source(), @@ -2732,18 +2679,16 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { }); } - private static boolean allDates(AnalyzerContext context, EsRelation relation, InvalidMappedField imf) { + private static boolean allDates(AnalyzerContext context, InvalidMappedField imf) { if (imf.types().stream().allMatch(DataType::isDate) == false) { return false; } - // If we need to load the fields from unmapped indices, we will treat it as a keyword, i.e., not all types are dates. - if (context.unmappedResolution() != UnmappedResolution.LOAD) { - return true; + // If the field is potentially unmapped (i.e. not mapped in all indices), we treat it as a keyword (not all dates), + // so that it can be resolved via the union types / MultiTypeEsField mechanism instead. + if (context.unmappedResolution() == UnmappedResolution.LOAD && imf.isPotentiallyUnmapped()) { + return false; } - // Since DateMillisToNanosInEsRelation runs before ResolveUnmapped, isPotentiallyUnmapped isn't set yet. - int mappedCount = imf.getTypesToIndices().values().stream().mapToInt(Set::size).sum(); - int totalCount = relation.concreteIndices().values().stream().mapToInt(List::size).sum(); - return mappedCount >= totalCount; + return true; } } From d3c8392fdaad6fd897be576e24ef5d4424312393 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 2 Apr 2026 17:39:07 +0200 Subject: [PATCH 33/69] Make PUNKs unsupported for now --- .../src/main/resources/unmapped-load.csv-spec | 12 +- .../unmapped-type-conflicts.csv-spec | 28 ++--- .../xpack/esql/analysis/Analyzer.java | 1 - .../analysis/AnalyzerUnmappedGoldenTests.java | 46 +++++++ .../esql/analysis/AnalyzerUnmappedTests.java | 113 ++---------------- .../load/analysis.expected | 2 +- .../load/analysis.expected | 2 +- .../load/analysis.expected | 4 +- .../load/analysis.expected | 2 +- .../load/analysis.expected | 3 + .../load/query.esql | 2 + .../nullify/analysis.expected | 3 + .../nullify/query.esql | 2 + .../load/analysis.expected | 3 + .../load/query.esql | 2 + .../nullify/analysis.expected | 3 + .../nullify/query.esql | 2 + .../load/analysis.expected | 4 + .../load/query.esql | 3 + .../nullify/analysis.expected | 4 + .../nullify/query.esql | 3 + .../load/analysis.expected | 3 + .../load/query.esql | 2 + .../nullify/analysis.expected | 3 + .../nullify/query.esql | 2 + .../load/analysis.expected | 3 + .../load/query.esql | 2 + .../nullify/analysis.expected | 3 + .../nullify/query.esql | 2 + .../load/analysis.expected | 2 +- .../load/analysis.expected | 2 +- 31 files changed, 136 insertions(+), 132 deletions(-) create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/query.esql diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index f536e092156df..bfdbfdb68c4e6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1208,11 +1208,11 @@ FROM employees_gender_text, employees_no_gender METADATA _index ; emp_no:integer | gender:text | _index:keyword -10001 | M | employees_gender_text +10001 | null | employees_gender_text 10001 | null | employees_no_gender -10002 | F | employees_gender_text +10002 | null | employees_gender_text 10002 | null | employees_no_gender -10003 | M | employees_gender_text +10003 | null | employees_gender_text 10003 | null | employees_no_gender ; @@ -1228,11 +1228,11 @@ FROM employees_gender_text, employees_no_gender METADATA _index ; emp_no:integer | gender:text | _index:keyword -10001 | M | employees_gender_text +10001 | null | employees_gender_text 10001 | null | employees_no_gender -10002 | F | employees_gender_text +10002 | null | employees_gender_text 10002 | null | employees_no_gender -10003 | M | employees_gender_text +10003 | null | employees_gender_text 10003 | null | employees_no_gender ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 0ef79c8ffbb41..f8a0247264e25 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -921,19 +921,19 @@ FROM sample_data, no_mapping_sample_data METADATA _index _index:keyword | event_duration:long no_mapping_sample_data | null -sample_data | 725448 +sample_data | null no_mapping_sample_data | null -sample_data | 1232382 +sample_data | null no_mapping_sample_data | null -sample_data | 1756467 +sample_data | null no_mapping_sample_data | null -sample_data | 2764889 +sample_data | null no_mapping_sample_data | null -sample_data | 3450233 +sample_data | null no_mapping_sample_data | null -sample_data | 5033755 +sample_data | null no_mapping_sample_data | null -sample_data | 8268153 +sample_data | null ; typeConflictLongUnmappedSortWithCastToKeyword @@ -947,19 +947,19 @@ FROM sample_data, no_mapping_sample_data METADATA _index _index:keyword | event_duration:long no_mapping_sample_data | null -sample_data | 1232382 +sample_data | null no_mapping_sample_data | null -sample_data | 1756467 +sample_data | null no_mapping_sample_data | null -sample_data | 2764889 +sample_data | null no_mapping_sample_data | null -sample_data | 3450233 +sample_data | null no_mapping_sample_data | null -sample_data | 5033755 +sample_data | null no_mapping_sample_data | null -sample_data | 725448 +sample_data | null no_mapping_sample_data | null -sample_data | 8268153 +sample_data | null ; ############################ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index b7f78175b3c70..7051fd2bffd53 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -15,7 +15,6 @@ import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Page; import org.elasticsearch.core.Nullable; -import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.IdFieldMapper; import org.elasticsearch.logging.Logger; diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java index a6db44ab182de..5c30a5186c2eb 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java @@ -675,6 +675,52 @@ public void testNoTypeConflictKeywordAndUnmappedWhere() throws Exception { """); } + public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() throws Exception { + // first_name and last_name are keyword, partially unmapped (missing in employees_no_names). + // They should appear as PotentiallyUnmappedKeywordEsField in the EsRelation without being explicitly referenced. + runTests(""" + FROM employees, employees_no_names + | SORT emp_no + | LIMIT 1 + """); + } + + public void testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped() throws Exception { + // event_duration is long, partially unmapped (missing in no_mapping_sample_data). + // It should appear as InvalidMappedField (unsupported) in the EsRelation. + runTests(""" + FROM sample_data, no_mapping_sample_data + | KEEP @timestamp, event_duration + """); + } + + public void testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped() throws Exception { + // gender is text in employees_gender_text but missing in employees_no_gender. + // It should appear as InvalidMappedField (unsupported) in the EsRelation. + runTests(""" + FROM employees_gender_text, employees_no_gender + | KEEP gender + """); + } + + public void testOnlyPartiallyMappedKeywordFieldLoaded() throws Exception { + // message (keyword) and event_duration (long) are both partially unmapped. + // Only message should become PotentiallyUnmappedKeywordEsField; event_duration should be InvalidMappedField. + runTests(""" + FROM sample_data, no_mapping_sample_data + | KEEP message, event_duration + """); + } + + public void testNonPartiallyMappedKeywordFieldNotLoadedFromSource() throws Exception { + // first_name (keyword, partially unmapped) should become PotentiallyUnmappedKeywordEsField. + // gender (keyword, fully mapped in both indices) should remain a regular KeywordEsField. + runTests(""" + FROM employees, employees_no_names + | KEEP first_name, gender + """); + } + public void testForkBranchesAfterStats1stBranch() throws Exception { runTestsNullifyOnly(""" FROM employees diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 16879e4ddc3f4..df4e88561dc25 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -25,7 +25,6 @@ import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.core.type.KeywordEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; -import org.elasticsearch.xpack.esql.core.type.TextEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; @@ -865,72 +864,12 @@ public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() { as(fieldAttr.field(), PotentiallyUnmappedKeywordEsField.class); } - /** - * Verify that a non-keyword partially-mapped field (e.g. integer) is NOT converted to - * {@link PotentiallyUnmappedKeywordEsField}, since the fix only applies to keyword fields. - */ - public void testPartiallyMappedNonKeywordFieldNotLoadedWithoutExplicitReference() { - var plan = analyzer().addIndex( - new EsIndex( - "test*", - EsqlTestUtils.loadMapping("mapping-basic.json"), - Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), - Map.of(), - Map.of(), - Map.of("salary", Set.of("test2")) // salary is integer, not keyword - ) - ).statement(setUnmappedLoad(""" - FROM test* - | SORT emp_no - """)); - - var limit = as(plan, Limit.class); - var order = as(limit.child(), OrderBy.class); - var relation = as(order.child(), EsRelation.class); - - var salaryAttr = relation.output().stream().filter(a -> a.name().equals("salary")).findFirst().orElseThrow(); - var fieldAttr = as(salaryAttr, FieldAttribute.class); - assertThat(fieldAttr.field(), not(instanceOf(PotentiallyUnmappedKeywordEsField.class))); - assertThat(fieldAttr.field().getClass(), is(EsField.class)); - assertThat(fieldAttr.field().getDataType(), is(DataType.INTEGER)); - } - - /** - * Verify that a partially-mapped text field is NOT converted to {@link PotentiallyUnmappedKeywordEsField}, - * since the fix only applies to keyword fields. Text fields should not be loaded from _source. - */ - public void testPartiallyMappedTextFieldNotLoaded() { - var plan = analyzer().addIndex( - new EsIndex( - "test*", - EsqlTestUtils.loadMapping("mapping-basic.json"), - Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), - Map.of(), - Map.of(), - Map.of("gender", Set.of("test2")) // gender is text, not keyword - ) - ).statement(setUnmappedLoad(""" - FROM test* - | SORT emp_no - """)); - - var limit = as(plan, Limit.class); - var order = as(limit.child(), OrderBy.class); - var relation = as(order.child(), EsRelation.class); - - var genderAttr = relation.output().stream().filter(a -> a.name().equals("gender")).findFirst().orElseThrow(); - var fieldAttr = as(genderAttr, FieldAttribute.class); - assertThat(fieldAttr.field(), not(instanceOf(PotentiallyUnmappedKeywordEsField.class))); - assertThat(fieldAttr.field(), instanceOf(TextEsField.class)); - assertThat(fieldAttr.field().getDataType(), is(DataType.TEXT)); - } - /** * Verify that partially-mapped fields of ALL non-keyword types are NOT converted to - * {@link PotentiallyUnmappedKeywordEsField}. Only keyword fields should be loaded from _source. + * {@link PotentiallyUnmappedKeywordEsField}, but are instead marked as potentially unmapped via {@link InvalidMappedField}. * This iterates over all {@link DataType} values that can appear as ES mapped field types. */ - public void testPartiallyMappedNonKeywordFieldsNotLoaded() { + public void testPartiallyMappedNonKeywordFieldsMarkedAsPotentiallyUnmapped() { // Types that cannot appear as regular ES mapped fields in an EsIndex mapping Set excludedTypes = Set.of( DataType.KEYWORD, // this is the type we DO convert — not a negative test case @@ -985,48 +924,13 @@ public void testPartiallyMappedNonKeywordFieldsNotLoaded() { not(instanceOf(PotentiallyUnmappedKeywordEsField.class)) ); assertThat( - "Partially-mapped " + dataType + " field should retain its original data type", - fieldAttr.dataType(), - is(dataType.widenSmallNumeric()) + "Partially-mapped " + dataType + " field should be marked as unsupported due to type conflict", + fieldAttr.field(), + instanceOf(UnsupportedEsField.class) ); } } - /** - * When both a keyword and non-keyword field are partially mapped, only the keyword field - * should be converted to {@link PotentiallyUnmappedKeywordEsField}. - */ - public void testOnlyPartiallyMappedKeywordFieldLoaded() { - var plan = analyzer().addIndex( - new EsIndex( - "test*", - EsqlTestUtils.loadMapping("mapping-basic.json"), - Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), - Map.of(), - Map.of(), - Map.of("first_name", Set.of("test2"), "salary", Set.of("test2")) // keyword + integer - ) - ).statement(setUnmappedLoad(""" - FROM test* - | SORT emp_no - """)); - - var limit = as(plan, Limit.class); - var order = as(limit.child(), OrderBy.class); - var relation = as(order.child(), EsRelation.class); - - // keyword field should be converted - var firstName = relation.output().stream().filter(a -> a.name().equals("first_name")).findFirst().orElseThrow(); - as(as(firstName, FieldAttribute.class).field(), PotentiallyUnmappedKeywordEsField.class); - - // integer field should NOT be loaded - var salary = relation.output().stream().filter(a -> a.name().equals("salary")).findFirst().orElseThrow(); - var salaryField = as(salary, FieldAttribute.class).field(); - assertThat(salaryField, not(instanceOf(PotentiallyUnmappedKeywordEsField.class))); - assertThat(salaryField.getClass(), is(EsField.class)); - assertThat(salaryField.getDataType(), is(DataType.INTEGER)); - } - /** * Verify that a keyword field that is NOT partially unmapped is not converted to * {@link PotentiallyUnmappedKeywordEsField}, even when another keyword field is. @@ -1527,9 +1431,10 @@ public void testDisallowLoadWithPartialUnionTimestampInWhere() { allOf( containsString("Found 1 problem"), containsString( - "line 1:116: Cannot use field [@timestamp] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [date_nanos] in [sample_data, sample_data_ts_nanos]" - ) + "line 1:116: Cannot use field [@timestamp] due to ambiguities being mapped as [3] incompatible types: "), + containsString("[keyword] due to loading from _source"), + containsString("[date_nanos] in [sample_data_ts_nanos]"), + containsString("[datetime] in [sample_data]") ) ); } diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnly/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnly/load/analysis.expected index 96ca49e813873..7ad32459802a0 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnly/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnly/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] \_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnlyCast/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnlyCast/load/analysis.expected index bc9513ec2778f..03374119f27cb 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnlyCast/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnlyCast/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] \_Eval[[TOLONG(message{f(PotentiallyUnmappedKeywordEsField)}#0) AS x#1]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected index 337900ed01ba1..e461a85d6a5a6 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] -\_Project[[event_duration{f}#0]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#0, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file +\_Project[[!event_duration]] + \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected index 1de4ceb4f12be..a22940451b157 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0]] \_Filter[LIKE(TOSTRING(message{f(PotentiallyUnmappedKeywordEsField)}#0), "Connected*", false)] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/analysis.expected new file mode 100644 index 0000000000000..72858027185a7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[first_name{f(PotentiallyUnmappedKeywordEsField)}#0, gender{f}#1]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#2, birth_date{f}#3, emp_no{f}#4, first_name{f(PotentiallyUnmappedKeywordEsField)}#0, gender{f}#1, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f(PotentiallyUnmappedKeywordEsField)}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/query.esql new file mode 100644 index 0000000000000..8b1fba87772c2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM employees, employees_no_names +| KEEP first_name, gender diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/analysis.expected new file mode 100644 index 0000000000000..e40244072d71a --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[first_name{f}#0, gender{f}#1]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#2, birth_date{f}#3, emp_no{f}#4, first_name{f}#0, gender{f}#1, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/query.esql new file mode 100644 index 0000000000000..db70e3133f6b2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM employees, employees_no_names +| KEEP first_name, gender diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/analysis.expected new file mode 100644 index 0000000000000..32e0128d01db3 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0, !event_duration]] + \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/query.esql new file mode 100644 index 0000000000000..c5a81e9d2b29f --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| KEEP message, event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/analysis.expected new file mode 100644 index 0000000000000..704395a2a87e0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[message{f}#0, event_duration{f}#1]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#1, message{f}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/query.esql new file mode 100644 index 0000000000000..251a66c30cb72 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| KEEP message, event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/analysis.expected new file mode 100644 index 0000000000000..d62949f5d6842 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/analysis.expected @@ -0,0 +1,4 @@ +Limit[10000[INTEGER],false,false] +\_Limit[1[INTEGER],false,false] + \_OrderBy[[Order[emp_no{f}#0,ASC,LAST]]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#1, birth_date{f}#2, emp_no{f}#0, first_name{f(PotentiallyUnmappedKeywordEsField)}#3, gender{f}#4, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f(PotentiallyUnmappedKeywordEsField)}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/query.esql new file mode 100644 index 0000000000000..b8c7481e3d4ce --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="load"; FROM employees, employees_no_names +| SORT emp_no +| LIMIT 1 diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/analysis.expected new file mode 100644 index 0000000000000..570e8eb95973f --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/analysis.expected @@ -0,0 +1,4 @@ +Limit[10000[INTEGER],false,false] +\_Limit[1[INTEGER],false,false] + \_OrderBy[[Order[emp_no{f}#0,ASC,LAST]]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#1, birth_date{f}#2, emp_no{f}#0, first_name{f}#3, gender{f}#4, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/query.esql new file mode 100644 index 0000000000000..2cfb9690b7de0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="nullify"; FROM employees, employees_no_names +| SORT emp_no +| LIMIT 1 diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected new file mode 100644 index 0000000000000..6ca9eaca96c5e --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[!@timestamp, !event_duration]] + \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/query.esql new file mode 100644 index 0000000000000..5d07e509e230e --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| KEEP @timestamp, event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected new file mode 100644 index 0000000000000..a8a40a8e5b479 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, event_duration{f}#1]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#2, event_duration{f}#1, message{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/query.esql new file mode 100644 index 0000000000000..beba8dcb67071 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| KEEP @timestamp, event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected new file mode 100644 index 0000000000000..78e3c85d6e32a --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[!gender]] + \_EsRelation[employees_gender_text,employees_no_gender][avg_worked_seconds{f}#0, birth_date{f}#1, emp_no{f}#2, first_name{f}#3, !gender, height{f}#4, height.float{f}#5, height.half_float{f}#6, height.scaled_float{f}#7, hire_date{f}#8, is_rehired{f}#9, job_positions{f}#10, languages{f}#11, languages.byte{f}#12, languages.long{f}#13, languages.short{f}#14, last_name{f}#15, salary{f}#16, salary_change{f}#17, salary_change.int{f}#18, salary_change.keyword{f}#19, salary_change.long{f}#20, still_hired{f}#21] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/query.esql new file mode 100644 index 0000000000000..cde80280471a3 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM employees_gender_text, employees_no_gender +| KEEP gender diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected new file mode 100644 index 0000000000000..c8bcb0b6753f4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[gender{f}#0]] + \_EsRelation[employees_gender_text,employees_no_gender][avg_worked_seconds{f}#1, birth_date{f}#2, emp_no{f}#3, first_name{f}#4, gender{f}#0, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/query.esql new file mode 100644 index 0000000000000..03bfa9100c87d --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM employees_gender_text, employees_no_gender +| KEEP gender diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected index f2d64fa28372c..955ed6552846c 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[event_duration{r}#0]] \_Eval[[$$event_duration$converted_to$long{f(MultiTypeEsField)$}#1 AS event_duration#0]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f(PotentiallyUnmappedKeywordEsField)}#5, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#2, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected index 68726db5f431b..c886799b5a73a 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[ts{r}#0]] \_Eval[[$$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1 AS ts#0]] - \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#4, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file + \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#2, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file From 88090a3262a987adc1784c4afb74835bbe346c91 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 2 Apr 2026 17:54:53 +0200 Subject: [PATCH 34/69] Remove redundant tests --- .../esql/analysis/AnalyzerUnmappedTests.java | 65 ------------------- 1 file changed, 65 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index df4e88561dc25..8895a29506745 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -834,36 +834,6 @@ public void testTbucketWithUnmappedTimestampWithFork() { } } - /** - * Reproducer for https://github.com/elastic/elasticsearch/issues/141994 - * When unmapped_fields="load" and a keyword field is partially unmapped (mapped in some concrete indices but not in others), - * the field should be resolved as {@link PotentiallyUnmappedKeywordEsField} in the EsRelation output, even if the field is - * not explicitly referenced in any downstream expression (e.g. KEEP, SORT, WHERE). - */ - public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() { - var plan = analyzer().addIndex( - new EsIndex( - "test*", - EsqlTestUtils.loadMapping("mapping-basic.json"), - Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), - Map.of(), - Map.of(), - Map.of("first_name", Set.of("test2")) - ) - ).statement(setUnmappedLoad(""" - FROM test* - | SORT emp_no - """)); - - var limit = as(plan, Limit.class); - var order = as(limit.child(), OrderBy.class); - var relation = as(order.child(), EsRelation.class); - - var firstNameAttr = relation.output().stream().filter(a -> a.name().equals("first_name")).findFirst().orElseThrow(); - var fieldAttr = as(firstNameAttr, FieldAttribute.class); - as(fieldAttr.field(), PotentiallyUnmappedKeywordEsField.class); - } - /** * Verify that partially-mapped fields of ALL non-keyword types are NOT converted to * {@link PotentiallyUnmappedKeywordEsField}, but are instead marked as potentially unmapped via {@link InvalidMappedField}. @@ -931,41 +901,6 @@ public void testPartiallyMappedNonKeywordFieldsMarkedAsPotentiallyUnmapped() { } } - /** - * Verify that a keyword field that is NOT partially unmapped is not converted to - * {@link PotentiallyUnmappedKeywordEsField}, even when another keyword field is. - */ - public void testNonPartiallyMappedKeywordFieldNotLoadedFromSource() { - var plan = analyzer().addIndex( - new EsIndex( - "test*", - EsqlTestUtils.loadMapping("mapping-basic.json"), - Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), - Map.of(), - Map.of(), - Map.of("first_name", Set.of("test2")) // only first_name is partially unmapped - ) - ).statement(setUnmappedLoad(""" - FROM test* - | SORT emp_no - """)); - - var limit = as(plan, Limit.class); - var order = as(limit.child(), OrderBy.class); - var relation = as(order.child(), EsRelation.class); - - // first_name (partially unmapped keyword) should be converted - var firstName = relation.output().stream().filter(a -> a.name().equals("first_name")).findFirst().orElseThrow(); - as(as(firstName, FieldAttribute.class).field(), PotentiallyUnmappedKeywordEsField.class); - - // last_name (keyword but NOT partially unmapped) should NOT be converted - var lastName = relation.output().stream().filter(a -> a.name().equals("last_name")).findFirst().orElseThrow(); - var lastNameField = as(lastName, FieldAttribute.class).field(); - assertThat(lastNameField, not(instanceOf(PotentiallyUnmappedKeywordEsField.class))); - assertThat(lastNameField, instanceOf(KeywordEsField.class)); - assertThat(lastNameField.getDataType(), is(DataType.KEYWORD)); - } - public void testTbucketWithUnmappedTimestampWithLookupJoin() { var query = """ FROM test From 7bf841f19ceb84754e96d70da01ddaf64a542a2e Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 2 Apr 2026 18:11:52 +0200 Subject: [PATCH 35/69] Update golden tests --- .../analysis/AnalyzerUnmappedGoldenTests.java | 57 +++++++++++++------ .../nullify/analysis.expected | 3 - .../load/analysis.expected | 2 + .../load/query.esql | 1 - .../nullify/analysis.expected | 2 + .../nullify/query.esql | 1 - .../load/analysis.expected | 3 + .../load/query.esql | 2 + .../nullify/analysis.expected | 3 + .../nullify/query.esql | 2 + .../load/analysis.expected | 3 + .../load/query.esql | 2 + .../nullify/analysis.expected | 3 + .../nullify/query.esql | 2 + .../load/analysis.expected | 2 +- .../load/query.esql | 2 + .../nullify/analysis.expected | 3 + .../nullify/query.esql | 2 + 18 files changed, 72 insertions(+), 23 deletions(-) delete mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected rename x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/{testOnlyPartiallyMappedKeywordFieldLoaded => testPartiallyMappedFieldsAutomaticallyFound}/load/query.esql (69%) create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/analysis.expected rename x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/{testOnlyPartiallyMappedKeywordFieldLoaded => testPartiallyMappedFieldsAutomaticallyFound}/nullify/query.esql (69%) create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/query.esql rename x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/{testOnlyPartiallyMappedKeywordFieldLoaded => testPartiallyMappedFieldsDropNonKeywordWithWildcards}/load/analysis.expected (69%) create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/query.esql diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java index 5c30a5186c2eb..c302c7962491c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java @@ -675,9 +675,34 @@ public void testNoTypeConflictKeywordAndUnmappedWhere() throws Exception { """); } + // All fields are partially unmapped (no_mapping_sample_data has no mapped fields). + // Keyword fields should become PotentiallyUnmappedKeywordEsField; non-keyword fields should become InvalidMappedField. + // No explicit field reference — all fields come from the implicit output of FROM. + public void testPartiallyMappedFieldsAutomaticallyFound() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + """); + } + + // Same as testPartiallyMappedFieldsAutomaticallyFound, but with an explicit KEEP * to verify wildcard expansion + // handles partially-mapped fields correctly. + public void testPartiallyMappedFieldsAutomaticallyFoundKeepStar() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | KEEP * + """); + } + + public void testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | KEEP @timestamp, event_duration + """); + } + + // first_name and last_name are keyword, partially unmapped (missing in employees_no_names). + // They should appear as PotentiallyUnmappedKeywordEsField in the EsRelation without being explicitly referenced. public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() throws Exception { - // first_name and last_name are keyword, partially unmapped (missing in employees_no_names). - // They should appear as PotentiallyUnmappedKeywordEsField in the EsRelation without being explicitly referenced. runTests(""" FROM employees, employees_no_names | SORT emp_no @@ -685,39 +710,37 @@ public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() thro """); } - public void testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped() throws Exception { - // event_duration is long, partially unmapped (missing in no_mapping_sample_data). - // It should appear as InvalidMappedField (unsupported) in the EsRelation. + // first_name (keyword, partially unmapped) should become PotentiallyUnmappedKeywordEsField. + // gender (keyword, fully mapped in both indices) should remain a regular KeywordEsField. + public void testNonPartiallyMappedKeywordFieldNotLoadedFromSource() throws Exception { runTests(""" - FROM sample_data, no_mapping_sample_data - | KEEP @timestamp, event_duration + FROM employees, employees_no_names + | KEEP first_name, gender """); } + // gender is text in employees_gender_text but missing in employees_no_gender. + // It should appear as InvalidMappedField (unsupported) in the EsRelation. public void testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped() throws Exception { - // gender is text in employees_gender_text but missing in employees_no_gender. - // It should appear as InvalidMappedField (unsupported) in the EsRelation. runTests(""" FROM employees_gender_text, employees_no_gender | KEEP gender """); } - public void testOnlyPartiallyMappedKeywordFieldLoaded() throws Exception { - // message (keyword) and event_duration (long) are both partially unmapped. - // Only message should become PotentiallyUnmappedKeywordEsField; event_duration should be InvalidMappedField. + // DROP with wildcards on partially-mapped non-keyword fields, leaving only the keyword field (message). + public void testPartiallyMappedFieldsDropNonKeywordWithWildcards() throws Exception { runTests(""" FROM sample_data, no_mapping_sample_data - | KEEP message, event_duration + | DROP *_ip, *_duration, @timestamp """); } - public void testNonPartiallyMappedKeywordFieldNotLoadedFromSource() throws Exception { - // first_name (keyword, partially unmapped) should become PotentiallyUnmappedKeywordEsField. - // gender (keyword, fully mapped in both indices) should remain a regular KeywordEsField. + // DROP with wildcards on partially-mapped keyword fields, leaving only a few non-keyword fields. + public void testPartiallyMappedFieldsDropKeywordWithWildcards() throws Exception { runTests(""" FROM employees, employees_no_names - | KEEP first_name, gender + | DROP *date*, gender, height*, languages*, *_hired, *_seconds, *_positions, salary_change* """); } diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/analysis.expected deleted file mode 100644 index 704395a2a87e0..0000000000000 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/analysis.expected +++ /dev/null @@ -1,3 +0,0 @@ -Limit[1000[INTEGER],false,false] -\_Project[[message{f}#0, event_duration{f}#1]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#1, message{f}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected new file mode 100644 index 0000000000000..f8f5a6980f7e7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected @@ -0,0 +1,2 @@ +Limit[1000[INTEGER],false,false] +\_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/query.esql similarity index 69% rename from x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/query.esql rename to x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/query.esql index c5a81e9d2b29f..74dc2ef0710ee 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/query.esql +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/query.esql @@ -1,2 +1 @@ SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data -| KEEP message, event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/analysis.expected new file mode 100644 index 0000000000000..25c9cb672505f --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/analysis.expected @@ -0,0 +1,2 @@ +Limit[1000[INTEGER],false,false] +\_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/query.esql similarity index 69% rename from x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/query.esql rename to x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/query.esql index 251a66c30cb72..f7fa817d65c34 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/nullify/query.esql +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/query.esql @@ -1,2 +1 @@ SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data -| KEEP message, event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected new file mode 100644 index 0000000000000..bf899a9bace3c --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0]] + \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/query.esql new file mode 100644 index 0000000000000..c4112b1def237 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| KEEP * diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/analysis.expected new file mode 100644 index 0000000000000..00eca33ef618d --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/query.esql new file mode 100644 index 0000000000000..55b5093450910 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| KEEP * diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/analysis.expected new file mode 100644 index 0000000000000..4909f0a70f5ae --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[emp_no{f}#0, first_name{f(PotentiallyUnmappedKeywordEsField)}#1, is_rehired{f}#2, last_name{f(PotentiallyUnmappedKeywordEsField)}#3, salary{f}#4]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#5, birth_date{f}#6, emp_no{f}#0, first_name{f(PotentiallyUnmappedKeywordEsField)}#1, gender{f}#7, height{f}#8, height.float{f}#9, height.half_float{f}#10, height.scaled_float{f}#11, hire_date{f}#12, is_rehired{f}#2, job_positions{f}#13, languages{f}#14, languages.byte{f}#15, languages.long{f}#16, languages.short{f}#17, last_name{f(PotentiallyUnmappedKeywordEsField)}#3, salary{f}#4, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/query.esql new file mode 100644 index 0000000000000..371f30a81bec3 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM employees, employees_no_names +| DROP *date*, gender, height*, languages*, *_hired, *_seconds, *_positions, salary_change* diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/analysis.expected new file mode 100644 index 0000000000000..e3d98f9e3811e --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[emp_no{f}#0, first_name{f}#1, is_rehired{f}#2, last_name{f}#3, salary{f}#4]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#5, birth_date{f}#6, emp_no{f}#0, first_name{f}#1, gender{f}#7, height{f}#8, height.float{f}#9, height.half_float{f}#10, height.scaled_float{f}#11, hire_date{f}#12, is_rehired{f}#2, job_positions{f}#13, languages{f}#14, languages.byte{f}#15, languages.long{f}#16, languages.short{f}#17, last_name{f}#3, salary{f}#4, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/query.esql new file mode 100644 index 0000000000000..de53dad8692eb --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM employees, employees_no_names +| DROP *date*, gender, height*, languages*, *_hired, *_seconds, *_positions, salary_change* diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected similarity index 69% rename from x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/analysis.expected rename to x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected index 32e0128d01db3..7ad32459802a0 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testOnlyPartiallyMappedKeywordFieldLoaded/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] -\_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0, !event_duration]] +\_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0]] \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/query.esql new file mode 100644 index 0000000000000..c474e51eadeee --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| DROP *_ip, *_duration, @timestamp diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/analysis.expected new file mode 100644 index 0000000000000..1b43c24ef2c44 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[message{f}#0]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/query.esql new file mode 100644 index 0000000000000..331f316fcf2cc --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| DROP *_ip, *_duration, @timestamp From b32837343769fa5f41a22661738cb56307e37642 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Thu, 2 Apr 2026 16:31:37 +0000 Subject: [PATCH 36/69] [CI] Auto commit changes from spotless --- .../xpack/esql/analysis/AnalyzerUnmappedTests.java | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 8895a29506745..892dc2913e909 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; -import org.elasticsearch.xpack.esql.EsqlTestUtils; import org.elasticsearch.xpack.esql.TestAnalyzer; import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; @@ -23,7 +22,6 @@ import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; -import org.elasticsearch.xpack.esql.core.type.KeywordEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -1365,8 +1363,7 @@ public void testDisallowLoadWithPartialUnionTimestampInWhere() { + "| WHERE @timestamp == \"2021-01-01\"::date_nanos", allOf( containsString("Found 1 problem"), - containsString( - "line 1:116: Cannot use field [@timestamp] due to ambiguities being mapped as [3] incompatible types: "), + containsString("line 1:116: Cannot use field [@timestamp] due to ambiguities being mapped as [3] incompatible types: "), containsString("[keyword] due to loading from _source"), containsString("[date_nanos] in [sample_data_ts_nanos]"), containsString("[datetime] in [sample_data]") From 8b985b9bc9c79bea6b42afc4c5100152c91de735 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 7 Apr 2026 16:23:45 +0200 Subject: [PATCH 37/69] Load only the mapped data for PUNKs for now. --- .../src/main/resources/unmapped-load.csv-spec | 12 ++++---- .../unmapped-type-conflicts.csv-spec | 28 +++++++++---------- .../xpack/esql/analysis/Analyzer.java | 4 +-- .../esql/core/expression/FieldAttribute.java | 11 ++++++++ .../load/analysis.expected | 2 +- .../load/analysis.expected | 2 +- .../load/analysis.expected | 4 +-- .../load/analysis.expected | 2 +- .../load/analysis.expected | 2 +- .../load/analysis.expected | 4 +-- .../load/analysis.expected | 2 +- .../load/analysis.expected | 4 +-- .../load/analysis.expected | 2 +- .../nullify/analysis.expected | 2 +- .../nullify/analysis.expected | 2 +- .../load/analysis.expected | 2 +- .../load/analysis.expected | 2 +- 17 files changed, 49 insertions(+), 38 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index bfdbfdb68c4e6..f536e092156df 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1208,11 +1208,11 @@ FROM employees_gender_text, employees_no_gender METADATA _index ; emp_no:integer | gender:text | _index:keyword -10001 | null | employees_gender_text +10001 | M | employees_gender_text 10001 | null | employees_no_gender -10002 | null | employees_gender_text +10002 | F | employees_gender_text 10002 | null | employees_no_gender -10003 | null | employees_gender_text +10003 | M | employees_gender_text 10003 | null | employees_no_gender ; @@ -1228,11 +1228,11 @@ FROM employees_gender_text, employees_no_gender METADATA _index ; emp_no:integer | gender:text | _index:keyword -10001 | null | employees_gender_text +10001 | M | employees_gender_text 10001 | null | employees_no_gender -10002 | null | employees_gender_text +10002 | F | employees_gender_text 10002 | null | employees_no_gender -10003 | null | employees_gender_text +10003 | M | employees_gender_text 10003 | null | employees_no_gender ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index f8a0247264e25..0ef79c8ffbb41 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -921,19 +921,19 @@ FROM sample_data, no_mapping_sample_data METADATA _index _index:keyword | event_duration:long no_mapping_sample_data | null -sample_data | null +sample_data | 725448 no_mapping_sample_data | null -sample_data | null +sample_data | 1232382 no_mapping_sample_data | null -sample_data | null +sample_data | 1756467 no_mapping_sample_data | null -sample_data | null +sample_data | 2764889 no_mapping_sample_data | null -sample_data | null +sample_data | 3450233 no_mapping_sample_data | null -sample_data | null +sample_data | 5033755 no_mapping_sample_data | null -sample_data | null +sample_data | 8268153 ; typeConflictLongUnmappedSortWithCastToKeyword @@ -947,19 +947,19 @@ FROM sample_data, no_mapping_sample_data METADATA _index _index:keyword | event_duration:long no_mapping_sample_data | null -sample_data | null +sample_data | 1232382 no_mapping_sample_data | null -sample_data | null +sample_data | 1756467 no_mapping_sample_data | null -sample_data | null +sample_data | 2764889 no_mapping_sample_data | null -sample_data | null +sample_data | 3450233 no_mapping_sample_data | null -sample_data | null +sample_data | 5033755 no_mapping_sample_data | null -sample_data | null +sample_data | 725448 no_mapping_sample_data | null -sample_data | null +sample_data | 8268153 ; ############################ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 7051fd2bffd53..0a26bff18b645 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -1255,8 +1255,8 @@ private Attribute resolveInsistAttribute(Attribute attribute, List ch } static FieldAttribute invalidInsistAttribute(FieldAttribute fa, EsIndex esIndex) { - InvalidMappedField field = InvalidMappedField.potentiallyUnmapped(fa.name(), getTypesToIndices(fa, esIndex)); - return new FieldAttribute(fa.source(), null, fa.qualifier(), fa.name(), field); + InvalidMappedField field = InvalidMappedField.potentiallyUnmapped(fa.field().getName(), getTypesToIndices(fa, esIndex)); + return new FieldAttribute(fa.source(), fa.parentName(), fa.qualifier(), fa.name(), field); } private static Map> getTypesToIndices(FieldAttribute fa, EsIndex esIndex) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index 1d2dcfeae18a7..de1c6243f3e76 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -236,6 +236,8 @@ public FieldName fieldName() { return lazyFieldName; } + // TODO: better method name + // TODO: reference https://github.com/elastic/elasticsearch/issues/141995 /** * If the underlying field is an {@link InvalidMappedField} (ambiguous type across indices), * converts this attribute into an {@link UnsupportedAttribute} with a descriptive error message @@ -243,6 +245,15 @@ public FieldName fieldName() { */ public Attribute checkUnresolved() { if (field instanceof InvalidMappedField imf) { + if (imf.isPotentiallyUnmapped() && imf.types().size() == 1) { + // The field is mapped to a single type in some indices but unmapped in others. Revert it to a regular field so that + // values are loaded from the indices where it is mapped (and null is returned from unmapped indices). + DataType type = imf.types().iterator().next(); + var restoredField = new EsField(imf.getName(), type, imf.getProperties(), false, imf.getTimeSeriesFieldType()); + // TODO: add test where not passing on the parent name fails the test + // TODO: add TS tests and tests with different time series field types + return new FieldAttribute(source(), parentName(), qualifier(), name(), restoredField, nullable(), id(), synthetic()); + } // Field has conflicting types across indices — build a user-facing error message. String unresolvedMessage = "Cannot use field [" + name() + "] due to ambiguities being " + imf.errorMessage(); List types = imf.getTypesToIndices().keySet().stream().toList(); diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnly/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnly/load/analysis.expected index 7ad32459802a0..96ca49e813873 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnly/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnly/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] \_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0]] - \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnlyCast/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnlyCast/load/analysis.expected index 03374119f27cb..bc9513ec2778f 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnlyCast/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedInOneIndexOnlyCast/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] \_Eval[[TOLONG(message{f(PotentiallyUnmappedKeywordEsField)}#0) AS x#1]] - \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected index e461a85d6a5a6..337900ed01ba1 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] -\_Project[[!event_duration]] - \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file +\_Project[[event_duration{f}#0]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#0, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected index a22940451b157..1de4ceb4f12be 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0]] \_Filter[LIKE(TOSTRING(message{f(PotentiallyUnmappedKeywordEsField)}#0), "Connected*", false)] - \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected index f8f5a6980f7e7..3afa3e37cfefa 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected @@ -1,2 +1,2 @@ Limit[1000[INTEGER],false,false] -\_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file +\_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected index bf899a9bace3c..6300f15f4c963 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] -\_Project[[!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0]] - \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file +\_Project[[@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f(PotentiallyUnmappedKeywordEsField)}#3]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected index 7ad32459802a0..96ca49e813873 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] \_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0]] - \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected index 6ca9eaca96c5e..c250aa9d2c04c 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] -\_Project[[!@timestamp, !event_duration]] - \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file +\_Project[[@timestamp{f}#0, event_duration{f}#1]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#2, event_duration{f}#1, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesFirstOverTimeUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesFirstOverTimeUnmapped/load/analysis.expected index c834a5f44b080..c3dc7a0a27f2f 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesFirstOverTimeUnmapped/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesFirstOverTimeUnmapped/load/analysis.expected @@ -1,3 +1,3 @@ Limit[10000[INTEGER],false,false] \_TimeSeriesAggregate[[TIMESERIESWITHOUT() AS _timeseries#0, TBUCKET(PT1H[TIME_DURATION],@timestamp{f}#1) AS tbucket(1 hour)#2],[VALUES(FIRSTOVERTIME(TODOUBLE(does_not_exist{f(PotentiallyUnmappedKeywordEsField)}#3),true[BOOLEAN],PT0S[TIME_DURATION],@timestamp{f}#1),true[BOOLEAN],PT0S[TIME_DURATION]) AS f#4, tbucket(1 hour){r}#2],null,null,@timestamp{f}#1] - \_EsRelation[k8s][TIME_SERIES][@timestamp{f}#1, client.ip{f}#5, cluster{f}#6, event{f}#7, event_city{f}#8, event_city_boundary{f}#9, event_location{f}#10, event_log{f}#11, event_shape{f}#12, events_received{f}#13, network.bytes_in{f}#14, network.cost{f}#15, network.eth0.currently_connected_clients{f}#16, network.eth0.firmware_version{f}#17, network.eth0.last_up{f}#18, network.eth0.rx{f}#19, network.eth0.tx{f}#20, network.eth0.up{f}#21, network.total_bytes_in{f}#22, network.total_bytes_out{f}#23, network.total_cost{f}#24, pod{f}#25, region{f}#26, does_not_exist{f(PotentiallyUnmappedKeywordEsField)}#3] + \_EsRelation[k8s][TIME_SERIES][@timestamp{f}#1, client.ip{f}#5, cluster{f}#6, event{f}#7, event_city{f}#8, event_city_boundary{f}#9, event_location{f}#10, event_log{f}#11, event_shape{f}#12, events_received{f}#13, network.bytes_in{f}#14, network.cost{f}#15, network.eth0.currently_connected_clients{f}#16, network.eth0.firmware_version{f}#17, network.eth0.last_up{f}#18, network.eth0.rx{f}#19, network.eth0.tx{f}#20, network.eth0.up{f}#21, network.total_bytes_in{f}#22, network.total_bytes_out{f}#23, network.total_cost{f}#24, pod{f}#25, region{f}#26, does_not_exist{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesFirstOverTimeUnmapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesFirstOverTimeUnmapped/nullify/analysis.expected index cbf895144525d..1f4759ddb2f76 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesFirstOverTimeUnmapped/nullify/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesFirstOverTimeUnmapped/nullify/analysis.expected @@ -1,3 +1,3 @@ Limit[10000[INTEGER],false,false] \_TimeSeriesAggregate[[TIMESERIESWITHOUT() AS _timeseries#0, TBUCKET(PT1H[TIME_DURATION],@timestamp{f}#1) AS tbucket(1 hour)#2],[VALUES(FIRSTOVERTIME(TODOUBLE(does_not_exist{f}#3),true[BOOLEAN],PT0S[TIME_DURATION],@timestamp{f}#1),true[BOOLEAN],PT0S[TIME_DURATION]) AS f#4, tbucket(1 hour){r}#2],null,null,@timestamp{f}#1] - \_EsRelation[k8s][TIME_SERIES][@timestamp{f}#1, client.ip{f}#5, cluster{f}#6, event{f}#7, event_city{f}#8, event_city_boundary{f}#9, event_location{f}#10, event_log{f}#11, event_shape{f}#12, events_received{f}#13, network.bytes_in{f}#14, network.cost{f}#15, network.eth0.currently_connected_clients{f}#16, network.eth0.firmware_version{f}#17, network.eth0.last_up{f}#18, network.eth0.rx{f}#19, network.eth0.tx{f}#20, network.eth0.up{f}#21, network.total_bytes_in{f}#22, network.total_bytes_out{f}#23, network.total_cost{f}#24, pod{f}#25, region{f}#26, does_not_exist{f}#3] + \_EsRelation[k8s][TIME_SERIES][@timestamp{f}#1, client.ip{f}#5, cluster{f}#6, event{f}#7, event_city{f}#8, event_city_boundary{f}#9, event_location{f}#10, event_log{f}#11, event_shape{f}#12, events_received{f}#13, network.bytes_in{f}#14, network.cost{f}#15, network.eth0.currently_connected_clients{f}#16, network.eth0.firmware_version{f}#17, network.eth0.last_up{f}#18, network.eth0.rx{f}#19, network.eth0.tx{f}#20, network.eth0.up{f}#21, network.total_bytes_in{f}#22, network.total_bytes_out{f}#23, network.total_cost{f}#24, pod{f}#25, region{f}#26, does_not_exist{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesRateUnmapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesRateUnmapped/nullify/analysis.expected index cdd3396b80e1a..512892be80bd8 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesRateUnmapped/nullify/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTimeSeriesRateUnmapped/nullify/analysis.expected @@ -1,3 +1,3 @@ Limit[10000[INTEGER],false,false] \_TimeSeriesAggregate[[TIMESERIESWITHOUT() AS _timeseries#0, TBUCKET(PT1H[TIME_DURATION],@timestamp{f}#1) AS tbucket(1 hour)#2],[VALUES(RATE(does_not_exist{f}#3,true[BOOLEAN],PT0S[TIME_DURATION],@timestamp{f}#1),true[BOOLEAN],PT0S[TIME_DURATION]) AS r#4, tbucket(1 hour){r}#2],null,null,@timestamp{f}#1] - \_EsRelation[k8s][TIME_SERIES][@timestamp{f}#1, client.ip{f}#5, cluster{f}#6, event{f}#7, event_city{f}#8, event_city_boundary{f}#9, event_location{f}#10, event_log{f}#11, event_shape{f}#12, events_received{f}#13, network.bytes_in{f}#14, network.cost{f}#15, network.eth0.currently_connected_clients{f}#16, network.eth0.firmware_version{f}#17, network.eth0.last_up{f}#18, network.eth0.rx{f}#19, network.eth0.tx{f}#20, network.eth0.up{f}#21, network.total_bytes_in{f}#22, network.total_bytes_out{f}#23, network.total_cost{f}#24, pod{f}#25, region{f}#26, does_not_exist{f}#3] + \_EsRelation[k8s][TIME_SERIES][@timestamp{f}#1, client.ip{f}#5, cluster{f}#6, event{f}#7, event_city{f}#8, event_city_boundary{f}#9, event_location{f}#10, event_log{f}#11, event_shape{f}#12, events_received{f}#13, network.bytes_in{f}#14, network.cost{f}#15, network.eth0.currently_connected_clients{f}#16, network.eth0.firmware_version{f}#17, network.eth0.last_up{f}#18, network.eth0.rx{f}#19, network.eth0.tx{f}#20, network.eth0.up{f}#21, network.total_bytes_in{f}#22, network.total_bytes_out{f}#23, network.total_cost{f}#24, pod{f}#25, region{f}#26, does_not_exist{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected index 955ed6552846c..f2d64fa28372c 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[event_duration{r}#0]] \_Eval[[$$event_duration$converted_to$long{f(MultiTypeEsField)$}#1 AS event_duration#0]] - \_EsRelation[sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#2, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f(PotentiallyUnmappedKeywordEsField)}#5, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected index c886799b5a73a..68726db5f431b 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[ts{r}#0]] \_Eval[[$$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1 AS ts#0]] - \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, !client_ip, !event_duration, message{f(PotentiallyUnmappedKeywordEsField)}#2, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file + \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#4, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file From cc4ba1b952ce304d678cc05ebee12b515232b897 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 7 Apr 2026 17:11:32 +0200 Subject: [PATCH 38/69] Put verification back in to avoid using PUNKs except for KEEP/DROP, as before. --- .../xpack/esql/analysis/Analyzer.java | 2 +- .../xpack/esql/analysis/Verifier.java | 103 +++++++++++++++++- .../esql/analysis/AnalyzerUnmappedTests.java | 101 ++++------------- 3 files changed, 126 insertions(+), 80 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 0a26bff18b645..5f359b78943a7 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -282,7 +282,7 @@ public LogicalPlan analyze(LogicalPlan plan) { } public LogicalPlan verify(LogicalPlan plan, BitSet partialMetrics) { - Collection failures = verifier.verify(plan, partialMetrics, context().unmappedResolution()); + Collection failures = verifier.verify(plan, partialMetrics, context()); if (failures.isEmpty() == false) { throw new VerificationException(failures); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 91fecb73044a4..266e8ac8feb49 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -19,15 +19,18 @@ import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -39,6 +42,9 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Fork; @@ -62,6 +68,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -102,11 +109,12 @@ public Verifier(Metrics metrics, XPackLicenseState licenseState, List verify(LogicalPlan plan, BitSet partialMetrics, UnmappedResolution unmappedResolution) { + Collection verify(LogicalPlan plan, BitSet partialMetrics, AnalyzerContext context) { assert partialMetrics != null; + UnmappedResolution unmappedResolution = context.unmappedResolution(); Failures failures = new Failures(); boolean unmappedTimestampHandled = unmappedResolution != UnmappedResolution.DEFAULT @@ -117,6 +125,14 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics, UnmappedReso ConfigurationAware.verifyNoMarkerConfiguration(plan, failures); + // Partially-unmapped non-keyword field references are checked before the bail-out so that a query with both + // an UnsupportedAttribute and a PUNK field produces both errors in one batch. + // TODO: test that shows that error message for PUNKs and using un-cast union types show up together + // TODO: mention that this is temporary, reference https://github.com/elastic/elasticsearch/issues/141995 + if (unmappedResolution == UnmappedResolution.LOAD) { + checkPartiallyUnmappedNonKeywordReferences(plan, failures, context); + } + // in case of failures bail-out as all other checks will be redundant if (failures.hasFailures()) { return failures.failures(); @@ -505,6 +521,89 @@ private static Set flattenedFieldNames(List attributes) { return names; } + /** + * Reject queries that refer to partially unmapped non-keyword fields (PUNKs) when {@code unmapped_fields="load"}. + * Any expression referencing a PUNK attribute is flagged as a verification failure unless it's within a conversion + * function or KEEP/DROP. + *

+ * Example + *

+ * With the two indices below + *

    + *
  1. index1 has foo(long) and bar(keyword)
    2. + *
  2. index2 has bar(keyword)
    3. + *
+ * + * The following queries should pass (assume unmapped_fields="load" and reading from both indices) + *
    + *
  • KEEP foo
    • + *
  • DROP foo*
    • + *
  • EVAL x = foo::long
    • + *
+ * The following queries should fail (assume unmapped_fields="load" and reading from both indices) + *
    + *
  • RENAME foo as x/li> + *
  • EVAL x = foo
    • + *
  • WHERE foo > 1
    • + *
+ */ + private static void checkPartiallyUnmappedNonKeywordReferences(LogicalPlan plan, Failures failures, AnalyzerContext context) { + final String errorMessage = "Using partially unmapped non-KEYWORD field [{}] is not supported with unmapped_fields=\"load\""; + + AttributeSet punks = partiallyUnmappedNonKeywords(plan, context.indexResolution()); + Consumer addFailureIfPunk = fa -> { + if (punks.contains(fa)) { + failures.add(fail(fa, errorMessage, fa.fieldName())); + } + }; + + plan.forEachUp(p -> { + if (p instanceof EsRelation) { + return; + } + + // Project represents KEEP/DROP/RENAME. We are consistent with UnsupportedAttribute (unsupported type/type conflict): + // - RENAME is forbidden. + // - KEEP/DROP are fine, with and without wildcards. + if (p instanceof Project project) { + for (NamedExpression projection : project.projections()) { + if (projection instanceof Attribute) { + continue; + } + projection.forEachDown(FieldAttribute.class, addFailureIfPunk); + } + return; + } + + p.forEachExpression(FieldAttribute.class, addFailureIfPunk); + }); + } + + /** + * Walks the plan's {@link EsRelation} nodes and collects partially unmapped non-keyword attributes. + */ + private static AttributeSet partiallyUnmappedNonKeywords(LogicalPlan plan, Map indexResolutions) { + AttributeSet.Builder punks = AttributeSet.builder(); + + plan.forEachUp(EsRelation.class, relation -> { + IndexResolution indexResolution = indexResolutions.get(new IndexPattern(relation.source(), relation.indexPattern())); + if (indexResolution != null && indexResolution.isValid()) { + EsIndex index = indexResolution.get(); + for (Attribute attr : relation.output()) { + if (attr instanceof FieldAttribute fa + && index.isPartiallyUnmappedField(fa.fieldName().string()) + && fa.dataType() != DataType.KEYWORD + // punk_field::long is fine; in this case, the FieldAttribute contains a MultiTypeEsField with the conversions. + && fa.field() instanceof MultiTypeEsField == false) { + punks.add(fa); + } + } + } + }); + + return punks.build(); + } + private void licenseCheck(LogicalPlan plan, Failures failures) { Consumer> licenseCheck = n -> { if (n instanceof LicenseAware la && la.licenseCheck(licenseState) == false) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 892dc2913e909..6f43ccd06978e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -736,7 +736,7 @@ public void testSameMappingHashWithUnmappedIndex() { ta.addIndex(entry.getKey().indexPattern(), entry.getValue()); } var e = expectThrows(VerificationException.class, () -> ta.statement(setUnmappedLoad("FROM foo, bar, baz | SORT message"))); - assertThat(e.getMessage(), allOf(containsString("Cannot use field [message]"), containsString("[long] in [bar, foo]"))); + assertThat(e.getMessage(), partiallyUnmappedNonKeywordError("message")); } private static final String UNMAPPED_TIMESTAMP_SUFFIX = UnresolvedTimestamp.UNRESOLVED_SUFFIX + Verifier.UNMAPPED_TIMESTAMP_SUFFIX; @@ -892,9 +892,9 @@ public void testPartiallyMappedNonKeywordFieldsMarkedAsPotentiallyUnmapped() { not(instanceOf(PotentiallyUnmappedKeywordEsField.class)) ); assertThat( - "Partially-mapped " + dataType + " field should be marked as unsupported due to type conflict", - fieldAttr.field(), - instanceOf(UnsupportedEsField.class) + "Partially-mapped " + dataType + " field should be reverted to a regular field with its original type", + fieldAttr.dataType(), + is(dataType.widenSmallNumeric()) ); } } @@ -1109,13 +1109,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeyword() { assertUnmappedLoadError( analyzer().addIndex(esIndex), "FROM idx* | WHERE partial_long > 0", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:47: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1131,14 +1125,8 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordReportsAllFields() { "FROM idx* | SORT partial_long, partial_double", allOf( containsString("Found 2 problems"), - containsString( - "line 1:46: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ), - containsString( - "line 1:60: Cannot use field [partial_double] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [double] in [idx_mapped]" - ) + partiallyUnmappedNonKeywordError("partial_long"), + partiallyUnmappedNonKeywordError("partial_double") ) ); } @@ -1193,13 +1181,7 @@ public void testDisallowLoadCommaSeparatedIndicesWhenPartialNonKeywordUsed() { assertUnmappedLoadError( analyzer().addIndex(pattern, IndexResolution.valid(merged)), "FROM idx_a, idx_b | WHERE partial_long > 0", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:55: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_a]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1220,28 +1202,12 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInRename() { ); var analyzer = analyzer().addIndex(esIndex); - assertUnmappedLoadError( - analyzer, - "FROM idx* | RENAME partial_long AS pl", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:48: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) - ); + assertUnmappedLoadError(analyzer, "FROM idx* | RENAME partial_long AS pl", partiallyUnmappedNonKeywordError("partial_long")); assertUnmappedLoadError( analyzer, "FROM idx* | RENAME common as c, partial_long AS pl", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:61: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1252,13 +1218,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInSort() { assertUnmappedLoadError( analyzer().addIndex(esIndex), "FROM idx* | SORT partial_long", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:46: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1283,13 +1243,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInChangePoint() { assertUnmappedLoadError( analyzer().addIndex(esIndex), "FROM idx* | CHANGE_POINT partial_long ON @timestamp AS type, pvalue", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:54: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1301,13 +1255,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInMvExpand() { assertUnmappedLoadError( analyzer().addIndex(esIndex), "FROM idx* | MV_EXPAND partial_long", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:51: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1317,17 +1265,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordDottedPath() { var sub = longField("sub"); var obj = new EsField("obj", DataType.OBJECT, Map.of("sub", sub), true, EsField.TimeSeriesFieldType.NONE); var esIndex = partialIndex(Map.of("obj", obj), Set.of("obj.sub")); - assertUnmappedLoadError( - analyzer().addIndex(esIndex), - "FROM idx* | SORT `obj.sub`", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:46: Cannot use field [obj.sub] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) - ); + assertUnmappedLoadError(analyzer().addIndex(esIndex), "FROM idx* | SORT `obj.sub`", partiallyUnmappedNonKeywordError("obj.sub")); } /** @@ -1464,7 +1402,12 @@ private void typeConflictVerificationFailure(String statement, Map ta.statement(statement)); - assertThat(e.getMessage(), containsString("Cannot use field [message]")); + // Single-type partially unmapped fields are caught explicitly by the Verifier; multi-type conflicts are caught by + // being marked with UnsupportedAttributes, whose error message mentions the type conflicts. + assertThat( + e.getMessage(), + Matchers.anyOf(partiallyUnmappedNonKeywordError("message"), containsString("Cannot use field [message]")) + ); } private static EsIndex partialIndex(Map mapping, Set partialFieldNames) { @@ -1573,6 +1516,10 @@ public void testUnmappedFieldsLoadWithFullTextSearchFails() { ); } + private static Matcher partiallyUnmappedNonKeywordError(String fieldName) { + return containsString("Using partially unmapped non-KEYWORD field [FieldName[string=" + fieldName + "]]"); + } + @Override protected List filteredWarnings() { return withInlinestatsWarning(withDefaultLimitWarning(super.filteredWarnings())); From 43f3b9e34b1db89917ee7c2d32d3b6f391455e23 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 7 Apr 2026 17:22:06 +0200 Subject: [PATCH 39/69] Fix golden tests --- .../elasticsearch/xpack/esql/optimizer/GoldenTestCase.java | 4 ++++ .../load/analysis.expected | 4 ++-- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java index 6ac3fdea044c8..64fa3c162e035 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java @@ -797,6 +797,10 @@ private static Map createMappingForIndex(CsvTestsDataLoader.Tes var mapping = new TreeMap<>(LoadMapping.loadMapping(dataset.streamMapping())); if (dataset.typeMapping() != null) { for (var entry : dataset.typeMapping().entrySet()) { + if (entry.getValue() == null) { + mapping.remove(entry.getKey()); + continue; + } if (mapping.containsKey(entry.getKey())) { DataType dataType = DataType.fromTypeName(entry.getValue()); EsField field = mapping.get(entry.getKey()); diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected index 78e3c85d6e32a..c8bcb0b6753f4 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] -\_Project[[!gender]] - \_EsRelation[employees_gender_text,employees_no_gender][avg_worked_seconds{f}#0, birth_date{f}#1, emp_no{f}#2, first_name{f}#3, !gender, height{f}#4, height.float{f}#5, height.half_float{f}#6, height.scaled_float{f}#7, hire_date{f}#8, is_rehired{f}#9, job_positions{f}#10, languages{f}#11, languages.byte{f}#12, languages.long{f}#13, languages.short{f}#14, last_name{f}#15, salary{f}#16, salary_change{f}#17, salary_change.int{f}#18, salary_change.keyword{f}#19, salary_change.long{f}#20, still_hired{f}#21] \ No newline at end of file +\_Project[[gender{f}#0]] + \_EsRelation[employees_gender_text,employees_no_gender][avg_worked_seconds{f}#1, birth_date{f}#2, emp_no{f}#3, first_name{f}#4, gender{f}#0, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file From c04ae5a814e5bfd4e62ba8cebb8c763c0380be5f Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 7 Apr 2026 17:37:45 +0200 Subject: [PATCH 40/69] Small cleanup --- .../xpack/esql/analysis/Analyzer.java | 30 ++++++++++++++++++- .../xpack/esql/analysis/Verifier.java | 2 +- .../esql/core/expression/Expressions.java | 4 +-- .../esql/core/expression/FieldAttribute.java | 13 +------- 4 files changed, 33 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 5f359b78943a7..3bdfa8d6dca10 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -2622,10 +2622,38 @@ public LogicalPlan apply(LogicalPlan plan) { // unsupported / unresolved fields can be explicitly retained return cleanPlan.transformUp( LogicalPlan.class, - p -> p.transformExpressionsOnly(FieldAttribute.class, FieldAttribute::checkUnresolved) + p -> p.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::cleanTypeConflicts) ); } + /** + * Return an {@link UnsupportedAttribute} so the verifier can flag illegal use of fields with type conflicts. + *

+ * If the field is mapped to a single type in some indices but unmapped in others: Instead return a regular field attribute with a + * single type so that values are loaded from the indices where it is mapped (and null is returned from unmapped indices). + * This is a temporary solution until https://github.com/elastic/elasticsearch/issues/141995 is implemented. + */ + private static Attribute cleanTypeConflicts(FieldAttribute fa) { + EsField field = fa.field(); + if (field instanceof InvalidMappedField imf && imf.isPotentiallyUnmapped() && imf.types().size() == 1) { + DataType type = imf.types().iterator().next(); + var restoredField = new EsField(imf.getName(), type, imf.getProperties(), false, imf.getTimeSeriesFieldType()); + // TODO: add test where not passing on the parent name fails the test + // TODO: add TS tests and tests with different time series field types + return new FieldAttribute( + fa.source(), + fa.parentName(), + fa.qualifier(), + fa.name(), + restoredField, + fa.nullable(), + fa.id(), + fa.synthetic() + ); + } + return fa.flagTypeConflicts(); + } + private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { List output = plan.output(); List newOutput = new ArrayList<>(output.size()); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 266e8ac8feb49..e1b7f1e15d027 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -125,10 +125,10 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics, AnalyzerCont ConfigurationAware.verifyNoMarkerConfiguration(plan, failures); + // Temporary check before we implement https://github.com/elastic/elasticsearch/issues/141995. // Partially-unmapped non-keyword field references are checked before the bail-out so that a query with both // an UnsupportedAttribute and a PUNK field produces both errors in one batch. // TODO: test that shows that error message for PUNKs and using un-cast union types show up together - // TODO: mention that this is temporary, reference https://github.com/elastic/elasticsearch/issues/141995 if (unmappedResolution == UnmappedResolution.LOAD) { checkPartiallyUnmappedNonKeywordReferences(plan, failures, context); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java index 04358c5ce332e..210c156ad1d82 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java @@ -42,7 +42,7 @@ public static List asAttributes(List named *

* Exception: a {@link FieldAttribute} backed by an {@link InvalidMappedField} (ambiguous type across indices) is instead * converted to an {@link org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute} via - * {@link FieldAttribute#checkUnresolved()}, so the analyzer can surface a clear user-facing error. + * {@link FieldAttribute#flagTypeConflicts()}, so the analyzer can surface a clear user-facing error. */ public static List toReferenceAttributesPreservingIds( List named, @@ -60,7 +60,7 @@ public static List toReferenceAttributesPreservingIds( Attribute existing = existingByName.get(exp.name()); NameId id = existing != null ? existing.id() : new NameId(); Attribute refAttr = switch (exp) { - case FieldAttribute fa when fa.field() instanceof InvalidMappedField -> fa.checkUnresolved(); + case FieldAttribute fa when fa.field() instanceof InvalidMappedField -> fa.flagTypeConflicts(); case ReferenceAttribute ra -> ra.withId(id); default -> new ReferenceAttribute(exp.source(), null, exp.name(), exp.dataType(), exp.nullable(), id, exp.synthetic()); }; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index de1c6243f3e76..cf3c8512ee2f3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -236,24 +236,13 @@ public FieldName fieldName() { return lazyFieldName; } - // TODO: better method name - // TODO: reference https://github.com/elastic/elasticsearch/issues/141995 /** * If the underlying field is an {@link InvalidMappedField} (ambiguous type across indices), * converts this attribute into an {@link UnsupportedAttribute} with a descriptive error message * so the analyzer can surface a clear user-facing error. */ - public Attribute checkUnresolved() { + public Attribute flagTypeConflicts() { if (field instanceof InvalidMappedField imf) { - if (imf.isPotentiallyUnmapped() && imf.types().size() == 1) { - // The field is mapped to a single type in some indices but unmapped in others. Revert it to a regular field so that - // values are loaded from the indices where it is mapped (and null is returned from unmapped indices). - DataType type = imf.types().iterator().next(); - var restoredField = new EsField(imf.getName(), type, imf.getProperties(), false, imf.getTimeSeriesFieldType()); - // TODO: add test where not passing on the parent name fails the test - // TODO: add TS tests and tests with different time series field types - return new FieldAttribute(source(), parentName(), qualifier(), name(), restoredField, nullable(), id(), synthetic()); - } // Field has conflicting types across indices — build a user-facing error message. String unresolvedMessage = "Cannot use field [" + name() + "] due to ambiguities being " + imf.errorMessage(); List types = imf.getTypesToIndices().keySet().stream().toList(); From 661515d724da8020a4310183ce29a940749c0378 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 8 Apr 2026 12:31:11 +0200 Subject: [PATCH 41/69] Add capability --- .../src/main/resources/unmapped-load.csv-spec | 32 ++++++------------- .../xpack/esql/action/EsqlCapabilities.java | 14 ++++++++ 2 files changed, 23 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index f536e092156df..8c37b5db15f36 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1051,7 +1051,6 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword ; employeesNoNamesLoad -required_capability: index_metadata_field required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -1072,7 +1071,6 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword employeesNoNamesLoadStats -required_capability: index_metadata_field required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -1110,7 +1108,6 @@ c:long | first_name:keyword ; employeesNoNamesLoadInlineStats -required_capability: index_metadata_field required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -1131,7 +1128,6 @@ emp_no:integer | last_name:keyword | _index:keyword | c:long | first_name:ke ; employeesNoNamesLoadInlineStatsCount -required_capability: index_metadata_field required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -1197,8 +1193,7 @@ c:long | last_name:keyword // employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while // employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. partiallyMappedTextFieldNotLoadedKeep -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM employees_gender_text, employees_no_gender METADATA _index @@ -1217,8 +1212,7 @@ emp_no:integer | gender:text | _index:keyword ; partiallyMappedTextFieldNotLoadedDrop -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM employees_gender_text, employees_no_gender METADATA _index @@ -1242,7 +1236,6 @@ emp_no:integer | gender:text | _index:keyword // message (keyword) should be loaded from _source; @timestamp (date), client_ip (ip), event_duration (long) should be // null. partiallyMappedNonKeywordFieldsNotLoaded -required_capability: index_metadata_field required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -1265,8 +1258,7 @@ null | null | null | Connected to 10. // A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded // from _source even when not explicitly referenced in a downstream expression like KEEP. fieldIsPartiallyUnmappedMultiIndexWithoutKeep -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -1294,8 +1286,7 @@ Connected to 10.1.0.1 | sample_data // EVAL on a fully mapped field (salary); the partially-mapped keyword fields (first_name, last_name) are not explicitly // referenced by any command — they remain in the output implicitly and get loaded. fieldIsPartiallyUnmappedWithEvalOnOtherMappedField -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1315,8 +1306,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | s ; fieldIsPartiallyUnmappedWithRenameOnOtherMappedField -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1338,8 +1328,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:k // EVAL on a partially mapped field (last_name). // first_name is not referenced by any command but must still be loaded. fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1359,8 +1348,7 @@ emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword ; fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM employees, employees_no_names METADATA _index @@ -1381,8 +1369,7 @@ emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index: // Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -1409,8 +1396,7 @@ Connected to 10.1.0.1 | sample_data // DROP the partially-mapped field itself — verify no errors from converting then pruning. fieldIsPartiallyUnmappedMultiIndexDropPartiallyMappedField -required_capability: index_metadata_field -required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index f06e77f9d4e33..084e9531b1cf9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -263,6 +263,20 @@ public enum Cap { */ OPTIONAL_FIELDS_V5, + /** + * Do not depend on mentioning fields in expressions to trigger loading them. + * Otherwise, {@code SET unmapped_fields="load"; FROM logs-*} will return nulls + * for partially mapped keyword fields, while {@code FROM logs-* | KEEP *} will + * trigger their loading. + *

+ * Also, always load values of partially mapped fields from the indices where + * they are mapped. + *

+ * Fixes https://github.com/elastic/elasticsearch/issues/141994 + * and https://github.com/elastic/elasticsearch/issues/145206 + */ + OPTIONAL_FIELDS_FIX_LOAD_PARTIALLY_MAPPED, + /** * Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently * supported. From 19813336a4c89a5f443e7aa6f1ca8adb8cba5e99 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 8 Apr 2026 12:31:20 +0200 Subject: [PATCH 42/69] Fix error message --- .../java/org/elasticsearch/xpack/esql/analysis/Verifier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index e1b7f1e15d027..00e9424661424 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -553,7 +553,7 @@ private static void checkPartiallyUnmappedNonKeywordReferences(LogicalPlan plan, AttributeSet punks = partiallyUnmappedNonKeywords(plan, context.indexResolution()); Consumer addFailureIfPunk = fa -> { if (punks.contains(fa)) { - failures.add(fail(fa, errorMessage, fa.fieldName())); + failures.add(fail(fa, errorMessage, fa.fieldName().string())); } }; From fd3b6d067ccc16031bceb6da29eed13e95eb6136 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 8 Apr 2026 12:31:28 +0200 Subject: [PATCH 43/69] Remove obsolete test Not going to fix INSIST tests, we'll remove them anyway. --- .../xpack/esql/analysis/AnalyzerTests.java | 22 ------------------- 1 file changed, 22 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 8c5be9189ffef..9ed225a8559e6 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3362,28 +3362,6 @@ public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordTyp assertThat(attribute.field(), is(new PotentiallyUnmappedKeywordEsField("message"))); } - public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeywordAndMissingCast_createsAnInvalidMappedField() { - assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); - - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), - fieldCapabilitiesIndexResponse("bar", Map.of()) - ), - List.of() - ); - IndexResolution resolution = mergedResolution("foo,bar", caps); - var plan = analyzer().addIndex(resolution).query("FROM foo, bar | INSIST_🐔 message"); - var limit = as(plan, Limit.class); - var insist = as(limit.child(), Insist.class); - var attribute = (UnsupportedAttribute) EsqlTestUtils.singleValue(insist.output()); - assertThat(attribute.name(), is("message")); - - String expected = "Cannot use field [message] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [foo]"; - assertThat(attribute.unresolvedMessage(), is(expected)); - } - public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyword_createsAnInvalidMappedField() { assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); From 712411633edbca9a543c5a1c52ceb8e5211adb93 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 8 Apr 2026 12:35:38 +0200 Subject: [PATCH 44/69] Fix test expectation --- .../xpack/esql/analysis/AnalyzerUnmappedTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 6f43ccd06978e..f44260e178a6d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -1517,7 +1517,7 @@ public void testUnmappedFieldsLoadWithFullTextSearchFails() { } private static Matcher partiallyUnmappedNonKeywordError(String fieldName) { - return containsString("Using partially unmapped non-KEYWORD field [FieldName[string=" + fieldName + "]]"); + return containsString("Using partially unmapped non-KEYWORD field [" + fieldName+ "]"); } @Override From 467d36bf5bb2e35524f2712d3f96783761188afb Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Wed, 8 Apr 2026 20:07:07 +0200 Subject: [PATCH 45/69] Fix yaml tests --- .../xpack/esql/analysis/Analyzer.java | 11 +- .../xpack/esql/analysis/Verifier.java | 4 +- .../esql/core/expression/FieldAttribute.java | 2 +- .../192_unmapped_load_partial_non_keyword.yml | 103 ++---------------- .../test/esql/260_flattened_subfield.yml | 10 +- 5 files changed, 28 insertions(+), 102 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 3bdfa8d6dca10..c07b776c9ed2a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -353,7 +353,7 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR attributes.addAll(metadata.stream().map(NamedExpression::toAttribute).toList()); if (context.unmappedResolution() == UnmappedResolution.LOAD) { - loadPartiallyMappedFields(attributes, esIndex); + loadPartiallyUnmappedFields(attributes, esIndex); } return new EsRelation( @@ -378,9 +378,9 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR * mechanism. * */ - private static void loadPartiallyMappedFields(List attributes, EsIndex esIndex) { + private static void loadPartiallyUnmappedFields(List attributes, EsIndex esIndex) { for (int i = 0; i < attributes.size(); i++) { - if (attributes.get(i) instanceof FieldAttribute fa && esIndex.isPartiallyUnmappedField(fa.fieldName().string())) { + if (attributes.get(i) instanceof FieldAttribute fa && isPartiallyUnmappedRegularField(fa, esIndex)) { if (fa.dataType() == KEYWORD) { attributes.set(i, ResolveRefs.insistKeyword(fa)); } else { @@ -390,6 +390,11 @@ private static void loadPartiallyMappedFields(List attributes, EsInde } } + private static boolean isPartiallyUnmappedRegularField(FieldAttribute fa, EsIndex esIndex) { + // We ignore proper subclasses of FieldAttribute; these represent unsupported or special attributes. + return fa.getClass().equals(FieldAttribute.class) && esIndex.isPartiallyUnmappedField(fa.fieldName().string()); + } + private List resolveMetadata(List metadata, AnalyzerContext context) { LinkedHashMap resolved = new LinkedHashMap<>(); Set allTags = null; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 00e9424661424..cff9866d6e535 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -128,7 +128,6 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics, AnalyzerCont // Temporary check before we implement https://github.com/elastic/elasticsearch/issues/141995. // Partially-unmapped non-keyword field references are checked before the bail-out so that a query with both // an UnsupportedAttribute and a PUNK field produces both errors in one batch. - // TODO: test that shows that error message for PUNKs and using un-cast union types show up together if (unmappedResolution == UnmappedResolution.LOAD) { checkPartiallyUnmappedNonKeywordReferences(plan, failures, context); } @@ -558,7 +557,8 @@ private static void checkPartiallyUnmappedNonKeywordReferences(LogicalPlan plan, }; plan.forEachUp(p -> { - if (p instanceof EsRelation) { + // Fork will have a PUNK in the output even if it wasn't actually used in the query, that's fine. + if (p instanceof EsRelation || p instanceof Fork) { return; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index cf3c8512ee2f3..8234ed9cd3ab6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -29,7 +29,7 @@ import java.util.Set; /** - * Attribute for an ES field. + * Attribute for an ES index field. * This class offers: * - name - the name of the attribute, but not necessarily of the field. * - The raw EsField representing the field; for parent.child.grandchild this is just grandchild. diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/192_unmapped_load_partial_non_keyword.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/192_unmapped_load_partial_non_keyword.yml index 49a80fa5d3ca5..3f86496af41d0 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/192_unmapped_load_partial_non_keyword.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/192_unmapped_load_partial_non_keyword.yml @@ -6,8 +6,8 @@ setup: - method: POST path: /_query parameters: [ ] - capabilities: [ optional_fields_v5 ] - reason: 'requires SET unmapped_fields="load"' + capabilities: [ optional_fields_fix_load_partially_mapped ] + reason: 'requires SET unmapped_fields="load" and current error message for partially unmapped non-keyword fields' # # The table below represents the field mappings for the three indices created in this test. # In `FROM index*`, all the fields except lambda are PUNKs: partially unmapped non-keyword fields. @@ -64,105 +64,19 @@ setup: d: type: boolean ---- -no-command-punks: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in DROP - query: 'SET unmapped_fields="load"; FROM index*' - - length: { values: 0 } - ---- -drop-punks: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in DROP - query: 'SET unmapped_fields="load"; FROM index2,index3 | DROP beta, a.b.c.d' - - length: { values: 0 } - ---- -drop-punks-wildcard: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in wildcard DROP - query: 'SET unmapped_fields="load"; FROM index2,index3 | DROP bet*, a.b*' - ---- -keep-punks: - - skip: - awaits_fix: "https://github.com/elastic/elasticsearch/issues/145206" - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in KEEP - query: 'SET unmapped_fields="load"; FROM index2,index3 | KEEP beta, a.b.c.d' - ---- -keep-punks-wildcard: - - skip: - awaits_fix: "https://github.com/elastic/elasticsearch/issues/145206" - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # PUNKs should not be rejected in wildcard KEEP - query: 'SET unmapped_fields="load"; FROM index1,index2,index3 | KEEP beta*' - --- rename-punks: - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" + catch: bad_request esql.query: body: # [beta] is double in index2 # PUNKs should be rejected in RENAME query: 'SET unmapped_fields="load"; FROM index1,index2,index3 | RENAME beta AS mu' - ---- -convert-punks: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in conversion functions (e.g.: punk::TYPE or TO_TYPE(punk)) - query: 'SET unmapped_fields="load"; FROM index1,index2,index3 | EVAL x = beta::long, y = a.b.c.d::boolean, m = TO_LONG(beta), n = TO_BOOLEAN(a.b.c.d) | SORT x ASC | KEEP y' - ---- -date-nanos: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [tau] is date index1 - # [tau] is date_nanos index2 - # Field is mapped differently in both indices. DATE/DATE_NANOS types shouldn't fail because we auto-cast. - query: 'SET unmapped_fields="load"; FROM index1,index2 | WHERE tau > 0::date' + - match: { status: 400 } + - match: { error.type: verification_exception } + - contains: { error.reason: 'Found 1 problem' } + - contains: { error.reason: '1:64: Using partially unmapped non-KEYWORD field [beta] is not supported with unmapped_fields="load"' } --- date-nanos-punks: @@ -228,5 +142,4 @@ stats-punks: - match: { status: 400 } - match: { error.type: verification_exception } - contains: { error.reason: 'Found 1 problem' } - - contains: { error.reason: 'line 1:87: Cannot use field [a.b.c.d] due to ambiguities being mapped as [2] incompatible types: [keyword] due to loading from _source, [boolean] in [index3]' } - + - contains: { error.reason: '1:87: Using partially unmapped non-KEYWORD field [a.b.c.d] is not supported with unmapped_fields="load"' } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml index 57c37bde5ed12..38e578ebf4bcb 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml @@ -114,8 +114,16 @@ setup: --- 'loading from two indices with different mappings 2': + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [optional_fields_fix_load_partially_mapped] + reason: "error message changed" - do: - catch: '/Cannot use field \[foo.bar\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] due to loading from _source.*\[unsupported\]/' + catch: bad_request esql.query: body: # [foo] is flattened in index1 From 846be26d47506364219fdb69b7ed8dc7d44b2ab2 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 8 Apr 2026 18:15:51 +0000 Subject: [PATCH 46/69] [CI] Auto commit changes from spotless --- .../xpack/esql/analysis/AnalyzerUnmappedTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index f44260e178a6d..77047a0c15955 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -1517,7 +1517,7 @@ public void testUnmappedFieldsLoadWithFullTextSearchFails() { } private static Matcher partiallyUnmappedNonKeywordError(String fieldName) { - return containsString("Using partially unmapped non-KEYWORD field [" + fieldName+ "]"); + return containsString("Using partially unmapped non-KEYWORD field [" + fieldName + "]"); } @Override From 2bdb1084fdc8c079d6ac310673f4b25f5965c4cb Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 9 Apr 2026 11:39:56 +0200 Subject: [PATCH 47/69] Fix capabilities in spec tests --- .../src/main/resources/unmapped-load.csv-spec | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 8c37b5db15f36..30c0a6cdf0b5d 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1193,6 +1193,7 @@ c:long | last_name:keyword // employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while // employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. partiallyMappedTextFieldNotLoadedKeep +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; @@ -1212,6 +1213,7 @@ emp_no:integer | gender:text | _index:keyword ; partiallyMappedTextFieldNotLoadedDrop +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; @@ -1258,6 +1260,7 @@ null | null | null | Connected to 10. // A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded // from _source even when not explicitly referenced in a downstream expression like KEEP. fieldIsPartiallyUnmappedMultiIndexWithoutKeep +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; @@ -1286,6 +1289,7 @@ Connected to 10.1.0.1 | sample_data // EVAL on a fully mapped field (salary); the partially-mapped keyword fields (first_name, last_name) are not explicitly // referenced by any command — they remain in the output implicitly and get loaded. fieldIsPartiallyUnmappedWithEvalOnOtherMappedField +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; @@ -1306,6 +1310,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | s ; fieldIsPartiallyUnmappedWithRenameOnOtherMappedField +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; @@ -1328,6 +1333,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:k // EVAL on a partially mapped field (last_name). // first_name is not referenced by any command but must still be loaded. fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; @@ -1348,6 +1354,7 @@ emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword ; fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; @@ -1369,6 +1376,7 @@ emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index: // Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; @@ -1396,6 +1404,7 @@ Connected to 10.1.0.1 | sample_data // DROP the partially-mapped field itself — verify no errors from converting then pruning. fieldIsPartiallyUnmappedMultiIndexDropPartiallyMappedField +required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; From b9d00c39607611d3a4db84bfe27e4026a1fb0de4 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 9 Apr 2026 11:42:55 +0200 Subject: [PATCH 48/69] Move new spec tests to unmapped-type-conflicts --- .../src/main/resources/unmapped-load.csv-spec | 391 ------------------ .../unmapped-type-conflicts.csv-spec | 390 +++++++++++++++++ 2 files changed, 390 insertions(+), 391 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 30c0a6cdf0b5d..04ab2d77b4b8c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1031,397 +1031,6 @@ FROM partial_mapping_sample_data 2024-10-23T13:51:54.732Z | Connection error? | 725449 ; -// Confirm that the employees_no_names index does not have first_name and last_name mapped. -employeesNoNamesFieldsRemoved -required_capability: index_metadata_field - -FROM employees, employees_no_names METADATA _index -| KEEP emp_no, first_name, last_name, _index -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword -10001 | Georgi | Facello | employees -10001 | null | null | employees_no_names -10002 | Bezalel | Simmel | employees -10002 | null | null | employees_no_names -10003 | Parto | Bamford | employees -10003 | null | null | employees_no_names -; - -employeesNoNamesLoad -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| KEEP emp_no, first_name, last_name, _index -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword -10001 | Georgi | Facello | employees -10001 | Georgi | Facello | employees_no_names -10002 | Bezalel | Simmel | employees -10002 | Bezalel | Simmel | employees_no_names -10003 | Parto | Bamford | employees -10003 | Parto | Bamford | employees_no_names -; - - -employeesNoNamesLoadStats -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| KEEP emp_no, first_name, last_name, _index -| SORT emp_no, _index -| LIMIT 6 -| STATS count(*) BY first_name -; -ignoreOrder:true - -count(*):long | first_name:keyword -2 | Bezalel -2 | Georgi -2 | Parto -; - -employeesNoNamesLoadStatsByFirstName -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM employees, employees_no_names -| KEEP emp_no, first_name, last_name -| STATS c=count(*) BY first_name -| SORT first_name ASC NULLS FIRST -| LIMIT 5 -; - -c:long | first_name:keyword -20 | null -2 | Alejandro -2 | Amabile -2 | Anneke -2 | Anoosh -; - -employeesNoNamesLoadInlineStats -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| KEEP emp_no, first_name, last_name, _index -| INLINE STATS c=count(*) BY first_name -| SORT first_name NULLS FIRST, emp_no, _index -| LIMIT 6 -; - -emp_no:integer | last_name:keyword | _index:keyword | c:long | first_name:keyword -10030 | Demeyer | employees | 20 | null -10030 | Demeyer | employees_no_names | 20 | null -10031 | Joslin | employees | 20 | null -10031 | Joslin | employees_no_names | 20 | null -10032 | Reistad | employees | 20 | null -10032 | Reistad | employees_no_names | 20 | null -; - -employeesNoNamesLoadInlineStatsCount -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| KEEP emp_no, first_name, last_name, _index -| INLINE STATS c=count(first_name) BY last_name -| SORT first_name NULLS FIRST, emp_no DESC, _index -| LIMIT 22 -; - -emp_no:integer | first_name:keyword | _index:keyword | c:long | last_name:keyword -10039 | null | employees | 0 | Brender -10039 | null | employees_no_names | 0 | Brender -10038 | null | employees | 2 | Lortz -10038 | null | employees_no_names | 2 | Lortz -10037 | null | employees | 0 | Makrucki -10037 | null | employees_no_names | 0 | Makrucki -10036 | null | employees | 0 | Portugali -10036 | null | employees_no_names | 0 | Portugali -10035 | null | employees | 0 | Chappelet -10035 | null | employees_no_names | 0 | Chappelet -10034 | null | employees | 0 | Swan -10034 | null | employees_no_names | 0 | Swan -10033 | null | employees | 0 | Merlo -10033 | null | employees_no_names | 0 | Merlo -10032 | null | employees | 2 | Reistad -10032 | null | employees_no_names | 2 | Reistad -10031 | null | employees | 0 | Joslin -10031 | null | employees_no_names | 0 | Joslin -10030 | null | employees | 0 | Demeyer -10030 | null | employees_no_names | 0 | Demeyer -10059 | Alejandro | employees | 2 | McAlpine -10059 | Alejandro | employees_no_names | 2 | McAlpine -; - -employeesNoNamesLoadStatsWhereStartsWith -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM employees, employees_no_names -| KEEP emp_no, first_name, last_name -| STATS c=count(first_name) WHERE STARTS_WITH(last_name, "M") BY last_name -| SORT c DESC, last_name -| LIMIT 12 -; - -c:long | last_name:keyword -2 | Malabarba -2 | Maliniak -2 | Mandell -2 | McAlpine -2 | McClurg -2 | McFarlin -2 | Meriste -2 | Mondadori -2 | Montemayor -2 | Morton -0 | Awdeh -0 | Azuma -; - -// Verify that a partially-mapped TEXT field is not automatically loaded from _source for the unmapped field index. -// employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while -// employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. -partiallyMappedTextFieldNotLoadedKeep -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM employees_gender_text, employees_no_gender METADATA _index -| KEEP emp_no, gender, _index -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | gender:text | _index:keyword -10001 | M | employees_gender_text -10001 | null | employees_no_gender -10002 | F | employees_gender_text -10002 | null | employees_no_gender -10003 | M | employees_gender_text -10003 | null | employees_no_gender -; - -partiallyMappedTextFieldNotLoadedDrop -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM employees_gender_text, employees_no_gender METADATA _index -| DROP first_name, last_name, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | gender:text | _index:keyword -10001 | M | employees_gender_text -10001 | null | employees_no_gender -10002 | F | employees_gender_text -10002 | null | employees_no_gender -10003 | M | employees_gender_text -10003 | null | employees_no_gender -; - -// Verify that partially-mapped non-keyword fields (date, ip, long) are NOT loaded from _source when not mentioned -// exlicitly, at least until we solve https://github.com/elastic/elasticsearch/issues/141995. -// All fields in sample_data are partially mapped (no_mapping_sample_data has no mapped fields). -// message (keyword) should be loaded from _source; @timestamp (date), client_ip (ip), event_duration (long) should be -// null. -partiallyMappedNonKeywordFieldsNotLoaded -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM sample_data, no_mapping_sample_data METADATA _index -| SORT _index, message DESC -| LIMIT 6 -; -ignoreOrder:true - -@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword -null | null | null | Connection error? | no_mapping_sample_data -null | null | null | Connection error? | no_mapping_sample_data -null | null | null | Connection error? | no_mapping_sample_data -null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data -null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data -null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data -; - -// https://github.com/elastic/elasticsearch/issues/141994 -// A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded -// from _source even when not explicitly referenced in a downstream expression like KEEP. -fieldIsPartiallyUnmappedMultiIndexWithoutKeep -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM sample_data, no_mapping_sample_data METADATA _index -| DROP @timestamp, client_ip, event_duration -; -ignoreOrder:true - -message:keyword | _index:keyword -Connection error? | no_mapping_sample_data -Connection error? | no_mapping_sample_data -Connection error? | no_mapping_sample_data -Connected to 10.1.0.3! | no_mapping_sample_data -Connected to 10.1.0.2! | no_mapping_sample_data -Connected to 10.1.0.1! | no_mapping_sample_data -42 | no_mapping_sample_data -Disconnected | sample_data -Connection error | sample_data -Connection error | sample_data -Connection error | sample_data -Connected to 10.1.0.3 | sample_data -Connected to 10.1.0.2 | sample_data -Connected to 10.1.0.1 | sample_data -; - -// EVAL on a fully mapped field (salary); the partially-mapped keyword fields (first_name, last_name) are not explicitly -// referenced by any command — they remain in the output implicitly and get loaded. -fieldIsPartiallyUnmappedWithEvalOnOtherMappedField -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| EVAL salary_k = salary / 1000 -| DROP gender, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | salary_k:integer -10001 | Georgi | Facello | employees | 57 -10001 | Georgi | Facello | employees_no_names | 57 -10002 | Bezalel | Simmel | employees | 56 -10002 | Bezalel | Simmel | employees_no_names | 56 -10003 | Parto | Bamford | employees | 61 -10003 | Parto | Bamford | employees_no_names | 61 -; - -fieldIsPartiallyUnmappedWithRenameOnOtherMappedField -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| RENAME salary AS pay -| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:keyword -10001 | Georgi | Facello | 57305 | employees -10001 | Georgi | Facello | 57305 | employees_no_names -10002 | Bezalel | Simmel | 56371 | employees -10002 | Bezalel | Simmel | 56371 | employees_no_names -10003 | Parto | Bamford | 61805 | employees -10003 | Parto | Bamford | 61805 | employees_no_names -; - -// EVAL on a partially mapped field (last_name). -// first_name is not referenced by any command but must still be loaded. -fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| EVAL last_name = TO_UPPER(last_name) -| DROP gender, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword -10001 | Georgi | employees | FACELLO -10001 | Georgi | employees_no_names | FACELLO -10002 | Bezalel | employees | SIMMEL -10002 | Bezalel | employees_no_names | SIMMEL -10003 | Parto | employees | BAMFORD -10003 | Parto | employees_no_names | BAMFORD -; - -fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| RENAME last_name AS surname -| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index:keyword -10001 | Georgi | Facello | 57305 | employees -10001 | Georgi | Facello | 57305 | employees_no_names -10002 | Bezalel | Simmel | 56371 | employees -10002 | Bezalel | Simmel | 56371 | employees_no_names -10003 | Parto | Bamford | 61805 | employees -10003 | Parto | Bamford | 61805 | employees_no_names -; - -// Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. -fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM sample_data, no_mapping_sample_data METADATA _index -| DROP *_ip, *_duration, @timestamp -; -ignoreOrder:true - -message:keyword | _index:keyword -Connection error? | no_mapping_sample_data -Connection error? | no_mapping_sample_data -Connection error? | no_mapping_sample_data -Connected to 10.1.0.3! | no_mapping_sample_data -Connected to 10.1.0.2! | no_mapping_sample_data -Connected to 10.1.0.1! | no_mapping_sample_data -42 | no_mapping_sample_data -Disconnected | sample_data -Connection error | sample_data -Connection error | sample_data -Connection error | sample_data -Connected to 10.1.0.3 | sample_data -Connected to 10.1.0.2 | sample_data -Connected to 10.1.0.1 | sample_data -; - -// DROP the partially-mapped field itself — verify no errors from converting then pruning. -fieldIsPartiallyUnmappedMultiIndexDropPartiallyMappedField -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM sample_data, no_mapping_sample_data METADATA _index -| DROP @timestamp, client_ip, event_duration, message -| SORT _index -| LIMIT 5 -; - -_index:keyword -no_mapping_sample_data -no_mapping_sample_data -no_mapping_sample_data -no_mapping_sample_data -no_mapping_sample_data -; - maxOverTimeOfKeyword required_capability: optional_fields_v5 required_capability: ts_command_v0 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 0ef79c8ffbb41..20a1750e84c00 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -165,6 +165,396 @@ sample_data | 5033755.0 sample_data | 8268153.0 ; +// Confirm that the employees_no_names index does not have first_name and last_name mapped. +employeesNoNamesFieldsRemoved +required_capability: index_metadata_field + +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword +10001 | Georgi | Facello | employees +10001 | null | null | employees_no_names +10002 | Bezalel | Simmel | employees +10002 | null | null | employees_no_names +10003 | Parto | Bamford | employees +10003 | null | null | employees_no_names +; + +employeesNoNamesLoad +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword +10001 | Georgi | Facello | employees +10001 | Georgi | Facello | employees_no_names +10002 | Bezalel | Simmel | employees +10002 | Bezalel | Simmel | employees_no_names +10003 | Parto | Bamford | employees +10003 | Parto | Bamford | employees_no_names +; + + +employeesNoNamesLoadStats +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +| STATS count(*) BY first_name +; +ignoreOrder:true + +count(*):long | first_name:keyword +2 | Bezalel +2 | Georgi +2 | Parto +; + +employeesNoNamesLoadStatsByFirstName +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names +| KEEP emp_no, first_name, last_name +| STATS c=count(*) BY first_name +| SORT first_name ASC NULLS FIRST +| LIMIT 5 +; + +c:long | first_name:keyword +20 | null +2 | Alejandro +2 | Amabile +2 | Anneke +2 | Anoosh +; + +employeesNoNamesLoadInlineStats +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| INLINE STATS c=count(*) BY first_name +| SORT first_name NULLS FIRST, emp_no, _index +| LIMIT 6 +; + +emp_no:integer | last_name:keyword | _index:keyword | c:long | first_name:keyword +10030 | Demeyer | employees | 20 | null +10030 | Demeyer | employees_no_names | 20 | null +10031 | Joslin | employees | 20 | null +10031 | Joslin | employees_no_names | 20 | null +10032 | Reistad | employees | 20 | null +10032 | Reistad | employees_no_names | 20 | null +; + +employeesNoNamesLoadInlineStatsCount +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| INLINE STATS c=count(first_name) BY last_name +| SORT first_name NULLS FIRST, emp_no DESC, _index +| LIMIT 22 +; + +emp_no:integer | first_name:keyword | _index:keyword | c:long | last_name:keyword +10039 | null | employees | 0 | Brender +10039 | null | employees_no_names | 0 | Brender +10038 | null | employees | 2 | Lortz +10038 | null | employees_no_names | 2 | Lortz +10037 | null | employees | 0 | Makrucki +10037 | null | employees_no_names | 0 | Makrucki +10036 | null | employees | 0 | Portugali +10036 | null | employees_no_names | 0 | Portugali +10035 | null | employees | 0 | Chappelet +10035 | null | employees_no_names | 0 | Chappelet +10034 | null | employees | 0 | Swan +10034 | null | employees_no_names | 0 | Swan +10033 | null | employees | 0 | Merlo +10033 | null | employees_no_names | 0 | Merlo +10032 | null | employees | 2 | Reistad +10032 | null | employees_no_names | 2 | Reistad +10031 | null | employees | 0 | Joslin +10031 | null | employees_no_names | 0 | Joslin +10030 | null | employees | 0 | Demeyer +10030 | null | employees_no_names | 0 | Demeyer +10059 | Alejandro | employees | 2 | McAlpine +10059 | Alejandro | employees_no_names | 2 | McAlpine +; + +employeesNoNamesLoadStatsWhereStartsWith +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names +| KEEP emp_no, first_name, last_name +| STATS c=count(first_name) WHERE STARTS_WITH(last_name, "M") BY last_name +| SORT c DESC, last_name +| LIMIT 12 +; + +c:long | last_name:keyword +2 | Malabarba +2 | Maliniak +2 | Mandell +2 | McAlpine +2 | McClurg +2 | McFarlin +2 | Meriste +2 | Mondadori +2 | Montemayor +2 | Morton +0 | Awdeh +0 | Azuma +; + +// Verify that a partially-mapped TEXT field is not automatically loaded from _source for the unmapped field index. +// employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while +// employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. +partiallyMappedTextFieldNotLoadedKeep +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees_gender_text, employees_no_gender METADATA _index +| KEEP emp_no, gender, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | gender:text | _index:keyword +10001 | M | employees_gender_text +10001 | null | employees_no_gender +10002 | F | employees_gender_text +10002 | null | employees_no_gender +10003 | M | employees_gender_text +10003 | null | employees_no_gender +; + +partiallyMappedTextFieldNotLoadedDrop +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees_gender_text, employees_no_gender METADATA _index +| DROP first_name, last_name, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | gender:text | _index:keyword +10001 | M | employees_gender_text +10001 | null | employees_no_gender +10002 | F | employees_gender_text +10002 | null | employees_no_gender +10003 | M | employees_gender_text +10003 | null | employees_no_gender +; + +// Verify that partially-mapped non-keyword fields (date, ip, long) are NOT loaded from _source when not mentioned +// exlicitly, at least until we solve https://github.com/elastic/elasticsearch/issues/141995. +// All fields in sample_data are partially mapped (no_mapping_sample_data has no mapped fields). +// message (keyword) should be loaded from _source; @timestamp (date), client_ip (ip), event_duration (long) should be +// null. +partiallyMappedNonKeywordFieldsNotLoaded +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| SORT _index, message DESC +| LIMIT 6 +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +; + +// https://github.com/elastic/elasticsearch/issues/141994 +// A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded +// from _source even when not explicitly referenced in a downstream expression like KEEP. +fieldIsPartiallyUnmappedMultiIndexWithoutKeep +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| DROP @timestamp, client_ip, event_duration +; +ignoreOrder:true + +message:keyword | _index:keyword +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connected to 10.1.0.3! | no_mapping_sample_data +Connected to 10.1.0.2! | no_mapping_sample_data +Connected to 10.1.0.1! | no_mapping_sample_data +42 | no_mapping_sample_data +Disconnected | sample_data +Connection error | sample_data +Connection error | sample_data +Connection error | sample_data +Connected to 10.1.0.3 | sample_data +Connected to 10.1.0.2 | sample_data +Connected to 10.1.0.1 | sample_data +; + +// EVAL on a fully mapped field (salary); the partially-mapped keyword fields (first_name, last_name) are not explicitly +// referenced by any command — they remain in the output implicitly and get loaded. +fieldIsPartiallyUnmappedWithEvalOnOtherMappedField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| EVAL salary_k = salary / 1000 +| DROP gender, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | salary_k:integer +10001 | Georgi | Facello | employees | 57 +10001 | Georgi | Facello | employees_no_names | 57 +10002 | Bezalel | Simmel | employees | 56 +10002 | Bezalel | Simmel | employees_no_names | 56 +10003 | Parto | Bamford | employees | 61 +10003 | Parto | Bamford | employees_no_names | 61 +; + +fieldIsPartiallyUnmappedWithRenameOnOtherMappedField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| RENAME salary AS pay +| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:keyword +10001 | Georgi | Facello | 57305 | employees +10001 | Georgi | Facello | 57305 | employees_no_names +10002 | Bezalel | Simmel | 56371 | employees +10002 | Bezalel | Simmel | 56371 | employees_no_names +10003 | Parto | Bamford | 61805 | employees +10003 | Parto | Bamford | 61805 | employees_no_names +; + +// EVAL on a partially mapped field (last_name). +// first_name is not referenced by any command but must still be loaded. +fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| EVAL last_name = TO_UPPER(last_name) +| DROP gender, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword +10001 | Georgi | employees | FACELLO +10001 | Georgi | employees_no_names | FACELLO +10002 | Bezalel | employees | SIMMEL +10002 | Bezalel | employees_no_names | SIMMEL +10003 | Parto | employees | BAMFORD +10003 | Parto | employees_no_names | BAMFORD +; + +fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| RENAME last_name AS surname +| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index:keyword +10001 | Georgi | Facello | 57305 | employees +10001 | Georgi | Facello | 57305 | employees_no_names +10002 | Bezalel | Simmel | 56371 | employees +10002 | Bezalel | Simmel | 56371 | employees_no_names +10003 | Parto | Bamford | 61805 | employees +10003 | Parto | Bamford | 61805 | employees_no_names +; + +// Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. +fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| DROP *_ip, *_duration, @timestamp +; +ignoreOrder:true + +message:keyword | _index:keyword +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connected to 10.1.0.3! | no_mapping_sample_data +Connected to 10.1.0.2! | no_mapping_sample_data +Connected to 10.1.0.1! | no_mapping_sample_data +42 | no_mapping_sample_data +Disconnected | sample_data +Connection error | sample_data +Connection error | sample_data +Connection error | sample_data +Connected to 10.1.0.3 | sample_data +Connected to 10.1.0.2 | sample_data +Connected to 10.1.0.1 | sample_data +; + +// DROP the partially-mapped field itself — verify no errors from converting then pruning. +fieldIsPartiallyUnmappedMultiIndexDropPartiallyMappedField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| DROP @timestamp, client_ip, event_duration, message +| SORT _index +| LIMIT 5 +; + +_index:keyword +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +; ########################### # Mapped and non-existent # From 8178c7fc8937f494d07b05e1d9f67194432d030c Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 9 Apr 2026 19:21:47 +0200 Subject: [PATCH 49/69] Align test names --- .../unmapped-type-conflicts.csv-spec | 35 ++++++++++--------- 1 file changed, 18 insertions(+), 17 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 20a1750e84c00..4460f1d4e8fd0 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -184,7 +184,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword 10003 | null | null | employees_no_names ; -employeesNoNamesLoad +noTypeConflictKeywordKeepTriggersLoad required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -204,7 +204,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword ; -employeesNoNamesLoadStats +noTypeConflictKeywordKeepTriggersLoadStats required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -222,7 +222,7 @@ count(*):long | first_name:keyword 2 | Parto ; -employeesNoNamesLoadStatsByFirstName +noTypeConflictKeywordKeepTriggersLoadStatsWithNulls required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -241,7 +241,7 @@ c:long | first_name:keyword 2 | Anoosh ; -employeesNoNamesLoadInlineStats +noTypeConflictKeywordKeepTriggersLoadInlineStats required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -261,7 +261,7 @@ emp_no:integer | last_name:keyword | _index:keyword | c:long | first_name:ke 10032 | Reistad | employees_no_names | 20 | null ; -employeesNoNamesLoadInlineStatsCount +noTypeConflictKeywordKeepTriggersLoadInlineStatsWithNulls required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -297,7 +297,7 @@ emp_no:integer | first_name:keyword | _index:keyword | c:long | last_name:ke 10059 | Alejandro | employees_no_names | 2 | McAlpine ; -employeesNoNamesLoadStatsWhereStartsWith +noTypeConflictKeepTriggersLoadInlineStatsWithFilter required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -326,7 +326,7 @@ c:long | last_name:keyword // Verify that a partially-mapped TEXT field is not automatically loaded from _source for the unmapped field index. // employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while // employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. -partiallyMappedTextFieldNotLoadedKeep +typeConflictPartiallyUnmappedTextFieldNotLoaded required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped @@ -346,7 +346,8 @@ emp_no:integer | gender:text | _index:keyword 10003 | null | employees_no_gender ; -partiallyMappedTextFieldNotLoadedDrop + +typeConflictPartiallyUnmappedTextFieldNotLoadedWithDrop required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped @@ -367,11 +368,11 @@ emp_no:integer | gender:text | _index:keyword ; // Verify that partially-mapped non-keyword fields (date, ip, long) are NOT loaded from _source when not mentioned -// exlicitly, at least until we solve https://github.com/elastic/elasticsearch/issues/141995. +// explicitly, at least until we solve https://github.com/elastic/elasticsearch/issues/141995. // All fields in sample_data are partially mapped (no_mapping_sample_data has no mapped fields). // message (keyword) should be loaded from _source; @timestamp (date), client_ip (ip), event_duration (long) should be // null. -partiallyMappedNonKeywordFieldsNotLoaded +typeConflictPartiallyUnmappedNonKeywordFieldsNotLoadedWithoutExplicitMention required_capability: optional_fields_v5 SET unmapped_fields="load"\; @@ -393,7 +394,7 @@ null | null | null | Connected to 10. // https://github.com/elastic/elasticsearch/issues/141994 // A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded // from _source even when not explicitly referenced in a downstream expression like KEEP. -fieldIsPartiallyUnmappedMultiIndexWithoutKeep +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithoutExplicitMention required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped @@ -422,7 +423,7 @@ Connected to 10.1.0.1 | sample_data // EVAL on a fully mapped field (salary); the partially-mapped keyword fields (first_name, last_name) are not explicitly // referenced by any command — they remain in the output implicitly and get loaded. -fieldIsPartiallyUnmappedWithEvalOnOtherMappedField +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithEvalOnOtherField required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped @@ -443,7 +444,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | s 10003 | Parto | Bamford | employees_no_names | 61 ; -fieldIsPartiallyUnmappedWithRenameOnOtherMappedField +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithRenameOnOtherField required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped @@ -466,7 +467,7 @@ emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:k // EVAL on a partially mapped field (last_name). // first_name is not referenced by any command but must still be loaded. -fieldIsPartiallyUnmappedWithEvalOnOtherPartiallyUnmappedField +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithEvalOnSameField required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped @@ -487,7 +488,7 @@ emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword 10003 | Parto | employees_no_names | BAMFORD ; -fieldIsPartiallyUnmappedWithRenameOnOtherPartiallyUnmappedField +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithRenameOnOtherField required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped @@ -509,7 +510,7 @@ emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index: ; // Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. -fieldIsPartiallyUnmappedMultiIndexWithWildcardDrop +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithWildcardDropOnOtherFields required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped @@ -537,7 +538,7 @@ Connected to 10.1.0.1 | sample_data ; // DROP the partially-mapped field itself — verify no errors from converting then pruning. -fieldIsPartiallyUnmappedMultiIndexDropPartiallyMappedField +typeConflictPartiallyUnmappedKeywordFieldDropped required_capability: optional_fields_v5 required_capability: optional_fields_fix_load_partially_mapped From cde54354c0e7cebc53f458981a547f7f6d9f2327 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Thu, 9 Apr 2026 19:31:57 +0200 Subject: [PATCH 50/69] Add reproducer for #145206 --- .../unmapped-type-conflicts.csv-spec | 70 +++++++++++++------ 1 file changed, 49 insertions(+), 21 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 4460f1d4e8fd0..3a1b12565cc82 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -465,6 +465,27 @@ emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:k 10003 | Parto | Bamford | 61805 | employees_no_names ; +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithRenameOnSameField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| RENAME last_name AS surname +| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index:keyword +10001 | Georgi | Facello | 57305 | employees +10001 | Georgi | Facello | 57305 | employees_no_names +10002 | Bezalel | Simmel | 56371 | employees +10002 | Bezalel | Simmel | 56371 | employees_no_names +10003 | Parto | Bamford | 61805 | employees +10003 | Parto | Bamford | 61805 | employees_no_names +; + // EVAL on a partially mapped field (last_name). // first_name is not referenced by any command but must still be loaded. noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithEvalOnSameField @@ -488,27 +509,6 @@ emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword 10003 | Parto | employees_no_names | BAMFORD ; -noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithRenameOnOtherField -required_capability: optional_fields_v5 -required_capability: optional_fields_fix_load_partially_mapped - -SET unmapped_fields="load"\; -FROM employees, employees_no_names METADATA _index -| RENAME last_name AS surname -| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* -| SORT emp_no, _index -| LIMIT 6 -; - -emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index:keyword -10001 | Georgi | Facello | 57305 | employees -10001 | Georgi | Facello | 57305 | employees_no_names -10002 | Bezalel | Simmel | 56371 | employees -10002 | Bezalel | Simmel | 56371 | employees_no_names -10003 | Parto | Bamford | 61805 | employees -10003 | Parto | Bamford | 61805 | employees_no_names -; - // Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithWildcardDropOnOtherFields required_capability: optional_fields_v5 @@ -557,6 +557,34 @@ no_mapping_sample_data no_mapping_sample_data ; +// reproducer for https://github.com/elastic/elasticsearch/issues/145206 +typeConflictPartiallyUnmappedKeywordFieldKept +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| KEEP event_duration +; +ignoreOrder:true + +event_duration:long +1756467 +5033755 +8268153 +725448 +1232382 +2764889 +3450233 +null +null +null +null +null +null +null +; + ########################### # Mapped and non-existent # ########################### From 12fc2334b3f3798baee4855de2357301f2e8ad85 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 17:31:40 +0200 Subject: [PATCH 51/69] Add tests with union types --- .../unmapped-type-conflicts.csv-spec | 142 ++++++++++++++++++ 1 file changed, 142 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 3a1b12565cc82..2b65a3869bdd5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -646,6 +646,34 @@ colors | null colors | null ; +typeConflictLongNonExistentNotCast +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data, colors METADATA _index +| KEEP _index, event_duration +| SORT _index DESC, event_duration +| LIMIT 15 +; + +_index:keyword | event_duration:long +sample_data | 725448 +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 8268153 +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + typeConflictLongNonExistentCastToKeyword required_capability: optional_fields_v5 @@ -830,6 +858,39 @@ sample_data_ts_long | 1698069235832 sample_data_ts_long | 1698069301543 ; +typeConflictLongDateUnmappedNotCast +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index +| KEEP _index, @timestamp +| SORT _index +; + +_index:keyword | @timestamp:unsupported +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +; + typeConflictLongDateUnmappedCastToDouble required_capability: optional_fields_v5 @@ -944,6 +1005,44 @@ sample_data_ts_long | 2023-10-23T13:55:01.543Z # Mapped x 2 and non-existent # ############################### +typeConflictLongDateNonExistentNotCast +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, colors METADATA _index +| KEEP _index, @timestamp +| SORT _index DESC +| LIMIT 25 +; + +_index:keyword | @timestamp:unsupported +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + typeConflictLongDateNonExistentCastToKeyword required_capability: optional_fields_v5 @@ -1065,6 +1164,49 @@ colors | null # Mapped x 2, unmapped, and non-existent # ########################################## +typeConflictLongDateUnmappedNonExistentNotCast +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data, colors METADATA _index +| KEEP _index, @timestamp +| SORT _index DESC +| LIMIT 30 +; + +_index:keyword | @timestamp:unsupported +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + typeConflictLongDateUnmappedNonExistentCastToKeyword required_capability: optional_fields_v5 From d7e680e6ade7a1052fe89ad7c91b070844a6955e Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 17:45:18 +0200 Subject: [PATCH 52/69] Add date/nanos tests --- .../unmapped-type-conflicts.csv-spec | 503 ++++++++++++++++-- 1 file changed, 468 insertions(+), 35 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 2b65a3869bdd5..8f7278c32f70b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -652,9 +652,10 @@ required_capability: optional_fields_v5 SET unmapped_fields="load"\; FROM sample_data, colors METADATA _index | KEEP _index, event_duration -| SORT _index DESC, event_duration +| SORT _index DESC | LIMIT 15 ; +ignoreOrder:true _index:keyword | event_duration:long sample_data | 725448 @@ -786,40 +787,6 @@ heights | null heights | null ; -typeConflictDateNanosDateUnmappedCastToDate -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM sample_data, sample_data_ts_nanos, no_mapping_sample_data METADATA _index -| EVAL ts = @timestamp::date -| KEEP _index, ts -| SORT _index, ts -; - -_index:keyword | ts:datetime -no_mapping_sample_data | 2024-10-23T12:15:03.360Z -no_mapping_sample_data | 2024-10-23T12:27:28.948Z -no_mapping_sample_data | 2024-10-23T13:33:34.937Z -no_mapping_sample_data | 2024-10-23T13:51:54.732Z -no_mapping_sample_data | 2024-10-23T13:52:55.015Z -no_mapping_sample_data | 2024-10-23T13:53:55.832Z -no_mapping_sample_data | 2024-10-23T13:55:01.543Z -sample_data | 2023-10-23T12:15:03.360Z -sample_data | 2023-10-23T12:27:28.948Z -sample_data | 2023-10-23T13:33:34.937Z -sample_data | 2023-10-23T13:51:54.732Z -sample_data | 2023-10-23T13:52:55.015Z -sample_data | 2023-10-23T13:53:55.832Z -sample_data | 2023-10-23T13:55:01.543Z -sample_data_ts_nanos | 2023-10-23T12:15:03.360Z -sample_data_ts_nanos | 2023-10-23T12:27:28.948Z -sample_data_ts_nanos | 2023-10-23T13:33:34.937Z -sample_data_ts_nanos | 2023-10-23T13:51:54.732Z -sample_data_ts_nanos | 2023-10-23T13:52:55.015Z -sample_data_ts_nanos | 2023-10-23T13:53:55.832Z -sample_data_ts_nanos | 2023-10-23T13:55:01.543Z -; - ########################### # Mapped x 2 and unmapped # ########################### @@ -1001,6 +968,142 @@ sample_data_ts_long | 2023-10-23T13:53:55.832Z sample_data_ts_long | 2023-10-23T13:55:01.543Z ; +typeConflictDateNanosDateUnmappedNotCast +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data METADATA _index +| KEEP _index, @timestamp +| SORT _index +; + +_index:keyword | @timestamp:unsupported +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +; + +typeConflictDateNanosDateUnmappedCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:keyword +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +; + + +typeConflictDateNanosDateUnmappedCastToDate +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data, sample_data_ts_nanos, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:datetime +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_nanos | 2023-10-23T12:15:03.360Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543Z +; + +typeConflictDateNanosDateUnmappedCastToDateNanos +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::date_nanos +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:date_nanos +no_mapping_sample_data | 2024-10-23T12:15:03.360000000Z +no_mapping_sample_data | 2024-10-23T12:27:28.948000000Z +no_mapping_sample_data | 2024-10-23T13:33:34.937000000Z +no_mapping_sample_data | 2024-10-23T13:51:54.732000000Z +no_mapping_sample_data | 2024-10-23T13:52:55.015000000Z +no_mapping_sample_data | 2024-10-23T13:53:55.832000000Z +no_mapping_sample_data | 2024-10-23T13:55:01.543000000Z +sample_data | 2023-10-23T12:15:03.360000000Z +sample_data | 2023-10-23T12:27:28.948000000Z +sample_data | 2023-10-23T13:33:34.937000000Z +sample_data | 2023-10-23T13:51:54.732000000Z +sample_data | 2023-10-23T13:52:55.015000000Z +sample_data | 2023-10-23T13:53:55.832000000Z +sample_data | 2023-10-23T13:55:01.543000000Z +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +; + ############################### # Mapped x 2 and non-existent # ############################### @@ -1160,6 +1263,161 @@ colors | null colors | null ; +typeConflictDateNanosDateNonExistentNotCast +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, colors METADATA _index +| KEEP _index, @timestamp +| SORT _index DESC +| LIMIT 25 +; + +_index:keyword | @timestamp:unsupported +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateNonExistentCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, colors METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:keyword +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateNonExistentCastToDate +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, colors METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:datetime +sample_data_ts_nanos | 2023-10-23T12:15:03.360Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateNonExistentCastToDateNanos +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, colors METADATA _index +| EVAL ts = @timestamp::date_nanos +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:date_nanos +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +sample_data | 2023-10-23T12:15:03.360000000Z +sample_data | 2023-10-23T12:27:28.948000000Z +sample_data | 2023-10-23T13:33:34.937000000Z +sample_data | 2023-10-23T13:51:54.732000000Z +sample_data | 2023-10-23T13:52:55.015000000Z +sample_data | 2023-10-23T13:53:55.832000000Z +sample_data | 2023-10-23T13:55:01.543000000Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + ########################################## # Mapped x 2, unmapped, and non-existent # ########################################## @@ -1386,6 +1644,181 @@ sample_data_ts_long | 2023-10-23T13:53:55.832Z sample_data_ts_long | 2023-10-23T13:55:01.543Z ; +typeConflictDateNanosDateUnmappedNonExistentNotCast +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index +| KEEP _index, @timestamp +| SORT _index DESC +| LIMIT 30 +; + +_index:keyword | @timestamp:unsupported +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateUnmappedNonExistentCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +_index:keyword | ts:keyword +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateUnmappedNonExistentCastToDate +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +_index:keyword | ts:datetime +sample_data_ts_nanos | 2023-10-23T12:15:03.360Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateUnmappedNonExistentCastToDateNanos +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::date_nanos +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +_index:keyword | ts:date_nanos +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +sample_data | 2023-10-23T12:15:03.360000000Z +sample_data | 2023-10-23T12:27:28.948000000Z +sample_data | 2023-10-23T13:33:34.937000000Z +sample_data | 2023-10-23T13:51:54.732000000Z +sample_data | 2023-10-23T13:52:55.015000000Z +sample_data | 2023-10-23T13:53:55.832000000Z +sample_data | 2023-10-23T13:55:01.543000000Z +no_mapping_sample_data | 2024-10-23T12:15:03.360000000Z +no_mapping_sample_data | 2024-10-23T12:27:28.948000000Z +no_mapping_sample_data | 2024-10-23T13:33:34.937000000Z +no_mapping_sample_data | 2024-10-23T13:51:54.732000000Z +no_mapping_sample_data | 2024-10-23T13:52:55.015000000Z +no_mapping_sample_data | 2024-10-23T13:53:55.832000000Z +no_mapping_sample_data | 2024-10-23T13:55:01.543000000Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + ############################## # WHERE with type casts # ############################## From b4ac2d34d623e9cf551e7af849d9803d47e1f9d2 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 17:58:22 +0200 Subject: [PATCH 53/69] Add tests without KEEP/DROP --- .../unmapped-type-conflicts.csv-spec | 26 +++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 8f7278c32f70b..1995bb40c7849 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -4,6 +4,32 @@ # Mapped and unmapped conflict tests # ###################################### +typeConflictWithUnmappedJustFrom +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | sample_data +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | sample_data +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | sample_data +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | sample_data +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | sample_data +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | sample_data +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | 42 | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +; + noTypeConflictKeywordUnmappedCastToKeyword required_capability: optional_fields_v5 From c3b4460c47697e0970d2f180c0178c61c0c62afa Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 18:00:31 +0200 Subject: [PATCH 54/69] Fix capability --- .../src/main/resources/unmapped-type-conflicts.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 1995bb40c7849..9a435336811f7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -674,6 +674,7 @@ colors | null typeConflictLongNonExistentNotCast required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data, colors METADATA _index From b39181b09efe4c34ca57c45a21b9f5f65b944b40 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 18:12:25 +0200 Subject: [PATCH 55/69] Add more golden tests --- .../analysis/AnalyzerUnmappedGoldenTests.java | 16 ++++++++++++++++ .../load/analysis.expected | 3 +++ .../load/query.esql | 2 ++ .../nullify/analysis.expected | 3 +++ .../nullify/query.esql | 2 ++ .../load/analysis.expected | 3 +++ .../load/query.esql | 2 ++ .../nullify/analysis.expected | 3 +++ .../nullify/query.esql | 2 ++ 9 files changed, 36 insertions(+) create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/query.esql create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/analysis.expected create mode 100644 x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/query.esql diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java index c8450064797cf..90d8259dc3764 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java @@ -731,6 +731,22 @@ public void testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped() throws Exc """); } + // DROP a single partially-mapped keyword field (message), leaving only non-keyword fields. + public void testPartiallyMappedFieldsDropOnePartiallyMapped() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | DROP message + """); + } + + // DROP a single partially-mapped non-keyword field (event_duration), leaving message and the other non-keyword fields. + public void testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | DROP event_duration + """); + } + // DROP with wildcards on partially-mapped non-keyword fields, leaving only the keyword field (message). public void testPartiallyMappedFieldsDropNonKeywordWithWildcards() throws Exception { runTests(""" diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/analysis.expected new file mode 100644 index 0000000000000..321582baaaa1e --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/query.esql new file mode 100644 index 0000000000000..0913b709d4160 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| DROP message diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/analysis.expected new file mode 100644 index 0000000000000..e40d25c344506 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/query.esql new file mode 100644 index 0000000000000..4226ac74785c0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| DROP message diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/analysis.expected new file mode 100644 index 0000000000000..4a31070ea7d9a --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, message{f(PotentiallyUnmappedKeywordEsField)}#2]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#2] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/query.esql new file mode 100644 index 0000000000000..83ae295113395 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| DROP event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/analysis.expected new file mode 100644 index 0000000000000..ff1b5a9e6460b --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, message{f}#2]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#3, message{f}#2] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/query.esql new file mode 100644 index 0000000000000..ac026bac0ad2a --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| DROP event_duration From 89f31af2ad6a6e2aa43883417e1c8d85e1dd1255 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 18:17:09 +0200 Subject: [PATCH 56/69] More tests with KEEP + wildcard --- .../unmapped-type-conflicts.csv-spec | 56 +++++++++++++++++++ 1 file changed, 56 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 9a435336811f7..6463a4a20c90e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -30,6 +30,62 @@ null | null | null | Connected to null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data ; + +typeConflictWithUnmappedJustKeepWildcard +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| KEEP * +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | sample_data +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | sample_data +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | sample_data +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | sample_data +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | sample_data +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | sample_data +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | 42 | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +; + + +typeConflictWithUnmappedJustKeepWildcardPrefix +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| KEEP @t*, cli*, ev*, me*, _in* +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | sample_data +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | sample_data +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | sample_data +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | sample_data +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | sample_data +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | sample_data +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | 42 | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +; + noTypeConflictKeywordUnmappedCastToKeyword required_capability: optional_fields_v5 From 26dadac118c9908db59ba77807fd83b52826fe17 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 18:31:16 +0200 Subject: [PATCH 57/69] Another TS test --- .../src/main/resources/unmapped-load.csv-spec | 23 ------- .../unmapped-type-conflicts.csv-spec | 65 +++++++++++++++++++ 2 files changed, 65 insertions(+), 23 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 04ab2d77b4b8c..b5742313a7f40 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1031,29 +1031,6 @@ FROM partial_mapping_sample_data 2024-10-23T13:51:54.732Z | Connection error? | 725449 ; -maxOverTimeOfKeyword -required_capability: optional_fields_v5 -required_capability: ts_command_v0 - -SET unmapped_fields="load"\; -TS k8s,k8s_unmapped METADATA _index -| STATS pod = min(max_over_time(region)) BY time_bucket = bucket(@timestamp,1minute), _index -| SORT time_bucket, _index -| LIMIT 10; - -pod:keyword | time_bucket:datetime | _index:keyword -us | 2024-05-10T00:00:00.000Z | k8s -us | 2024-05-10T00:00:00.000Z | k8s_unmapped -us | 2024-05-10T00:01:00.000Z | k8s -us | 2024-05-10T00:01:00.000Z | k8s_unmapped -us | 2024-05-10T00:02:00.000Z | k8s -us | 2024-05-10T00:02:00.000Z | k8s_unmapped -us | 2024-05-10T00:03:00.000Z | k8s -us | 2024-05-10T00:03:00.000Z | k8s_unmapped -us | 2024-05-10T00:04:00.000Z | k8s -us | 2024-05-10T00:04:00.000Z | k8s_unmapped -; - // Copied over from max_over_time_with_filtering, except bytes_in is unmapped. maxOverTimeWithFiltering required_capability: optional_fields_v5 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 6463a4a20c90e..1eb3682768e20 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -2326,3 +2326,68 @@ sample_data | 3450233.0 sample_data | 5033755.0 sample_data | 8268153.0 ; + +############################ +# TS with partially mapped # +############################ + +// region is keyword, mapped in k8s but unmapped in k8s_unmapped. +// As a PotentiallyUnmappedKeywordEsField, it is loaded from _source for k8s_unmapped. +tsPartiallyMappedKeywordField +required_capability: optional_fields_v5 +required_capability: ts_command_v0 + +SET unmapped_fields="load"\; +TS k8s, k8s_unmapped METADATA _index +| STATS r = min(max_over_time(region)) BY time_bucket = bucket(@timestamp, 1minute), _index +| SORT time_bucket, _index +| LIMIT 10 +; + +r:keyword | time_bucket:datetime | _index:keyword +us | 2024-05-10T00:00:00.000Z | k8s +us | 2024-05-10T00:00:00.000Z | k8s_unmapped +us | 2024-05-10T00:01:00.000Z | k8s +us | 2024-05-10T00:01:00.000Z | k8s_unmapped +us | 2024-05-10T00:02:00.000Z | k8s +us | 2024-05-10T00:02:00.000Z | k8s_unmapped +us | 2024-05-10T00:03:00.000Z | k8s +us | 2024-05-10T00:03:00.000Z | k8s_unmapped +us | 2024-05-10T00:04:00.000Z | k8s +us | 2024-05-10T00:04:00.000Z | k8s_unmapped +; + +// network.bytes_in is long, mapped in k8s but unmapped in k8s_unmapped. +// Cast to long to resolve it via MultiTypeEsField. +tsPartiallyMappedNonKeywordFieldWithCast +required_capability: optional_fields_v5 +required_capability: ts_command_v0 + +SET unmapped_fields="load"\; +TS k8s, k8s_unmapped METADATA _index +| WHERE pod == "one" +| STATS tx = sum(max_over_time(network.bytes_in :: long)) BY cluster, time_bucket = bucket(@timestamp, 10minute), _index +| SORT time_bucket, cluster, _index +| LIMIT 20 +; + +tx:long | cluster:keyword | time_bucket:datetime | _index:keyword +970 | prod | 2024-05-10T00:00:00.000Z | k8s +970 | prod | 2024-05-10T00:00:00.000Z | k8s_unmapped +842 | qa | 2024-05-10T00:00:00.000Z | k8s +842 | qa | 2024-05-10T00:00:00.000Z | k8s_unmapped +753 | staging | 2024-05-10T00:00:00.000Z | k8s +753 | staging | 2024-05-10T00:00:00.000Z | k8s_unmapped +990 | prod | 2024-05-10T00:10:00.000Z | k8s +990 | prod | 2024-05-10T00:10:00.000Z | k8s_unmapped +1006 | qa | 2024-05-10T00:10:00.000Z | k8s +1006 | qa | 2024-05-10T00:10:00.000Z | k8s_unmapped +947 | staging | 2024-05-10T00:10:00.000Z | k8s +947 | staging | 2024-05-10T00:10:00.000Z | k8s_unmapped +953 | prod | 2024-05-10T00:20:00.000Z | k8s +953 | prod | 2024-05-10T00:20:00.000Z | k8s_unmapped +917 | qa | 2024-05-10T00:20:00.000Z | k8s +917 | qa | 2024-05-10T00:20:00.000Z | k8s_unmapped +749 | staging | 2024-05-10T00:20:00.000Z | k8s +749 | staging | 2024-05-10T00:20:00.000Z | k8s_unmapped +; From d41f435e0606f1e5850986999647ec2204ba75af Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 19:11:04 +0200 Subject: [PATCH 58/69] Tests with subfields --- .../xpack/esql/CsvTestsDataLoader.java | 45 ++++++- .../unmapped-type-conflicts.csv-spec | 127 ++++++++++++++++++ .../xpack/esql/optimizer/GoldenTestCase.java | 22 ++- 3 files changed, 185 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index baeb7b4515226..36a76ed3a772b 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -183,6 +183,30 @@ public class CsvTestsDataLoader { new TestDataset("k8s-downsampled", "k8s-downsampled-mappings.json", "k8s-downsampled.csv", "k8s-downsampled-settings.json"), new TestDataset("distances"), new TestDataset("addresses"), + new TestDataset("addresses").withIndex("addresses_no_continent") + .withTypeMapping(removeFields("city.country.continent")) + .withDynamic("false"), + new TestDataset("addresses").withIndex("addresses_text") + .withTypeMapping( + Map.of( + "street", + "text", + "number", + "text", + "zip_code", + "text", + "city.name", + "text", + "city.country.name", + "text", + "city.country.continent.name", + "text", + "city.country.continent.planet.name", + "text", + "city.country.continent.planet.galaxy", + "text" + ) + ), new TestDataset("books").withSetting("books-settings.json"), new TestDataset("semantic_text").withInferenceEndpoints("test_sparse_inference", "test_dense_inference"), new TestDataset("logs"), @@ -813,19 +837,30 @@ public static String readMappingFile(TestDataset dataset) throws IOException { JsonNode mappingNode = mapper.readTree(mappingJsonText); if (hasTypeMappingOverrides) { - ObjectNode propertiesNode = (ObjectNode) mappingNode.path("properties"); for (Map.Entry entry : dataset.typeMapping.entrySet()) { String key = entry.getKey(); String newType = entry.getValue(); + // Navigate dotted paths to find the parent properties node and leaf field name. + String[] segments = key.split("\\."); + ObjectNode propertiesNode = (ObjectNode) mappingNode.path("properties"); + for (int i = 0; i < segments.length - 1 && propertiesNode != null; i++) { + JsonNode child = propertiesNode.get(segments[i]); + propertiesNode = child != null ? (ObjectNode) child.path("properties") : null; + } + String leafName = segments[segments.length - 1]; + + if (propertiesNode == null) { + continue; + } if (newType == null) { // null value means remove the field from the mapping - if (propertiesNode.has(key)) { - propertiesNode.remove(key); + if (propertiesNode.has(leafName)) { + propertiesNode.remove(leafName); modified = true; } - } else if (propertiesNode.has(key)) { - ((ObjectNode) propertiesNode.get(key)).put("type", newType); + } else if (propertiesNode.has(leafName)) { + ((ObjectNode) propertiesNode.get(leafName)).put("type", newType); modified = true; } } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 1eb3682768e20..51f3bc39a9853 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -2327,6 +2327,133 @@ sample_data | 5033755.0 sample_data | 8268153.0 ; +############################################ +# Deeply nested partially unmapped fields # +############################################ + +// city.country.continent.name (keyword, 4-level deep) is mapped in addresses but not in addresses_no_continent. +// city.country.continent.planet.name and city.country.continent.planet.galaxy (5-level deep) are also unmapped. +// All fields are keyword, so they are loaded via PotentiallyUnmappedKeywordEsField. +deeplyNestedSubfieldPartiallyUnmappedKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses, addresses_no_continent METADATA _index +| KEEP _index, city.name, city.country.name, city.country.continent.name, city.country.continent.planet.name +| SORT _index, city.name +; + +_index:keyword | city.name:keyword | city.country.name:keyword | city.country.continent.name:keyword | city.country.continent.planet.name:keyword +addresses | Amsterdam | Netherlands | Europe | Earth +addresses | San Francisco | United States of America | North America | Earth +addresses | Tokyo | Japan | Asia | Earth +addresses_no_continent | Amsterdam | Netherlands | Europe | Earth +addresses_no_continent | San Francisco | United States of America | North America | Earth +addresses_no_continent | Tokyo | Japan | Asia | Earth +; + +// Cast city.country.continent.planet.name (keyword, 5-level deep, partially unmapped) to keyword. +deeplyNestedSubfieldPartiallyUnmappedCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses, addresses_no_continent METADATA _index +| EVAL planet = city.country.continent.planet.name::keyword +| KEEP _index, city.name, planet +| SORT _index, city.name +; + +_index:keyword | city.name:keyword | planet:keyword +addresses | Amsterdam | Earth +addresses | San Francisco | Earth +addresses | Tokyo | Earth +addresses_no_continent | Amsterdam | Earth +addresses_no_continent | San Francisco | Earth +addresses_no_continent | Tokyo | Earth +; + +// city.country.continent.planet.galaxy is keyword, 5-level deep, and sits under continent which is entirely stripped +// from addresses_no_continent. It is loaded via PotentiallyUnmappedKeywordEsField from both indices. +deeplyNestedSubfieldUnderPartiallyUnmappedParent +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses, addresses_no_continent METADATA _index +| KEEP _index, city.name, city.country.continent.planet.galaxy +| SORT _index, city.name +; + +_index:keyword | city.name:keyword | city.country.continent.planet.galaxy:keyword +addresses | Amsterdam | Milky Way +addresses | San Francisco | Milky Way +addresses | Tokyo | Milky Way +addresses_no_continent | Amsterdam | Milky Way +addresses_no_continent | San Francisco | Milky Way +addresses_no_continent | Tokyo | Milky Way +; + +// Same as above, but using addresses_text where galaxy is mapped as text (non-keyword). +// Without a cast, galaxy is a regular text field: values from addresses_text, null from addresses_no_continent. +deeplyNestedNonKeywordSubfieldUnderPartiallyUnmappedParentNotCast +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses_text, addresses_no_continent METADATA _index +| KEEP _index, city.country.continent.planet.galaxy +| SORT _index +; + +_index:keyword | city.country.continent.planet.galaxy:text +addresses_no_continent | null +addresses_no_continent | null +addresses_no_continent | null +addresses_text | Milky Way +addresses_text | Milky Way +addresses_text | Milky Way +; + +// city.country.continent.name is text (non-keyword) in addresses_text, unmapped in addresses_no_continent. +// Without a cast, it's a regular field: values from addresses_text, null from addresses_no_continent. +// Note: city.name has a type conflict (keyword vs text) across these indices, so we avoid referencing it. +deeplyNestedNonKeywordSubfieldPartiallyUnmappedNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM addresses_text, addresses_no_continent METADATA _index +| KEEP _index, city.country.continent.name +| SORT _index +; + +_index:keyword | city.country.continent.name:text +addresses_no_continent | null +addresses_no_continent | null +addresses_no_continent | null +addresses_text | North America +addresses_text | Asia +addresses_text | Europe +; + +// Cast city.country.continent.name (text, partially unmapped) to keyword via MultiTypeEsField. +deeplyNestedNonKeywordSubfieldPartiallyUnmappedCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses_text, addresses_no_continent METADATA _index +| EVAL continent = city.country.continent.name::keyword +| KEEP _index, continent +| SORT _index, continent +; + +_index:keyword | continent:keyword +addresses_no_continent | Asia +addresses_no_continent | Europe +addresses_no_continent | North America +addresses_text | Asia +addresses_text | Europe +addresses_text | North America +; + ############################ # TS with partially mapped # ############################ diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java index 64fa3c162e035..e1393b37235d4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java @@ -793,17 +793,31 @@ public static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndexT ); } + //TODO should de-duplicate, strong overlap with CsvTestsDataLoader#readMappingFile private static Map createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) { var mapping = new TreeMap<>(LoadMapping.loadMapping(dataset.streamMapping())); if (dataset.typeMapping() != null) { for (var entry : dataset.typeMapping().entrySet()) { + String key = entry.getKey(); + String[] segments = key.split("\\."); + // Navigate to the parent map containing the leaf field. + Map targetMap = mapping; + for (int i = 0; i < segments.length - 1 && targetMap != null; i++) { + EsField parent = targetMap.get(segments[i]); + targetMap = parent != null ? parent.getProperties() : null; + } + String leafName = segments[segments.length - 1]; + if (targetMap == null) { + continue; + } + if (entry.getValue() == null) { - mapping.remove(entry.getKey()); + targetMap.remove(leafName); continue; } - if (mapping.containsKey(entry.getKey())) { + if (targetMap.containsKey(leafName)) { DataType dataType = DataType.fromTypeName(entry.getValue()); - EsField field = mapping.get(entry.getKey()); + EsField field = targetMap.get(leafName); EsField editedField = new EsField( field.getName(), dataType, @@ -811,7 +825,7 @@ private static Map createMappingForIndex(CsvTestsDataLoader.Tes field.isAggregatable(), field.getTimeSeriesFieldType() ); - mapping.put(entry.getKey(), editedField); + targetMap.put(leafName, editedField); } } } From 69b8b2bf999911346f428397f5662da3a0d8c963 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Fri, 10 Apr 2026 17:19:07 +0000 Subject: [PATCH 59/69] [CI] Auto commit changes from spotless --- .../org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java index e1393b37235d4..cba4d3930c1a9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java @@ -793,7 +793,7 @@ public static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndexT ); } - //TODO should de-duplicate, strong overlap with CsvTestsDataLoader#readMappingFile + // TODO should de-duplicate, strong overlap with CsvTestsDataLoader#readMappingFile private static Map createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) { var mapping = new TreeMap<>(LoadMapping.loadMapping(dataset.streamMapping())); if (dataset.typeMapping() != null) { From e05eccd09093abb268e850c3c620fb361ac8ea25 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Fri, 10 Apr 2026 19:21:04 +0200 Subject: [PATCH 60/69] Add test: conflict + PUNK in same EVAL --- .../esql/analysis/AnalyzerUnmappedTests.java | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index a4d9dbbe254b5..10bc31dd6e63a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -1135,6 +1135,36 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordReportsAllFields() { ); } + /** + * An EVAL referencing both a partially unmapped non-keyword field and a field with a genuine type conflict + * should report errors for both fields. + */ + public void testDisallowLoadWithPartialNonKeywordAndTypeConflictInSameEval() { + assumeTrue("Requires OPTIONAL_FIELDS_V5", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V5.isEnabled()); + + var conflicted = new InvalidMappedField( + "conflicted", + Map.of(DataType.LONG.typeName(), Set.of("idx_a"), DataType.DOUBLE.typeName(), Set.of("idx_b")) + ); + var merged = new EsIndex( + "idx*", + Map.of("partial_long", longField("partial_long"), "conflicted", conflicted), + Map.of("idx_a", IndexMode.STANDARD, "idx_b", IndexMode.STANDARD, "idx_unmapped", IndexMode.STANDARD), + Map.of(), + Map.of(), + Map.of("partial_long", Set.of("idx_unmapped")) + ); + assertUnmappedLoadError( + analyzer().addIndex("idx*", IndexResolution.valid(merged)), + "FROM idx* | EVAL x = partial_long + 1, y = conflicted + 1", + allOf( + containsString("Found 2 problems"), + partiallyUnmappedNonKeywordError("partial_long"), + containsString("Cannot use field [conflicted]") + ) + ); + } + public void testAllowLoadWithPartialNonKeywordWhenFieldNotReferenced() { assumeTrue("Requires OPTIONAL_FIELDS_V5", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V5.isEnabled()); From 4efaf41c7faa908fdb45b9134a8c3391a4f4581d Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Mon, 13 Apr 2026 08:36:01 +0200 Subject: [PATCH 61/69] Update docs/changelog/144228.yaml --- docs/changelog/144228.yaml | 8 ++++++++ 1 file changed, 8 insertions(+) create mode 100644 docs/changelog/144228.yaml diff --git a/docs/changelog/144228.yaml b/docs/changelog/144228.yaml new file mode 100644 index 0000000000000..3bfcadb4184b0 --- /dev/null +++ b/docs/changelog/144228.yaml @@ -0,0 +1,8 @@ +area: ES|QL +issues: + - 145206 + - 141994 +pr: 144228 +summary: "Make unmapped_fields=\"load\" automatically trigger for partially mapped\ + \ KEYWORD fields, and allow projecting partially mapped non-KEYWORD fields" +type: bug From 5262284cded79182446e092e4120bfa891bb969e Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Mon, 13 Apr 2026 08:44:27 +0200 Subject: [PATCH 62/69] Improve comments --- .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 8 ++------ .../org/elasticsearch/xpack/esql/analysis/Verifier.java | 2 +- 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index a7d27445c2149..fc7d50ec5bb4b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -264,13 +264,9 @@ public enum Cap { OPTIONAL_FIELDS_V5, /** - * Do not depend on mentioning fields in expressions to trigger loading them. - * Otherwise, {@code SET unmapped_fields="load"; FROM logs-*} will return nulls - * for partially mapped keyword fields, while {@code FROM logs-* | KEEP *} will - * trigger their loading. + * Unconditionally load partially mapped keyword fields, whether they are mentioned in expressions or not. *

- * Also, always load values of partially mapped fields from the indices where - * they are mapped. + * Also, always load values of partially mapped fields from the indices where they are mapped. *

* Fixes https://github.com/elastic/elasticsearch/issues/141994 * and https://github.com/elastic/elasticsearch/issues/145206 diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 4953fcd62c62f..4914eb9d30ab1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -541,7 +541,7 @@ private static Set flattenedFieldNames(List attributes) { * * The following queries should fail (assume unmapped_fields="load" and reading from both indices) *

    - *
  • RENAME foo as x/li> + *
  • RENAME foo as x
    • *
  • EVAL x = foo
    • *
  • WHERE foo > 1
    • *
From 9a0e6b56eb23c7fb484a9f453411b3a86bd4e004 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 14 Apr 2026 12:04:34 +0200 Subject: [PATCH 63/69] Fix some capability requirements --- .../src/main/resources/unmapped-type-conflicts.csv-spec | 3 +++ 1 file changed, 3 insertions(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 51f3bc39a9853..f8e6f0e9577a2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -1193,6 +1193,7 @@ sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z typeConflictLongDateNonExistentNotCast required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data_ts_long, sample_data, colors METADATA _index @@ -1729,6 +1730,7 @@ sample_data_ts_long | 2023-10-23T13:55:01.543Z typeConflictDateNanosDateUnmappedNonExistentNotCast required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index @@ -2396,6 +2398,7 @@ addresses_no_continent | Tokyo | Milky Way // Without a cast, galaxy is a regular text field: values from addresses_text, null from addresses_no_continent. deeplyNestedNonKeywordSubfieldUnderPartiallyUnmappedParentNotCast required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM addresses_text, addresses_no_continent METADATA _index From e46e9429cf5acbe872c3cc01be2e37010249b011 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 14 Apr 2026 13:12:19 +0200 Subject: [PATCH 64/69] Fix some more tests --- .../src/main/resources/unmapped-type-conflicts.csv-spec | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index f8e6f0e9577a2..75484fd2dd128 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -910,6 +910,7 @@ sample_data_ts_long | 1698069301543 typeConflictLongDateUnmappedNotCast required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index @@ -2427,14 +2428,15 @@ FROM addresses_text, addresses_no_continent METADATA _index | KEEP _index, city.country.continent.name | SORT _index ; +ignoreOrder:true _index:keyword | city.country.continent.name:text addresses_no_continent | null addresses_no_continent | null addresses_no_continent | null -addresses_text | North America addresses_text | Asia addresses_text | Europe +addresses_text | North America ; // Cast city.country.continent.name (text, partially unmapped) to keyword via MultiTypeEsField. From 88e23ad9a574aaadeccad9a96c7eaa8b86ff0f32 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 14 Apr 2026 13:34:20 +0200 Subject: [PATCH 65/69] Fix last tests --- .../src/main/resources/inlinestats.csv-spec | 2 +- .../qa/testFixtures/src/main/resources/limit.csv-spec | 2 +- .../src/main/resources/lookup-join.csv-spec | 10 +++++----- .../logical/PushDownAndCombineLimitByGoldenTests.java | 2 +- .../local_physical_optimization.expected | 2 +- .../testLimitByNotPushedPastEval/query.esql | 2 +- 6 files changed, 10 insertions(+), 10 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index 26f7f7f371cc5..49917e0a56788 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -4284,7 +4284,7 @@ inlineStatsAfterPruningAggregate6 required_capability: inline_stats_double_release_fix required_capability: join_lookup_v12 -from d*,* +from d*,*,-addresses_* | rename scalerank as other2 | rename author.keyword as name_str | rename intersects as is_active_bool diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/limit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/limit.csv-spec index 31f72490214b2..d9682afb1b9a4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/limit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/limit.csv-spec @@ -1090,7 +1090,7 @@ limitByWithEnrich required_capability: enrich_load required_capability: limit_by_enrich_fix -FROM * +FROM *,-addresses_* | ENRICH languages_policy on street | KEEP abbrev, integer, year | LIMIT 1 BY abbrev diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 8a8279b0fd841..d644bbebc52b7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1778,7 +1778,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | sort language_code, birth_date @@ -1795,7 +1795,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | keep birth_date, language_code @@ -1813,7 +1813,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | keep birth_date, language_code @@ -1829,7 +1829,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | keep birth_date, language_code @@ -1847,7 +1847,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | keep birth_date, language_code diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimitByGoldenTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimitByGoldenTests.java index ea60b6c7110fa..565fa6002717b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimitByGoldenTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimitByGoldenTests.java @@ -35,7 +35,7 @@ public static void checkLimitByCapability() {} public void testLimitByNotPushedPastEval() { runGoldenTest( """ - FROM * + FROM airport_city_boundaries, addresses, all_types, books | ENRICH languages on street | KEEP abbrev, integer, year | LIMIT 1 BY abbrev diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/local_physical_optimization.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/local_physical_optimization.expected index ac89d2840635a..9386f463f4f78 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/local_physical_optimization.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/local_physical_optimization.expected @@ -7,4 +7,4 @@ ProjectExec[[abbrev{f}#0, integer{f}#1, year{f}#2]] \_FieldExtractExec[integer{f}#1, street{f}#3, year{f}#2]<[],[],[]> \_LimitByExec[1[INTEGER],[abbrev{f}#0],70] \_EvalExec[[null[KEYWORD] AS abbrev#0]] - \_EsQueryExec[*], indexMode[standard], [_doc{f}#5], limit[], sort[] estimatedRowSize[112] queryBuilderAndTags [[QueryBuilderAndTags[query=null, tags=[]]]] \ No newline at end of file + \_EsQueryExec[airport_city_boundaries,addresses,all_types,books], indexMode[standard], [_doc{f}#5], limit[], sort[] estimatedRowSize[112] queryBuilderAndTags [[QueryBuilderAndTags[query=null, tags=[]]]] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/query.esql index 2dfe3d5911bcc..15b884b6c6ea9 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/query.esql +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/query.esql @@ -1,4 +1,4 @@ -FROM * +FROM airport_city_boundaries, addresses, all_types, books | ENRICH languages on street | KEEP abbrev, integer, year | LIMIT 1 BY abbrev From 143a74c9e0a93093e6ebe1823c2b0966bca3ab93 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 14 Apr 2026 17:06:41 +0200 Subject: [PATCH 66/69] Fix one more spec test --- .../esql/qa/testFixtures/src/main/resources/enrich.csv-spec | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec index eb11a897ce23f..1dd9370c425d4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec @@ -652,7 +652,7 @@ required_capability: enrich_load required_capability: fix_replace_missing_field_with_null_duplicate_name_id_in_layout required_capability: dense_vector_agg_metric_double_if_fns -from * +from *,-addresses_* | keep author.keyword, book_no, scalerank, street, bytes_in, @timestamp, abbrev, city_location, distance, description, birth_date, language_code, intersects, client_ip, event_duration, version | enrich languages_policy on author.keyword | sort book_no From 724c562587d5629859e058780bec2a8034957e7a Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 14 Apr 2026 17:38:28 +0200 Subject: [PATCH 67/69] Whack one more test --- .../src/main/resources/unmapped-type-conflicts.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 75484fd2dd128..1272cc211ffa7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -1054,6 +1054,7 @@ sample_data_ts_long | 2023-10-23T13:55:01.543Z typeConflictDateNanosDateUnmappedNotCast required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data METADATA _index From d6791ae8b6d93e25d021e97230453fca3571b606 Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 14 Apr 2026 17:40:08 +0200 Subject: [PATCH 68/69] And one more --- .../src/main/resources/unmapped-type-conflicts.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 1272cc211ffa7..787c807190153 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -1351,6 +1351,7 @@ colors | null typeConflictDateNanosDateNonExistentNotCast required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data_ts_nanos, sample_data, colors METADATA _index From 08321aed38336cbf80f1f3077b3b26d0aeed5dff Mon Sep 17 00:00:00 2001 From: Alexander Spies Date: Tue, 14 Apr 2026 17:54:27 +0200 Subject: [PATCH 69/69] Last one, hopefully --- .../src/main/resources/unmapped-type-conflicts.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 787c807190153..b7daca0503fdd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -1511,6 +1511,7 @@ colors | null typeConflictLongDateUnmappedNonExistentNotCast required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped SET unmapped_fields="load"\; FROM sample_data_ts_long, sample_data, no_mapping_sample_data, colors METADATA _index