diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java index 9c53c778f10f8..d6280faf38be9 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java @@ -54,7 +54,6 @@ import java.util.LinkedHashMap; import java.util.Locale; import java.util.Map; -import java.util.Set; import java.util.concurrent.TimeUnit; import static java.util.Collections.emptyMap; @@ -108,7 +107,7 @@ public void setup() { mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE)); } - var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Set.of()); + var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Map.of()); var functionRegistry = new EsqlFunctionRegistry(); parser = new EsqlParser(new EsqlConfig(functionRegistry)); diff --git a/benchmarks/src/main/java/org/elasticsearch/benchmark/esql/ViewResolutionBenchmarkBase.java b/benchmarks/src/main/java/org/elasticsearch/benchmark/esql/ViewResolutionBenchmarkBase.java index a2d047b2fd7b3..3e0c5fd480137 100644 --- a/benchmarks/src/main/java/org/elasticsearch/benchmark/esql/ViewResolutionBenchmarkBase.java +++ b/benchmarks/src/main/java/org/elasticsearch/benchmark/esql/ViewResolutionBenchmarkBase.java @@ -167,7 +167,7 @@ public void setup() { String name = "col" + i; mapping.put(name, new EsField(name, KEYWORD, emptyMap(), true, EsField.TimeSeriesFieldType.NONE)); } - EsIndex esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Set.of()); + EsIndex esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Map.of()); Configuration config = new Configuration( DateUtils.UTC, diff --git a/docs/changelog/143693.yaml b/docs/changelog/143693.yaml new file mode 100644 index 0000000000000..a65437a2d2caa --- /dev/null +++ b/docs/changelog/143693.yaml @@ -0,0 +1,7 @@ +area: ES|QL +issues: + - 142004 + - 141912 +pr: 143693 +summary: Type conflict resolution in unmapped-fields load +type: feature diff --git a/server/src/main/resources/transport/definitions/referable/esql_potentially_unmapped_expression.csv b/server/src/main/resources/transport/definitions/referable/esql_potentially_unmapped_expression.csv new file mode 100644 index 0000000000000..0bdfdf267306f --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_potentially_unmapped_expression.csv @@ -0,0 +1 @@ +9334000 diff --git a/server/src/main/resources/transport/upper_bounds/9.4.csv b/server/src/main/resources/transport/upper_bounds/9.4.csv index cab3358803045..9741bd6b30b5e 100644 --- a/server/src/main/resources/transport/upper_bounds/9.4.csv +++ b/server/src/main/resources/transport/upper_bounds/9.4.csv @@ -1 +1 @@ -chunked_fetch_phase,9333000 +esql_potentially_unmapped_expression,9334000 diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java index 77faf747bc0be..b9e3f6809e5c0 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java @@ -18,7 +18,7 @@ import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.APPROXIMATION_V5; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METRICS_GROUP_BY_ALL; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING; @@ -66,7 +66,7 @@ protected void shouldSkipTest(String testName) throws IOException { // FORK is not supported with unmapped_fields="load", see https://github.com/elastic/elasticsearch/issues/142033 assumeFalse( "FORK is not supported with unmapped_fields=\"load\"", - testCase.requiredCapabilities.contains(OPTIONAL_FIELDS_V3.capabilityName()) + testCase.requiredCapabilities.contains(OPTIONAL_FIELDS_V4.capabilityName()) ); assumeFalse( diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index a03b8719fca85..cb5445d8d8113 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -107,6 +107,7 @@ public class CsvTestsDataLoader { new TestDataset("languages_mixed_numerics").withSetting("lookup-settings.json"), new TestDataset("ul_logs"), new TestDataset("sample_data"), + new TestDataset("sample_data").withIndex("cloned_sample_data"), new TestDataset("partial_mapping_sample_data"), new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv").withTypeMapping( Stream.of("timestamp", "client_ip", "event_duration").collect(toMap(k -> k, k -> "keyword")) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java index 8b4c5035c9744..bf5aa5acfac0a 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/TestAnalyzer.java @@ -49,7 +49,6 @@ import java.util.LinkedHashMap; import java.util.List; import java.util.Map; -import java.util.Set; import java.util.function.Supplier; import java.util.stream.Collectors; @@ -156,7 +155,7 @@ public TestAnalyzer addNoFieldsIndex() { Map.of(noFieldsIndexName, IndexMode.STANDARD), Map.of("", List.of(noFieldsIndexName)), Map.of("", List.of(noFieldsIndexName)), - Set.of() + Map.of() ); addIndex(noFieldsIndexName, IndexResolution.valid(noFieldsIndex)); return this; @@ -770,7 +769,7 @@ public AnalyzerContext buildContext() { */ public static IndexResolution loadMapping(String resource, String indexName, IndexMode indexMode) { return IndexResolution.valid( - new EsIndex(indexName, EsqlTestUtils.loadMapping(resource), Map.of(indexName, indexMode), Map.of(), Map.of(), Set.of()) + new EsIndex(indexName, EsqlTestUtils.loadMapping(resource), Map.of(indexName, indexMode), Map.of(), Map.of(), Map.of()) ); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec index b9020b7e666aa..71dc0d8b156de 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec @@ -1,6 +1,6 @@ flattened root fields required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -17,7 +17,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened KEEP subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -42,7 +42,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened DROP subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -68,7 +68,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened DROP subfield after KEEP-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -94,7 +94,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -111,7 +111,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE and KEEP *-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -129,7 +129,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE and KEEP specific subfields-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -147,7 +147,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with EVAL TO_UPPER-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -173,7 +173,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with EVAL LENGTH-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -199,7 +199,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed SORT on flattened subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -224,7 +224,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed RENAME flattened subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -250,7 +250,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with STATS count-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -265,7 +265,7 @@ count(*):long | resource.attributes.agent.type:keyword # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with STATS count_distinct-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -280,7 +280,7 @@ count_distinct(resource.attributes.host.name):long # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with INLINE STATS-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -302,7 +302,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with MV_EXPAND-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM flattened_otel_logs diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 356b23deb8090..a61663fb95de1 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -3,7 +3,7 @@ #################### // These ones verify we don't do anything extra when unmapped_fields is not set to "load" doesNotLoadUnmappedFieldsSort -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 FROM partial_mapping_sample_data | SORT @timestamp DESC @@ -20,7 +20,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsStats -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 FROM partial_mapping_sample_data | STATS count(*) BY message @@ -36,7 +36,7 @@ count(*):long | message:keyword ; doesNotLoadUnmappedFieldsInlineStats -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 FROM partial_mapping_sample_data | INLINE STATS count = COUNT(*) BY message @@ -51,7 +51,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsInlineStatsNoGrouping -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 FROM partial_mapping_sample_data | INLINE STATS max_duration = MAX(event_duration) @@ -66,7 +66,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsEnrich -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: enrich_load FROM partial_mapping_sample_data @@ -84,7 +84,7 @@ FROM partial_mapping_sample_data doesNotLoadUnmappedFieldsLookupJoin-Ignore // temporarily forbidding "load" with LOOKUP JOIN -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: join_lookup_v12 FROM partial_mapping_sample_data @@ -105,7 +105,7 @@ FROM partial_mapping_sample_data ###################### unmappedFieldDoesNotAppearLast -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -118,7 +118,7 @@ FROM partial_mapping_sample_data ; fieldDoesNotExistSingleIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -137,7 +137,7 @@ FROM partial_mapping_sample_data ; fieldIsUnmappedSingleIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -156,7 +156,7 @@ FROM partial_mapping_sample_data ; fieldUnmappedInSourceButExcludedSingleIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -176,7 +176,7 @@ FROM partial_mapping_excluded_source_sample_data ; fieldUnmappedInSourceButSourceDisabledSingleIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -196,7 +196,7 @@ FROM partial_mapping_no_source_sample_data ; statsByUnmappedFieldExistsInSource -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -213,7 +213,7 @@ s:long | c:long | unmapped_message:keyword ; inlineStatsByUnmappedFieldExistsInSource -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -231,7 +231,7 @@ FROM partial_mapping_sample_data ; unmappedFieldNonExistentTsIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -250,7 +250,7 @@ FROM k8s_unmapped ; unmappedNumericFromK8sSyntheticTsIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -269,7 +269,7 @@ TS k8s_unmapped ; statsImplicitLastOverTimeUnmappedTsIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -287,7 +287,7 @@ time_bucket:datetime | x:keyword ; statsByUnmappedFieldTsIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -312,7 +312,7 @@ s:long | c:long | event_log:keyword ; explicitLastOverTimeUnmappedTsIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -332,7 +332,7 @@ qa | 2024-05-10T00:10:00.000Z | volutpat hac ; fieldIsNestedAndUnmapped -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -351,7 +351,7 @@ FROM partial_mapping_sample_data ; fieldIsNestedAndNonExistent -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -374,7 +374,7 @@ FROM partial_mapping_sample_data ############################### noFieldExistsMultiParametersSingleIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -393,7 +393,7 @@ FROM partial_mapping_sample_data ; mixedFieldsMultiParametersSingleIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -416,7 +416,7 @@ FROM partial_mapping_sample_data ##################### mixedFieldsMultiParametersMultiIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -442,7 +442,7 @@ sample_data | 2023-10-23T12:15:03.360Z | null | Connected ; fieldDoesNotExistMultiIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -468,7 +468,7 @@ sample_data | 2023-10-23T12:15:03.360Z | null ; fieldIsUnmappedMultiIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -494,7 +494,7 @@ FROM partial_mapping_sample_data, sample_data METADATA _index ; fieldIsPartiallyUnmappedMultiIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -520,7 +520,7 @@ sample_data | Connected to 10.1.0.1 ; fieldIsPartiallyUnmappedAndRenamedMultiIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data @@ -547,7 +547,7 @@ Connected to 10.1.0.1 ; fieldIsPartiallyUnmappedPartiallySourceIsDisabledMultiIndex -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -573,8 +573,146 @@ partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | Connection er partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! ; +threeIndicesKeepMessageUnmappedThird +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM partial_mapping_sample_data, sample_data, partial_mapping_no_source_sample_data METADATA _index +| KEEP _index, @timestamp, message +| SORT _index, @timestamp DESC +; + +_index:keyword | @timestamp:date | message:keyword +partial_mapping_no_source_sample_data | 2024-10-23T13:55:01.543Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:53:55.832Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:52:55.015Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:51:54.732Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:33:34.937Z | null +partial_mapping_no_source_sample_data | 2024-10-23T12:27:28.948Z | null +partial_mapping_no_source_sample_data | 2024-10-23T12:15:03.360Z | null +partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! +partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | 42 +partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | Connected to 10.1.0.2! +partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | Connected to 10.1.0.3! +sample_data | 2023-10-23T13:55:01.543Z | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | Connection error +sample_data | 2023-10-23T13:52:55.015Z | Connection error +sample_data | 2023-10-23T13:51:54.732Z | Connection error +sample_data | 2023-10-23T13:33:34.937Z | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | Connected to 10.1.0.3 +; + +threeIndicesDropClientIpUnmappedThird +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM partial_mapping_sample_data, sample_data, partial_mapping_no_source_sample_data METADATA _index +| KEEP _index, @timestamp, message, client_ip +| DROP client_ip +| SORT _index, @timestamp DESC +; + +_index:keyword | @timestamp:date | message:keyword +partial_mapping_no_source_sample_data | 2024-10-23T13:55:01.543Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:53:55.832Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:52:55.015Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:51:54.732Z | null +partial_mapping_no_source_sample_data | 2024-10-23T13:33:34.937Z | null +partial_mapping_no_source_sample_data | 2024-10-23T12:27:28.948Z | null +partial_mapping_no_source_sample_data | 2024-10-23T12:15:03.360Z | null +partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! +partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | 42 +partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | Connected to 10.1.0.2! +partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | Connected to 10.1.0.3! +sample_data | 2023-10-23T13:55:01.543Z | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | Connection error +sample_data | 2023-10-23T13:52:55.015Z | Connection error +sample_data | 2023-10-23T13:51:54.732Z | Connection error +sample_data | 2023-10-23T13:33:34.937Z | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | Connected to 10.1.0.3 +; + +threeIndicesKeepMessageExcludedThird +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM partial_mapping_sample_data, sample_data, partial_mapping_excluded_source_sample_data METADATA _index +| KEEP _index, @timestamp, message +| SORT _index, @timestamp DESC +; + +_index:keyword | @timestamp:date | message:keyword +partial_mapping_excluded_source_sample_data | 2024-10-23T13:55:01.543Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T13:53:55.832Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T13:52:55.015Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T13:51:54.732Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T13:33:34.937Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T12:27:28.948Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T12:15:03.360Z | null +partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! +partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | 42 +partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | Connected to 10.1.0.2! +partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | Connected to 10.1.0.3! +sample_data | 2023-10-23T13:55:01.543Z | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | Connection error +sample_data | 2023-10-23T13:52:55.015Z | Connection error +sample_data | 2023-10-23T13:51:54.732Z | Connection error +sample_data | 2023-10-23T13:33:34.937Z | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | Connected to 10.1.0.3 +; + +threeIndicesDropClientIpExcludedThird +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM partial_mapping_sample_data, sample_data, partial_mapping_excluded_source_sample_data METADATA _index +| KEEP _index, @timestamp, message, client_ip +| DROP client_ip +| SORT _index, @timestamp DESC +; + +_index:keyword | @timestamp:date | message:keyword +partial_mapping_excluded_source_sample_data | 2024-10-23T13:55:01.543Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T13:53:55.832Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T13:52:55.015Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T13:51:54.732Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T13:33:34.937Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T12:27:28.948Z | null +partial_mapping_excluded_source_sample_data | 2024-10-23T12:15:03.360Z | null +partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to 10.1.0.1! +partial_mapping_sample_data | 2024-10-23T13:53:55.832Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:52:55.015Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:51:54.732Z | Connection error? +partial_mapping_sample_data | 2024-10-23T13:33:34.937Z | 42 +partial_mapping_sample_data | 2024-10-23T12:27:28.948Z | Connected to 10.1.0.2! +partial_mapping_sample_data | 2024-10-23T12:15:03.360Z | Connected to 10.1.0.3! +sample_data | 2023-10-23T13:55:01.543Z | Connected to 10.1.0.1 +sample_data | 2023-10-23T13:53:55.832Z | Connection error +sample_data | 2023-10-23T13:52:55.015Z | Connection error +sample_data | 2023-10-23T13:51:54.732Z | Connection error +sample_data | 2023-10-23T13:33:34.937Z | Disconnected +sample_data | 2023-10-23T12:27:28.948Z | Connected to 10.1.0.2 +sample_data | 2023-10-23T12:15:03.360Z | Connected to 10.1.0.3 +; + partialMappingStats -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -593,7 +731,7 @@ max(@timestamp):date | count(*):long | message:keyword ; partialMappingCoalesce -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -622,7 +760,7 @@ FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data MET ; partialMappingKeywordCast -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -650,7 +788,7 @@ FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data MET ; partialMappingStatsAfterCast -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -666,7 +804,7 @@ count(*):long | message::INT:integer ; filterOnPartiallyUnmappedField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -685,7 +823,7 @@ sample_data | Connection error ; sortOnPartiallyUnmappedField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -703,7 +841,7 @@ no_mapping_sample_data | Connected to 10.1.0.2! ; filterOnFullyUnmappedField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -719,7 +857,7 @@ FROM partial_mapping_sample_data ; filterOnConjunctionMappedAndUnmapped -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -735,7 +873,7 @@ FROM partial_mapping_sample_data ; filterOnConjunctionMappedAndNonExistent -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -747,7 +885,7 @@ _index:keyword | message:keyword | does_not_exist:keyword ; filterOnDisjunctionMappedAndUnmapped -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -764,7 +902,7 @@ FROM partial_mapping_sample_data ; filterOnDisjunctionMappedAndNonExistent -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -780,7 +918,7 @@ sample_data | Connection error | null ; sortOnNonExistentField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM no_mapping_sample_data METADATA _index @@ -798,7 +936,7 @@ no_mapping_sample_data | null ; sortOnUnmappedField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -815,7 +953,7 @@ FROM partial_mapping_sample_data ; sortOnExpressionOnUnmappedField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -833,7 +971,7 @@ FROM partial_mapping_sample_data ; sortOnMultipleFieldsWithNonExistent -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -851,7 +989,7 @@ no_mapping_sample_data | Connected to 10.1.0.2! | null ; sortOnMultipleFieldsWithUnmapped -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -869,7 +1007,7 @@ FROM partial_mapping_sample_data ; filterOnNonExistentFieldCastToLong -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -881,7 +1019,7 @@ _index:keyword | message:keyword | does_not_exist:keyword ; filterOnUnmappedFieldCastToLong -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -894,7 +1032,7 @@ FROM partial_mapping_sample_data ; maxOverTimeOfKeyword -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -918,7 +1056,7 @@ us | 2024-05-10T00:04:00.000Z | k8s_unmapped // Copied over from max_over_time_with_filtering, except bytes_in is unmapped. maxOverTimeWithFiltering -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -943,7 +1081,7 @@ tx:long | cluster:keyword | time_bucket:datetime // Copied over from eval_on_max_over_time, except bytes_in is unmapped. evalOnMaxOverTime -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -968,7 +1106,7 @@ max_bytes:double | cluster:keyword | time_bucket:datetime | kb_minus_offset // https://github.com/elastic/elasticsearch/issues/143991 statsFilteredAggAfterEvalWithDottedUnmappedField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: optional_fields_detect_unmapped_fields_in_agg_filters SET unmapped_fields="load"\; @@ -983,7 +1121,7 @@ null | foo // https://github.com/elastic/elasticsearch/issues/143991 statsFilteredAggAfterEvalWithDottedUnmappedFieldLoadsFromSource -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: optional_fields_detect_unmapped_fields_in_agg_filters SET unmapped_fields="load"\; @@ -999,3 +1137,44 @@ FROM partial_mapping_sample_data msg_count:integer | unmapped_messag:keyword 5 | foo ; + +// Regression test: indices with the same mapping hash must not be treated as partially unmapped. +sameMappingHashFullyMapped +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, cloned_sample_data +| STATS c = COUNT(*) BY message +| SORT message +; + +c:long | message:keyword +2 | Connected to 10.1.0.1 +2 | Connected to 10.1.0.2 +2 | Connected to 10.1.0.3 +6 | Connection error +2 | Disconnected +; + +// Regression test: two same-hash indices + an unmapped index should still treat the field as partially unmapped. +sameMappingHashWithUnmappedIndex +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, cloned_sample_data, no_mapping_sample_data +| STATS c = COUNT(*) BY message +| SORT message +; + +c:long | message:keyword +1 | 42 +2 | Connected to 10.1.0.1 +1 | Connected to 10.1.0.1! +2 | Connected to 10.1.0.2 +1 | Connected to 10.1.0.2! +2 | Connected to 10.1.0.3 +1 | Connected to 10.1.0.3! +6 | Connection error +3 | Connection error? +2 | Disconnected +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec new file mode 100644 index 0000000000000..879a16c860c12 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -0,0 +1,1251 @@ +// This test file covers type conflict resolution with unmapped fields, which is similar to union types. + +###################################### +# Mapped and unmapped conflict tests # +###################################### + +noTypeConflictKeywordUnmappedCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| KEEP _index, message +| SORT _index, message +; + +_index:keyword | message:keyword +no_mapping_sample_data | 42 +no_mapping_sample_data | Connected to 10.1.0.1! +no_mapping_sample_data | Connected to 10.1.0.2! +no_mapping_sample_data | Connected to 10.1.0.3! +no_mapping_sample_data | Connection error? +no_mapping_sample_data | Connection error? +no_mapping_sample_data | Connection error? +sample_data | Connected to 10.1.0.1 +sample_data | Connected to 10.1.0.2 +sample_data | Connected to 10.1.0.3 +sample_data | Connection error +sample_data | Connection error +sample_data | Connection error +sample_data | Disconnected +; + +noTypeConflictKeywordUnmappedCastToKeywordExplicitEval +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| EVAL message = message::keyword +| KEEP _index, message +| SORT _index, message +; + +_index:keyword | message:keyword +no_mapping_sample_data | 42 +no_mapping_sample_data | Connected to 10.1.0.1! +no_mapping_sample_data | Connected to 10.1.0.2! +no_mapping_sample_data | Connected to 10.1.0.3! +no_mapping_sample_data | Connection error? +no_mapping_sample_data | Connection error? +no_mapping_sample_data | Connection error? +sample_data | Connected to 10.1.0.1 +sample_data | Connected to 10.1.0.2 +sample_data | Connected to 10.1.0.3 +sample_data | Connection error +sample_data | Connection error +sample_data | Connection error +sample_data | Disconnected +; + +typeConflictLongUnmappedCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| EVAL event_duration = event_duration::keyword +| KEEP _index, event_duration +| SORT _index, event_duration +; + +_index:keyword | event_duration:keyword +no_mapping_sample_data | 1232381 +no_mapping_sample_data | 1756466 +no_mapping_sample_data | 2764888 +no_mapping_sample_data | 3450232 +no_mapping_sample_data | 5033754 +no_mapping_sample_data | 725447 +no_mapping_sample_data | 8268152 +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 725448 +sample_data | 8268153 +; + +typeConflictLongUnmappedCastToLong +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| EVAL event_duration = event_duration::long +| KEEP _index, event_duration +| SORT _index, event_duration +; + +_index:keyword | event_duration:long +no_mapping_sample_data | 725447 +no_mapping_sample_data | 1232381 +no_mapping_sample_data | 1756466 +no_mapping_sample_data | 2764888 +no_mapping_sample_data | 3450232 +no_mapping_sample_data | 5033754 +no_mapping_sample_data | 8268152 +sample_data | 725448 +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 8268153 +; + +typeConflictLongUnmappedCastToDouble +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| EVAL event_duration = event_duration::double +| KEEP _index, event_duration +| SORT _index, event_duration +; + +_index:keyword | event_duration:double +no_mapping_sample_data | 725447.0 +no_mapping_sample_data | 1232381.0 +no_mapping_sample_data | 1756466.0 +no_mapping_sample_data | 2764888.0 +no_mapping_sample_data | 3450232.0 +no_mapping_sample_data | 5033754.0 +no_mapping_sample_data | 8268152.0 +sample_data | 725448.0 +sample_data | 1232382.0 +sample_data | 1756467.0 +sample_data | 2764889.0 +sample_data | 3450233.0 +sample_data | 5033755.0 +sample_data | 8268153.0 +; + +typeConflictLongUnmappedCastToDoubleNoSort +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| EVAL event_duration = event_duration::double +| KEEP _index, event_duration +; +ignoreOrder:true + +_index:keyword | event_duration:double +no_mapping_sample_data | 725447.0 +no_mapping_sample_data | 1232381.0 +no_mapping_sample_data | 1756466.0 +no_mapping_sample_data | 2764888.0 +no_mapping_sample_data | 3450232.0 +no_mapping_sample_data | 5033754.0 +no_mapping_sample_data | 8268152.0 +sample_data | 725448.0 +sample_data | 1232382.0 +sample_data | 1756467.0 +sample_data | 2764889.0 +sample_data | 3450233.0 +sample_data | 5033755.0 +sample_data | 8268153.0 +; + + +########################### +# Mapped and non-existent # +########################### + +noTypeConflictKeywordNonExistentNoCast +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, colors METADATA _index +| KEEP _index, message +| SORT _index DESC, message +| LIMIT 15 +; + +_index:keyword | message:keyword +sample_data | Connected to 10.1.0.1 +sample_data | Connected to 10.1.0.2 +sample_data | Connected to 10.1.0.3 +sample_data | Connection error +sample_data | Connection error +sample_data | Connection error +sample_data | Disconnected +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +noTypeConflictKeywordNonExistentCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, colors METADATA _index +| EVAL message = message::keyword +| KEEP _index, message +| SORT _index DESC, message +| LIMIT 15 +; + +_index:keyword | message:keyword +sample_data | Connected to 10.1.0.1 +sample_data | Connected to 10.1.0.2 +sample_data | Connected to 10.1.0.3 +sample_data | Connection error +sample_data | Connection error +sample_data | Connection error +sample_data | Disconnected +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongNonExistentCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, colors METADATA _index +| EVAL event_duration = event_duration::keyword +| KEEP _index, event_duration +| SORT _index DESC, event_duration +| LIMIT 15 +; + +_index:keyword | event_duration:keyword +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 725448 +sample_data | 8268153 +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongNonExistentCastToLong +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, colors METADATA _index +| EVAL event_duration = event_duration::long +| KEEP _index, event_duration +| SORT _index DESC, event_duration +| LIMIT 15 +; + +_index:keyword | event_duration:long +sample_data | 725448 +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 8268153 +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongNonExistentCastToDouble +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, colors METADATA _index +| EVAL event_duration = event_duration::double +| KEEP _index, event_duration +| SORT _index DESC, event_duration +| LIMIT 15 +; + +_index:keyword | event_duration:double +sample_data | 725448.0 +sample_data | 1232382.0 +sample_data | 1756467.0 +sample_data | 2764889.0 +sample_data | 3450233.0 +sample_data | 5033755.0 +sample_data | 8268153.0 +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongNonExistentCastToDoubleNoSort +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, heights METADATA _index +| EVAL event_duration = event_duration::double +| KEEP _index, event_duration +; +ignoreOrder:true + +_index:keyword | event_duration:double +sample_data | 725448.0 +sample_data | 1232382.0 +sample_data | 1756467.0 +sample_data | 2764889.0 +sample_data | 3450233.0 +sample_data | 5033755.0 +sample_data | 8268153.0 +heights | null +heights | null +heights | null +heights | null +heights | null +; + +typeConflictDateNanosDateUnmappedCastToDate +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, sample_data_ts_nanos, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:datetime +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_nanos | 2023-10-23T12:15:03.360Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543Z +; + +########################### +# Mapped x 2 and unmapped # +########################### + +typeConflictLongDateUnmappedCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:keyword +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_long | 1698063303360 +sample_data_ts_long | 1698064048948 +sample_data_ts_long | 1698068014937 +sample_data_ts_long | 1698069114732 +sample_data_ts_long | 1698069175015 +sample_data_ts_long | 1698069235832 +sample_data_ts_long | 1698069301543 +; + +typeConflictLongDateUnmappedCastToDouble +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::double +| KEEP _index, ts +| SORT _index, ts +; + +warning:Line 3:13: evaluation of [@timestamp::double] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T12:15:03.360Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T12:27:28.948Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:33:34.937Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:51:54.732Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:52:55.015Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:53:55.832Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:55:01.543Z] +_index:keyword | ts:double +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +sample_data | 1.69806330336E12 +sample_data | 1.698064048948E12 +sample_data | 1.698068014937E12 +sample_data | 1.698069114732E12 +sample_data | 1.698069175015E12 +sample_data | 1.698069235832E12 +sample_data | 1.698069301543E12 +sample_data_ts_long | 1.69806330336E12 +sample_data_ts_long | 1.698064048948E12 +sample_data_ts_long | 1.698068014937E12 +sample_data_ts_long | 1.698069114732E12 +sample_data_ts_long | 1.698069175015E12 +sample_data_ts_long | 1.698069235832E12 +sample_data_ts_long | 1.698069301543E12 +; + +typeConflictLongDateUnmappedCastToDate +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:date +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_long | 2023-10-23T12:15:03.360Z +sample_data_ts_long | 2023-10-23T12:27:28.948Z +sample_data_ts_long | 2023-10-23T13:33:34.937Z +sample_data_ts_long | 2023-10-23T13:51:54.732Z +sample_data_ts_long | 2023-10-23T13:52:55.015Z +sample_data_ts_long | 2023-10-23T13:53:55.832Z +sample_data_ts_long | 2023-10-23T13:55:01.543Z +; + +typeConflictLongDateUnmappedCastToDateNoSort +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +; +ignoreOrder:true + +_index:keyword | ts:date +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_long | 2023-10-23T12:15:03.360Z +sample_data_ts_long | 2023-10-23T12:27:28.948Z +sample_data_ts_long | 2023-10-23T13:33:34.937Z +sample_data_ts_long | 2023-10-23T13:51:54.732Z +sample_data_ts_long | 2023-10-23T13:52:55.015Z +sample_data_ts_long | 2023-10-23T13:53:55.832Z +sample_data_ts_long | 2023-10-23T13:55:01.543Z +; + +############################### +# Mapped x 2 and non-existent # +############################### + +typeConflictLongDateNonExistentCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, colors METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:keyword +sample_data_ts_long | 1698063303360 +sample_data_ts_long | 1698064048948 +sample_data_ts_long | 1698068014937 +sample_data_ts_long | 1698069114732 +sample_data_ts_long | 1698069175015 +sample_data_ts_long | 1698069235832 +sample_data_ts_long | 1698069301543 +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongDateNonExistentCastToDouble +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, colors METADATA _index +| EVAL ts = @timestamp::double +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:double +sample_data_ts_long | 1.69806330336E12 +sample_data_ts_long | 1.698064048948E12 +sample_data_ts_long | 1.698068014937E12 +sample_data_ts_long | 1.698069114732E12 +sample_data_ts_long | 1.698069175015E12 +sample_data_ts_long | 1.698069235832E12 +sample_data_ts_long | 1.698069301543E12 +sample_data | 1.69806330336E12 +sample_data | 1.698064048948E12 +sample_data | 1.698068014937E12 +sample_data | 1.698069114732E12 +sample_data | 1.698069175015E12 +sample_data | 1.698069235832E12 +sample_data | 1.698069301543E12 +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongDateNonExistentCastToDate +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, colors METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:datetime +sample_data_ts_long | 2023-10-23T12:15:03.360Z +sample_data_ts_long | 2023-10-23T12:27:28.948Z +sample_data_ts_long | 2023-10-23T13:33:34.937Z +sample_data_ts_long | 2023-10-23T13:51:54.732Z +sample_data_ts_long | 2023-10-23T13:52:55.015Z +sample_data_ts_long | 2023-10-23T13:53:55.832Z +sample_data_ts_long | 2023-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +########################################## +# Mapped x 2, unmapped, and non-existent # +########################################## + +typeConflictLongDateUnmappedNonExistentCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +_index:keyword | ts:keyword +sample_data_ts_long | 1698063303360 +sample_data_ts_long | 1698064048948 +sample_data_ts_long | 1698068014937 +sample_data_ts_long | 1698069114732 +sample_data_ts_long | 1698069175015 +sample_data_ts_long | 1698069235832 +sample_data_ts_long | 1698069301543 +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongDateUnmappedNonExistentCastToDouble +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::double +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +warning:Line 3:13: evaluation of [@timestamp::double] failed, treating result as null. Only first 20 failures recorded. +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T12:15:03.360Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T12:27:28.948Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:33:34.937Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:51:54.732Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:52:55.015Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:53:55.832Z] +warning:Line 3:13: org.elasticsearch.xpack.esql.core.InvalidArgumentException: Cannot parse number [2024-10-23T13:55:01.543Z] +_index:keyword | ts:double +sample_data_ts_long | 1.69806330336E12 +sample_data_ts_long | 1.698064048948E12 +sample_data_ts_long | 1.698068014937E12 +sample_data_ts_long | 1.698069114732E12 +sample_data_ts_long | 1.698069175015E12 +sample_data_ts_long | 1.698069235832E12 +sample_data_ts_long | 1.698069301543E12 +sample_data | 1.69806330336E12 +sample_data | 1.698064048948E12 +sample_data | 1.698068014937E12 +sample_data | 1.698069114732E12 +sample_data | 1.698069175015E12 +sample_data | 1.698069235832E12 +sample_data | 1.698069301543E12 +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongDateUnmappedNonExistentCastToDate +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +_index:keyword | ts:datetime +sample_data_ts_long | 2023-10-23T12:15:03.360Z +sample_data_ts_long | 2023-10-23T12:27:28.948Z +sample_data_ts_long | 2023-10-23T13:33:34.937Z +sample_data_ts_long | 2023-10-23T13:51:54.732Z +sample_data_ts_long | 2023-10-23T13:52:55.015Z +sample_data_ts_long | 2023-10-23T13:53:55.832Z +sample_data_ts_long | 2023-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictLongDateUnmappedNonExistentCastToDateNoSort +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data, heights METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +; +ignoreOrder:true + +_index:keyword | ts:datetime +heights | null +heights | null +heights | null +heights | null +heights | null +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_long | 2023-10-23T12:15:03.360Z +sample_data_ts_long | 2023-10-23T12:27:28.948Z +sample_data_ts_long | 2023-10-23T13:33:34.937Z +sample_data_ts_long | 2023-10-23T13:51:54.732Z +sample_data_ts_long | 2023-10-23T13:52:55.015Z +sample_data_ts_long | 2023-10-23T13:53:55.832Z +sample_data_ts_long | 2023-10-23T13:55:01.543Z +; + +############################## +# WHERE with type casts # +############################## + +typeConflictLongUnmappedWhereWithCastToLong +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| WHERE event_duration::long > 3000000 +| EVAL event_duration = event_duration::keyword +| KEEP _index, event_duration +| SORT _index, event_duration +; + +_index:keyword | event_duration:keyword +no_mapping_sample_data | 3450232 +no_mapping_sample_data | 5033754 +no_mapping_sample_data | 8268152 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 8268153 +; + +noTypeConflictKeywordUnmappedWhereWithCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| WHERE message::keyword LIKE "Connected*" +| KEEP _index, message +| SORT _index, message +; + +_index:keyword | message:keyword +no_mapping_sample_data | Connected to 10.1.0.1! +no_mapping_sample_data | Connected to 10.1.0.2! +no_mapping_sample_data | Connected to 10.1.0.3! +sample_data | Connected to 10.1.0.1 +sample_data | Connected to 10.1.0.2 +sample_data | Connected to 10.1.0.3 +; + +typeConflictLongNonExistentWhereWithCastToLong +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, colors METADATA _index +| WHERE event_duration::long > 3000000 +| EVAL event_duration = event_duration::keyword +| KEEP _index, event_duration +| SORT _index DESC, event_duration +; + +_index:keyword | event_duration:keyword +sample_data | 3450233 +sample_data | 5033755 +sample_data | 8268153 +; + +typeConflictLongDateUnmappedWhereWithCastToDate +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index +| WHERE @timestamp::date > "2024-01-01T00:00:00.000Z" +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:keyword +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +; + +############################# +# SORT with type casts # +############################# + +typeConflictLongUnmappedSortWithCastToLong +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| SORT event_duration::long, _index +| KEEP _index, event_duration +; + +_index:keyword | event_duration:long +no_mapping_sample_data | null +sample_data | 725448 +no_mapping_sample_data | null +sample_data | 1232382 +no_mapping_sample_data | null +sample_data | 1756467 +no_mapping_sample_data | null +sample_data | 2764889 +no_mapping_sample_data | null +sample_data | 3450233 +no_mapping_sample_data | null +sample_data | 5033755 +no_mapping_sample_data | null +sample_data | 8268153 +; + +typeConflictLongUnmappedSortWithCastToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| SORT event_duration::keyword, _index +| KEEP _index, event_duration +; + +_index:keyword | event_duration:long +no_mapping_sample_data | null +sample_data | 1232382 +no_mapping_sample_data | null +sample_data | 1756467 +no_mapping_sample_data | null +sample_data | 2764889 +no_mapping_sample_data | null +sample_data | 3450233 +no_mapping_sample_data | null +sample_data | 5033755 +no_mapping_sample_data | null +sample_data | 725448 +no_mapping_sample_data | null +sample_data | 8268153 +; + +############################ +# Chained type casts # +############################ + +typeConflictLongUnmappedChainedCastLongToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| EVAL event_duration = event_duration::long::keyword +| KEEP _index, event_duration +| SORT _index, event_duration +; + +_index:keyword | event_duration:keyword +no_mapping_sample_data | 1232381 +no_mapping_sample_data | 1756466 +no_mapping_sample_data | 2764888 +no_mapping_sample_data | 3450232 +no_mapping_sample_data | 5033754 +no_mapping_sample_data | 725447 +no_mapping_sample_data | 8268152 +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 725448 +sample_data | 8268153 +; + +typeConflictLongUnmappedChainedCastLongToDouble +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| EVAL event_duration = event_duration::long::double +| KEEP _index, event_duration +| SORT _index, event_duration +; + +_index:keyword | event_duration:double +no_mapping_sample_data | 725447.0 +no_mapping_sample_data | 1232381.0 +no_mapping_sample_data | 1756466.0 +no_mapping_sample_data | 2764888.0 +no_mapping_sample_data | 3450232.0 +no_mapping_sample_data | 5033754.0 +no_mapping_sample_data | 8268152.0 +sample_data | 725448.0 +sample_data | 1232382.0 +sample_data | 1756467.0 +sample_data | 2764889.0 +sample_data | 3450233.0 +sample_data | 5033755.0 +sample_data | 8268153.0 +; + +typeConflictLongDateUnmappedChainedCastDateToKeyword +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::date::keyword +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:keyword +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_long | 2023-10-23T12:15:03.360Z +sample_data_ts_long | 2023-10-23T12:27:28.948Z +sample_data_ts_long | 2023-10-23T13:33:34.937Z +sample_data_ts_long | 2023-10-23T13:51:54.732Z +sample_data_ts_long | 2023-10-23T13:52:55.015Z +sample_data_ts_long | 2023-10-23T13:53:55.832Z +sample_data_ts_long | 2023-10-23T13:55:01.543Z +; + +typeConflictLongUnmappedChainedCastInWhere +required_capability: optional_fields_v4 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| WHERE event_duration::long::double > 3000000.0 +| EVAL event_duration = event_duration::keyword +| KEEP _index, event_duration +| SORT _index, event_duration +; + +_index:keyword | event_duration:keyword +no_mapping_sample_data | 3450232 +no_mapping_sample_data | 5033754 +no_mapping_sample_data | 8268152 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 8268153 +; + +#################################### +# Mapped and _source excluded # +#################################### + +noTypeConflictKeywordSourceExcludedNoCast +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM sample_data, partial_mapping_no_source_sample_data METADATA _index +| KEEP _index, message +| SORT _index DESC, message +| LIMIT 15 +; + +_index:keyword | message:keyword +sample_data | Connected to 10.1.0.1 +sample_data | Connected to 10.1.0.2 +sample_data | Connected to 10.1.0.3 +sample_data | Connection error +sample_data | Connection error +sample_data | Connection error +sample_data | Disconnected +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +; + +noTypeConflictKeywordSourceExcludedCastToKeyword +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM sample_data, partial_mapping_no_source_sample_data METADATA _index +| EVAL message = message::keyword +| KEEP _index, message +| SORT _index DESC, message +| LIMIT 15 +; + +_index:keyword | message:keyword +sample_data | Connected to 10.1.0.1 +sample_data | Connected to 10.1.0.2 +sample_data | Connected to 10.1.0.3 +sample_data | Connection error +sample_data | Connection error +sample_data | Connection error +sample_data | Disconnected +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +; + +typeConflictLongSourceExcludedCastToKeyword +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM sample_data, partial_mapping_no_source_sample_data METADATA _index +| EVAL event_duration = event_duration::keyword +| KEEP _index, event_duration +| SORT _index DESC, event_duration +| LIMIT 15 +; + +_index:keyword | event_duration:keyword +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 725448 +sample_data | 8268153 +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +; + +typeConflictLongSourceExcludedCastToLong +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM sample_data, partial_mapping_no_source_sample_data METADATA _index +| EVAL event_duration = event_duration::long +| KEEP _index, event_duration +| SORT _index DESC, event_duration +| LIMIT 15 +; + +_index:keyword | event_duration:long +sample_data | 725448 +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 8268153 +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +; + +typeConflictLongSourceExcludedCastToDouble +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM sample_data, partial_mapping_no_source_sample_data METADATA _index +| EVAL event_duration = event_duration::double +| KEEP _index, event_duration +| SORT _index DESC, event_duration +| LIMIT 15 +; + +_index:keyword | event_duration:double +sample_data | 725448.0 +sample_data | 1232382.0 +sample_data | 1756467.0 +sample_data | 2764889.0 +sample_data | 3450233.0 +sample_data | 5033755.0 +sample_data | 8268153.0 +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +; + +typeConflictLongSourceExcludedCastToDoubleNoSort +required_capability: optional_fields_v4 +required_capability: source_field_mapping + +SET unmapped_fields="load"\; +FROM sample_data, partial_mapping_no_source_sample_data METADATA _index +| EVAL event_duration = event_duration::double +| KEEP _index, event_duration +; +ignoreOrder:true + +_index:keyword | event_duration:double +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +partial_mapping_no_source_sample_data | null +sample_data | 725448.0 +sample_data | 1232382.0 +sample_data | 1756467.0 +sample_data | 2764889.0 +sample_data | 3450233.0 +sample_data | 5033755.0 +sample_data | 8268153.0 +; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec index 113e072271b0f..f6e0df5f3d350 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec @@ -459,7 +459,7 @@ total:long | country:keyword ; unmappedFields_Load_with_Count -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -475,7 +475,7 @@ rehired_count:long ; unmappedFields_Load_with_WithNonExistentField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -491,7 +491,7 @@ rehired_count:long | null_count:long ; unmappedFields_Load_with_EvalOnNonExistentField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -509,7 +509,7 @@ c:long ; unmappedFields_Load_with_WhereOnNonExistentField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -526,7 +526,7 @@ c:long ; unmappedFields_Load_with_StatsByNonExistentField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -542,7 +542,7 @@ c:long | nonexistent_field:keyword ; unmappedFields_Load_with_SortByNonExistentField -required_capability: optional_fields_v3 +required_capability: optional_fields_v4 required_capability: views_with_branching required_capability: views_crud_as_index_actions diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java index 52ed907eb12ea..1951ef29f3e81 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/EsqlActionIT.java @@ -2655,4 +2655,50 @@ public void testExplainWithApproximation() { } } } + + /** + * Verifies that unmapped fields are loaded from _source (not replaced with constant nulls) when shards are processed one at a time. + * Reproducer for a bug where single-shard concurrency causes potentiallyUnmappedExpression to be lost during shard-level planning, + * resulting in null values instead of the actual _source values for unmapped fields. + */ + public void testUnmappedFieldsLoadWithSingleShardConcurrency() { + assumeTrue("requires unmapped fields load support", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled()); + assertAcked(prepareCreate("test_mapped").setMapping("event_duration", "type=long")); + assertAcked(prepareCreate("test_unmapped").setMapping(""" + {"dynamic": false, "properties": {}}""")); + + client().prepareBulk() + .setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE) + .add(prepareIndex("test_mapped").setSource(Map.of("event_duration", 100))) + .add(prepareIndex("test_mapped").setSource(Map.of("event_duration", 200))) + .add(prepareIndex("test_unmapped").setSource(Map.of("event_duration", 10))) + .add(prepareIndex("test_unmapped").setSource(Map.of("event_duration", 20))) + .get(); + + var pragmas = new QueryPragmas(Settings.builder().put(QueryPragmas.MAX_CONCURRENT_SHARDS_PER_NODE.getKey(), 1).build()); + try (var resp = run(syncEsqlQueryRequest(""" + SET unmapped_fields="load"; + FROM test_mapped, test_unmapped METADATA _index + | EVAL event_duration = event_duration::long + | KEEP _index, event_duration + | SORT _index, event_duration""").pragmas(pragmas))) { + + assertThat( + resp.columns(), + equalTo(List.of(new ColumnInfoImpl("_index", "keyword", null), new ColumnInfoImpl("event_duration", "long", null))) + ); + + assertThat( + getValuesList(resp), + equalTo( + List.of( + List.of("test_mapped", 100L), + List.of("test_mapped", 200L), + List.of("test_unmapped", 10L), + List.of("test_unmapped", 20L) + ) + ) + ); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index fba170f44fec8..0074f8f9b6146 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -226,7 +226,7 @@ public enum Cap { /** * Support for optional fields (might or might not be present in the mappings) using DEFAULT/NULLIFY only. - * Compared to {@link #OPTIONAL_FIELDS_V3}, this does not enable support for LOAD. + * Compared to {@link #OPTIONAL_FIELDS_V4}, this does not enable support for LOAD. */ OPTIONAL_FIELDS_NULLIFY_TECH_PREVIEW, @@ -251,8 +251,9 @@ public enum Cap { * Support for optional fields (might or might not be present in the mappings) using DEFAULT/NULLIFY/LOAD. * V2: Prevent pushing down filters and sorts to Lucene of potentially unmapped fields. * V3: Fix synthetic _source numeric load bug (#143916) + * V4: Support for union type like resolution for load. */ - OPTIONAL_FIELDS_V3(Build.current().isSnapshot()), + OPTIONAL_FIELDS_V4(Build.current().isSnapshot()), /** * Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently @@ -2375,7 +2376,7 @@ public enum Cap { * Reject loading sub-fields of flattened fields when {@code unmapped_fields="load"} * See https://github.com/elastic/elasticsearch/issues/143494 */ - REJECT_LOADING_FLATTENED_SUBFIELDS(OPTIONAL_FIELDS_V3.isEnabled()), + REJECT_LOADING_FLATTENED_SUBFIELDS(OPTIONAL_FIELDS_V4.isEnabled()), FIX_DIV_ERROR_MESSAGE, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index eed5b18f29e5a..53706fa9837f2 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -14,6 +14,7 @@ import org.elasticsearch.compute.data.AggregateMetricDoubleBlockBuilder; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.Page; +import org.elasticsearch.core.Nullable; import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexMode; import org.elasticsearch.index.mapper.IdFieldMapper; @@ -180,6 +181,7 @@ import java.util.List; import java.util.Map; import java.util.Set; +import java.util.TreeSet; import java.util.function.Function; import java.util.function.Predicate; import java.util.regex.Pattern; @@ -1207,7 +1209,8 @@ private Attribute resolveInsistAttribute(Attribute attribute, List ch // Field is partially unmapped. // TODO: Should the check for partially unmapped fields be done specific to each sub-query in a fork? if (resolvedCol instanceof FieldAttribute fa && indices.stream().anyMatch(r -> r.get().isPartiallyUnmappedField(fa.name()))) { - return fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa); + // NOTE: We use indices.getFirst() here as a temporary solution. INSIST will be removed after load is in GA anyway. + return fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa, indices.getFirst().get()); } // Either the field is mapped everywhere and we can just use the resolved column, or the INSIST clause isn't on top of a FROM @@ -1215,18 +1218,19 @@ private Attribute resolveInsistAttribute(Attribute attribute, List ch return resolvedCol; } - private static FieldAttribute invalidInsistAttribute(FieldAttribute fa) { - var name = fa.name(); - EsField field = fa.field() instanceof InvalidMappedField imf - ? new InvalidMappedField(name, InvalidMappedField.makeErrorsMessageIncludingInsistKeyword(imf.getTypesToIndices())) - : new InvalidMappedField( - name, - Strings.format( - "mapped as [2] incompatible types: [keyword] enforced by INSIST command, and [%s] in index mappings", - fa.dataType().typeName() - ) - ); - return new FieldAttribute(fa.source(), null, fa.qualifier(), name, field); + private static FieldAttribute invalidInsistAttribute(FieldAttribute fa, EsIndex esIndex) { + InvalidMappedField field = InvalidMappedField.potentiallyUnmapped(fa.name(), getTypesToIndices(fa, esIndex)); + return new FieldAttribute(fa.source(), null, fa.qualifier(), fa.name(), field); + } + + private static Map> getTypesToIndices(FieldAttribute fa, EsIndex esIndex) { + if (fa.field() instanceof InvalidMappedField imf) { + return imf.getTypesToIndices(); + } + // Field isn't currently invalid, meaning it's mapped to a single type in all the indices where it's actually mapped. + TreeSet indicesWithField = new TreeSet<>(esIndex.concreteQualifiedIndices()); + indicesWithField.removeAll(esIndex.getUnmappedIndices(fa.name())); + return Map.of(fa.dataType().typeName(), indicesWithField); } public static FieldAttribute insistKeyword(Attribute attribute) { @@ -1248,18 +1252,28 @@ private static LogicalPlan resolvePartiallyMapped(LogicalPlan plan, AnalyzerCont Map insistedMap = new HashMap<>(); var transformed = plan.transformExpressionsOnly(FieldAttribute.class, fa -> { var esField = fa.field(); - var isInsisted = esField instanceof PotentiallyUnmappedKeywordEsField || esField instanceof InvalidMappedField; - if (isInsisted == false) { - var existing = insistedMap.get(fa); - if (existing != null) { // field shows up multiple times in the node; return first processing - return existing; - } - // Field is partially unmapped. - if (indexResolutions.stream().anyMatch(r -> r.get().isPartiallyUnmappedField(fa.name()))) { - FieldAttribute newFA = fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa); - insistedMap.put(fa, newFA); - return newFA; - } + if (esField instanceof PotentiallyUnmappedKeywordEsField + || esField instanceof InvalidMappedField imf && imf.isPotentiallyUnmapped()) { + return fa; + } + var existing = insistedMap.get(fa); + if (existing != null) { // field shows up multiple times in the node; return first processing + return existing; + } + + if (indexResolutions.isEmpty()) { + throw new IllegalStateException("Unmapped fields with empty index resolutions."); + } + if (indexResolutions.size() > 1) { + throw new IllegalStateException( + Strings.format("Multiple index patterns should be disabled with unmapped fields", indexResolutions) + ); + } + EsIndex esIndex = indexResolutions.getFirst().get(); + if (esIndex.isPartiallyUnmappedField(fa.name())) { + FieldAttribute newFA = fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa, esIndex); + insistedMap.put(fa, newFA); + return newFA; } return fa; }); @@ -2452,9 +2466,12 @@ private Expression resolveConvertFunction(ConvertFunction convert, List typeResolutions + Map typeResolutions, + @Nullable Expression potentiallyUnmappedConversion ) { Map typesToConversionExpressions = new HashMap<>(); InvalidMappedField imf = (InvalidMappedField) fa.field(); @@ -2542,7 +2567,8 @@ private static MultiTypeEsField resolvedMultiTypeEsField( typesToConversionExpressions.put(typeName, typeResolutions.get(key)); } }); - return MultiTypeEsField.resolveFrom(imf, typesToConversionExpressions); + return MultiTypeEsField.resolveFrom(imf, typesToConversionExpressions) + .withPotentiallyUnmappedExpression(potentiallyUnmappedConversion); } private static boolean canConvertOriginalTypes(MultiTypeEsField multiTypeEsField, Set supportedTypes) { @@ -2648,11 +2674,18 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { return relation; } return relation.transformExpressionsUp(FieldAttribute.class, f -> { - if (f.field() instanceof InvalidMappedField imf && imf.types().stream().allMatch(DataType::isDate)) { + if (f.field() instanceof InvalidMappedField imf && allDates(context, relation, imf)) { HashMap typeResolutions = new HashMap<>(); var convert = new ToDateNanos(f.source(), f, context.configuration()); imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions)); - var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions); + // This rule runs in the "Initialize" batch, before ResolveUnmapped. The isFieldMappedInAllIndices + // check above should prevent reaching here for fields that are unmapped in any index when in LOAD mode. + if (imf.isPotentiallyUnmapped()) { + throw new IllegalStateException( + "Unexpected potentially unmapped field [" + imf.getName() + "] in DateMillisToNanosInEsRelation" + ); + } + var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions, null); return new FieldAttribute( f.source(), f.parentName(), @@ -2668,6 +2701,20 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { }); }); } + + private static boolean allDates(AnalyzerContext context, EsRelation relation, InvalidMappedField imf) { + if (imf.types().stream().allMatch(DataType::isDate) == false) { + return false; + } + // If we need to load the fields from unmapped indices, we will treat it as a keyword, i.e., not all types are dates. + if (context.unmappedResolution() != UnmappedResolution.LOAD) { + return true; + } + // Since DateMillisToNanosInEsRelation runs before ResolveUnmapped, isPotentiallyUnmapped isn't set yet. + int mappedCount = imf.getTypesToIndices().values().stream().mapToInt(Set::size).sum(); + int totalCount = relation.concreteIndices().values().stream().mapToInt(List::size).sum(); + return mappedCount >= totalCount; + } } private static void typeResolutions( diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java index 8223d83b9b6df..45f3a9c59b3d8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java @@ -22,30 +22,45 @@ import java.util.stream.Collectors; /** - * Representation of field mapped differently across indices. + * Representation of field mapped differently across indices; or being potentially unmapped in some, in which case it is treated as + * {@link DataType#KEYWORD} in the indices where it is unmapped. * Used during mapping discovery only. - * Note that the field typesToIndices is not serialized because that information is + * Note that the fields typesToIndices and isPotentiallyUnmapped are not serialized because that information is * not required through the cluster, only surviving as long as the Analyser phase of query planning. - * It is used specifically for the 'union types' feature in ES|QL. + * It is used specifically for the 'union types' and 'unmapped fields' feature in ES|QL. */ public class InvalidMappedField extends EsField { private final String errorMessage; private final Map> typesToIndices; + private final boolean isPotentiallyUnmapped; public InvalidMappedField(String name, String errorMessage, Map properties) { - this(name, errorMessage, properties, Map.of(), TimeSeriesFieldType.UNKNOWN); + this(name, errorMessage, properties, Map.of(), false, TimeSeriesFieldType.UNKNOWN); } public InvalidMappedField(String name, String errorMessage) { this(name, errorMessage, new TreeMap<>()); } + public InvalidMappedField(String name, Map> typesToIndices) { + this(name, makeErrorMessage(typesToIndices, false), new TreeMap<>(), typesToIndices, false, TimeSeriesFieldType.UNKNOWN); + } + /** - * Constructor supporting union types, used in ES|QL. + * An {@link InvalidMappedField} is potentially unmapped if at least one index does not contain a mapping for the field, and the user + * requested we load the values from {@code _source}. In that case, there is (possibly) an additional type conflict since we treat + * unmapped fields as {@link DataType#KEYWORD}. */ - public InvalidMappedField(String name, Map> typesToIndices) { - this(name, makeErrorMessage(typesToIndices, false), new TreeMap<>(), typesToIndices, TimeSeriesFieldType.UNKNOWN); + public static InvalidMappedField potentiallyUnmapped(String name, Map> typesToIndices) { + return new InvalidMappedField( + name, + makeErrorMessage(typesToIndices, true), + new TreeMap<>(), + typesToIndices, + true, + TimeSeriesFieldType.UNKNOWN + ); } private InvalidMappedField( @@ -53,11 +68,13 @@ private InvalidMappedField( String errorMessage, Map properties, Map> typesToIndices, + boolean isPotentiallyUnmapped, TimeSeriesFieldType type ) { super(name, DataType.UNSUPPORTED, properties, false, type); this.errorMessage = errorMessage; this.typesToIndices = typesToIndices; + this.isPotentiallyUnmapped = isPotentiallyUnmapped; } protected InvalidMappedField(StreamInput in) throws IOException { @@ -66,6 +83,7 @@ protected InvalidMappedField(StreamInput in) throws IOException { in.readString(), in.readImmutableMap(StreamInput::readString, EsField::readFrom), Map.of(), + false, readTimeSeriesFieldType(in) ); } @@ -120,8 +138,8 @@ public Map> getTypesToIndices() { return typesToIndices; } - public static String makeErrorsMessageIncludingInsistKeyword(Map> typesToIndices) { - return makeErrorMessage(typesToIndices, true); + public boolean isPotentiallyUnmapped() { + return isPotentiallyUnmapped; } private static String makeErrorMessage(Map> typesToIndices, boolean includeInsistKeyword) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java index 97303bf74c939..46a2b5afa17c3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java @@ -10,6 +10,8 @@ import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; +import org.elasticsearch.core.Nullable; +import org.elasticsearch.core.Strings; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamOutput; @@ -30,18 +32,30 @@ * type conversion is done at the data node level. */ public class MultiTypeEsField extends EsField { + private static final TransportVersion POTENTIALLY_UNMAPPED_EXPRESSION = TransportVersion.fromName( + "esql_potentially_unmapped_expression" + ); private final Map indexToConversionExpressions; + /** + * If this is not {@code null}, then this expression should be used to convert the field value in case the field is not mapped in an + * index from {@link DataType#KEYWORD} to the target type. + */ + @Nullable + private final Expression potentiallyUnmappedExpression; + public MultiTypeEsField( String name, DataType dataType, boolean aggregatable, Map indexToConversionExpressions, - TimeSeriesFieldType timeSeriesFieldType + TimeSeriesFieldType timeSeriesFieldType, + @Nullable Expression potentiallyUnmappedExpression ) { super(name, dataType, Map.of(), aggregatable, timeSeriesFieldType); this.indexToConversionExpressions = indexToConversionExpressions; + this.potentiallyUnmappedExpression = potentiallyUnmappedExpression; } protected MultiTypeEsField(StreamInput in) throws IOException { @@ -50,7 +64,8 @@ protected MultiTypeEsField(StreamInput in) throws IOException { DataType.readFrom(in), in.readBoolean(), in.readImmutableMap(i -> i.readNamedWriteable(Expression.class)), - readTimeSeriesFieldType(in) + readTimeSeriesFieldType(in), + in.getTransportVersion().supports(POTENTIALLY_UNMAPPED_EXPRESSION) ? in.readOptionalNamedWriteable(Expression.class) : null ); } @@ -61,6 +76,9 @@ public void writeContent(StreamOutput out) throws IOException { out.writeBoolean(isAggregatable()); out.writeMap(getIndexToConversionExpressions(), (o, v) -> out.writeNamedWriteable(v)); writeTimeSeriesFieldType(out); + if (out.getTransportVersion().supports(POTENTIALLY_UNMAPPED_EXPRESSION)) { + out.writeOptionalNamedWriteable(potentiallyUnmappedExpression); + } } public String getWriteableName(TransportVersion transportVersion) { @@ -72,14 +90,29 @@ public String getNodeStringName() { return "MultiTypeEsField"; } + public @Nullable Expression getPotentiallyUnmappedExpression() { + return potentiallyUnmappedExpression; + } + public Map getIndexToConversionExpressions() { return indexToConversionExpressions; } - public Expression getConversionExpressionForIndex(String indexName) { + public @Nullable Expression getConversionExpressionForIndex(String indexName) { return indexToConversionExpressions.get(indexName); } + public MultiTypeEsField withPotentiallyUnmappedExpression(@Nullable Expression potentiallyUnmappedExpression) { + return new MultiTypeEsField( + getName(), + getDataType(), + isAggregatable(), + indexToConversionExpressions, + getTimeSeriesFieldType(), + potentiallyUnmappedExpression + ); + } + public static MultiTypeEsField resolveFrom( InvalidMappedField invalidMappedField, Map typesToConversionExpressions @@ -104,7 +137,8 @@ public static MultiTypeEsField resolveFrom( resolvedDataType, false, indexToConversionExpressions, - invalidMappedField.getTimeSeriesFieldType() + invalidMappedField.getTimeSeriesFieldType(), + null ); } @@ -114,18 +148,20 @@ public boolean equals(Object obj) { return false; } if (obj instanceof MultiTypeEsField other) { - return super.equals(other) && indexToConversionExpressions.equals(other.indexToConversionExpressions); + return super.equals(other) + && indexToConversionExpressions.equals(other.indexToConversionExpressions) + && Objects.equals(potentiallyUnmappedExpression, other.potentiallyUnmappedExpression); } return false; } @Override public int hashCode() { - return Objects.hash(super.hashCode(), indexToConversionExpressions); + return Objects.hash(super.hashCode(), indexToConversionExpressions, potentiallyUnmappedExpression); } @Override public String toString() { - return super.toString() + " (" + indexToConversionExpressions + ")"; + return Strings.format("%s (%s, %s)", super.toString(), indexToConversionExpressions, potentiallyUnmappedExpression); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/SingleFieldFullTextFunction.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/SingleFieldFullTextFunction.java index aa45a44eeff76..fd0899585ac37 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/SingleFieldFullTextFunction.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/function/fulltext/SingleFieldFullTextFunction.java @@ -240,6 +240,6 @@ protected String expectedQueryTypesString() { } static String expectedTypesAsString(Set dataTypes) { - return String.join(", ", dataTypes.stream().map(dt -> dt.name().toLowerCase(Locale.ROOT)).toList()); + return String.join(", ", dataTypes.stream().map(dt -> dt.name().toLowerCase(Locale.ROOT)).sorted().toList()); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java index dcd9a048a9f86..351139095cd46 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/EsIndex.java @@ -9,6 +9,7 @@ import org.elasticsearch.index.IndexMode; import org.elasticsearch.xpack.esql.core.type.EsField; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -19,17 +20,22 @@ public record EsIndex( Map indexNameWithModes, Map> originalIndices, // keyed by cluster alias Map> concreteIndices, // keyed by cluster alias - Set partiallyUnmappedFields + Map> fieldToUnmappedIndices // keyed by field name; Set are concrete index names. ) { public EsIndex { assert name != null; assert mapping != null; - assert partiallyUnmappedFields != null; + assert fieldToUnmappedIndices != null; + assert fieldToUnmappedIndices.values().stream().noneMatch(Set::isEmpty); } public boolean isPartiallyUnmappedField(String fieldName) { - return partiallyUnmappedFields.contains(fieldName); + return fieldToUnmappedIndices.containsKey(fieldName); + } + + public Set getUnmappedIndices(String fieldName) { + return fieldToUnmappedIndices.getOrDefault(fieldName, Collections.emptySet()); } public Set concreteQualifiedIndices() { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java index 1bea2ad837c99..752e95bc1e20f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/index/IndexResolution.java @@ -40,7 +40,7 @@ public static IndexResolution valid(EsIndex index) { } public static IndexResolution empty(String indexPattern) { - return valid(new EsIndex(indexPattern, Map.of(), Map.of(), Map.of(), Map.of(), Set.of())); + return valid(new EsIndex(indexPattern, Map.of(), Map.of(), Map.of(), Map.of(), Map.of())); } public static IndexResolution invalid(String invalid) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java index 27fe30ae590ea..72c97e02fdb88 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/local/ReplaceFieldWithConstantOrNull.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.Literal; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TimeSeriesMetadataAttribute; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.optimizer.LocalLogicalOptimizerContext; import org.elasticsearch.xpack.esql.optimizer.rules.RuleUtils; @@ -80,7 +81,8 @@ else if (esRelation.indexMode() == IndexMode.STANDARD) { // Do not use the attribute name, this can deviate from the field name for union types; use fieldName() instead. // Also retain fields from lookup indices and external sources because we do not have stats for these. Predicate shouldBeRetained = f -> f instanceof TimeSeriesMetadataAttribute - || f.field() instanceof PotentiallyUnmappedKeywordEsField + // We should still attempt to load potentially unmapped fields if they're unmapped; that's the whole point! + || isPotentiallyUnmapped(f) // The source (or doc) field is added to the relation output as a hack to enable late materialization in the reduce driver. || EsQueryExec.isDocAttribute(f) || localLogicalOptimizerContext.searchStats().exists(f.fieldName()) @@ -90,6 +92,11 @@ else if (esRelation.indexMode() == IndexMode.STANDARD) { return plan.transformUp(p -> replaceWithNullOrConstant(p, shouldBeRetained, attrToConstant)); } + private static boolean isPotentiallyUnmapped(FieldAttribute f) { + return f.field() instanceof PotentiallyUnmappedKeywordEsField + || (f.field() instanceof MultiTypeEsField mtf && mtf.getPotentiallyUnmappedExpression() != null); + } + private LogicalPlan replaceWithNullOrConstant( LogicalPlan plan, Predicate shouldBeRetained, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java index 1513dccd8e850..b643264d9e04c 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java @@ -96,12 +96,12 @@ public class QuerySettings { String resolution = Foldables.stringLiteralValueOf(value, "Unexpected value"); try { UnmappedResolution res = UnmappedResolution.valueOf(resolution.toUpperCase(Locale.ROOT)); - if (res == UnmappedResolution.LOAD && EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled() == false) { + if (res == UnmappedResolution.LOAD && EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled() == false) { throw new IllegalArgumentException("'LOAD' is only supported in snapshot builds"); } return res; } catch (Exception exc) { - var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled() + var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled() ? UnmappedResolution.values() : Arrays.stream(UnmappedResolution.values()).filter(e -> e != UnmappedResolution.LOAD).toArray(); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java index c8290438238d2..05a05e32a5cb6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java @@ -86,6 +86,7 @@ import org.elasticsearch.xpack.esql.expression.function.BlockLoaderWarnings; import org.elasticsearch.xpack.esql.expression.function.blockloader.BlockLoaderExpression; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec.Sort; import org.elasticsearch.xpack.esql.plan.physical.EstimatesRowSize; @@ -263,7 +264,13 @@ private ValuesSourceReaderOperator.LoaderAndConverter blockLoaderAndConverter( String indexName = shardContext.ctx.getFullyQualifiedIndex().getName(); Expression conversion = unionTypes.getConversionExpressionForIndex(indexName); if (conversion == null) { - return ValuesSourceReaderOperator.LOAD_CONSTANT_NULLS; + Expression potentiallyUnmapped = unionTypes.getPotentiallyUnmappedExpression(); + if (!(potentiallyUnmapped instanceof AbstractConvertFunction convert)) { + return ValuesSourceReaderOperator.LOAD_CONSTANT_NULLS; + } + fieldName = getFieldName((Attribute) convert.field()); + shardContext = wrapWithUnmappedFieldContext(shardContext, new PotentiallyUnmappedKeywordEsField(fieldName)); + conversion = potentiallyUnmapped; } if (conversion instanceof BlockLoaderExpression ble) { BlockLoaderExpression.PushedBlockLoaderExpression e = ble.tryPushToFieldLoading(SearchStats.EMPTY); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 91f6d142ea075..4040173473255 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -80,8 +80,6 @@ import java.util.stream.Collectors; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.DOC_VALUES; -import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_BOUNDS; -import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.EXTRACT_SPATIAL_CENTROID; import static org.elasticsearch.index.mapper.MappedFieldType.FieldExtractPreference.NONE; import static org.elasticsearch.index.query.QueryBuilders.boolQuery; import static org.elasticsearch.xpack.esql.capabilities.TranslationAware.translatable; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java index 1b91feb12d893..c7bdef0ea14f3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java @@ -1185,7 +1185,7 @@ private IndexResolution checkSingleIndex( Map.of(indexName, IndexMode.LOOKUP), Map.of(), Map.of(), - Set.of() + Map.of() ); return IndexResolution.valid(newIndex, newIndex.concreteQualifiedIndices(), lookupIndexResolution.failures()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 94f92e7fc89f8..1318a7f4e7351 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -50,6 +50,7 @@ import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.stream.Collectors; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; @@ -298,7 +299,8 @@ public static IndexResolution mergedMappings( OriginalIndexExtractor originalIndexExtractor ) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker - int numberOfIndices = fieldsInfo.caps.getIndexResponses().size(); + List indexResponses = fieldsInfo.caps.getIndexResponses(); + int numberOfIndices = indexResponses.size(); if (numberOfIndices == 0) { return allowEmpty ? IndexResolution.empty(indexPattern) : IndexResolution.notFound(indexPattern); } @@ -307,13 +309,15 @@ public static IndexResolution mergedMappings( var collectedFieldCaps = collectFieldCaps(fieldsInfo.caps); Map fieldsCaps = collectedFieldCaps.fieldsCaps; Map indexMappingHashDuplicates = collectedFieldCaps.indexMappingHashDuplicates; + Map> fieldToMappedIndices = collectedFieldCaps.fieldToMappedIndices; + Set allIndexNames = indexResponses.stream().map(FieldCapabilitiesIndexResponse::getIndexName).collect(Collectors.toSet()); // Build hierarchical fields - it's easier to do it in sorted order so the object fields come first. // TODO flattened is simpler - could we get away with that? String[] names = fieldsCaps.keySet().toArray(new String[0]); Arrays.sort(names); Map rootFields = new HashMap<>(); - Set partiallyUnmappedFields = new HashSet<>(); + Map> fieldToUnmappedIndices = new HashMap<>(); for (String name : names) { Map fields = rootFields; String fullName = name; @@ -350,16 +354,17 @@ public static IndexResolution mergedMappings( new HashMap<>() ); fields.put(name, field); - var isPartiallyUnmapped = fcs.size() + indexMappingHashDuplicates.getOrDefault(fieldCap.indexMappingHash, 0) < numberOfIndices; - if (isPartiallyUnmapped) { - partiallyUnmappedFields.add(fullName); + Set unmappedIndices = new TreeSet<>(allIndexNames); + unmappedIndices.removeAll(fieldToMappedIndices.getOrDefault(fullName, Set.of())); + if (unmappedIndices.isEmpty() == false) { + fieldToUnmappedIndices.put(fullName, unmappedIndices); } } boolean allEmpty = true; - Map indexNameWithModes = Maps.newMapWithExpectedSize(fieldsInfo.caps.getIndexResponses().size()); + Map indexNameWithModes = Maps.newMapWithExpectedSize(indexResponses.size()); Map> concreteIndices = Maps.newHashMapWithExpectedSize(8); - for (FieldCapabilitiesIndexResponse ir : fieldsInfo.caps.getIndexResponses()) { + for (FieldCapabilitiesIndexResponse ir : indexResponses) { allEmpty &= ir.get().isEmpty(); indexNameWithModes.put(ir.getIndexName(), ir.getIndexMode()); var parts = RemoteClusterAware.splitIndexName(ir.getIndexName()); @@ -382,7 +387,7 @@ public static IndexResolution mergedMappings( // once all remotes support it (v9.3+) originalIndexExtractor.apply(indexPattern, fieldsInfo.caps), concreteIndices, - partiallyUnmappedFields + fieldToUnmappedIndices ); var failures = EsqlCCSUtils.groupFailuresPerCluster(fieldsInfo.caps.getFailures()); return IndexResolution.valid(index, indexNameWithModes.keySet(), failures); @@ -393,27 +398,31 @@ private record IndexFieldCapabilitiesWithSourceHash(List private record CollectedFieldCaps( Map fieldsCaps, // The map won't contain entries without duplicates, i.e., it's number of occurrences - 1. - Map indexMappingHashDuplicates + Map indexMappingHashDuplicates, + Map> fieldToMappedIndices ) {} private static CollectedFieldCaps collectFieldCaps(FieldCapabilitiesResponse fieldCapsResponse) { Map indexMappingHashToDuplicateCount = new HashMap<>(); Map fieldsCaps = new HashMap<>(); + Map> fieldToMappedIndices = new HashMap<>(); for (FieldCapabilitiesIndexResponse response : fieldCapsResponse.getIndexResponses()) { - if (indexMappingHashToDuplicateCount.compute(response.getIndexMappingHash(), (k, v) -> v == null ? 1 : v + 1) > 1) { - continue; - } + boolean isNew = indexMappingHashToDuplicateCount.compute(response.getIndexMappingHash(), (k, v) -> v == null ? 1 : v + 1) <= 1; + String indexName = response.getIndexName(); for (IndexFieldCapabilities fc : response.get().values()) { if (fc.isMetadatafield()) { // ESQL builds the metadata fields if they are asked for without using the resolution. continue; } - List all = fieldsCaps.computeIfAbsent( - fc.name(), - (_key) -> new IndexFieldCapabilitiesWithSourceHash(new ArrayList<>(), response.getIndexMappingHash()) - ).fieldCapabilities; - all.add(fc); + if (isNew) { + List all = fieldsCaps.computeIfAbsent( + fc.name(), + (_key) -> new IndexFieldCapabilitiesWithSourceHash(new ArrayList<>(), response.getIndexMappingHash()) + ).fieldCapabilities; + all.add(fc); + } + fieldToMappedIndices.computeIfAbsent(fc.name(), k -> new HashSet<>()).add(indexName); } } @@ -427,7 +436,7 @@ private static CollectedFieldCaps collectFieldCaps(FieldCapabilitiesResponse fie } } - return new CollectedFieldCaps(fieldsCaps, indexMappingHashToDuplicateCount); + return new CollectedFieldCaps(fieldsCaps, indexMappingHashToDuplicateCount, fieldToMappedIndices); } private static EsField createField( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java index 7f9a7fb38ca78..7aa64c66520d9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/CsvTests.java @@ -324,6 +324,10 @@ public final void test() throws Throwable { "can't load flattened field values in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.LOAD_FLATTENED_FIELD.capabilityName()) ); + assumeFalseLogging( + "Can't simulate _source loading for unmapped fields in csv tests", + testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.capabilityName()) + ); assumeFalseLogging( "can't use rereank in csv tests", testCase.requiredCapabilities.contains(EsqlCapabilities.Cap.RERANK.capabilityName()) @@ -496,7 +500,7 @@ public static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndexT indexModes, Map.of(), Map.of(), - mergedMappings.partiallyUnmappedFields + mergedMappings.fieldToUnmappedIndices ) ); } @@ -534,10 +538,11 @@ private static Map createMappingForIndex(CsvTestsDataLoader.Tes record MappingPerIndex(String index, Map mapping) {} - record MergedResult(Map mapping, Set partiallyUnmappedFields) {} + record MergedResult(Map mapping, Map> fieldToUnmappedIndices) {} private static MergedResult mergeMappings(List mappingsPerIndex) { int numberOfIndices = mappingsPerIndex.size(); + Set allIndices = mappingsPerIndex.stream().map(MappingPerIndex::index).collect(toSet()); Map> columnNamesToFieldByIndices = new HashMap<>(); for (var mappingPerIndex : mappingsPerIndex) { for (var entry : mappingPerIndex.mapping().entrySet()) { @@ -547,15 +552,19 @@ private static MergedResult mergeMappings(List mappingsPerIndex } } - var partiallyUnmappedFields = columnNamesToFieldByIndices.entrySet() - .stream() - .filter(e -> e.getValue().size() < numberOfIndices) - .map(Map.Entry::getKey) - .collect(toSet()); + Map> fieldToUnmappedIndices = new HashMap<>(); + for (var e : columnNamesToFieldByIndices.entrySet()) { + if (e.getValue().size() < numberOfIndices) { + Set unmappedIndices = allIndices.stream().filter(i -> e.getValue().containsKey(i) == false).collect(toSet()); + if (unmappedIndices.isEmpty() == false) { + fieldToUnmappedIndices.put(e.getKey(), unmappedIndices); + } + } + } var mappings = columnNamesToFieldByIndices.entrySet() .stream() .collect(Collectors.toMap(Map.Entry::getKey, e -> mergeFields(e.getKey(), e.getValue()))); - return new MergedResult(mappings, partiallyUnmappedFields); + return new MergedResult(mappings, fieldToUnmappedIndices); } private static EsField mergeFields(String index, Map columnNameToField) { @@ -726,7 +735,8 @@ public static Map testDa private static TestPhysicalOperationProviders testOperationProviders( FoldContext foldCtx, - Map allDatasets + Map allDatasets, + UnmappedResolution unmappedResolution ) throws Exception { var indexPages = new ArrayList(); for (CsvTestsDataLoader.MultiIndexTestDataset datasets : allDatasets.values()) { @@ -738,7 +748,7 @@ private static TestPhysicalOperationProviders testOperationProviders( ); } } - return TestPhysicalOperationProviders.create(foldCtx, indexPages); + return TestPhysicalOperationProviders.create(foldCtx, indexPages, unmappedResolution); } private ActualResults executePlan(BigArrays bigArrays) throws Exception { @@ -820,7 +830,7 @@ private ActualResults executePlan(BigArrays bigArrays) throws Exception { PlannerSettings.DEFAULTS, EsqlTestUtils.MOCK_TRANSPORT_ACTION_SERVICES ); - TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(foldCtx, testDatasets); + TestPhysicalOperationProviders physicalOperationProviders = testOperationProviders(foldCtx, testDatasets, unmappedResolution); PlainActionFuture listener = new PlainActionFuture<>(); var logicalPlanPreOptimizer = new LogicalPlanPreOptimizer( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index a642dc509781e..7632b5fa8aff3 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -7,6 +7,12 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.TransportVersion; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesIndexResponse; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; +import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities; +import org.elasticsearch.action.fieldcaps.IndexFieldCapabilitiesBuilder; +import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.core.tree.Source; @@ -16,7 +22,9 @@ import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; +import org.elasticsearch.xpack.esql.session.IndexResolver; +import java.nio.charset.StandardCharsets; import java.util.Arrays; import java.util.HashMap; import java.util.LinkedHashMap; @@ -109,8 +117,39 @@ public static IndexResolution indexWithDateDateNanosUnionType() { Map.of("index1", IndexMode.STANDARD, "index2", IndexMode.STANDARD, "index3", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of() + Map.of() ); return IndexResolution.valid(index); } + + public static FieldCapabilitiesIndexResponse fieldCapabilitiesIndexResponse( + String indexName, + Map fields + ) { + String indexMappingHash = new String( + MessageDigests.sha256().digest(fields.toString().getBytes(StandardCharsets.UTF_8)), + StandardCharsets.UTF_8 + ); + return new FieldCapabilitiesIndexResponse(indexName, indexMappingHash, fields, false, IndexMode.STANDARD); + } + + public static Map fieldResponseMap(String fieldName, String type) { + return Map.of(fieldName, new IndexFieldCapabilitiesBuilder(fieldName, type).build()); + } + + public static Map fieldResponseMap(Map fieldTypes) { + Map result = new HashMap<>(); + for (Map.Entry entry : fieldTypes.entrySet()) { + result.putAll(fieldResponseMap(entry.getKey(), entry.getValue())); + } + return result; + } + + public static IndexResolver.FieldsInfo fieldsInfoOnCurrentVersion(FieldCapabilitiesResponse caps) { + return new IndexResolver.FieldsInfo(caps, TransportVersion.current(), false, false, false, false); + } + + public static IndexResolution mergedResolution(String indexPattern, FieldCapabilitiesResponse caps) { + return IndexResolver.mergedMappings(indexPattern, false, fieldsInfoOnCurrentVersion(caps), IndexResolver.DO_NOT_GROUP); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index f92846e2b2c2c..14bbeeafb0f33 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -14,7 +14,6 @@ import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.action.fieldcaps.IndexFieldCapabilities; import org.elasticsearch.action.fieldcaps.IndexFieldCapabilitiesBuilder; -import org.elasticsearch.common.hash.MessageDigests; import org.elasticsearch.common.lucene.BytesRefs; import org.elasticsearch.common.settings.Settings; import org.elasticsearch.index.IndexMode; @@ -121,7 +120,6 @@ import org.elasticsearch.xpack.esql.session.IndexResolver; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.time.Instant; import java.time.Period; import java.util.ArrayList; @@ -160,7 +158,10 @@ import static org.elasticsearch.xpack.esql.TestAnalyzer.loadMapping; import static org.elasticsearch.xpack.esql.analysis.Analyzer.NO_FIELDS; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.TEXT_EMBEDDING_INFERENCE_ID; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.fieldCapabilitiesIndexResponse; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.fieldResponseMap; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.mergedResolution; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.randomInferenceIdOtherThan; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.unresolvedRelation; import static org.elasticsearch.xpack.esql.core.tree.Source.EMPTY; @@ -3342,20 +3343,14 @@ public void testResolveInsist_fieldDoesNotExist_createsUnmappedField() { public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordType_createsUnmappedField() { assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); - IndexResolution resolution = IndexResolver.mergedMappings( - "foo,bar", - false, - fieldsInfoOnCurrentVersion( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("keyword")), - fieldCapabilitiesIndexResponse("bar", Map.of()) - ), - List.of() - ) + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "keyword")), + fieldCapabilitiesIndexResponse("bar", Map.of()) ), - IndexResolver.DO_NOT_GROUP + List.of() ); + IndexResolution resolution = mergedResolution("foo,bar", caps); String query = "FROM foo, bar | INSIST_🐔 message"; var plan = analyzer().addIndex(resolution).query(query); @@ -3369,20 +3364,14 @@ public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordTyp public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeywordAndMissingCast_createsAnInvalidMappedField() { assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); - IndexResolution resolution = IndexResolver.mergedMappings( - "foo,bar", - false, - fieldsInfoOnCurrentVersion( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", Map.of()) - ), - List.of() - ) + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("bar", Map.of()) ), - IndexResolver.DO_NOT_GROUP + List.of() ); + IndexResolution resolution = mergedResolution("foo,bar", caps); var plan = analyzer().addIndex(resolution).query("FROM foo, bar | INSIST_🐔 message"); var limit = as(plan, Limit.class); var insist = as(limit.child(), Insist.class); @@ -3390,28 +3379,22 @@ public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeyword assertThat(attribute.name(), is("message")); String expected = "Cannot use field [message] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] enforced by INSIST command, and [long] in index mappings"; + + "[keyword] enforced by INSIST command, [long] in [foo]"; assertThat(attribute.unresolvedMessage(), is(expected)); } public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyword_createsAnInvalidMappedField() { assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); - IndexResolution resolution = IndexResolver.mergedMappings( - "foo,bar", - false, - fieldsInfoOnCurrentVersion( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", Map.of()) - ), - List.of() - ) + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("bar", fieldResponseMap("message", "date")), + fieldCapabilitiesIndexResponse("bazz", Map.of()) ), - IndexResolver.DO_NOT_GROUP + List.of() ); + IndexResolution resolution = mergedResolution("foo,bar", caps); var plan = analyzer().addIndex(resolution).query("FROM foo, bar | INSIST_🐔 message"); var limit = as(plan, Limit.class); var insist = as(limit.child(), Insist.class); @@ -3422,50 +3405,19 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyw assertThat(attr.unresolvedMessage(), is(expected)); } - public void testResolveInsist_multiIndexSameMapping_fieldIsMapped() { - assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); - - IndexResolution resolution = IndexResolver.mergedMappings( - "foo,bar", - false, - fieldsInfoOnCurrentVersion( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("long")) - ), - List.of() - ) - ), - IndexResolver.DO_NOT_GROUP - ); - var plan = analyzer().addIndex(resolution).query("FROM foo, bar | INSIST_🐔 message"); - var limit = as(plan, Limit.class); - var insist = as(limit.child(), Insist.class); - var attribute = (FieldAttribute) EsqlTestUtils.singleValue(insist.output()); - assertThat(attribute.name(), is("message")); - assertThat(attribute.dataType(), is(DataType.LONG)); - } - public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithKeyword_createsAnInvalidMappedField() { assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); - IndexResolution resolution = IndexResolver.mergedMappings( - "foo,bar", - false, - fieldsInfoOnCurrentVersion( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", messageResponseMap("keyword")), - fieldCapabilitiesIndexResponse("qux", Map.of()) - ), - List.of() - ) + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("bar", fieldResponseMap("message", "date")), + fieldCapabilitiesIndexResponse("bazz", fieldResponseMap("message", "keyword")), + fieldCapabilitiesIndexResponse("qux", Map.of()) ), - IndexResolver.DO_NOT_GROUP + List.of() ); + IndexResolution resolution = mergedResolution("foo,bar", caps); var plan = analyzer().addIndex(resolution).query("FROM foo, bar | INSIST_🐔 message"); var limit = as(plan, Limit.class); var insist = as(limit.child(), Insist.class); @@ -3476,35 +3428,6 @@ public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithKe assertThat(attr.unresolvedMessage(), is(expected)); } - public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesWithCast_castsAreNotSupported() { - assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); - - IndexResolution resolution = IndexResolver.mergedMappings( - "foo,bar", - false, - fieldsInfoOnCurrentVersion( - new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", messageResponseMap("long")), - fieldCapabilitiesIndexResponse("bar", messageResponseMap("date")), - fieldCapabilitiesIndexResponse("bazz", Map.of()) - ), - List.of() - ) - ), - IndexResolver.DO_NOT_GROUP - ); - VerificationException e = expectThrows( - VerificationException.class, - () -> analyzer().addIndex(resolution).query("FROM foo, bar | INSIST_🐔 message | EVAL message = message :: keyword") - ); - // This isn't the most informative error, but it'll do for now. - assertThat( - e.getMessage(), - containsString("EVAL does not support type [unsupported] as the return data type of expression [message]") - ); - } - public void testResolveDenseVector() { FieldCapabilitiesResponse caps = FieldCapabilitiesResponse.builder() .withIndexResponses( @@ -4013,22 +3936,6 @@ public void testFuseError() { """, containsString("FUSE requires a key column, default [_id] column not found")); } - // TODO There's too much boilerplate involved here! We need a better way of creating FieldCapabilitiesResponses from a mapping or index. - private static FieldCapabilitiesIndexResponse fieldCapabilitiesIndexResponse( - String indexName, - Map fields - ) { - String indexMappingHash = new String( - MessageDigests.sha256().digest(fields.toString().getBytes(StandardCharsets.UTF_8)), - StandardCharsets.UTF_8 - ); - return new FieldCapabilitiesIndexResponse(indexName, indexMappingHash, fields, false, IndexMode.STANDARD); - } - - private static Map messageResponseMap(String date) { - return Map.of("message", new IndexFieldCapabilitiesBuilder("message", date).build()); - } - private void assertProjection(LogicalPlan plan, String... names) { var limit = as(plan, Limit.class); assertThat(Expressions.names(limit.output()), contains(names)); @@ -4055,8 +3962,7 @@ private static LogicalPlan analyzeWithEmptyFieldCapsResponse(String query) throw List idxResponses = List.of( new FieldCapabilitiesIndexResponse("idx", "idx", Map.of(), true, IndexMode.STANDARD) ); - IndexResolver.FieldsInfo caps = fieldsInfoOnCurrentVersion(new FieldCapabilitiesResponse(idxResponses, List.of())); - IndexResolution resolution = IndexResolver.mergedMappings("test*", false, caps, IndexResolver.DO_NOT_GROUP); + IndexResolution resolution = mergedResolution("test*", new FieldCapabilitiesResponse(idxResponses, List.of())); return analyzer().addIndex(resolution).query(query); } @@ -4697,7 +4603,7 @@ public void testProjectionForUnionTypeResolution() { Map.of("union_index_1", IndexMode.STANDARD, "union_index_2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of() + Map.of() ); IndexResolution resolution = IndexResolution.valid(index); @@ -4740,7 +4646,7 @@ public void testExplicitRetainOriginalFieldWithCast() { Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of() + Map.of() ); IndexResolution resolution = IndexResolution.valid(index); @@ -5013,7 +4919,7 @@ public void testImplicitCastingForAggregateMetricDouble() { Map.of("k8s", IndexMode.TIME_SERIES, "k8s-downsampled", IndexMode.TIME_SERIES), Map.of(), Map.of(), - Set.of() + Map.of() ); var testAnalyzer = analyzer().addIndex(esIndex); var stddevPlan = testAnalyzer.query(""" diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java index 1850bcefc88a3..a6db44ab182de 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java @@ -651,6 +651,30 @@ public void testMappedToNonKeywordInOneIndexOnly() throws Exception { """); } + public void testTypeConflictMappedAndUnmappedWithCast() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | EVAL event_duration = event_duration::long + | KEEP event_duration + """); + } + + public void testTypeConflictMappedTimesTwoAndUnmapped() throws Exception { + runTests(""" + FROM sample_data_ts_long, sample_data, no_mapping_sample_data + | EVAL ts = @timestamp::date + | KEEP ts + """); + } + + public void testNoTypeConflictKeywordAndUnmappedWhere() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | WHERE message::keyword LIKE "Connected*" + | KEEP message + """); + } + public void testForkBranchesAfterStats1stBranch() throws Exception { runTestsNullifyOnly(""" FROM employees diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index b350cceef8b9d..3d74312757335 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -7,18 +7,23 @@ package org.elasticsearch.xpack.esql.analysis; +import org.elasticsearch.action.fieldcaps.FieldCapabilitiesResponse; import org.elasticsearch.common.collect.Iterators; import org.elasticsearch.index.IndexMode; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.TestAnalyzer; +import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Filter; @@ -31,11 +36,14 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; import static org.elasticsearch.xpack.esql.EsqlTestUtils.analyzer; import static org.elasticsearch.xpack.esql.EsqlTestUtils.as; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.fieldCapabilitiesIndexResponse; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.fieldResponseMap; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexResolutions; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.mergedResolution; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTests.withInlinestatsWarning; import static org.hamcrest.Matchers.allOf; import static org.hamcrest.Matchers.containsString; @@ -46,6 +54,16 @@ // @TestLogging(value = "org.elasticsearch.xpack.esql:TRACE", reason = "debug") public class AnalyzerUnmappedTests extends ESTestCase { + + /** + * Query suffixes that use the unsupported type-conflict field [message] in different commands. + * Each type conflict test iterates over these to verify the error is raised regardless of how the field is used. + */ + private static final String[] TYPE_CONFLICT_QUERY_SUFFIXES = new String[] { + "| SORT message", + "| EVAL x = message", + "| WHERE message IS NOT NULL" }; + public void testFailKeepAndNonMatchingStar() { assertUnmappedFailure(test(), """ FROM test @@ -590,6 +608,114 @@ public void testLoadModeDisallowsBranchingViewEquivalentWithUnmappedField() { ); } + public void testTypeConflictLongUnmappedNoCast() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("bar", Map.of()) + ), + List.of() + ); + var resolutions = indexResolutions(mergedResolution("foo,bar", caps)); + for (String suffix : TYPE_CONFLICT_QUERY_SUFFIXES) { + typeConflictVerificationFailure(setUnmappedLoad("FROM foo, bar " + suffix), resolutions); + } + } + + public void testTypeConflictLongKeywordUnmappedNoCast() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("bar", fieldResponseMap("message", "keyword")), + fieldCapabilitiesIndexResponse("baz", Map.of()) + ), + List.of() + ); + var resolutions = indexResolutions(mergedResolution("foo,bar,baz", caps)); + for (String suffix : TYPE_CONFLICT_QUERY_SUFFIXES) { + typeConflictVerificationFailure(setUnmappedLoad("FROM foo, bar, baz " + suffix), resolutions); + } + } + + public void testTypeConflictLongIntUnmappedNoCast() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("bar", fieldResponseMap("message", "integer")), + fieldCapabilitiesIndexResponse("baz", Map.of()) + ), + List.of() + ); + var resolutions = indexResolutions(mergedResolution("foo,bar,baz", caps)); + for (String suffix : TYPE_CONFLICT_QUERY_SUFFIXES) { + typeConflictVerificationFailure(setUnmappedLoad("FROM foo, bar, baz " + suffix), resolutions); + } + } + + public void testTypeConflictTextUnmappedNoCast() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "text")), + fieldCapabilitiesIndexResponse("bar", Map.of()) + ), + List.of() + ); + var resolutions = indexResolutions(mergedResolution("foo,bar", caps)); + for (String suffix : TYPE_CONFLICT_QUERY_SUFFIXES) { + typeConflictVerificationFailure(setUnmappedLoad("FROM foo, bar " + suffix), resolutions); + } + } + + public void testSameMappingHashNotPartiallyUnmapped() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("bar", fieldResponseMap("message", "long")) + ), + List.of() + ); + var resolutions = indexResolutions(mergedResolution("foo,bar", caps)); + TestAnalyzer ta = analyzer(); + for (var entry : resolutions.entrySet()) { + ta.addIndex(entry.getKey().indexPattern(), entry.getValue()); + } + var plan = ta.statement(setUnmappedLoad("FROM foo, bar | EVAL x = message + 1")); + var limit = as(plan, Limit.class); + var eval = as(limit.child(), org.elasticsearch.xpack.esql.plan.logical.Eval.class); + var attr = eval.output().stream().filter(a -> a.name().equals("message")).findFirst().orElseThrow(); + assertThat(attr.dataType(), is(DataType.LONG)); + } + + public void testSameMappingHashWithUnmappedIndex() { + assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); + + FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( + List.of( + fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("bar", fieldResponseMap("message", "long")), + fieldCapabilitiesIndexResponse("baz", Map.of()) + ), + List.of() + ); + var resolutions = indexResolutions(mergedResolution("foo,bar,baz", caps)); + TestAnalyzer ta = analyzer(); + for (var entry : resolutions.entrySet()) { + ta.addIndex(entry.getKey().indexPattern(), entry.getValue()); + } + var e = expectThrows(VerificationException.class, () -> ta.statement(setUnmappedLoad("FROM foo, bar, baz | SORT message"))); + assertThat(e.getMessage(), allOf(containsString("Cannot use field [message]"), containsString("[long] in [bar, foo]"))); + } + private static final String UNMAPPED_TIMESTAMP_SUFFIX = UnresolvedTimestamp.UNRESOLVED_SUFFIX + Verifier.UNMAPPED_TIMESTAMP_SUFFIX; public void testTbucketWithUnmappedTimestamp() { @@ -897,20 +1023,29 @@ private static TestAnalyzer test() { private static TestAnalyzer index1() { Map mapping = Map.of("field", new UnsupportedEsField("field", List.of("flattened"))); - return analyzer().addIndex(new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Set.of())); + return analyzer().addIndex(new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Map.of())); } private static void assertUnmappedLoadError(TestAnalyzer analyzer, String query, Matcher matcher) { analyzer.statementError(setUnmappedLoad(query), matcher); } + private void typeConflictVerificationFailure(String statement, Map indexResolutions) { + TestAnalyzer ta = analyzer(); + for (var entry : indexResolutions.entrySet()) { + ta.addIndex(entry.getKey().indexPattern(), entry.getValue()); + } + var e = expectThrows(VerificationException.class, () -> ta.statement(statement)); + assertThat(e.getMessage(), containsString("Cannot use field [message]")); + } + private static String setUnmappedNullify(String query) { assumeTrue("Requires OPTIONAL_FIELDS_NULLIFY_TECH_PREVIEW", EsqlCapabilities.Cap.OPTIONAL_FIELDS_NULLIFY_TECH_PREVIEW.isEnabled()); return "SET unmapped_fields=\"nullify\"; " + query; } private static String setUnmappedLoad(String query) { - assumeTrue("Requires OPTIONAL_FIELDS_V3", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled()); + assumeTrue("Requires OPTIONAL_FIELDS_V4", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled()); return "SET unmapped_fields=\"load\"; " + query; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java index 9e10fa2eb2cc9..2a686260e804a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/index/EsIndexGenerator.java @@ -15,7 +15,6 @@ import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import static org.elasticsearch.core.Tuple.tuple; import static org.elasticsearch.test.ESTestCase.randomFrom; @@ -26,19 +25,19 @@ public class EsIndexGenerator { public static EsIndex esIndex(String name) { - return new EsIndex(name, Map.of(), Map.of(), Map.of(), Map.of(), Set.of()); + return new EsIndex(name, Map.of(), Map.of(), Map.of(), Map.of(), Map.of()); } public static EsIndex esIndex(String name, Map mapping) { - return new EsIndex(name, mapping, Map.of(), Map.of(), Map.of(), Set.of()); + return new EsIndex(name, mapping, Map.of(), Map.of(), Map.of(), Map.of()); } public static EsIndex esIndex(String name, Map mapping, Map indexNameWithModes) { - return new EsIndex(name, mapping, indexNameWithModes, Map.of(), Map.of(), Set.of()); + return new EsIndex(name, mapping, indexNameWithModes, Map.of(), Map.of(), Map.of()); } public static EsIndex randomEsIndex() { - return new EsIndex(randomIdentifier(), randomMapping(), randomIndexNameWithModes(), Map.of(), Map.of(), Set.of()); + return new EsIndex(randomIdentifier(), randomMapping(), randomIndexNameWithModes(), Map.of(), Map.of(), Map.of()); } public static Map randomMapping() { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java index 09aed790cfc35..37872589e6819 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/AbstractLogicalPlanOptimizerTests.java @@ -118,7 +118,7 @@ protected static TestAnalyzer multiIndexAnalyzer() { Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of("partial_type_keyword") + Map.of("partial_type_keyword", Set.of("test2")) ); return analyzerWithEnrichPolicies().addIndex(multiIndex); } @@ -151,7 +151,7 @@ protected static TestAnalyzer unionIndexAnalyzer() { Map.of("union_types_index", IndexMode.STANDARD, "union_types_index_incompatible", IndexMode.STANDARD), Map.of("", List.of("union_types_index*")), Map.of("", List.of("union_types_index_incompatible", "union_types_index")), - Set.of() + Map.of() ); return analyzerWithEnrichPolicies().addAnalysisTestsInferenceResolution() .addIndex(unionIndex) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java index 46f1072b2e5be..ba87fa103ce05 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LogicalPlanOptimizerTests.java @@ -162,7 +162,6 @@ import java.util.List; import java.util.Locale; import java.util.Map; -import java.util.Set; import java.util.function.BiFunction; import java.util.function.Function; import java.util.stream.Collectors; @@ -10677,7 +10676,7 @@ public void testTsWildcardStatsWithMixedIndexModes() { Map.of("ts_index", IndexMode.TIME_SERIES, "standard_index", IndexMode.STANDARD), Map.of(), Map.of(), - Set.of() + Map.of() ); var testAnalyzer = EsqlTestUtils.analyzer().addIndex(IndexResolution.valid(mixedIndex)); var plan = logicalOptimizerWithLatestVersion.optimize(testAnalyzer.query("TS * | STATS count(events_received)")); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java index f2fa46ed8a90c..4641abdb2c1fe 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java @@ -41,7 +41,7 @@ private Optional tryRunTestsNullifyOnly(String query, EnumSet } private Optional tryRunTestsLoadOnly(String query, EnumSet stages, String... nestedPaths) { - return EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled() + return EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled() ? builder(setUnmappedLoad(query)).nestedPath(ArrayUtils.prepend("load", nestedPaths)).stages(stages).tryRun() : Optional.empty(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java index ce2315027aaba..818aca9314c48 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java @@ -199,7 +199,7 @@ private Object settingValue(EsqlStatement query, int position) { } public void testSetUnmappedFields_snapshot() { - assumeTrue("OPTIONAL_FIELDS_V3 option required", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled()); + assumeTrue("OPTIONAL_FIELDS_V4 option required", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled()); var modes = List.of("DEFAULT", "NULLIFY", "LOAD"); verifySetUnmappedFields(modes); @@ -236,7 +236,7 @@ public void testSetUnmappedFieldsWrongValue() { v -> Arrays.stream(UnmappedResolution.values()).anyMatch(x -> x.name().equalsIgnoreCase(v)), () -> randomAlphaOfLengthBetween(0, 10) ); - var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled() + var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled() ? UnmappedResolution.values() : Arrays.stream(UnmappedResolution.values()).filter(e -> e != UnmappedResolution.LOAD).toArray(); expectValidationError( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java index 6c4a18521ef1e..ef45c11504310 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java @@ -114,7 +114,7 @@ public void testValidate_UnmappedFields_techPreview() { } public void testValidate_UnmappedFields_allValues() { - assumeTrue("Requires unmapped fields", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled()); + assumeTrue("Requires unmapped fields", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V4.isEnabled()); validateUnmappedFields("DEFAULT", "NULLIFY", "LOAD"); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 297c5f042c69c..4cce7aa85e9b7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -45,6 +45,7 @@ import org.elasticsearch.plugins.scanners.StablePluginsRegistry; import org.elasticsearch.xpack.cluster.routing.allocation.mapper.DataTierFieldMapper; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; +import org.elasticsearch.xpack.esql.analysis.UnmappedResolution; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; @@ -81,14 +82,25 @@ public class TestPhysicalOperationProviders extends AbstractPhysicalOperationProviders { private final List indexPages; + private final UnmappedResolution unmappedResolution; - private TestPhysicalOperationProviders(FoldContext foldContext, List indexPages, AnalysisRegistry analysisRegistry) { + private TestPhysicalOperationProviders( + FoldContext foldContext, + List indexPages, + UnmappedResolution unmappedResolution, + AnalysisRegistry analysisRegistry + ) { super(foldContext, analysisRegistry); this.indexPages = indexPages; + this.unmappedResolution = unmappedResolution; } - public static TestPhysicalOperationProviders create(FoldContext foldContext, List indexPages) throws IOException { - return new TestPhysicalOperationProviders(foldContext, indexPages, createAnalysisRegistry()); + public static TestPhysicalOperationProviders create( + FoldContext foldContext, + List indexPages, + UnmappedResolution unmappedResolution + ) throws IOException { + return new TestPhysicalOperationProviders(foldContext, indexPages, unmappedResolution, createAnalysisRegistry()); } public record IndexPage(String index, Page page, List columnNames, Set mappedFields) { @@ -301,11 +313,12 @@ private BiFunction getBlockExtraction(DriverCo } } return (indexDoc, blockCopier) -> switch (extractBlockForSingleDoc(indexDoc, attribute.name(), blockCopier)) { - case BlockResultMissing missing -> throw new EsqlIllegalArgumentException( - "Cannot find column named [{}] in {}", - missing.columnName, - missing.columnNames - ); + case BlockResultMissing missing -> { + if (unmappedResolution == UnmappedResolution.NULLIFY) { + yield getNullsBlock(indexDoc); + } + throw new EsqlIllegalArgumentException("Cannot find column named [{}] in {}", missing.columnName, missing.columnNames); + } case BlockResultSuccess success -> success.block; }; } @@ -316,13 +329,18 @@ private Block getBlockForMultiType( MultiTypeEsField multiTypeEsField, TestBlockCopier blockCopier ) { - var conversion = (AbstractConvertFunction) multiTypeEsField.getConversionExpressionForIndex(getIndexPage(indexDoc).index); + var conversion = getConversion(multiTypeEsField, getIndexPage(indexDoc)); if (conversion == null) { return getNullsBlock(indexDoc); } return switch (extractBlockForSingleDoc(indexDoc, ((FieldAttribute) conversion.field()).fieldName().string(), blockCopier)) { case BlockResultMissing unused -> getNullsBlock(indexDoc); case BlockResultSuccess success -> { + if (success.block.elementType() != PlannerUtils.toElementType(conversion.field().dataType().widenSmallNumeric()) + && success.block.elementType() == PlannerUtils.toElementType(conversion.dataType().widenSmallNumeric())) { + // Block is already in the correct type, we can skip the conversion. + yield success.block; + } try (var converter = new TypeConverter(conversion).build(context)) { yield converter.convert(success.block); } @@ -330,6 +348,15 @@ private Block getBlockForMultiType( }; } + @Nullable + private static AbstractConvertFunction getConversion(MultiTypeEsField multiTypeEsField, IndexPage indexPage) { + var conversion = (AbstractConvertFunction) multiTypeEsField.getConversionExpressionForIndex(indexPage.index); + boolean isPotentiallyUnmapped = conversion == null + && multiTypeEsField.getPotentiallyUnmappedExpression() != null + && indexPage.mappedFields().contains(multiTypeEsField.getName()) == false; + return isPotentiallyUnmapped ? (AbstractConvertFunction) multiTypeEsField.getPotentiallyUnmappedExpression() : conversion; + } + private IndexPage getIndexPage(DocBlock indexDoc) { return indexPages.get(indexDoc.asVector().shards().getInt(0)); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java index 4529eb495acaf..e4fe0177c736e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/MultiTypeEsFieldTests.java @@ -64,8 +64,11 @@ protected MultiTypeEsField createTestInstance() { DataType dataType = randomFrom(types()); DataType toType = toString ? DataType.KEYWORD : dataType; Map indexToConvertExpressions = randomConvertExpressions(name, toString, dataType); + Expression potentiallyUnmappedExpression = randomBoolean() ? null : createToString(name, dataType); + EsField.TimeSeriesFieldType tsType = randomFrom(EsField.TimeSeriesFieldType.values()); - return new MultiTypeEsField(name, toType, false, indexToConvertExpressions, tsType); + + return new MultiTypeEsField(name, toType, false, indexToConvertExpressions, tsType, potentiallyUnmappedExpression); } @Override @@ -74,14 +77,16 @@ protected MultiTypeEsField mutateInstance(MultiTypeEsField instance) throws IOEx DataType dataType = instance.getDataType(); Map indexToConvertExpressions = instance.getIndexToConversionExpressions(); EsField.TimeSeriesFieldType tsType = instance.getTimeSeriesFieldType(); - switch (between(0, 3)) { + Expression potentiallyUnmappedExpression = instance.getPotentiallyUnmappedExpression(); + switch (between(0, 4)) { case 0 -> name = randomAlphaOfLength(name.length() + 1); case 1 -> dataType = randomValueOtherThan(dataType, () -> randomFrom(DataType.types())); case 2 -> indexToConvertExpressions = mutateConvertExpressions(name, dataType, indexToConvertExpressions); case 3 -> tsType = randomValueOtherThan(tsType, () -> randomFrom(EsField.TimeSeriesFieldType.values())); + case 4 -> potentiallyUnmappedExpression = potentiallyUnmappedExpression != null ? null : createToString(name, dataType); default -> throw new IllegalArgumentException(); } - return new MultiTypeEsField(name, dataType, false, indexToConvertExpressions, tsType); + return new MultiTypeEsField(name, dataType, false, indexToConvertExpressions, tsType, potentiallyUnmappedExpression); } @Override @@ -94,11 +99,8 @@ protected final NamedWriteableRegistry getNamedWriteableRegistry() { private Map randomConvertExpressions(String name, boolean toString, DataType dataType) { Map indexToConvertExpressions = new HashMap<>(); if (toString) { - indexToConvertExpressions.put(randomAlphaOfLength(4), new ToString(Source.EMPTY, fieldAttribute(name, dataType), config())); - indexToConvertExpressions.put( - randomAlphaOfLength(4), - new ToString(Source.EMPTY, fieldAttribute(name, DataType.KEYWORD), config()) - ); + indexToConvertExpressions.put(randomAlphaOfLength(4), createToString(name, dataType)); + indexToConvertExpressions.put(randomAlphaOfLength(4), createToString(name, DataType.KEYWORD)); } else { indexToConvertExpressions.put(randomAlphaOfLength(4), testConvertExpression(name, DataType.KEYWORD, dataType)); indexToConvertExpressions.put(randomAlphaOfLength(4), testConvertExpression(name, dataType, dataType)); @@ -157,4 +159,8 @@ private Expression testConvertExpression(String name, DataType fromType, DataTyp private static FieldAttribute fieldAttribute(String name, DataType dataType) { return new FieldAttribute(Source.EMPTY, name, new EsField(name, dataType, Map.of(), true, EsField.TimeSeriesFieldType.NONE)); } + + private ToString createToString(String name, DataType dataType) { + return new ToString(Source.EMPTY, fieldAttribute(name, dataType), config()); + } } diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected new file mode 100644 index 0000000000000..1de4ceb4f12be --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/analysis.expected @@ -0,0 +1,4 @@ +Limit[1000[INTEGER],false,false] +\_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0]] + \_Filter[LIKE(TOSTRING(message{f(PotentiallyUnmappedKeywordEsField)}#0), "Connected*", false)] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/query.esql new file mode 100644 index 0000000000000..f101e34ada0eb --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/load/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| WHERE message::keyword LIKE "Connected*" +| KEEP message diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/nullify/analysis.expected new file mode 100644 index 0000000000000..7a3a5b952a724 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/nullify/analysis.expected @@ -0,0 +1,4 @@ +Limit[1000[INTEGER],false,false] +\_Project[[message{f}#0]] + \_Filter[LIKE(TOSTRING(message{f}#0), "Connected*", false)] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/nullify/query.esql new file mode 100644 index 0000000000000..05a5e51d2255b --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNoTypeConflictKeywordAndUnmappedWhere/nullify/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| WHERE message::keyword LIKE "Connected*" +| KEEP message diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected new file mode 100644 index 0000000000000..fba946502a112 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected @@ -0,0 +1,4 @@ +Limit[1000[INTEGER],false,false] +\_Project[[event_duration{r}#0]] + \_Eval[[$$event_duration$converted_to$long{f(MultiTypeEsField)$}#1 AS event_duration#0]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f}#5, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/query.esql new file mode 100644 index 0000000000000..b8c78ed145ba9 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| EVAL event_duration = event_duration::long +| KEEP event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/nullify/analysis.expected new file mode 100644 index 0000000000000..a58bbac2e73d4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/nullify/analysis.expected @@ -0,0 +1,4 @@ +Limit[1000[INTEGER],false,false] +\_Project[[event_duration{r}#0]] + \_Eval[[TOLONG(event_duration{f}#1) AS event_duration#0]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#1, message{f}#4] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/nullify/query.esql new file mode 100644 index 0000000000000..81beb6a23a976 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/nullify/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| EVAL event_duration = event_duration::long +| KEEP event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected new file mode 100644 index 0000000000000..4160e1709b996 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected @@ -0,0 +1,4 @@ +Limit[1000[INTEGER],false,false] +\_Project[[ts{r}#0]] + \_Eval[[$$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1 AS ts#0]] + \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, client_ip{f}#2, event_duration{f}#3, message{f}#4, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/query.esql new file mode 100644 index 0000000000000..7d2c51e708ad1 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="load"; FROM sample_data_ts_long, sample_data, no_mapping_sample_data +| EVAL ts = @timestamp::date +| KEEP ts diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/nullify/analysis.expected new file mode 100644 index 0000000000000..4160e1709b996 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/nullify/analysis.expected @@ -0,0 +1,4 @@ +Limit[1000[INTEGER],false,false] +\_Project[[ts{r}#0]] + \_Eval[[$$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1 AS ts#0]] + \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, client_ip{f}#2, event_duration{f}#3, message{f}#4, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/nullify/query.esql new file mode 100644 index 0000000000000..f9a3756c3bb8e --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/nullify/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="nullify"; FROM sample_data_ts_long, sample_data, no_mapping_sample_data +| EVAL ts = @timestamp::date +| KEEP ts diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml index 8a3532b5fc3c0..73c375084390c 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml @@ -120,7 +120,7 @@ setup: - match: { error.type: verification_exception } - contains: { error.reason: 'Found 1 problem' } - - contains: { error.reason: 'line 1:56: Cannot use field [foo.bar] due to ambiguities being mapped as [2] incompatible types: [keyword] enforced by INSIST command, and [unsupported] in index mappings' } + - contains: { error.reason: 'line 1:56: Cannot use field [foo.bar] due to ambiguities being mapped as [2] incompatible types: [keyword] enforced by INSIST command, [unsupported] in [index4]' } --- 'loading from two indices with different mappings 3':