diff --git a/docs/changelog/144228.yaml b/docs/changelog/144228.yaml new file mode 100644 index 0000000000000..3bfcadb4184b0 --- /dev/null +++ b/docs/changelog/144228.yaml @@ -0,0 +1,8 @@ +area: ES|QL +issues: + - 145206 + - 141994 +pr: 144228 +summary: "Make unmapped_fields=\"load\" automatically trigger for partially mapped\ + \ KEYWORD fields, and allow projecting partially mapped non-KEYWORD fields" +type: bug diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java index e0b58b5dc132c..fbc66c83701ff 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestUtils.java @@ -413,7 +413,8 @@ public void close() { for (int i = 0; i < entries.length; i++) { String[] header = entries[i].split(":"); String name = header[0].trim(); - String typeName = (typeMapping != null && typeMapping.containsKey(name)) ? typeMapping.get(name) + String typeName = (typeMapping != null && typeMapping.containsKey(name) && typeMapping.get(name) != null) + ? typeMapping.get(name) : header.length > 1 ? header[1].trim() : null; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 397d5b1bc2e4a..1b91119f9852f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -44,6 +44,7 @@ import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.Collection; +import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Locale; @@ -89,6 +90,17 @@ public class CsvTestsDataLoader { new TestDataset("employees", "mapping-default.json", "employees.csv").noSubfields(), new TestDataset("voyager", "mapping-voyager.json", "voyager.csv").noSubfields(), new TestDataset("employees_incompatible", "mapping-default-incompatible.json", "employees_incompatible.csv").noSubfields(), + new TestDataset("employees", "mapping-default.json", "employees.csv").withIndex("employees_no_names") + .withTypeMapping(removeFields("first_name", "last_name")) + .withDynamic("false") + .noSubfields(), + new TestDataset("employees", "mapping-default.json", "employees.csv").withIndex("employees_gender_text") + .withTypeMapping(Map.of("gender", "text")) + .noSubfields(), + new TestDataset("employees", "mapping-default.json", "employees.csv").withIndex("employees_no_gender") + .withTypeMapping(removeFields("gender")) + .withDynamic("false") + .noSubfields(), new TestDataset("all_types", "mapping-all-types.json", "all-types.csv"), new TestDataset("hosts"), new TestDataset("apps"), @@ -110,9 +122,7 @@ public class CsvTestsDataLoader { new TestDataset("sample_data"), new TestDataset("sample_data").withIndex("cloned_sample_data"), new TestDataset("partial_mapping_sample_data"), - new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv").withTypeMapping( - Stream.of("timestamp", "client_ip", "event_duration").collect(toMap(k -> k, k -> "keyword")) - ), + new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv"), new TestDataset( "partial_mapping_no_source_sample_data", "mapping-partial_mapping_no_source_sample_data.json", @@ -173,6 +183,30 @@ public class CsvTestsDataLoader { new TestDataset("k8s-downsampled", "k8s-downsampled-mappings.json", "k8s-downsampled.csv", "k8s-downsampled-settings.json"), new TestDataset("distances"), new TestDataset("addresses"), + new TestDataset("addresses").withIndex("addresses_no_continent") + .withTypeMapping(removeFields("city.country.continent")) + .withDynamic("false"), + new TestDataset("addresses").withIndex("addresses_text") + .withTypeMapping( + Map.of( + "street", + "text", + "number", + "text", + "zip_code", + "text", + "city.name", + "text", + "city.country.name", + "text", + "city.country.continent.name", + "text", + "city.country.continent.planet.name", + "text", + "city.country.continent.planet.galaxy", + "text" + ) + ), new TestDataset("books").withSetting("books-settings.json"), new TestDataset("semantic_text").withInferenceEndpoints("test_sparse_inference", "test_dense_inference"), new TestDataset("logs"), @@ -781,25 +815,63 @@ private static void load(RestClient client, TestDataset dataset, IndexCreator in } } + /** + * Creates a type mapping that removes the given fields from the mapping. + * For use with {@link TestDataset#withTypeMapping}. + */ + private static Map removeFields(String... fields) { + var map = new HashMap(); + for (String field : fields) { + map.put(field, null); + } + return map; + } + public static String readMappingFile(TestDataset dataset) throws IOException { String mappingJsonText = dataset.loadMappings(); - if (dataset.typeMapping == null || dataset.typeMapping.isEmpty()) { + boolean hasTypeMappingOverrides = dataset.typeMapping != null && dataset.typeMapping.isEmpty() == false; + if (hasTypeMappingOverrides == false && dataset.dynamic == null) { return mappingJsonText; } boolean modified = false; ObjectMapper mapper = new ObjectMapper(); JsonNode mappingNode = mapper.readTree(mappingJsonText); - JsonNode propertiesNode = mappingNode.path("properties"); - for (Map.Entry entry : dataset.typeMapping.entrySet()) { - String key = entry.getKey(); - String newType = entry.getValue(); + if (hasTypeMappingOverrides) { + for (Map.Entry entry : dataset.typeMapping.entrySet()) { + String key = entry.getKey(); + String newType = entry.getValue(); + + // Navigate dotted paths to find the parent properties node and leaf field name. + String[] segments = key.split("\\."); + ObjectNode propertiesNode = (ObjectNode) mappingNode.path("properties"); + for (int i = 0; i < segments.length - 1 && propertiesNode != null; i++) { + JsonNode child = propertiesNode.get(segments[i]); + propertiesNode = child != null ? (ObjectNode) child.path("properties") : null; + } + String leafName = segments[segments.length - 1]; - if (propertiesNode.has(key)) { - modified = true; - ((ObjectNode) propertiesNode.get(key)).put("type", newType); + if (propertiesNode == null) { + continue; + } + if (newType == null) { + // null value means remove the field from the mapping + if (propertiesNode.has(leafName)) { + propertiesNode.remove(leafName); + modified = true; + } + } else if (propertiesNode.has(leafName)) { + ((ObjectNode) propertiesNode.get(leafName)).put("type", newType); + modified = true; + } } } + + if (dataset.dynamic != null) { + ((ObjectNode) mappingNode).put("dynamic", dataset.dynamic); + modified = true; + } + if (modified) { return mapper.writerWithDefaultPrettyPrinter().writeValueAsString(mappingNode); } @@ -1084,22 +1156,23 @@ public record TestDataset( String dataFileName, String settingFileName, boolean allowSubFields, - @Nullable Map typeMapping, // Override mappings read from mappings file - @Nullable Map dynamicTypeMapping, // Define mappings not in the mapping files, but available from field-caps + @Nullable Map typeMapping, + @Nullable Map dynamicTypeMapping, + @Nullable String dynamic, List inferenceEndpoints, List requiredCapabilities ) { public TestDataset(String indexName) { - this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, null, List.of(), List.of()); + this(indexName, "mapping-" + indexName + ".json", indexName + ".csv", null, true, null, null, null, List.of(), List.of()); } public TestDataset(String indexName, String mappingFileName, String dataFileName) { - this(indexName, mappingFileName, dataFileName, null, true, null, null, List.of(), List.of()); + this(indexName, mappingFileName, dataFileName, null, true, null, null, null, List.of(), List.of()); } public TestDataset(String indexName, String mappingFileName, String dataFileName, String settingFileName) { - this(indexName, mappingFileName, dataFileName, settingFileName, true, null, null, List.of(), List.of()); + this(indexName, mappingFileName, dataFileName, settingFileName, true, null, null, null, List.of(), List.of()); } public TestDataset withIndex(String indexName) { @@ -1111,6 +1184,7 @@ public TestDataset withIndex(String indexName) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1125,6 +1199,7 @@ public TestDataset withData(String dataFileName) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1139,6 +1214,7 @@ public TestDataset noData() { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1153,6 +1229,7 @@ public TestDataset withSetting(String settingFileName) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1167,11 +1244,20 @@ public TestDataset noSubfields() { false, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); } + /** + * Overrides the types of fields in the mapping file. Each entry maps a field name to its new type + * (e.g. {@code Map.of("client_ip", "keyword")} changes client_ip from ip to keyword). + * A {@code null} value removes the field from the mapping entirely, making it unmapped for this index. + *

+ * This affects both the Elasticsearch index mapping (via {@link CsvTestsDataLoader#readMappingFile}) and + * the in-memory mapping used by CSV unit tests. + */ public TestDataset withTypeMapping(Map typeMapping) { return new TestDataset( indexName, @@ -1181,11 +1267,16 @@ public TestDataset withTypeMapping(Map typeMapping) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); } + /** + * Adds field mappings that are not present in the mapping file, but will be in the field caps response, e.g. because during + * ingestion more fields are added dynamically. Required for csv tests which do not ingest the csvs into real indices. + */ public TestDataset withDynamicTypeMapping(Map dynamicTypeMapping) { return new TestDataset( indexName, @@ -1195,6 +1286,26 @@ public TestDataset withDynamicTypeMapping(Map dynamicTypeMapping allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, + inferenceEndpoints, + requiredCapabilities + ); + } + + /** + * Sets the "dynamic" mapping parameter (e.g. "false", "strict", "runtime"). + * This prevents unmapped fields in the data from being automatically indexed. + */ + public TestDataset withDynamic(String dynamic) { + return new TestDataset( + indexName, + mappingFileName, + dataFileName, + settingFileName, + allowSubFields, + typeMapping, + dynamicTypeMapping, + dynamic, inferenceEndpoints, requiredCapabilities ); @@ -1209,6 +1320,7 @@ public TestDataset withInferenceEndpoints(String... inferenceEndpoints) { allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, List.of(inferenceEndpoints), requiredCapabilities ); @@ -1223,6 +1335,7 @@ public TestDataset withRequiredCapabilities(EsqlCapabilities.Cap... requiredCapa allowSubFields, typeMapping, dynamicTypeMapping, + dynamic, inferenceEndpoints, List.of(requiredCapabilities) ); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec index eb11a897ce23f..1dd9370c425d4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/enrich.csv-spec @@ -652,7 +652,7 @@ required_capability: enrich_load required_capability: fix_replace_missing_field_with_null_duplicate_name_id_in_layout required_capability: dense_vector_agg_metric_double_if_fns -from * +from *,-addresses_* | keep author.keyword, book_no, scalerank, street, bytes_in, @timestamp, abbrev, city_location, distance, description, birth_date, language_code, intersects, client_ip, event_duration, version | enrich languages_policy on author.keyword | sort book_no diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec index 26f7f7f371cc5..49917e0a56788 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec @@ -4284,7 +4284,7 @@ inlineStatsAfterPruningAggregate6 required_capability: inline_stats_double_release_fix required_capability: join_lookup_v12 -from d*,* +from d*,*,-addresses_* | rename scalerank as other2 | rename author.keyword as name_str | rename intersects as is_active_bool diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/limit.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/limit.csv-spec index 31f72490214b2..d9682afb1b9a4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/limit.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/limit.csv-spec @@ -1090,7 +1090,7 @@ limitByWithEnrich required_capability: enrich_load required_capability: limit_by_enrich_fix -FROM * +FROM *,-addresses_* | ENRICH languages_policy on street | KEEP abbrev, integer, year | LIMIT 1 BY abbrev diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 8a8279b0fd841..d644bbebc52b7 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1778,7 +1778,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | sort language_code, birth_date @@ -1795,7 +1795,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | keep birth_date, language_code @@ -1813,7 +1813,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | keep birth_date, language_code @@ -1829,7 +1829,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | keep birth_date, language_code @@ -1847,7 +1847,7 @@ required_capability: join_lookup_v12 required_capability: remove_redundant_sort required_capability: make_number_of_channels_consistent_with_layout -from * +from *,-addresses_* | rename city.country.continent.planet.name as message | lookup join message_types_lookup on message | keep birth_date, language_code diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index 04ab2d77b4b8c..b5742313a7f40 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -1031,29 +1031,6 @@ FROM partial_mapping_sample_data 2024-10-23T13:51:54.732Z | Connection error? | 725449 ; -maxOverTimeOfKeyword -required_capability: optional_fields_v5 -required_capability: ts_command_v0 - -SET unmapped_fields="load"\; -TS k8s,k8s_unmapped METADATA _index -| STATS pod = min(max_over_time(region)) BY time_bucket = bucket(@timestamp,1minute), _index -| SORT time_bucket, _index -| LIMIT 10; - -pod:keyword | time_bucket:datetime | _index:keyword -us | 2024-05-10T00:00:00.000Z | k8s -us | 2024-05-10T00:00:00.000Z | k8s_unmapped -us | 2024-05-10T00:01:00.000Z | k8s -us | 2024-05-10T00:01:00.000Z | k8s_unmapped -us | 2024-05-10T00:02:00.000Z | k8s -us | 2024-05-10T00:02:00.000Z | k8s_unmapped -us | 2024-05-10T00:03:00.000Z | k8s -us | 2024-05-10T00:03:00.000Z | k8s_unmapped -us | 2024-05-10T00:04:00.000Z | k8s -us | 2024-05-10T00:04:00.000Z | k8s_unmapped -; - // Copied over from max_over_time_with_filtering, except bytes_in is unmapped. maxOverTimeWithFiltering required_capability: optional_fields_v5 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec index 0ef79c8ffbb41..b7daca0503fdd 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-type-conflicts.csv-spec @@ -4,6 +4,88 @@ # Mapped and unmapped conflict tests # ###################################### +typeConflictWithUnmappedJustFrom +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | sample_data +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | sample_data +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | sample_data +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | sample_data +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | sample_data +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | sample_data +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | 42 | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +; + + +typeConflictWithUnmappedJustKeepWildcard +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| KEEP * +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | sample_data +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | sample_data +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | sample_data +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | sample_data +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | sample_data +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | sample_data +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | 42 | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +; + + +typeConflictWithUnmappedJustKeepWildcardPrefix +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| KEEP @t*, cli*, ev*, me*, _in* +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +2023-10-23T13:55:01.543Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 | sample_data +2023-10-23T13:53:55.832Z | 172.21.3.15 | 5033755 | Connection error | sample_data +2023-10-23T13:52:55.015Z | 172.21.3.15 | 8268153 | Connection error | sample_data +2023-10-23T13:51:54.732Z | 172.21.3.15 | 725448 | Connection error | sample_data +2023-10-23T13:33:34.937Z | 172.21.0.5 | 1232382 | Disconnected | sample_data +2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | sample_data +2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | 42 | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +; + noTypeConflictKeywordUnmappedCastToKeyword required_capability: optional_fields_v5 @@ -165,6 +247,425 @@ sample_data | 5033755.0 sample_data | 8268153.0 ; +// Confirm that the employees_no_names index does not have first_name and last_name mapped. +employeesNoNamesFieldsRemoved +required_capability: index_metadata_field + +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword +10001 | Georgi | Facello | employees +10001 | null | null | employees_no_names +10002 | Bezalel | Simmel | employees +10002 | null | null | employees_no_names +10003 | Parto | Bamford | employees +10003 | null | null | employees_no_names +; + +noTypeConflictKeywordKeepTriggersLoad +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword +10001 | Georgi | Facello | employees +10001 | Georgi | Facello | employees_no_names +10002 | Bezalel | Simmel | employees +10002 | Bezalel | Simmel | employees_no_names +10003 | Parto | Bamford | employees +10003 | Parto | Bamford | employees_no_names +; + + +noTypeConflictKeywordKeepTriggersLoadStats +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| SORT emp_no, _index +| LIMIT 6 +| STATS count(*) BY first_name +; +ignoreOrder:true + +count(*):long | first_name:keyword +2 | Bezalel +2 | Georgi +2 | Parto +; + +noTypeConflictKeywordKeepTriggersLoadStatsWithNulls +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names +| KEEP emp_no, first_name, last_name +| STATS c=count(*) BY first_name +| SORT first_name ASC NULLS FIRST +| LIMIT 5 +; + +c:long | first_name:keyword +20 | null +2 | Alejandro +2 | Amabile +2 | Anneke +2 | Anoosh +; + +noTypeConflictKeywordKeepTriggersLoadInlineStats +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| INLINE STATS c=count(*) BY first_name +| SORT first_name NULLS FIRST, emp_no, _index +| LIMIT 6 +; + +emp_no:integer | last_name:keyword | _index:keyword | c:long | first_name:keyword +10030 | Demeyer | employees | 20 | null +10030 | Demeyer | employees_no_names | 20 | null +10031 | Joslin | employees | 20 | null +10031 | Joslin | employees_no_names | 20 | null +10032 | Reistad | employees | 20 | null +10032 | Reistad | employees_no_names | 20 | null +; + +noTypeConflictKeywordKeepTriggersLoadInlineStatsWithNulls +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| KEEP emp_no, first_name, last_name, _index +| INLINE STATS c=count(first_name) BY last_name +| SORT first_name NULLS FIRST, emp_no DESC, _index +| LIMIT 22 +; + +emp_no:integer | first_name:keyword | _index:keyword | c:long | last_name:keyword +10039 | null | employees | 0 | Brender +10039 | null | employees_no_names | 0 | Brender +10038 | null | employees | 2 | Lortz +10038 | null | employees_no_names | 2 | Lortz +10037 | null | employees | 0 | Makrucki +10037 | null | employees_no_names | 0 | Makrucki +10036 | null | employees | 0 | Portugali +10036 | null | employees_no_names | 0 | Portugali +10035 | null | employees | 0 | Chappelet +10035 | null | employees_no_names | 0 | Chappelet +10034 | null | employees | 0 | Swan +10034 | null | employees_no_names | 0 | Swan +10033 | null | employees | 0 | Merlo +10033 | null | employees_no_names | 0 | Merlo +10032 | null | employees | 2 | Reistad +10032 | null | employees_no_names | 2 | Reistad +10031 | null | employees | 0 | Joslin +10031 | null | employees_no_names | 0 | Joslin +10030 | null | employees | 0 | Demeyer +10030 | null | employees_no_names | 0 | Demeyer +10059 | Alejandro | employees | 2 | McAlpine +10059 | Alejandro | employees_no_names | 2 | McAlpine +; + +noTypeConflictKeepTriggersLoadInlineStatsWithFilter +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM employees, employees_no_names +| KEEP emp_no, first_name, last_name +| STATS c=count(first_name) WHERE STARTS_WITH(last_name, "M") BY last_name +| SORT c DESC, last_name +| LIMIT 12 +; + +c:long | last_name:keyword +2 | Malabarba +2 | Maliniak +2 | Mandell +2 | McAlpine +2 | McClurg +2 | McFarlin +2 | Meriste +2 | Mondadori +2 | Montemayor +2 | Morton +0 | Awdeh +0 | Azuma +; + +// Verify that a partially-mapped TEXT field is not automatically loaded from _source for the unmapped field index. +// employees_no_gender doesn't have the gender field in its mappings, but it does have it in its _source, while +// employees_gender_text has gender mapped as a TEXT field. Don't load gender from _source for employees_no_gender. +typeConflictPartiallyUnmappedTextFieldNotLoaded +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees_gender_text, employees_no_gender METADATA _index +| KEEP emp_no, gender, _index +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | gender:text | _index:keyword +10001 | M | employees_gender_text +10001 | null | employees_no_gender +10002 | F | employees_gender_text +10002 | null | employees_no_gender +10003 | M | employees_gender_text +10003 | null | employees_no_gender +; + + +typeConflictPartiallyUnmappedTextFieldNotLoadedWithDrop +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees_gender_text, employees_no_gender METADATA _index +| DROP first_name, last_name, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | gender:text | _index:keyword +10001 | M | employees_gender_text +10001 | null | employees_no_gender +10002 | F | employees_gender_text +10002 | null | employees_no_gender +10003 | M | employees_gender_text +10003 | null | employees_no_gender +; + +// Verify that partially-mapped non-keyword fields (date, ip, long) are NOT loaded from _source when not mentioned +// explicitly, at least until we solve https://github.com/elastic/elasticsearch/issues/141995. +// All fields in sample_data are partially mapped (no_mapping_sample_data has no mapped fields). +// message (keyword) should be loaded from _source; @timestamp (date), client_ip (ip), event_duration (long) should be +// null. +typeConflictPartiallyUnmappedNonKeywordFieldsNotLoadedWithoutExplicitMention +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| SORT _index, message DESC +| LIMIT 6 +; +ignoreOrder:true + +@timestamp:date | client_ip:ip | event_duration:long | message:keyword | _index:keyword +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connection error? | no_mapping_sample_data +null | null | null | Connected to 10.1.0.3! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.2! | no_mapping_sample_data +null | null | null | Connected to 10.1.0.1! | no_mapping_sample_data +; + +// https://github.com/elastic/elasticsearch/issues/141994 +// A partially-mapped keyword field (message, mapped in sample_data but not in no_mapping_sample_data) should be loaded +// from _source even when not explicitly referenced in a downstream expression like KEEP. +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithoutExplicitMention +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| DROP @timestamp, client_ip, event_duration +; +ignoreOrder:true + +message:keyword | _index:keyword +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connected to 10.1.0.3! | no_mapping_sample_data +Connected to 10.1.0.2! | no_mapping_sample_data +Connected to 10.1.0.1! | no_mapping_sample_data +42 | no_mapping_sample_data +Disconnected | sample_data +Connection error | sample_data +Connection error | sample_data +Connection error | sample_data +Connected to 10.1.0.3 | sample_data +Connected to 10.1.0.2 | sample_data +Connected to 10.1.0.1 | sample_data +; + +// EVAL on a fully mapped field (salary); the partially-mapped keyword fields (first_name, last_name) are not explicitly +// referenced by any command — they remain in the output implicitly and get loaded. +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithEvalOnOtherField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| EVAL salary_k = salary / 1000 +| DROP gender, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | _index:keyword | salary_k:integer +10001 | Georgi | Facello | employees | 57 +10001 | Georgi | Facello | employees_no_names | 57 +10002 | Bezalel | Simmel | employees | 56 +10002 | Bezalel | Simmel | employees_no_names | 56 +10003 | Parto | Bamford | employees | 61 +10003 | Parto | Bamford | employees_no_names | 61 +; + +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithRenameOnOtherField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| RENAME salary AS pay +| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | last_name:keyword | pay:integer | _index:keyword +10001 | Georgi | Facello | 57305 | employees +10001 | Georgi | Facello | 57305 | employees_no_names +10002 | Bezalel | Simmel | 56371 | employees +10002 | Bezalel | Simmel | 56371 | employees_no_names +10003 | Parto | Bamford | 61805 | employees +10003 | Parto | Bamford | 61805 | employees_no_names +; + +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithRenameOnSameField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| RENAME last_name AS surname +| DROP gender, birth_date, hire_date, languages*, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | surname:keyword | salary:integer | _index:keyword +10001 | Georgi | Facello | 57305 | employees +10001 | Georgi | Facello | 57305 | employees_no_names +10002 | Bezalel | Simmel | 56371 | employees +10002 | Bezalel | Simmel | 56371 | employees_no_names +10003 | Parto | Bamford | 61805 | employees +10003 | Parto | Bamford | 61805 | employees_no_names +; + +// EVAL on a partially mapped field (last_name). +// first_name is not referenced by any command but must still be loaded. +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithEvalOnSameField +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM employees, employees_no_names METADATA _index +| EVAL last_name = TO_UPPER(last_name) +| DROP gender, birth_date, hire_date, languages*, salary, height*, still_hired, avg_worked_seconds, job_positions, is_rehired, salary_change* +| SORT emp_no, _index +| LIMIT 6 +; + +emp_no:integer | first_name:keyword | _index:keyword | last_name:keyword +10001 | Georgi | employees | FACELLO +10001 | Georgi | employees_no_names | FACELLO +10002 | Bezalel | employees | SIMMEL +10002 | Bezalel | employees_no_names | SIMMEL +10003 | Parto | employees | BAMFORD +10003 | Parto | employees_no_names | BAMFORD +; + +// Wildcard DROP leaving the partially-mapped keyword field (message) implicit — no explicit reference. +noTypeConflictPartiallyUnmappedKeywordFieldLoadedWithWildcardDropOnOtherFields +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| DROP *_ip, *_duration, @timestamp +; +ignoreOrder:true + +message:keyword | _index:keyword +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connection error? | no_mapping_sample_data +Connected to 10.1.0.3! | no_mapping_sample_data +Connected to 10.1.0.2! | no_mapping_sample_data +Connected to 10.1.0.1! | no_mapping_sample_data +42 | no_mapping_sample_data +Disconnected | sample_data +Connection error | sample_data +Connection error | sample_data +Connection error | sample_data +Connected to 10.1.0.3 | sample_data +Connected to 10.1.0.2 | sample_data +Connected to 10.1.0.1 | sample_data +; + +// DROP the partially-mapped field itself — verify no errors from converting then pruning. +typeConflictPartiallyUnmappedKeywordFieldDropped +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| DROP @timestamp, client_ip, event_duration, message +| SORT _index +| LIMIT 5 +; + +_index:keyword +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +no_mapping_sample_data +; + +// reproducer for https://github.com/elastic/elasticsearch/issues/145206 +typeConflictPartiallyUnmappedKeywordFieldKept +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, no_mapping_sample_data METADATA _index +| KEEP event_duration +; +ignoreOrder:true + +event_duration:long +1756467 +5033755 +8268153 +725448 +1232382 +2764889 +3450233 +null +null +null +null +null +null +null +; ########################### # Mapped and non-existent # @@ -227,6 +728,36 @@ colors | null colors | null ; +typeConflictLongNonExistentNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data, colors METADATA _index +| KEEP _index, event_duration +| SORT _index DESC +| LIMIT 15 +; +ignoreOrder:true + +_index:keyword | event_duration:long +sample_data | 725448 +sample_data | 1232382 +sample_data | 1756467 +sample_data | 2764889 +sample_data | 3450233 +sample_data | 5033755 +sample_data | 8268153 +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + typeConflictLongNonExistentCastToKeyword required_capability: optional_fields_v5 @@ -339,40 +870,6 @@ heights | null heights | null ; -typeConflictDateNanosDateUnmappedCastToDate -required_capability: optional_fields_v5 - -SET unmapped_fields="load"\; -FROM sample_data, sample_data_ts_nanos, no_mapping_sample_data METADATA _index -| EVAL ts = @timestamp::date -| KEEP _index, ts -| SORT _index, ts -; - -_index:keyword | ts:datetime -no_mapping_sample_data | 2024-10-23T12:15:03.360Z -no_mapping_sample_data | 2024-10-23T12:27:28.948Z -no_mapping_sample_data | 2024-10-23T13:33:34.937Z -no_mapping_sample_data | 2024-10-23T13:51:54.732Z -no_mapping_sample_data | 2024-10-23T13:52:55.015Z -no_mapping_sample_data | 2024-10-23T13:53:55.832Z -no_mapping_sample_data | 2024-10-23T13:55:01.543Z -sample_data | 2023-10-23T12:15:03.360Z -sample_data | 2023-10-23T12:27:28.948Z -sample_data | 2023-10-23T13:33:34.937Z -sample_data | 2023-10-23T13:51:54.732Z -sample_data | 2023-10-23T13:52:55.015Z -sample_data | 2023-10-23T13:53:55.832Z -sample_data | 2023-10-23T13:55:01.543Z -sample_data_ts_nanos | 2023-10-23T12:15:03.360Z -sample_data_ts_nanos | 2023-10-23T12:27:28.948Z -sample_data_ts_nanos | 2023-10-23T13:33:34.937Z -sample_data_ts_nanos | 2023-10-23T13:51:54.732Z -sample_data_ts_nanos | 2023-10-23T13:52:55.015Z -sample_data_ts_nanos | 2023-10-23T13:53:55.832Z -sample_data_ts_nanos | 2023-10-23T13:55:01.543Z -; - ########################### # Mapped x 2 and unmapped # ########################### @@ -411,6 +908,40 @@ sample_data_ts_long | 1698069235832 sample_data_ts_long | 1698069301543 ; +typeConflictLongDateUnmappedNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data METADATA _index +| KEEP _index, @timestamp +| SORT _index +; + +_index:keyword | @timestamp:unsupported +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +; + typeConflictLongDateUnmappedCastToDouble required_capability: optional_fields_v5 @@ -521,10 +1052,186 @@ sample_data_ts_long | 2023-10-23T13:53:55.832Z sample_data_ts_long | 2023-10-23T13:55:01.543Z ; +typeConflictDateNanosDateUnmappedNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data METADATA _index +| KEEP _index, @timestamp +| SORT _index +; + +_index:keyword | @timestamp:unsupported +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +; + +typeConflictDateNanosDateUnmappedCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:keyword +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +; + + +typeConflictDateNanosDateUnmappedCastToDate +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data, sample_data_ts_nanos, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:datetime +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +sample_data_ts_nanos | 2023-10-23T12:15:03.360Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543Z +; + +typeConflictDateNanosDateUnmappedCastToDateNanos +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data METADATA _index +| EVAL ts = @timestamp::date_nanos +| KEEP _index, ts +| SORT _index, ts +; + +_index:keyword | ts:date_nanos +no_mapping_sample_data | 2024-10-23T12:15:03.360000000Z +no_mapping_sample_data | 2024-10-23T12:27:28.948000000Z +no_mapping_sample_data | 2024-10-23T13:33:34.937000000Z +no_mapping_sample_data | 2024-10-23T13:51:54.732000000Z +no_mapping_sample_data | 2024-10-23T13:52:55.015000000Z +no_mapping_sample_data | 2024-10-23T13:53:55.832000000Z +no_mapping_sample_data | 2024-10-23T13:55:01.543000000Z +sample_data | 2023-10-23T12:15:03.360000000Z +sample_data | 2023-10-23T12:27:28.948000000Z +sample_data | 2023-10-23T13:33:34.937000000Z +sample_data | 2023-10-23T13:51:54.732000000Z +sample_data | 2023-10-23T13:52:55.015000000Z +sample_data | 2023-10-23T13:53:55.832000000Z +sample_data | 2023-10-23T13:55:01.543000000Z +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +; + ############################### # Mapped x 2 and non-existent # ############################### +typeConflictLongDateNonExistentNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, colors METADATA _index +| KEEP _index, @timestamp +| SORT _index DESC +| LIMIT 25 +; + +_index:keyword | @timestamp:unsupported +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + typeConflictLongDateNonExistentCastToKeyword required_capability: optional_fields_v5 @@ -642,10 +1349,210 @@ colors | null colors | null ; +typeConflictDateNanosDateNonExistentNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, colors METADATA _index +| KEEP _index, @timestamp +| SORT _index DESC +| LIMIT 25 +; + +_index:keyword | @timestamp:unsupported +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateNonExistentCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, colors METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:keyword +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateNonExistentCastToDate +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, colors METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:datetime +sample_data_ts_nanos | 2023-10-23T12:15:03.360Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateNonExistentCastToDateNanos +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, colors METADATA _index +| EVAL ts = @timestamp::date_nanos +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 25 +; + +_index:keyword | ts:date_nanos +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +sample_data | 2023-10-23T12:15:03.360000000Z +sample_data | 2023-10-23T12:27:28.948000000Z +sample_data | 2023-10-23T13:33:34.937000000Z +sample_data | 2023-10-23T13:51:54.732000000Z +sample_data | 2023-10-23T13:52:55.015000000Z +sample_data | 2023-10-23T13:53:55.832000000Z +sample_data | 2023-10-23T13:55:01.543000000Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + ########################################## # Mapped x 2, unmapped, and non-existent # ########################################## +typeConflictLongDateUnmappedNonExistentNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data_ts_long, sample_data, no_mapping_sample_data, colors METADATA _index +| KEEP _index, @timestamp +| SORT _index DESC +| LIMIT 30 +; + +_index:keyword | @timestamp:unsupported +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data_ts_long | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + typeConflictLongDateUnmappedNonExistentCastToKeyword required_capability: optional_fields_v5 @@ -825,6 +1732,182 @@ sample_data_ts_long | 2023-10-23T13:53:55.832Z sample_data_ts_long | 2023-10-23T13:55:01.543Z ; +typeConflictDateNanosDateUnmappedNonExistentNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index +| KEEP _index, @timestamp +| SORT _index DESC +| LIMIT 30 +; + +_index:keyword | @timestamp:unsupported +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data_ts_nanos | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +no_mapping_sample_data | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateUnmappedNonExistentCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::keyword +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +_index:keyword | ts:keyword +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateUnmappedNonExistentCastToDate +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::date +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +_index:keyword | ts:datetime +sample_data_ts_nanos | 2023-10-23T12:15:03.360Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543Z +sample_data | 2023-10-23T12:15:03.360Z +sample_data | 2023-10-23T12:27:28.948Z +sample_data | 2023-10-23T13:33:34.937Z +sample_data | 2023-10-23T13:51:54.732Z +sample_data | 2023-10-23T13:52:55.015Z +sample_data | 2023-10-23T13:53:55.832Z +sample_data | 2023-10-23T13:55:01.543Z +no_mapping_sample_data | 2024-10-23T12:15:03.360Z +no_mapping_sample_data | 2024-10-23T12:27:28.948Z +no_mapping_sample_data | 2024-10-23T13:33:34.937Z +no_mapping_sample_data | 2024-10-23T13:51:54.732Z +no_mapping_sample_data | 2024-10-23T13:52:55.015Z +no_mapping_sample_data | 2024-10-23T13:53:55.832Z +no_mapping_sample_data | 2024-10-23T13:55:01.543Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + +typeConflictDateNanosDateUnmappedNonExistentCastToDateNanos +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM sample_data_ts_nanos, sample_data, no_mapping_sample_data, colors METADATA _index +| EVAL ts = @timestamp::date_nanos +| KEEP _index, ts +| SORT _index DESC, ts +| LIMIT 30 +; + +_index:keyword | ts:date_nanos +sample_data_ts_nanos | 2023-10-23T12:15:03.360123456Z +sample_data_ts_nanos | 2023-10-23T12:27:28.948123456Z +sample_data_ts_nanos | 2023-10-23T13:33:34.937123456Z +sample_data_ts_nanos | 2023-10-23T13:51:54.732123456Z +sample_data_ts_nanos | 2023-10-23T13:52:55.015123456Z +sample_data_ts_nanos | 2023-10-23T13:53:55.832123456Z +sample_data_ts_nanos | 2023-10-23T13:55:01.543123456Z +sample_data | 2023-10-23T12:15:03.360000000Z +sample_data | 2023-10-23T12:27:28.948000000Z +sample_data | 2023-10-23T13:33:34.937000000Z +sample_data | 2023-10-23T13:51:54.732000000Z +sample_data | 2023-10-23T13:52:55.015000000Z +sample_data | 2023-10-23T13:53:55.832000000Z +sample_data | 2023-10-23T13:55:01.543000000Z +no_mapping_sample_data | 2024-10-23T12:15:03.360000000Z +no_mapping_sample_data | 2024-10-23T12:27:28.948000000Z +no_mapping_sample_data | 2024-10-23T13:33:34.937000000Z +no_mapping_sample_data | 2024-10-23T13:51:54.732000000Z +no_mapping_sample_data | 2024-10-23T13:52:55.015000000Z +no_mapping_sample_data | 2024-10-23T13:53:55.832000000Z +no_mapping_sample_data | 2024-10-23T13:55:01.543000000Z +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +colors | null +; + ############################## # WHERE with type casts # ############################## @@ -1249,3 +2332,197 @@ sample_data | 3450233.0 sample_data | 5033755.0 sample_data | 8268153.0 ; + +############################################ +# Deeply nested partially unmapped fields # +############################################ + +// city.country.continent.name (keyword, 4-level deep) is mapped in addresses but not in addresses_no_continent. +// city.country.continent.planet.name and city.country.continent.planet.galaxy (5-level deep) are also unmapped. +// All fields are keyword, so they are loaded via PotentiallyUnmappedKeywordEsField. +deeplyNestedSubfieldPartiallyUnmappedKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses, addresses_no_continent METADATA _index +| KEEP _index, city.name, city.country.name, city.country.continent.name, city.country.continent.planet.name +| SORT _index, city.name +; + +_index:keyword | city.name:keyword | city.country.name:keyword | city.country.continent.name:keyword | city.country.continent.planet.name:keyword +addresses | Amsterdam | Netherlands | Europe | Earth +addresses | San Francisco | United States of America | North America | Earth +addresses | Tokyo | Japan | Asia | Earth +addresses_no_continent | Amsterdam | Netherlands | Europe | Earth +addresses_no_continent | San Francisco | United States of America | North America | Earth +addresses_no_continent | Tokyo | Japan | Asia | Earth +; + +// Cast city.country.continent.planet.name (keyword, 5-level deep, partially unmapped) to keyword. +deeplyNestedSubfieldPartiallyUnmappedCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses, addresses_no_continent METADATA _index +| EVAL planet = city.country.continent.planet.name::keyword +| KEEP _index, city.name, planet +| SORT _index, city.name +; + +_index:keyword | city.name:keyword | planet:keyword +addresses | Amsterdam | Earth +addresses | San Francisco | Earth +addresses | Tokyo | Earth +addresses_no_continent | Amsterdam | Earth +addresses_no_continent | San Francisco | Earth +addresses_no_continent | Tokyo | Earth +; + +// city.country.continent.planet.galaxy is keyword, 5-level deep, and sits under continent which is entirely stripped +// from addresses_no_continent. It is loaded via PotentiallyUnmappedKeywordEsField from both indices. +deeplyNestedSubfieldUnderPartiallyUnmappedParent +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses, addresses_no_continent METADATA _index +| KEEP _index, city.name, city.country.continent.planet.galaxy +| SORT _index, city.name +; + +_index:keyword | city.name:keyword | city.country.continent.planet.galaxy:keyword +addresses | Amsterdam | Milky Way +addresses | San Francisco | Milky Way +addresses | Tokyo | Milky Way +addresses_no_continent | Amsterdam | Milky Way +addresses_no_continent | San Francisco | Milky Way +addresses_no_continent | Tokyo | Milky Way +; + +// Same as above, but using addresses_text where galaxy is mapped as text (non-keyword). +// Without a cast, galaxy is a regular text field: values from addresses_text, null from addresses_no_continent. +deeplyNestedNonKeywordSubfieldUnderPartiallyUnmappedParentNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM addresses_text, addresses_no_continent METADATA _index +| KEEP _index, city.country.continent.planet.galaxy +| SORT _index +; + +_index:keyword | city.country.continent.planet.galaxy:text +addresses_no_continent | null +addresses_no_continent | null +addresses_no_continent | null +addresses_text | Milky Way +addresses_text | Milky Way +addresses_text | Milky Way +; + +// city.country.continent.name is text (non-keyword) in addresses_text, unmapped in addresses_no_continent. +// Without a cast, it's a regular field: values from addresses_text, null from addresses_no_continent. +// Note: city.name has a type conflict (keyword vs text) across these indices, so we avoid referencing it. +deeplyNestedNonKeywordSubfieldPartiallyUnmappedNotCast +required_capability: optional_fields_v5 +required_capability: optional_fields_fix_load_partially_mapped + +SET unmapped_fields="load"\; +FROM addresses_text, addresses_no_continent METADATA _index +| KEEP _index, city.country.continent.name +| SORT _index +; +ignoreOrder:true + +_index:keyword | city.country.continent.name:text +addresses_no_continent | null +addresses_no_continent | null +addresses_no_continent | null +addresses_text | Asia +addresses_text | Europe +addresses_text | North America +; + +// Cast city.country.continent.name (text, partially unmapped) to keyword via MultiTypeEsField. +deeplyNestedNonKeywordSubfieldPartiallyUnmappedCastToKeyword +required_capability: optional_fields_v5 + +SET unmapped_fields="load"\; +FROM addresses_text, addresses_no_continent METADATA _index +| EVAL continent = city.country.continent.name::keyword +| KEEP _index, continent +| SORT _index, continent +; + +_index:keyword | continent:keyword +addresses_no_continent | Asia +addresses_no_continent | Europe +addresses_no_continent | North America +addresses_text | Asia +addresses_text | Europe +addresses_text | North America +; + +############################ +# TS with partially mapped # +############################ + +// region is keyword, mapped in k8s but unmapped in k8s_unmapped. +// As a PotentiallyUnmappedKeywordEsField, it is loaded from _source for k8s_unmapped. +tsPartiallyMappedKeywordField +required_capability: optional_fields_v5 +required_capability: ts_command_v0 + +SET unmapped_fields="load"\; +TS k8s, k8s_unmapped METADATA _index +| STATS r = min(max_over_time(region)) BY time_bucket = bucket(@timestamp, 1minute), _index +| SORT time_bucket, _index +| LIMIT 10 +; + +r:keyword | time_bucket:datetime | _index:keyword +us | 2024-05-10T00:00:00.000Z | k8s +us | 2024-05-10T00:00:00.000Z | k8s_unmapped +us | 2024-05-10T00:01:00.000Z | k8s +us | 2024-05-10T00:01:00.000Z | k8s_unmapped +us | 2024-05-10T00:02:00.000Z | k8s +us | 2024-05-10T00:02:00.000Z | k8s_unmapped +us | 2024-05-10T00:03:00.000Z | k8s +us | 2024-05-10T00:03:00.000Z | k8s_unmapped +us | 2024-05-10T00:04:00.000Z | k8s +us | 2024-05-10T00:04:00.000Z | k8s_unmapped +; + +// network.bytes_in is long, mapped in k8s but unmapped in k8s_unmapped. +// Cast to long to resolve it via MultiTypeEsField. +tsPartiallyMappedNonKeywordFieldWithCast +required_capability: optional_fields_v5 +required_capability: ts_command_v0 + +SET unmapped_fields="load"\; +TS k8s, k8s_unmapped METADATA _index +| WHERE pod == "one" +| STATS tx = sum(max_over_time(network.bytes_in :: long)) BY cluster, time_bucket = bucket(@timestamp, 10minute), _index +| SORT time_bucket, cluster, _index +| LIMIT 20 +; + +tx:long | cluster:keyword | time_bucket:datetime | _index:keyword +970 | prod | 2024-05-10T00:00:00.000Z | k8s +970 | prod | 2024-05-10T00:00:00.000Z | k8s_unmapped +842 | qa | 2024-05-10T00:00:00.000Z | k8s +842 | qa | 2024-05-10T00:00:00.000Z | k8s_unmapped +753 | staging | 2024-05-10T00:00:00.000Z | k8s +753 | staging | 2024-05-10T00:00:00.000Z | k8s_unmapped +990 | prod | 2024-05-10T00:10:00.000Z | k8s +990 | prod | 2024-05-10T00:10:00.000Z | k8s_unmapped +1006 | qa | 2024-05-10T00:10:00.000Z | k8s +1006 | qa | 2024-05-10T00:10:00.000Z | k8s_unmapped +947 | staging | 2024-05-10T00:10:00.000Z | k8s +947 | staging | 2024-05-10T00:10:00.000Z | k8s_unmapped +953 | prod | 2024-05-10T00:20:00.000Z | k8s +953 | prod | 2024-05-10T00:20:00.000Z | k8s_unmapped +917 | qa | 2024-05-10T00:20:00.000Z | k8s +917 | qa | 2024-05-10T00:20:00.000Z | k8s_unmapped +749 | staging | 2024-05-10T00:20:00.000Z | k8s +749 | staging | 2024-05-10T00:20:00.000Z | k8s_unmapped +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 73dfeb4a2ab87..b31f3afb5e617 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -269,6 +269,16 @@ public enum Cap { */ OPTIONAL_FIELDS_V5, + /** + * Unconditionally load partially mapped keyword fields, whether they are mentioned in expressions or not. + *

+ * Also, always load values of partially mapped fields from the indices where they are mapped. + *

+ * Fixes https://github.com/elastic/elasticsearch/issues/141994 + * and https://github.com/elastic/elasticsearch/issues/145206 + */ + OPTIONAL_FIELDS_FIX_LOAD_PARTIALLY_MAPPED, + /** * Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently * supported. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 156e3e35f8d7a..6024862df267e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -282,7 +282,7 @@ public LogicalPlan analyze(LogicalPlan plan) { } public LogicalPlan verify(LogicalPlan plan, BitSet partialMetrics) { - Collection failures = verifier.verify(plan, partialMetrics, context().unmappedResolution()); + Collection failures = verifier.verify(plan, partialMetrics, context()); if (failures.isEmpty() == false) { throw new VerificationException(failures); } @@ -351,6 +351,11 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR var attributes = mappingAsAttributes(plan.source(), esIndex.mapping()); attributes.addAll(metadata.stream().map(NamedExpression::toAttribute).toList()); + + if (context.unmappedResolution() == UnmappedResolution.LOAD) { + loadPartiallyUnmappedFields(attributes, esIndex); + } + return new EsRelation( plan.source(), esIndex.name(), @@ -362,6 +367,34 @@ private LogicalPlan resolveIndex(UnresolvedRelation plan, IndexResolution indexR ); } + /** + * When {@code SET unmapped_fields="load"}, convert partially-mapped fields so they can be used across indices where they + * may not exist: + *

    + *
  • Keyword fields are converted to use {@link PotentiallyUnmappedKeywordEsField} so they are loaded from + * {@code _source} on shards where they are unmapped.
    • + *
  • Non-keyword fields are marked as {@link InvalidMappedField#potentiallyUnmapped potentially unmapped} so that + * explicit casts (e.g. {@code field::long}) can resolve them via the union types / {@code MultiTypeEsField} + * mechanism.
    • + *
+ */ + private static void loadPartiallyUnmappedFields(List attributes, EsIndex esIndex) { + for (int i = 0; i < attributes.size(); i++) { + if (attributes.get(i) instanceof FieldAttribute fa && isPartiallyUnmappedRegularField(fa, esIndex)) { + if (fa.dataType() == KEYWORD) { + attributes.set(i, ResolveRefs.insistKeyword(fa)); + } else { + attributes.set(i, ResolveRefs.invalidInsistAttribute(fa, esIndex)); + } + } + } + } + + private static boolean isPartiallyUnmappedRegularField(FieldAttribute fa, EsIndex esIndex) { + // We ignore proper subclasses of FieldAttribute; these represent unsupported or special attributes. + return fa.getClass().equals(FieldAttribute.class) && esIndex.isPartiallyUnmappedField(fa.fieldName().string()); + } + private List resolveMetadata(List metadata, AnalyzerContext context) { LinkedHashMap resolved = new LinkedHashMap<>(); Set allTags = null; @@ -666,7 +699,7 @@ protected LogicalPlan rule(LogicalPlan plan, AnalyzerContext context) { default -> plan.transformExpressionsOnly(UnresolvedAttribute.class, ua -> maybeResolveAttribute(ua, childrenOutput)); }; - return context.unmappedResolution() == UnmappedResolution.LOAD ? resolvePartiallyMapped(resolved, context) : resolved; + return resolved; } private LogicalPlan resolveAggregate(Aggregate aggregate, List childrenOutput) { @@ -1226,9 +1259,9 @@ private Attribute resolveInsistAttribute(Attribute attribute, List ch return resolvedCol; } - private static FieldAttribute invalidInsistAttribute(FieldAttribute fa, EsIndex esIndex) { - InvalidMappedField field = InvalidMappedField.potentiallyUnmapped(fa.name(), getTypesToIndices(fa, esIndex)); - return new FieldAttribute(fa.source(), null, fa.qualifier(), fa.name(), field); + static FieldAttribute invalidInsistAttribute(FieldAttribute fa, EsIndex esIndex) { + InvalidMappedField field = InvalidMappedField.potentiallyUnmapped(fa.field().getName(), getTypesToIndices(fa, esIndex)); + return new FieldAttribute(fa.source(), fa.parentName(), fa.qualifier(), fa.name(), field); } private static Map> getTypesToIndices(FieldAttribute fa, EsIndex esIndex) { @@ -1251,59 +1284,6 @@ public static FieldAttribute insistKeyword(Attribute attribute) { ); } - /** - * This will inspect current node/{@code plan}'s expressions and check if any of the {@code FieldAttribute}s refer to fields that - * are partially unmapped across the indices involved in the plan fragment. If so, replace their field with an "insisted" EsField. - */ - private static LogicalPlan resolvePartiallyMapped(LogicalPlan plan, AnalyzerContext context) { - var indexResolutions = collectIndexResolutions(plan, context); - Map insistedMap = new HashMap<>(); - var transformed = plan.transformExpressionsOnly(FieldAttribute.class, fa -> { - var esField = fa.field(); - if (esField instanceof PotentiallyUnmappedKeywordEsField - || esField instanceof InvalidMappedField imf && imf.isPotentiallyUnmapped()) { - return fa; - } - var existing = insistedMap.get(fa); - if (existing != null) { // field shows up multiple times in the node; return first processing - return existing; - } - - if (indexResolutions.isEmpty()) { - throw new IllegalStateException("Unmapped fields with empty index resolutions."); - } - EsIndex esIndex = indexResolutions.getFirst().get(); - if (esIndex.isPartiallyUnmappedField(fa.name())) { - FieldAttribute newFA = fa.dataType() == KEYWORD ? insistKeyword(fa) : invalidInsistAttribute(fa, esIndex); - insistedMap.put(fa, newFA); - return newFA; - } - return fa; - }); - return insistedMap.isEmpty() ? transformed : propagateInsistedFields(transformed, insistedMap); - } - - /** - * Push only those fields from the {@code insistedMap} into {@code EsRelation}s in the {@code plan} that wrap a - * {@code PotentiallyUnmappedKeywordEsField}. - */ - private static LogicalPlan propagateInsistedFields(LogicalPlan plan, Map insistedMap) { - return plan.transformUp(EsRelation.class, esr -> { - var newOutput = new ArrayList(); - boolean updated = false; - for (Attribute attr : esr.output()) { - var newFA = insistedMap.get(attr); - if (newFA != null && newFA.field() instanceof PotentiallyUnmappedKeywordEsField) { - newOutput.add(newFA); - updated = true; - } else { - newOutput.add(attr); - } - } - return updated ? esr.withAttributes(newOutput) : esr; - }); - } - private LogicalPlan resolveFuse(Fuse fuse, List childrenOutput) { Source source = fuse.source(); Attribute score = fuse.score(); @@ -1980,7 +1960,7 @@ public LogicalPlan apply(LogicalPlan logicalPlan, AnalyzerContext context) { } } else { limit = context.configuration().resultTruncationMaxSize(isTsAggregate); // user provided a limit: cap result - // entries to the max + // entries to the max } var source = logicalPlan.source(); return new Limit(source, new Literal(source, limit, DataType.INTEGER), logicalPlan); @@ -2647,10 +2627,38 @@ public LogicalPlan apply(LogicalPlan plan) { // unsupported / unresolved fields can be explicitly retained return cleanPlan.transformUp( LogicalPlan.class, - p -> p.transformExpressionsOnly(FieldAttribute.class, FieldAttribute::checkUnresolved) + p -> p.transformExpressionsOnly(FieldAttribute.class, UnionTypesCleanup::cleanTypeConflicts) ); } + /** + * Return an {@link UnsupportedAttribute} so the verifier can flag illegal use of fields with type conflicts. + *

+ * If the field is mapped to a single type in some indices but unmapped in others: Instead return a regular field attribute with a + * single type so that values are loaded from the indices where it is mapped (and null is returned from unmapped indices). + * This is a temporary solution until https://github.com/elastic/elasticsearch/issues/141995 is implemented. + */ + private static Attribute cleanTypeConflicts(FieldAttribute fa) { + EsField field = fa.field(); + if (field instanceof InvalidMappedField imf && imf.isPotentiallyUnmapped() && imf.types().size() == 1) { + DataType type = imf.types().iterator().next(); + var restoredField = new EsField(imf.getName(), type, imf.getProperties(), false, imf.getTimeSeriesFieldType()); + // TODO: add test where not passing on the parent name fails the test + // TODO: add TS tests and tests with different time series field types + return new FieldAttribute( + fa.source(), + fa.parentName(), + fa.qualifier(), + fa.name(), + restoredField, + fa.nullable(), + fa.id(), + fa.synthetic() + ); + } + return fa.flagTypeConflicts(); + } + private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { List output = plan.output(); List newOutput = new ArrayList<>(output.size()); @@ -2678,17 +2686,14 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { return relation; } return relation.transformExpressionsUp(FieldAttribute.class, f -> { - if (f.field() instanceof InvalidMappedField imf && allDates(context, relation, imf)) { + if (f.field() instanceof InvalidMappedField imf && allDates(context, imf)) { HashMap typeResolutions = new HashMap<>(); var convert = new ToDateNanos(f.source(), f, context.configuration()); imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions)); - // This rule runs in the "Initialize" batch, before ResolveUnmapped. The isFieldMappedInAllIndices - // check above should prevent reaching here for fields that are unmapped in any index when in LOAD mode. - if (imf.isPotentiallyUnmapped()) { - throw new IllegalStateException( - "Unexpected potentially unmapped field [" + imf.getName() + "] in DateMillisToNanosInEsRelation" - ); - } + // The allDates check filters out fields that are not mapped in all indices, which includes + // potentiallyUnmapped fields. This assertion guards against future changes breaking that invariant. + assert imf.isPotentiallyUnmapped() == false + : "Unexpected potentially unmapped field [" + imf.getName() + "] in DateMillisToNanosInEsRelation"; var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions, null); return new FieldAttribute( f.source(), @@ -2706,18 +2711,16 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { }); } - private static boolean allDates(AnalyzerContext context, EsRelation relation, InvalidMappedField imf) { + private static boolean allDates(AnalyzerContext context, InvalidMappedField imf) { if (imf.types().stream().allMatch(DataType::isDate) == false) { return false; } - // If we need to load the fields from unmapped indices, we will treat it as a keyword, i.e., not all types are dates. - if (context.unmappedResolution() != UnmappedResolution.LOAD) { - return true; + // If the field is potentially unmapped (i.e. not mapped in all indices), we treat it as a keyword (not all dates), + // so that it can be resolved via the union types / MultiTypeEsField mechanism instead. + if (context.unmappedResolution() == UnmappedResolution.LOAD && imf.isPotentiallyUnmapped()) { + return false; } - // Since DateMillisToNanosInEsRelation runs before ResolveUnmapped, isPotentiallyUnmapped isn't set yet. - int mappedCount = imf.getTypesToIndices().values().stream().mapToInt(Set::size).sum(); - int totalCount = relation.concreteIndices().values().stream().mapToInt(List::size).sum(); - return mappedCount >= totalCount; + return true; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index b33cbab4f8675..4914eb9d30ab1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -19,9 +19,11 @@ import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.expression.AttributeSet; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; import org.elasticsearch.xpack.esql.core.expression.UnsupportedAttribute; @@ -29,6 +31,7 @@ import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.core.util.Holder; @@ -39,6 +42,9 @@ import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.EsqlBinaryComparison; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.NotEquals; +import org.elasticsearch.xpack.esql.index.EsIndex; +import org.elasticsearch.xpack.esql.index.IndexResolution; +import org.elasticsearch.xpack.esql.plan.IndexPattern; import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Fork; @@ -62,6 +68,7 @@ import java.util.Collections; import java.util.HashSet; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.function.BiConsumer; import java.util.function.Consumer; @@ -102,11 +109,12 @@ public Verifier(Metrics metrics, XPackLicenseState licenseState, List verify(LogicalPlan plan, BitSet partialMetrics, UnmappedResolution unmappedResolution) { + Collection verify(LogicalPlan plan, BitSet partialMetrics, AnalyzerContext context) { assert partialMetrics != null; + UnmappedResolution unmappedResolution = context.unmappedResolution(); Failures failures = new Failures(); boolean unmappedTimestampHandled = unmappedResolution != UnmappedResolution.DEFAULT @@ -117,6 +125,13 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics, UnmappedReso ConfigurationAware.verifyNoMarkerConfiguration(plan, failures); + // Temporary check before we implement https://github.com/elastic/elasticsearch/issues/141995. + // Partially-unmapped non-keyword field references are checked before the bail-out so that a query with both + // an UnsupportedAttribute and a PUNK field produces both errors in one batch. + if (unmappedResolution == UnmappedResolution.LOAD) { + checkPartiallyUnmappedNonKeywordReferences(plan, failures, context); + } + // in case of failures bail-out as all other checks will be redundant if (failures.hasFailures()) { return failures.failures(); @@ -505,6 +520,90 @@ private static Set flattenedFieldNames(List attributes) { return names; } + /** + * Reject queries that refer to partially unmapped non-keyword fields (PUNKs) when {@code unmapped_fields="load"}. + * Any expression referencing a PUNK attribute is flagged as a verification failure unless it's within a conversion + * function or KEEP/DROP. + *

+ * Example + *

+ * With the two indices below + *

    + *
  1. index1 has foo(long) and bar(keyword)
    2. + *
  2. index2 has bar(keyword)
    3. + *
+ * + * The following queries should pass (assume unmapped_fields="load" and reading from both indices) + *
    + *
  • KEEP foo
    • + *
  • DROP foo*
    • + *
  • EVAL x = foo::long
    • + *
+ * The following queries should fail (assume unmapped_fields="load" and reading from both indices) + *
    + *
  • RENAME foo as x
    • + *
  • EVAL x = foo
    • + *
  • WHERE foo > 1
    • + *
+ */ + private static void checkPartiallyUnmappedNonKeywordReferences(LogicalPlan plan, Failures failures, AnalyzerContext context) { + final String errorMessage = "Using partially unmapped non-KEYWORD field [{}] is not supported with unmapped_fields=\"load\""; + + AttributeSet punks = partiallyUnmappedNonKeywords(plan, context.indexResolution()); + Consumer addFailureIfPunk = fa -> { + if (punks.contains(fa)) { + failures.add(fail(fa, errorMessage, fa.fieldName().string())); + } + }; + + plan.forEachUp(p -> { + // Fork will have a PUNK in the output even if it wasn't actually used in the query, that's fine. + if (p instanceof EsRelation || p instanceof Fork) { + return; + } + + // Project represents KEEP/DROP/RENAME. We are consistent with UnsupportedAttribute (unsupported type/type conflict): + // - RENAME is forbidden. + // - KEEP/DROP are fine, with and without wildcards. + if (p instanceof Project project) { + for (NamedExpression projection : project.projections()) { + if (projection instanceof Attribute) { + continue; + } + projection.forEachDown(FieldAttribute.class, addFailureIfPunk); + } + return; + } + + p.forEachExpression(FieldAttribute.class, addFailureIfPunk); + }); + } + + /** + * Walks the plan's {@link EsRelation} nodes and collects partially unmapped non-keyword attributes. + */ + private static AttributeSet partiallyUnmappedNonKeywords(LogicalPlan plan, Map indexResolutions) { + AttributeSet.Builder punks = AttributeSet.builder(); + + plan.forEachUp(EsRelation.class, relation -> { + IndexResolution indexResolution = indexResolutions.get(new IndexPattern(relation.source(), relation.indexPattern())); + if (indexResolution != null && indexResolution.isValid()) { + EsIndex index = indexResolution.get(); + for (Attribute attr : relation.output()) { + if (attr instanceof FieldAttribute fa + && index.isPartiallyUnmappedField(fa.fieldName().string()) + && fa.dataType() != DataType.KEYWORD + // punk_field::long is fine; in this case, the FieldAttribute contains a MultiTypeEsField with the conversions. + && fa.field() instanceof MultiTypeEsField == false) { + punks.add(fa); + } + } + } + }); + + return punks.build(); + } + private void licenseCheck(LogicalPlan plan, Failures failures) { Consumer> licenseCheck = n -> { if (n instanceof LicenseAware la && la.licenseCheck(licenseState) == false) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java index 29b8b943b5076..c0d274bb51ece 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/Expressions.java @@ -42,7 +42,7 @@ public static List asAttributes(List named *

* Exception: a {@link FieldAttribute} backed by an {@link InvalidMappedField} (ambiguous type across indices) is instead * converted to an {@link UnsupportedAttribute} via - * {@link FieldAttribute#checkUnresolved()}, so the analyzer can surface a clear user-facing error. + * {@link FieldAttribute#flagTypeConflicts()}, so the analyzer can surface a clear user-facing error. */ public static List toReferenceAttributesPreservingIds( List named, @@ -60,7 +60,7 @@ public static List toReferenceAttributesPreservingIds( Attribute existing = existingByName.get(exp.name()); NameId id = existing != null ? existing.id() : new NameId(); Attribute refAttr = switch (exp) { - case FieldAttribute fa when fa.field() instanceof InvalidMappedField -> fa.checkUnresolved(); + case FieldAttribute fa when fa.field() instanceof InvalidMappedField -> fa.flagTypeConflicts(); case ReferenceAttribute ra -> ra.withId(id); default -> new ReferenceAttribute(exp.source(), null, exp.name(), exp.dataType(), exp.nullable(), id, exp.synthetic()); }; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java index 856f352d7cf1d..f027cda423cba 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/expression/FieldAttribute.java @@ -31,7 +31,7 @@ * Treat this as final - avoid creating new subclasses. This is used extensively throughout the codebase, making new subclasses risky. * Consider using a new {@link EsField} subclass if needed. *

- * Attribute for an ES field. May actually stand for a field with the same name across different indices (a union), e.g. when + * Attribute for an ES index field. May actually stand for a field with the same name across different indices (a union), e.g. when * index pattern were used in the query: {@code FROM logs-* | KEEP field}. *

* This class offers: @@ -247,7 +247,7 @@ public FieldName fieldName() { * converts this attribute into an {@link UnsupportedAttribute} with a descriptive error message * so the analyzer can surface a clear user-facing error. */ - public Attribute checkUnresolved() { + public Attribute flagTypeConflicts() { if (field instanceof InvalidMappedField imf) { // Field has conflicting types across indices — build a user-facing error message. String unresolvedMessage = "Cannot use field [" + name() + "] due to ambiguities being " + imf.errorMessage(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 3f1db2b71804f..1c4af4362b09f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -3363,28 +3363,6 @@ public void testResolveInsist_multiIndexFieldPartiallyMappedWithSingleKeywordTyp assertThat(attribute.field(), is(new PotentiallyUnmappedKeywordEsField("message"))); } - public void testResolveInsist_multiIndexFieldExistsWithSingleTypeButIsNotKeywordAndMissingCast_createsAnInvalidMappedField() { - assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); - - FieldCapabilitiesResponse caps = new FieldCapabilitiesResponse( - List.of( - fieldCapabilitiesIndexResponse("foo", fieldResponseMap("message", "long")), - fieldCapabilitiesIndexResponse("bar", Map.of()) - ), - List.of() - ); - IndexResolution resolution = mergedResolution("foo,bar", caps, true); - var plan = analyzer().addIndex(resolution).query("FROM foo, bar | INSIST_🐔 message"); - var limit = as(plan, Limit.class); - var insist = as(limit.child(), Insist.class); - var attribute = (UnsupportedAttribute) EsqlTestUtils.singleValue(insist.output()); - assertThat(attribute.name(), is("message")); - - String expected = "Cannot use field [message] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [foo]"; - assertThat(attribute.unresolvedMessage(), is(expected)); - } - public void testResolveInsist_multiIndexFieldPartiallyExistsWithMultiTypesNoKeyword_createsAnInvalidMappedField() { assumeTrue("Requires UNMAPPED FIELDS", EsqlCapabilities.Cap.UNMAPPED_FIELDS.isEnabled()); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java index fed10913c99fa..90d8259dc3764 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedGoldenTests.java @@ -678,6 +678,91 @@ public void testNoTypeConflictKeywordAndUnmappedWhere() throws Exception { """); } + // All fields are partially unmapped (no_mapping_sample_data has no mapped fields). + // Keyword fields should become PotentiallyUnmappedKeywordEsField; non-keyword fields should become InvalidMappedField. + // No explicit field reference — all fields come from the implicit output of FROM. + public void testPartiallyMappedFieldsAutomaticallyFound() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + """); + } + + // Same as testPartiallyMappedFieldsAutomaticallyFound, but with an explicit KEEP * to verify wildcard expansion + // handles partially-mapped fields correctly. + public void testPartiallyMappedFieldsAutomaticallyFoundKeepStar() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | KEEP * + """); + } + + public void testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | KEEP @timestamp, event_duration + """); + } + + // first_name and last_name are keyword, partially unmapped (missing in employees_no_names). + // They should appear as PotentiallyUnmappedKeywordEsField in the EsRelation without being explicitly referenced. + public void testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference() throws Exception { + runTests(""" + FROM employees, employees_no_names + | SORT emp_no + | LIMIT 1 + """); + } + + // first_name (keyword, partially unmapped) should become PotentiallyUnmappedKeywordEsField. + // gender (keyword, fully mapped in both indices) should remain a regular KeywordEsField. + public void testNonPartiallyMappedKeywordFieldNotLoadedFromSource() throws Exception { + runTests(""" + FROM employees, employees_no_names + | KEEP first_name, gender + """); + } + + // gender is text in employees_gender_text but missing in employees_no_gender. + // It should appear as InvalidMappedField (unsupported) in the EsRelation. + public void testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped() throws Exception { + runTests(""" + FROM employees_gender_text, employees_no_gender + | KEEP gender + """); + } + + // DROP a single partially-mapped keyword field (message), leaving only non-keyword fields. + public void testPartiallyMappedFieldsDropOnePartiallyMapped() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | DROP message + """); + } + + // DROP a single partially-mapped non-keyword field (event_duration), leaving message and the other non-keyword fields. + public void testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | DROP event_duration + """); + } + + // DROP with wildcards on partially-mapped non-keyword fields, leaving only the keyword field (message). + public void testPartiallyMappedFieldsDropNonKeywordWithWildcards() throws Exception { + runTests(""" + FROM sample_data, no_mapping_sample_data + | DROP *_ip, *_duration, @timestamp + """); + } + + // DROP with wildcards on partially-mapped keyword fields, leaving only a few non-keyword fields. + public void testPartiallyMappedFieldsDropKeywordWithWildcards() throws Exception { + runTests(""" + FROM employees, employees_no_names + | DROP *date*, gender, height*, languages*, *_hired, *_seconds, *_positions, salary_change* + """); + } + public void testForkBranchesAfterStats1stBranch() throws Exception { runTestsNullifyOnly(""" FROM employees diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 43fb1d7836d8a..10bc31dd6e63a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -15,12 +15,14 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expressions; +import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; +import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.type.UnsupportedEsField; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; @@ -30,6 +32,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; +import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.hamcrest.Matcher; import org.hamcrest.Matchers; @@ -737,7 +740,7 @@ public void testSameMappingHashWithUnmappedIndex() { ta.addIndex(entry.getKey().indexPattern(), entry.getValue()); } var e = expectThrows(VerificationException.class, () -> ta.statement(setUnmappedLoad("FROM foo, bar, baz | SORT message"))); - assertThat(e.getMessage(), allOf(containsString("Cannot use field [message]"), containsString("[long] in [bar, foo]"))); + assertThat(e.getMessage(), partiallyUnmappedNonKeywordError("message")); } private static final String UNMAPPED_TIMESTAMP_SUFFIX = UnresolvedTimestamp.UNRESOLVED_SUFFIX + Verifier.UNMAPPED_TIMESTAMP_SUFFIX; @@ -833,6 +836,73 @@ public void testTbucketWithUnmappedTimestampWithFork() { } } + /** + * Verify that partially-mapped fields of ALL non-keyword types are NOT converted to + * {@link PotentiallyUnmappedKeywordEsField}, but are instead marked as potentially unmapped via {@link InvalidMappedField}. + * This iterates over all {@link DataType} values that can appear as ES mapped field types. + */ + public void testPartiallyMappedNonKeywordFieldsMarkedAsPotentiallyUnmapped() { + // Types that cannot appear as regular ES mapped fields in an EsIndex mapping + Set excludedTypes = Set.of( + DataType.KEYWORD, // this is the type we DO convert — not a negative test case + DataType.NULL, // not a real mapped field type + DataType.UNSUPPORTED, // not a real mapped field type + DataType.DOC_DATA_TYPE, // internal _doc type + DataType.TSID_DATA_TYPE, // internal _tsid type + DataType.SOURCE, // internal _source type + DataType.DATE_PERIOD, // ESQL-internal, not an ES mapping type + DataType.TIME_DURATION, // ESQL-internal, not an ES mapping type + DataType.OBJECT, // not a leaf field type + DataType.GEOHASH, // ESQL-internal grid type, not a real ES mapped field type + DataType.GEOTILE, // ESQL-internal grid type, not a real ES mapped field type + DataType.GEOHEX // ESQL-internal grid type, not a real ES mapped field type + ); + + for (DataType dataType : DataType.values()) { + if (excludedTypes.contains(dataType)) { + continue; + } + // Build a minimal mapping: one keyword field (emp_no stand-in for SORT) and one field of the type under test + Map mapping = Map.of( + "sort_field", + new EsField("sort_field", DataType.INTEGER, Map.of(), true, EsField.TimeSeriesFieldType.NONE), + "test_field", + new EsField("test_field", dataType, Map.of(), true, EsField.TimeSeriesFieldType.NONE) + ); + + var plan = analyzer().addIndex( + new EsIndex( + "test*", + mapping, + Map.of("test1", IndexMode.STANDARD, "test2", IndexMode.STANDARD), + Map.of(), + Map.of(), + Map.of("test_field", Set.of("test2")) // partially unmapped + ) + ).statement(setUnmappedLoad(""" + FROM test* + | SORT sort_field + """)); + + var limit = as(plan, Limit.class); + var order = as(limit.child(), OrderBy.class); + var relation = as(order.child(), EsRelation.class); + + var testFieldAttr = relation.output().stream().filter(a -> a.name().equals("test_field")).findFirst().orElseThrow(); + var fieldAttr = as(testFieldAttr, FieldAttribute.class); + assertThat( + "Partially-mapped " + dataType + " field should not be converted to PotentiallyUnmappedKeywordEsField", + fieldAttr.field(), + not(instanceOf(PotentiallyUnmappedKeywordEsField.class)) + ); + assertThat( + "Partially-mapped " + dataType + " field should be reverted to a regular field with its original type", + fieldAttr.dataType(), + is(dataType.widenSmallNumeric()) + ); + } + } + public void testTbucketWithUnmappedTimestampWithLookupJoin() { var query = """ FROM test @@ -1043,13 +1113,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeyword() { assertUnmappedLoadError( analyzer().addIndex(esIndex), "FROM idx* | WHERE partial_long > 0", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:47: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1065,14 +1129,38 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordReportsAllFields() { "FROM idx* | SORT partial_long, partial_double", allOf( containsString("Found 2 problems"), - containsString( - "line 1:46: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ), - containsString( - "line 1:60: Cannot use field [partial_double] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [double] in [idx_mapped]" - ) + partiallyUnmappedNonKeywordError("partial_long"), + partiallyUnmappedNonKeywordError("partial_double") + ) + ); + } + + /** + * An EVAL referencing both a partially unmapped non-keyword field and a field with a genuine type conflict + * should report errors for both fields. + */ + public void testDisallowLoadWithPartialNonKeywordAndTypeConflictInSameEval() { + assumeTrue("Requires OPTIONAL_FIELDS_V5", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V5.isEnabled()); + + var conflicted = new InvalidMappedField( + "conflicted", + Map.of(DataType.LONG.typeName(), Set.of("idx_a"), DataType.DOUBLE.typeName(), Set.of("idx_b")) + ); + var merged = new EsIndex( + "idx*", + Map.of("partial_long", longField("partial_long"), "conflicted", conflicted), + Map.of("idx_a", IndexMode.STANDARD, "idx_b", IndexMode.STANDARD, "idx_unmapped", IndexMode.STANDARD), + Map.of(), + Map.of(), + Map.of("partial_long", Set.of("idx_unmapped")) + ); + assertUnmappedLoadError( + analyzer().addIndex("idx*", IndexResolution.valid(merged)), + "FROM idx* | EVAL x = partial_long + 1, y = conflicted + 1", + allOf( + containsString("Found 2 problems"), + partiallyUnmappedNonKeywordError("partial_long"), + containsString("Cannot use field [conflicted]") ) ); } @@ -1127,13 +1215,7 @@ public void testDisallowLoadCommaSeparatedIndicesWhenPartialNonKeywordUsed() { assertUnmappedLoadError( analyzer().addIndex(pattern, IndexResolution.valid(merged)), "FROM idx_a, idx_b | WHERE partial_long > 0", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:55: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_a]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1154,28 +1236,12 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInRename() { ); var analyzer = analyzer().addIndex(esIndex); - assertUnmappedLoadError( - analyzer, - "FROM idx* | RENAME partial_long AS pl", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:48: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) - ); + assertUnmappedLoadError(analyzer, "FROM idx* | RENAME partial_long AS pl", partiallyUnmappedNonKeywordError("partial_long")); assertUnmappedLoadError( analyzer, "FROM idx* | RENAME common as c, partial_long AS pl", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:61: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1186,13 +1252,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInSort() { assertUnmappedLoadError( analyzer().addIndex(esIndex), "FROM idx* | SORT partial_long", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:46: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1217,13 +1277,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInChangePoint() { assertUnmappedLoadError( analyzer().addIndex(esIndex), "FROM idx* | CHANGE_POINT partial_long ON @timestamp AS type, pvalue", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:54: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1235,13 +1289,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordInMvExpand() { assertUnmappedLoadError( analyzer().addIndex(esIndex), "FROM idx* | MV_EXPAND partial_long", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:51: Cannot use field [partial_long] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) + partiallyUnmappedNonKeywordError("partial_long") ); } @@ -1251,17 +1299,7 @@ public void testDisallowLoadWithPartiallyMappedNonKeywordDottedPath() { var sub = longField("sub"); var obj = new EsField("obj", DataType.OBJECT, Map.of("sub", sub), true, EsField.TimeSeriesFieldType.NONE); var esIndex = partialIndex(Map.of("obj", obj), Set.of("obj.sub")); - assertUnmappedLoadError( - analyzer().addIndex(esIndex), - "FROM idx* | SORT `obj.sub`", - allOf( - containsString("Found 1 problem"), - containsString( - "line 1:46: Cannot use field [obj.sub] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [long] in [idx_mapped]" - ) - ) - ); + assertUnmappedLoadError(analyzer().addIndex(esIndex), "FROM idx* | SORT `obj.sub`", partiallyUnmappedNonKeywordError("obj.sub")); } /** @@ -1297,10 +1335,10 @@ public void testDisallowLoadWithPartialUnionTimestampInWhere() { + "| WHERE @timestamp == \"2021-01-01\"::date_nanos", allOf( containsString("Found 1 problem"), - containsString( - "line 1:116: Cannot use field [@timestamp] due to ambiguities being mapped as [2] incompatible types: " - + "[keyword] due to loading from _source, [date_nanos] in [sample_data, sample_data_ts_nanos]" - ) + containsString("line 1:116: Cannot use field [@timestamp] due to ambiguities being mapped as [3] incompatible types: "), + containsString("[keyword] due to loading from _source"), + containsString("[date_nanos] in [sample_data_ts_nanos]"), + containsString("[datetime] in [sample_data]") ) ); } @@ -1398,7 +1436,12 @@ private void typeConflictVerificationFailure(String statement, Map ta.statement(statement)); - assertThat(e.getMessage(), containsString("Cannot use field [message]")); + // Single-type partially unmapped fields are caught explicitly by the Verifier; multi-type conflicts are caught by + // being marked with UnsupportedAttributes, whose error message mentions the type conflicts. + assertThat( + e.getMessage(), + Matchers.anyOf(partiallyUnmappedNonKeywordError("message"), containsString("Cannot use field [message]")) + ); } private static EsIndex partialIndex(Map mapping, Set partialFieldNames) { @@ -1503,9 +1546,14 @@ public void testUnmappedFieldsLoadWithFullTextSearchFails() { + "use \"default\" or \"nullify\"" ) ) + ); } + private static Matcher partiallyUnmappedNonKeywordError(String fieldName) { + return containsString("Using partially unmapped non-KEYWORD field [" + fieldName + "]"); + } + @Override protected List filteredWarnings() { return withInlinestatsWarning(withDefaultLimitWarning(super.filteredWarnings())); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java index 6ac3fdea044c8..cba4d3930c1a9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/GoldenTestCase.java @@ -793,13 +793,31 @@ public static IndexResolution loadIndexResolution(CsvTestsDataLoader.MultiIndexT ); } + // TODO should de-duplicate, strong overlap with CsvTestsDataLoader#readMappingFile private static Map createMappingForIndex(CsvTestsDataLoader.TestDataset dataset) { var mapping = new TreeMap<>(LoadMapping.loadMapping(dataset.streamMapping())); if (dataset.typeMapping() != null) { for (var entry : dataset.typeMapping().entrySet()) { - if (mapping.containsKey(entry.getKey())) { + String key = entry.getKey(); + String[] segments = key.split("\\."); + // Navigate to the parent map containing the leaf field. + Map targetMap = mapping; + for (int i = 0; i < segments.length - 1 && targetMap != null; i++) { + EsField parent = targetMap.get(segments[i]); + targetMap = parent != null ? parent.getProperties() : null; + } + String leafName = segments[segments.length - 1]; + if (targetMap == null) { + continue; + } + + if (entry.getValue() == null) { + targetMap.remove(leafName); + continue; + } + if (targetMap.containsKey(leafName)) { DataType dataType = DataType.fromTypeName(entry.getValue()); - EsField field = mapping.get(entry.getKey()); + EsField field = targetMap.get(leafName); EsField editedField = new EsField( field.getName(), dataType, @@ -807,7 +825,7 @@ private static Map createMappingForIndex(CsvTestsDataLoader.Tes field.isAggregatable(), field.getTimeSeriesFieldType() ); - mapping.put(entry.getKey(), editedField); + targetMap.put(leafName, editedField); } } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimitByGoldenTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimitByGoldenTests.java index ea60b6c7110fa..565fa6002717b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimitByGoldenTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/rules/logical/PushDownAndCombineLimitByGoldenTests.java @@ -35,7 +35,7 @@ public static void checkLimitByCapability() {} public void testLimitByNotPushedPastEval() { runGoldenTest( """ - FROM * + FROM airport_city_boundaries, addresses, all_types, books | ENRICH languages on street | KEEP abbrev, integer, year | LIMIT 1 BY abbrev diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected index 812e7c610bd25..337900ed01ba1 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testMappedToNonKeywordInOneIndexOnly/load/analysis.expected @@ -1,3 +1,3 @@ Limit[1000[INTEGER],false,false] -\_Project[[!event_duration]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3] \ No newline at end of file +\_Project[[event_duration{f}#0]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#0, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/analysis.expected new file mode 100644 index 0000000000000..72858027185a7 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[first_name{f(PotentiallyUnmappedKeywordEsField)}#0, gender{f}#1]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#2, birth_date{f}#3, emp_no{f}#4, first_name{f(PotentiallyUnmappedKeywordEsField)}#0, gender{f}#1, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f(PotentiallyUnmappedKeywordEsField)}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/query.esql new file mode 100644 index 0000000000000..8b1fba87772c2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM employees, employees_no_names +| KEEP first_name, gender diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/analysis.expected new file mode 100644 index 0000000000000..e40244072d71a --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[first_name{f}#0, gender{f}#1]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#2, birth_date{f}#3, emp_no{f}#4, first_name{f}#0, gender{f}#1, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/query.esql new file mode 100644 index 0000000000000..db70e3133f6b2 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testNonPartiallyMappedKeywordFieldNotLoadedFromSource/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM employees, employees_no_names +| KEEP first_name, gender diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected new file mode 100644 index 0000000000000..3afa3e37cfefa --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/analysis.expected @@ -0,0 +1,2 @@ +Limit[1000[INTEGER],false,false] +\_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/query.esql new file mode 100644 index 0000000000000..74dc2ef0710ee --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/load/query.esql @@ -0,0 +1 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/analysis.expected new file mode 100644 index 0000000000000..25c9cb672505f --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/analysis.expected @@ -0,0 +1,2 @@ +Limit[1000[INTEGER],false,false] +\_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/query.esql new file mode 100644 index 0000000000000..f7fa817d65c34 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFound/nullify/query.esql @@ -0,0 +1 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected new file mode 100644 index 0000000000000..6300f15f4c963 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f(PotentiallyUnmappedKeywordEsField)}#3]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/query.esql new file mode 100644 index 0000000000000..c4112b1def237 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| KEEP * diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/analysis.expected new file mode 100644 index 0000000000000..00eca33ef618d --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/query.esql new file mode 100644 index 0000000000000..55b5093450910 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsAutomaticallyFoundKeepStar/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| KEEP * diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/analysis.expected new file mode 100644 index 0000000000000..4909f0a70f5ae --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[emp_no{f}#0, first_name{f(PotentiallyUnmappedKeywordEsField)}#1, is_rehired{f}#2, last_name{f(PotentiallyUnmappedKeywordEsField)}#3, salary{f}#4]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#5, birth_date{f}#6, emp_no{f}#0, first_name{f(PotentiallyUnmappedKeywordEsField)}#1, gender{f}#7, height{f}#8, height.float{f}#9, height.half_float{f}#10, height.scaled_float{f}#11, hire_date{f}#12, is_rehired{f}#2, job_positions{f}#13, languages{f}#14, languages.byte{f}#15, languages.long{f}#16, languages.short{f}#17, last_name{f(PotentiallyUnmappedKeywordEsField)}#3, salary{f}#4, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/query.esql new file mode 100644 index 0000000000000..371f30a81bec3 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM employees, employees_no_names +| DROP *date*, gender, height*, languages*, *_hired, *_seconds, *_positions, salary_change* diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/analysis.expected new file mode 100644 index 0000000000000..e3d98f9e3811e --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[emp_no{f}#0, first_name{f}#1, is_rehired{f}#2, last_name{f}#3, salary{f}#4]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#5, birth_date{f}#6, emp_no{f}#0, first_name{f}#1, gender{f}#7, height{f}#8, height.float{f}#9, height.half_float{f}#10, height.scaled_float{f}#11, hire_date{f}#12, is_rehired{f}#2, job_positions{f}#13, languages{f}#14, languages.byte{f}#15, languages.long{f}#16, languages.short{f}#17, last_name{f}#3, salary{f}#4, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/query.esql new file mode 100644 index 0000000000000..de53dad8692eb --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropKeywordWithWildcards/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM employees, employees_no_names +| DROP *date*, gender, height*, languages*, *_hired, *_seconds, *_positions, salary_change* diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected new file mode 100644 index 0000000000000..96ca49e813873 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[message{f(PotentiallyUnmappedKeywordEsField)}#0]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/query.esql new file mode 100644 index 0000000000000..c474e51eadeee --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| DROP *_ip, *_duration, @timestamp diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/analysis.expected new file mode 100644 index 0000000000000..1b43c24ef2c44 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[message{f}#0]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#1, client_ip{f}#2, event_duration{f}#3, message{f}#0] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/query.esql new file mode 100644 index 0000000000000..331f316fcf2cc --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropNonKeywordWithWildcards/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| DROP *_ip, *_duration, @timestamp diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/analysis.expected new file mode 100644 index 0000000000000..321582baaaa1e --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/query.esql new file mode 100644 index 0000000000000..0913b709d4160 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| DROP message diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/analysis.expected new file mode 100644 index 0000000000000..e40d25c344506 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#2, message{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/query.esql new file mode 100644 index 0000000000000..4226ac74785c0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMapped/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| DROP message diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/analysis.expected new file mode 100644 index 0000000000000..4a31070ea7d9a --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, message{f(PotentiallyUnmappedKeywordEsField)}#2]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#2] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/query.esql new file mode 100644 index 0000000000000..83ae295113395 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| DROP event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/analysis.expected new file mode 100644 index 0000000000000..ff1b5a9e6460b --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, client_ip{f}#1, message{f}#2]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#1, event_duration{f}#3, message{f}#2] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/query.esql new file mode 100644 index 0000000000000..ac026bac0ad2a --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedFieldsDropOnePartiallyMappedNonKeyword/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| DROP event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/analysis.expected new file mode 100644 index 0000000000000..d62949f5d6842 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/analysis.expected @@ -0,0 +1,4 @@ +Limit[10000[INTEGER],false,false] +\_Limit[1[INTEGER],false,false] + \_OrderBy[[Order[emp_no{f}#0,ASC,LAST]]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#1, birth_date{f}#2, emp_no{f}#0, first_name{f(PotentiallyUnmappedKeywordEsField)}#3, gender{f}#4, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f(PotentiallyUnmappedKeywordEsField)}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/query.esql new file mode 100644 index 0000000000000..b8c7481e3d4ce --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/load/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="load"; FROM employees, employees_no_names +| SORT emp_no +| LIMIT 1 diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/analysis.expected new file mode 100644 index 0000000000000..570e8eb95973f --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/analysis.expected @@ -0,0 +1,4 @@ +Limit[10000[INTEGER],false,false] +\_Limit[1[INTEGER],false,false] + \_OrderBy[[Order[emp_no{f}#0,ASC,LAST]]] + \_EsRelation[employees,employees_no_names][avg_worked_seconds{f}#1, birth_date{f}#2, emp_no{f}#0, first_name{f}#3, gender{f}#4, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/query.esql new file mode 100644 index 0000000000000..2cfb9690b7de0 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedKeywordFieldLoadedWithoutExplicitReference/nullify/query.esql @@ -0,0 +1,3 @@ +SET unmapped_fields="nullify"; FROM employees, employees_no_names +| SORT emp_no +| LIMIT 1 diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected new file mode 100644 index 0000000000000..c250aa9d2c04c --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, event_duration{f}#1]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#2, event_duration{f}#1, message{f(PotentiallyUnmappedKeywordEsField)}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/query.esql new file mode 100644 index 0000000000000..5d07e509e230e --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM sample_data, no_mapping_sample_data +| KEEP @timestamp, event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected new file mode 100644 index 0000000000000..a8a40a8e5b479 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[@timestamp{f}#0, event_duration{f}#1]] + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#0, client_ip{f}#2, event_duration{f}#1, message{f}#3] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/query.esql new file mode 100644 index 0000000000000..beba8dcb67071 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedNonKeywordFieldMarkedAsPotentiallyUnmapped/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM sample_data, no_mapping_sample_data +| KEEP @timestamp, event_duration diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected new file mode 100644 index 0000000000000..c8bcb0b6753f4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[gender{f}#0]] + \_EsRelation[employees_gender_text,employees_no_gender][avg_worked_seconds{f}#1, birth_date{f}#2, emp_no{f}#3, first_name{f}#4, gender{f}#0, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/query.esql new file mode 100644 index 0000000000000..cde80280471a3 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/load/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="load"; FROM employees_gender_text, employees_no_gender +| KEEP gender diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected new file mode 100644 index 0000000000000..c8bcb0b6753f4 --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/analysis.expected @@ -0,0 +1,3 @@ +Limit[1000[INTEGER],false,false] +\_Project[[gender{f}#0]] + \_EsRelation[employees_gender_text,employees_no_gender][avg_worked_seconds{f}#1, birth_date{f}#2, emp_no{f}#3, first_name{f}#4, gender{f}#0, height{f}#5, height.float{f}#6, height.half_float{f}#7, height.scaled_float{f}#8, hire_date{f}#9, is_rehired{f}#10, job_positions{f}#11, languages{f}#12, languages.byte{f}#13, languages.long{f}#14, languages.short{f}#15, last_name{f}#16, salary{f}#17, salary_change{f}#18, salary_change.int{f}#19, salary_change.keyword{f}#20, salary_change.long{f}#21, still_hired{f}#22] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/query.esql new file mode 100644 index 0000000000000..03bfa9100c87d --- /dev/null +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testPartiallyMappedTextFieldMarkedAsPotentiallyUnmapped/nullify/query.esql @@ -0,0 +1,2 @@ +SET unmapped_fields="nullify"; FROM employees_gender_text, employees_no_gender +| KEEP gender diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected index fba946502a112..f2d64fa28372c 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedAndUnmappedWithCast/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[event_duration{r}#0]] \_Eval[[$$event_duration$converted_to$long{f(MultiTypeEsField)$}#1 AS event_duration#0]] - \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f}#5, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file + \_EsRelation[sample_data,no_mapping_sample_data][@timestamp{f}#2, client_ip{f}#3, event_duration{f}#4, message{f(PotentiallyUnmappedKeywordEsField)}#5, $$event_duration$converted_to$long{f(MultiTypeEsField)$}#1] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected index 4160e1709b996..68726db5f431b 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/analysis/golden_tests/AnalyzerUnmappedGoldenTests/testTypeConflictMappedTimesTwoAndUnmapped/load/analysis.expected @@ -1,4 +1,4 @@ Limit[1000[INTEGER],false,false] \_Project[[ts{r}#0]] \_Eval[[$$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1 AS ts#0]] - \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, client_ip{f}#2, event_duration{f}#3, message{f}#4, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file + \_EsRelation[sample_data_ts_long,sample_data,no_mapping_sample_data][!@timestamp, client_ip{f}#2, event_duration{f}#3, message{f(PotentiallyUnmappedKeywordEsField)}#4, $$@timestamp$converted_to$datetime{f(MultiTypeEsField)$}#1] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/local_physical_optimization.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/local_physical_optimization.expected index ac89d2840635a..9386f463f4f78 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/local_physical_optimization.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/local_physical_optimization.expected @@ -7,4 +7,4 @@ ProjectExec[[abbrev{f}#0, integer{f}#1, year{f}#2]] \_FieldExtractExec[integer{f}#1, street{f}#3, year{f}#2]<[],[],[]> \_LimitByExec[1[INTEGER],[abbrev{f}#0],70] \_EvalExec[[null[KEYWORD] AS abbrev#0]] - \_EsQueryExec[*], indexMode[standard], [_doc{f}#5], limit[], sort[] estimatedRowSize[112] queryBuilderAndTags [[QueryBuilderAndTags[query=null, tags=[]]]] \ No newline at end of file + \_EsQueryExec[airport_city_boundaries,addresses,all_types,books], indexMode[standard], [_doc{f}#5], limit[], sort[] estimatedRowSize[112] queryBuilderAndTags [[QueryBuilderAndTags[query=null, tags=[]]]] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/query.esql b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/query.esql index 2dfe3d5911bcc..15b884b6c6ea9 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/query.esql +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/rules/logical/golden_tests/PushDownAndCombineLimitByGoldenTests/testLimitByNotPushedPastEval/query.esql @@ -1,4 +1,4 @@ -FROM * +FROM airport_city_boundaries, addresses, all_types, books | ENRICH languages on street | KEEP abbrev, integer, year | LIMIT 1 BY abbrev diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/192_unmapped_load_partial_non_keyword.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/192_unmapped_load_partial_non_keyword.yml index 49a80fa5d3ca5..3f86496af41d0 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/192_unmapped_load_partial_non_keyword.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/192_unmapped_load_partial_non_keyword.yml @@ -6,8 +6,8 @@ setup: - method: POST path: /_query parameters: [ ] - capabilities: [ optional_fields_v5 ] - reason: 'requires SET unmapped_fields="load"' + capabilities: [ optional_fields_fix_load_partially_mapped ] + reason: 'requires SET unmapped_fields="load" and current error message for partially unmapped non-keyword fields' # # The table below represents the field mappings for the three indices created in this test. # In `FROM index*`, all the fields except lambda are PUNKs: partially unmapped non-keyword fields. @@ -64,105 +64,19 @@ setup: d: type: boolean ---- -no-command-punks: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in DROP - query: 'SET unmapped_fields="load"; FROM index*' - - length: { values: 0 } - ---- -drop-punks: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in DROP - query: 'SET unmapped_fields="load"; FROM index2,index3 | DROP beta, a.b.c.d' - - length: { values: 0 } - ---- -drop-punks-wildcard: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in wildcard DROP - query: 'SET unmapped_fields="load"; FROM index2,index3 | DROP bet*, a.b*' - ---- -keep-punks: - - skip: - awaits_fix: "https://github.com/elastic/elasticsearch/issues/145206" - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in KEEP - query: 'SET unmapped_fields="load"; FROM index2,index3 | KEEP beta, a.b.c.d' - ---- -keep-punks-wildcard: - - skip: - awaits_fix: "https://github.com/elastic/elasticsearch/issues/145206" - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # PUNKs should not be rejected in wildcard KEEP - query: 'SET unmapped_fields="load"; FROM index1,index2,index3 | KEEP beta*' - --- rename-punks: - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" + catch: bad_request esql.query: body: # [beta] is double in index2 # PUNKs should be rejected in RENAME query: 'SET unmapped_fields="load"; FROM index1,index2,index3 | RENAME beta AS mu' - ---- -convert-punks: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [beta] is double in index2 - # [a.b.c.d] is boolean in index3 - # PUNKs should not be rejected in conversion functions (e.g.: punk::TYPE or TO_TYPE(punk)) - query: 'SET unmapped_fields="load"; FROM index1,index2,index3 | EVAL x = beta::long, y = a.b.c.d::boolean, m = TO_LONG(beta), n = TO_BOOLEAN(a.b.c.d) | SORT x ASC | KEEP y' - ---- -date-nanos: - - do: - allowed_warnings_regex: - - "No limit defined, adding default limit of \\[.*\\]" - esql.query: - body: - # [tau] is date index1 - # [tau] is date_nanos index2 - # Field is mapped differently in both indices. DATE/DATE_NANOS types shouldn't fail because we auto-cast. - query: 'SET unmapped_fields="load"; FROM index1,index2 | WHERE tau > 0::date' + - match: { status: 400 } + - match: { error.type: verification_exception } + - contains: { error.reason: 'Found 1 problem' } + - contains: { error.reason: '1:64: Using partially unmapped non-KEYWORD field [beta] is not supported with unmapped_fields="load"' } --- date-nanos-punks: @@ -228,5 +142,4 @@ stats-punks: - match: { status: 400 } - match: { error.type: verification_exception } - contains: { error.reason: 'Found 1 problem' } - - contains: { error.reason: 'line 1:87: Cannot use field [a.b.c.d] due to ambiguities being mapped as [2] incompatible types: [keyword] due to loading from _source, [boolean] in [index3]' } - + - contains: { error.reason: '1:87: Using partially unmapped non-KEYWORD field [a.b.c.d] is not supported with unmapped_fields="load"' } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml index 18b09cbab70ad..38e578ebf4bcb 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/260_flattened_subfield.yml @@ -114,14 +114,26 @@ setup: --- 'loading from two indices with different mappings 2': + - requires: + test_runner_features: [capabilities] + capabilities: + - method: POST + path: /_query + parameters: [] + capabilities: [optional_fields_fix_load_partially_mapped] + reason: "error message changed" - do: - catch: '/Cannot use field \[foo.bar\] due to ambiguities being mapped as \[2\] incompatible types: \[keyword\] due to loading from _source.*\[unsupported\]/' + catch: bad_request esql.query: body: # [foo] is flattened in index1 # [foo.bar] is keyword in index4 query: 'SET unmapped_fields="load"; FROM index1,index4 | WHERE foo.bar == "something"' + - match: { error.type: verification_exception } + - contains: { error.reason: 'Found 1 problem' } + - contains: { error.reason: 'line 1:56: Cannot use field [foo.bar] with unsupported type [flattened]' } + --- 'loading from two indices with different mappings 3': - do: