From 581ac34a255898ff294c3750ed3722709ee62909 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Wed, 31 Jul 2024 16:09:49 -0400 Subject: [PATCH 1/7] ESQL: Remove the `NESTED` DataType We don't support nested fields at the moment and when we do I'm not sure it'll *be* a DataType. Maybe. Probably. But let's deal with that when we support it. --- .../org/elasticsearch/xpack/esql/core/type/DataType.java | 4 +--- .../main/java/org/elasticsearch/xpack/esql/LoadMapping.java | 3 +-- .../elasticsearch/xpack/esql/action/PositionToXContent.java | 2 +- .../elasticsearch/xpack/esql/action/ResponseValueUtils.java | 4 ++-- .../java/org/elasticsearch/xpack/esql/analysis/Analyzer.java | 5 ++--- .../xpack/esql/planner/LocalExecutionPlanner.java | 2 +- .../org/elasticsearch/xpack/esql/planner/PlannerUtils.java | 4 ++-- .../org/elasticsearch/xpack/esql/session/IndexResolver.java | 1 + .../esql/expression/function/AbstractFunctionTestCase.java | 2 +- .../expression/function/AbstractScalarFunctionTestCase.java | 2 +- 10 files changed, 13 insertions(+), 16 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 889c7b9fa9a07..4d6eb683480c6 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -105,7 +105,6 @@ public enum DataType { // 8.15.2-SNAPSHOT is 15 bytes, most are shorter, some can be longer VERSION(builder().esType("version").estimatedSize(15).docValues()), OBJECT(builder().esType("object").unknownSize()), - NESTED(builder().esType("nested").unknownSize()), SOURCE(builder().esType(SourceFieldMapper.NAME).unknownSize()), DATE_PERIOD(builder().typeName("DATE_PERIOD").estimatedSize(3 * Integer.BYTES)), TIME_DURATION(builder().typeName("TIME_DURATION").estimatedSize(Integer.BYTES + Long.BYTES)), @@ -287,7 +286,7 @@ public static boolean isPrimitiveAndSupported(DataType t) { } public static boolean isPrimitive(DataType t) { - return t != OBJECT && t != NESTED; + return t != OBJECT; } public static boolean isNull(DataType t) { @@ -335,7 +334,6 @@ public static boolean areCompatible(DataType left, DataType right) { */ public static boolean isRepresentable(DataType t) { return t != OBJECT - && t != NESTED && t != UNSUPPORTED && t != DATE_PERIOD && t != TIME_DURATION diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java index 5e3b8904d2a98..3b8259f3092d9 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java @@ -29,7 +29,6 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; -import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED; import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; @@ -82,7 +81,7 @@ private static void walkMapping(String name, Object value, Map // extract field type DataType esDataType = getType(content); final Map properties; - if (esDataType == OBJECT || esDataType == NESTED) { + if (esDataType == OBJECT) { properties = fromEs(content); } else if (content.containsKey("fields")) { // Check for multifields diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index 0bc1eb46abefe..6b82a18197fe1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -165,7 +165,7 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa } } }; - case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, NESTED, FLOAT, HALF_FLOAT, SCALED_FLOAT, + case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT, PARTIAL_AGG -> throw new IllegalArgumentException("can't convert values of type [" + columnInfo.type() + "]"); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index 67c6e1f48a47a..8c6a61a5bffbe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -140,8 +140,8 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR throw new UncheckedIOException(e); } } - case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, NESTED, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, - NULL, PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType); + case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, NULL, + PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 75e494fe9671e..bcc3c4fb91670 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -113,7 +113,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; -import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; @@ -245,8 +244,8 @@ private static void mappingAsAttributes(List list, Source source, Fie if (DataType.isPrimitive(type)) { list.add(attribute); } - // allow compound object even if they are unknown (but not NESTED) - if (type != NESTED && fieldProperties.isEmpty() == false) { + // allow compound object even if they are unknown + if (fieldProperties.isEmpty() == false) { mappingAsAttributes(list, source, attribute, fieldProperties); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 28855abfff73c..0708986ea9e41 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -351,7 +351,7 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte case TEXT, KEYWORD -> TopNEncoder.UTF8; case VERSION -> TopNEncoder.VERSION; case BOOLEAN, NULL, BYTE, SHORT, INTEGER, LONG, DOUBLE, FLOAT, HALF_FLOAT, DATETIME, DATE_PERIOD, TIME_DURATION, OBJECT, - NESTED, SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE; + SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE; case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE -> TopNEncoder.DEFAULT_UNSORTABLE; // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 57d8d9748aa01..1a6bda9ffc484 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -253,8 +253,8 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field case GEO_POINT, CARTESIAN_POINT -> fieldExtractPreference == DOC_VALUES ? ElementType.LONG : ElementType.BYTES_REF; case GEO_SHAPE, CARTESIAN_SHAPE -> ElementType.BYTES_REF; case PARTIAL_AGG -> ElementType.COMPOSITE; - case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, NESTED, FLOAT, HALF_FLOAT, SCALED_FLOAT -> - throw EsqlIllegalArgumentException.illegalDataType(dataType); + case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT -> throw EsqlIllegalArgumentException + .illegalDataType(dataType); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 65f47580103d5..30e3db3681528 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -81,6 +81,7 @@ public void resolveAsMergedMapping(String indexWildcard, Set fieldNames, ); } + // public for testing only public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker if (fieldCapsResponse.getIndexResponses().isEmpty()) { diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 20c583d3ac898..cc853cc458720 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -168,7 +168,7 @@ public static Literal randomLiteral(DataType type) { throw new UncheckedIOException(e); } } - case UNSUPPORTED, OBJECT, NESTED, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( + case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( "can't make random values for [" + type.typeName() + "]" ); }, type); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java index 1caea78e79ad5..712b3c1fd09fc 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractScalarFunctionTestCase.java @@ -717,7 +717,7 @@ public static Stream validFunctionParameters() { */ return false; } - if (t == DataType.OBJECT || t == DataType.NESTED) { + if (t == DataType.OBJECT) { // Object and nested fields aren't supported by any functions yet return false; } From d0405b2f6cdfa4b0aa8d658d9e4457a086b836d1 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 1 Aug 2024 12:26:36 -0400 Subject: [PATCH 2/7] More tests for nested --- .../test/esql/40_unsupported_types.yml | 39 +++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index c34666bb12b02..979f3926457ba 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -390,3 +390,42 @@ unsupported with sort: - match: { values.0.26: xy } - match: { values.0.27: "foo bar" } - match: { values.0.28: 3 } + +--- +nested: + - do: + bulk: + index: test + refresh: true + body: + - { "index": { } } + - { + "find_me": 1, + "nested": { + "foo": 1, + "bar": "bar", + "baz": 1.9 + } + } + + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test | WHERE find_me == 1 | KEEP nested, nested.*' + + - match: { columns.0.name: nested } + - match: { columns.0.type: unsupported } + - match: { columns.1.name: nested.bar } + - match: { columns.1.type: unsupported } + - match: { columns.2.name: nested.bar.keyword } + - match: { columns.2.type: unsupported } + - match: { columns.3.name: nested.baz } + - match: { columns.3.type: unsupported } + - match: { columns.4.name: nested.foo } + - match: { columns.4.type: unsupported } + - match: + values: + - [null, null, null, null, null] From 4a941555723576893ccb395c05d9498577413072 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 1 Aug 2024 13:03:17 -0400 Subject: [PATCH 3/7] Maybe this --- .../elasticsearch/xpack/esql/LoadMapping.java | 4 ++++ .../xpack/esql/session/IndexResolver.java | 24 +++++++++++++++++-- .../test/esql/40_unsupported_types.yml | 2 +- 3 files changed, 27 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java index 3b8259f3092d9..e750502cad198 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java @@ -78,6 +78,10 @@ private static void walkMapping(String name, Object value, Map if (value instanceof Map) { Map content = (Map) value; + if ("nested".equals(content.get("type"))) { + // Nested fields are entirely removed by IndexResolver so we mimic it. + return; + } // extract field type DataType esDataType = getType(content); final Map properties; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 30e3db3681528..ec3132f8b25fa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -94,8 +94,9 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp // TODO flattened is simpler - could we get away with that? String[] names = fieldsCaps.keySet().toArray(new String[0]); Arrays.sort(names); + Set forbiddenFields = new HashSet<>(); Map rootFields = new HashMap<>(); - for (String name : names) { + name: for (String name : names) { Map fields = rootFields; String fullName = name; boolean isAlias = false; @@ -106,6 +107,9 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp break; } String parent = name.substring(0, nextDot); + if (forbiddenFields.contains(parent)) { + continue name; + } EsField obj = fields.get(parent); if (obj == null) { obj = new EsField(parent, OBJECT, new HashMap<>(), false, true); @@ -117,9 +121,16 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp fields = obj.getProperties(); name = name.substring(nextDot + 1); } + + List caps = fieldsCaps.get(fullName); + if (allNested(caps)) { + forbiddenFields.add(name); + continue; + } // TODO we're careful to make isAlias match IndexResolver - but do we use it? + EsField field = firstUnsupportedParent == null - ? createField(fieldCapsResponse, name, fullName, fieldsCaps.get(fullName), isAlias) + ? createField(fieldCapsResponse, name, fullName, caps, isAlias) : new UnsupportedEsField( fullName, firstUnsupportedParent.getOriginalType(), @@ -145,6 +156,15 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices)); } + private boolean allNested(List caps) { + for (IndexFieldCapabilities cap : caps) { + if (false == cap.type().equalsIgnoreCase("nested")) { + return false; + } + } + return true; + } + private static Map> collectFieldCaps(FieldCapabilitiesResponse fieldCapsResponse) { Set seenHashes = new HashSet<>(); Map> fieldsCaps = new HashMap<>(); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index 979f3926457ba..99240c75a43bf 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -414,7 +414,7 @@ nested: - "No limit defined, adding default limit of \\[.*\\]" esql.query: body: - query: 'FROM test | WHERE find_me == 1 | KEEP nested, nested.*' + query: 'FROM test | WHERE find_me == 1 | KEEP n*' - match: { columns.0.name: nested } - match: { columns.0.type: unsupported } From 2fa37c1c4cf71f3f6bbde602870c8ad66e5508c3 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Thu, 1 Aug 2024 13:08:57 -0400 Subject: [PATCH 4/7] Test --- .../test/esql/40_unsupported_types.yml | 17 ++++------------- 1 file changed, 4 insertions(+), 13 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index 99240c75a43bf..5bb7b5e535656 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -416,16 +416,7 @@ nested: body: query: 'FROM test | WHERE find_me == 1 | KEEP n*' - - match: { columns.0.name: nested } - - match: { columns.0.type: unsupported } - - match: { columns.1.name: nested.bar } - - match: { columns.1.type: unsupported } - - match: { columns.2.name: nested.bar.keyword } - - match: { columns.2.type: unsupported } - - match: { columns.3.name: nested.baz } - - match: { columns.3.type: unsupported } - - match: { columns.4.name: nested.foo } - - match: { columns.4.type: unsupported } - - match: - values: - - [null, null, null, null, null] + # The `nested` field is not visible, nor are any of it's subfields. + - match: { columns.0.name: name } + - match: { columns.0.type: keyword } + - match: { values: [[null]] } From 47dac6070936c09bd2563393a3367b595ad318c8 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Mon, 5 Aug 2024 13:01:16 -0400 Subject: [PATCH 5/7] more tests --- .../test/esql/40_unsupported_types.yml | 72 ++++++++++++++++++- 1 file changed, 69 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index 5bb7b5e535656..e67a76e27ce15 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -392,7 +392,7 @@ unsupported with sort: - match: { values.0.28: 3 } --- -nested: +nested declared inline: - do: bulk: index: test @@ -417,6 +417,72 @@ nested: query: 'FROM test | WHERE find_me == 1 | KEEP n*' # The `nested` field is not visible, nor are any of it's subfields. - - match: { columns.0.name: name } - - match: { columns.0.type: keyword } + - match: { columns: [{name: name, type: keyword}] } - match: { values: [[null]] } + +--- +nested declared in mapping: + - do: + indices.create: + index: test_nested + body: + settings: + number_of_shards: 5 + mappings: + properties: + name: + type: keyword + nested: + type: nested + properties: + foo: + type: keyword + bar: + type: keyword + + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test_nested' + + # The `nested` field is not visible, nor are any of it's subfields. + - match: { columns: [{name: name, type: keyword}] } + +--- +double nested declared in mapping: + - do: + indices.create: + index: test_nested + body: + settings: + number_of_shards: 5 + mappings: + properties: + name: + type: keyword + nested: + type: nested + properties: + bort: + type: keyword + nested: + type: nested + properties: + foo: + type: keyword + bar: + type: keyword + + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test_nested' + + # The `nested` field is not visible, nor are any of it's subfields. + - match: { columns: [{name: name, type: keyword}] } From 98c316db41650d8dd7bfa2c4a0a6160c462bb587 Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 6 Aug 2024 08:42:00 -0400 Subject: [PATCH 6/7] Transport change..... --- .../org/elasticsearch/TransportVersions.java | 1 + .../xpack/esql/core/type/EsField.java | 28 +++++++++++++++++-- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 53e6d04821717..d3d61108c584d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -184,6 +184,7 @@ static TransportVersion def(int id) { public static final TransportVersion INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN = def(8_714_00_0); public static final TransportVersion ESQL_ATTRIBUTE_CACHED_SERIALIZATION = def(8_715_00_0); public static final TransportVersion REGISTER_SLM_STATS = def(8_716_00_0); + public static final TransportVersion ESQL_NESTED_UNSUPPORTED = def(8_717_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index 4ef20a724ab3c..eb3fb5e64d443 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -54,16 +55,39 @@ public EsField(String name, DataType esDataType, Map properties public EsField(StreamInput in) throws IOException { this.name = in.readString(); - this.esDataType = DataType.readFrom(in); + this.esDataType = readDataType(in); this.properties = in.readImmutableMap(i -> i.readNamedWriteable(EsField.class)); this.aggregatable = in.readBoolean(); this.isAlias = in.readBoolean(); } + private DataType readDataType(StreamInput in) throws IOException { + String name = in.readString(); + if (in.getTransportVersion().before(TransportVersions.ESQL_NESTED_UNSUPPORTED) && name.equalsIgnoreCase("NESTED")) { + /* + * The "nested" data type existed in older versions of ESQL but was + * entirely used to filter mappings away. Those versions will still + * sometimes send it inside EsField when hitting `nested` fields in + * indices. But the rest of ESQL will never see that type. Thus, we + * translate it here. We translate to UNSUPPORTED because that seems + * to work. We've already performed any required filtering. + */ + return DataType.UNSUPPORTED; + } + if (name.equalsIgnoreCase(DataType.DOC_DATA_TYPE.nameUpper())) { + return DataType.DOC_DATA_TYPE; + } + DataType dataType = DataType.fromTypeName(name); + if (dataType == null) { + throw new IOException("Unknown DataType for type name: " + name); + } + return dataType; + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); - out.writeString(esDataType.typeName()); + esDataType.writeTo(out); out.writeMap(properties, StreamOutput::writeNamedWriteable); out.writeBoolean(aggregatable); out.writeBoolean(isAlias); From 25460e950d859d42a89c7a20d2d7308f690ad82a Mon Sep 17 00:00:00 2001 From: Nik Everett Date: Tue, 6 Aug 2024 10:49:57 -0400 Subject: [PATCH 7/7] Explain the read more --- .../xpack/esql/core/type/DataType.java | 19 ++++++++++++++++++- .../xpack/esql/core/type/EsField.java | 9 +-------- 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 4d6eb683480c6..ad25c44eb9362 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -226,6 +226,11 @@ public static Collection types() { return TYPES; } + /** + * Resolve a type from a name. This name is sometimes user supplied, + * like in the case of {@code ::} and is sometimes the name + * used over the wire, like in {@link #readFrom(String)}. + */ public static DataType fromTypeName(String name) { return NAME_TO_TYPE.get(name.toLowerCase(Locale.ROOT)); } @@ -440,8 +445,20 @@ public void writeTo(StreamOutput out) throws IOException { public static DataType readFrom(StreamInput in) throws IOException { // TODO: Use our normal enum serialization pattern - String name = in.readString(); + return readFrom(in.readString()); + } + + /** + * Resolve a {@link DataType} from a name read from a {@link StreamInput}. + * @throws IOException on an unknown dataType + */ + public static DataType readFrom(String name) throws IOException { if (name.equalsIgnoreCase(DataType.DOC_DATA_TYPE.nameUpper())) { + /* + * DOC is not declared in fromTypeName because fromTypeName is + * exposed to users for things like `::` and we don't + * want folks to be able to convert to `DOC`. + */ return DataType.DOC_DATA_TYPE; } DataType dataType = DataType.fromTypeName(name); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index eb3fb5e64d443..eb17d720d2140 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -74,14 +74,7 @@ private DataType readDataType(StreamInput in) throws IOException { */ return DataType.UNSUPPORTED; } - if (name.equalsIgnoreCase(DataType.DOC_DATA_TYPE.nameUpper())) { - return DataType.DOC_DATA_TYPE; - } - DataType dataType = DataType.fromTypeName(name); - if (dataType == null) { - throw new IOException("Unknown DataType for type name: " + name); - } - return dataType; + return DataType.readFrom(name); } @Override