diff --git a/server/src/main/java/org/elasticsearch/TransportVersions.java b/server/src/main/java/org/elasticsearch/TransportVersions.java index 53e6d04821717..d3d61108c584d 100644 --- a/server/src/main/java/org/elasticsearch/TransportVersions.java +++ b/server/src/main/java/org/elasticsearch/TransportVersions.java @@ -184,6 +184,7 @@ static TransportVersion def(int id) { public static final TransportVersion INDEX_REQUEST_UPDATE_BY_DOC_ORIGIN = def(8_714_00_0); public static final TransportVersion ESQL_ATTRIBUTE_CACHED_SERIALIZATION = def(8_715_00_0); public static final TransportVersion REGISTER_SLM_STATS = def(8_716_00_0); + public static final TransportVersion ESQL_NESTED_UNSUPPORTED = def(8_717_00_0); /* * STOP! READ THIS FIRST! No, really, diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java index 889c7b9fa9a07..ad25c44eb9362 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/DataType.java @@ -105,7 +105,6 @@ public enum DataType { // 8.15.2-SNAPSHOT is 15 bytes, most are shorter, some can be longer VERSION(builder().esType("version").estimatedSize(15).docValues()), OBJECT(builder().esType("object").unknownSize()), - NESTED(builder().esType("nested").unknownSize()), SOURCE(builder().esType(SourceFieldMapper.NAME).unknownSize()), DATE_PERIOD(builder().typeName("DATE_PERIOD").estimatedSize(3 * Integer.BYTES)), TIME_DURATION(builder().typeName("TIME_DURATION").estimatedSize(Integer.BYTES + Long.BYTES)), @@ -227,6 +226,11 @@ public static Collection types() { return TYPES; } + /** + * Resolve a type from a name. This name is sometimes user supplied, + * like in the case of {@code ::} and is sometimes the name + * used over the wire, like in {@link #readFrom(String)}. + */ public static DataType fromTypeName(String name) { return NAME_TO_TYPE.get(name.toLowerCase(Locale.ROOT)); } @@ -287,7 +291,7 @@ public static boolean isPrimitiveAndSupported(DataType t) { } public static boolean isPrimitive(DataType t) { - return t != OBJECT && t != NESTED; + return t != OBJECT; } public static boolean isNull(DataType t) { @@ -335,7 +339,6 @@ public static boolean areCompatible(DataType left, DataType right) { */ public static boolean isRepresentable(DataType t) { return t != OBJECT - && t != NESTED && t != UNSUPPORTED && t != DATE_PERIOD && t != TIME_DURATION @@ -442,8 +445,20 @@ public void writeTo(StreamOutput out) throws IOException { public static DataType readFrom(StreamInput in) throws IOException { // TODO: Use our normal enum serialization pattern - String name = in.readString(); + return readFrom(in.readString()); + } + + /** + * Resolve a {@link DataType} from a name read from a {@link StreamInput}. + * @throws IOException on an unknown dataType + */ + public static DataType readFrom(String name) throws IOException { if (name.equalsIgnoreCase(DataType.DOC_DATA_TYPE.nameUpper())) { + /* + * DOC is not declared in fromTypeName because fromTypeName is + * exposed to users for things like `::` and we don't + * want folks to be able to convert to `DOC`. + */ return DataType.DOC_DATA_TYPE; } DataType dataType = DataType.fromTypeName(name); diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index 4ef20a724ab3c..eb17d720d2140 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersions; import org.elasticsearch.common.io.stream.NamedWriteable; import org.elasticsearch.common.io.stream.NamedWriteableRegistry; import org.elasticsearch.common.io.stream.StreamInput; @@ -54,16 +55,32 @@ public EsField(String name, DataType esDataType, Map properties public EsField(StreamInput in) throws IOException { this.name = in.readString(); - this.esDataType = DataType.readFrom(in); + this.esDataType = readDataType(in); this.properties = in.readImmutableMap(i -> i.readNamedWriteable(EsField.class)); this.aggregatable = in.readBoolean(); this.isAlias = in.readBoolean(); } + private DataType readDataType(StreamInput in) throws IOException { + String name = in.readString(); + if (in.getTransportVersion().before(TransportVersions.ESQL_NESTED_UNSUPPORTED) && name.equalsIgnoreCase("NESTED")) { + /* + * The "nested" data type existed in older versions of ESQL but was + * entirely used to filter mappings away. Those versions will still + * sometimes send it inside EsField when hitting `nested` fields in + * indices. But the rest of ESQL will never see that type. Thus, we + * translate it here. We translate to UNSUPPORTED because that seems + * to work. We've already performed any required filtering. + */ + return DataType.UNSUPPORTED; + } + return DataType.readFrom(name); + } + @Override public void writeTo(StreamOutput out) throws IOException { out.writeString(name); - out.writeString(esDataType.typeName()); + esDataType.writeTo(out); out.writeMap(properties, StreamOutput::writeNamedWriteable); out.writeBoolean(aggregatable); out.writeBoolean(isAlias); diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java index 5e3b8904d2a98..e750502cad198 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/LoadMapping.java @@ -29,7 +29,6 @@ import static java.util.Collections.emptyMap; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; -import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED; import static org.elasticsearch.xpack.esql.core.type.DataType.OBJECT; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; @@ -79,10 +78,14 @@ private static void walkMapping(String name, Object value, Map if (value instanceof Map) { Map content = (Map) value; + if ("nested".equals(content.get("type"))) { + // Nested fields are entirely removed by IndexResolver so we mimic it. + return; + } // extract field type DataType esDataType = getType(content); final Map properties; - if (esDataType == OBJECT || esDataType == NESTED) { + if (esDataType == OBJECT) { properties = fromEs(content); } else if (content.containsKey("fields")) { // Check for multifields diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java index 0bc1eb46abefe..6b82a18197fe1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/PositionToXContent.java @@ -165,7 +165,7 @@ protected XContentBuilder valueToXContent(XContentBuilder builder, ToXContent.Pa } } }; - case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, NESTED, FLOAT, HALF_FLOAT, SCALED_FLOAT, + case DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, SHORT, BYTE, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT, PARTIAL_AGG -> throw new IllegalArgumentException("can't convert values of type [" + columnInfo.type() + "]"); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java index 67c6e1f48a47a..8c6a61a5bffbe 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/ResponseValueUtils.java @@ -140,8 +140,8 @@ private static Object valueAt(DataType dataType, Block block, int offset, BytesR throw new UncheckedIOException(e); } } - case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, NESTED, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, - NULL, PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType); + case SHORT, BYTE, FLOAT, HALF_FLOAT, SCALED_FLOAT, OBJECT, DATE_PERIOD, TIME_DURATION, DOC_DATA_TYPE, TSID_DATA_TYPE, NULL, + PARTIAL_AGG -> throw EsqlIllegalArgumentException.illegalDataType(dataType); }; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 75e494fe9671e..bcc3c4fb91670 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -113,7 +113,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; -import static org.elasticsearch.xpack.esql.core.type.DataType.NESTED; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; @@ -245,8 +244,8 @@ private static void mappingAsAttributes(List list, Source source, Fie if (DataType.isPrimitive(type)) { list.add(attribute); } - // allow compound object even if they are unknown (but not NESTED) - if (type != NESTED && fieldProperties.isEmpty() == false) { + // allow compound object even if they are unknown + if (fieldProperties.isEmpty() == false) { mappingAsAttributes(list, source, attribute, fieldProperties); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java index 28855abfff73c..0708986ea9e41 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/LocalExecutionPlanner.java @@ -351,7 +351,7 @@ private PhysicalOperation planTopN(TopNExec topNExec, LocalExecutionPlannerConte case TEXT, KEYWORD -> TopNEncoder.UTF8; case VERSION -> TopNEncoder.VERSION; case BOOLEAN, NULL, BYTE, SHORT, INTEGER, LONG, DOUBLE, FLOAT, HALF_FLOAT, DATETIME, DATE_PERIOD, TIME_DURATION, OBJECT, - NESTED, SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE; + SCALED_FLOAT, UNSIGNED_LONG, DOC_DATA_TYPE, TSID_DATA_TYPE -> TopNEncoder.DEFAULT_SORTABLE; case GEO_POINT, CARTESIAN_POINT, GEO_SHAPE, CARTESIAN_SHAPE, COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE -> TopNEncoder.DEFAULT_UNSORTABLE; // unsupported fields are encoded as BytesRef, we'll use the same encoder; all values should be null at this point diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java index 57d8d9748aa01..1a6bda9ffc484 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java @@ -253,8 +253,8 @@ public static ElementType toElementType(DataType dataType, MappedFieldType.Field case GEO_POINT, CARTESIAN_POINT -> fieldExtractPreference == DOC_VALUES ? ElementType.LONG : ElementType.BYTES_REF; case GEO_SHAPE, CARTESIAN_SHAPE -> ElementType.BYTES_REF; case PARTIAL_AGG -> ElementType.COMPOSITE; - case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, NESTED, FLOAT, HALF_FLOAT, SCALED_FLOAT -> - throw EsqlIllegalArgumentException.illegalDataType(dataType); + case SHORT, BYTE, DATE_PERIOD, TIME_DURATION, OBJECT, FLOAT, HALF_FLOAT, SCALED_FLOAT -> throw EsqlIllegalArgumentException + .illegalDataType(dataType); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java index 65f47580103d5..ec3132f8b25fa 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/IndexResolver.java @@ -81,6 +81,7 @@ public void resolveAsMergedMapping(String indexWildcard, Set fieldNames, ); } + // public for testing only public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResponse fieldCapsResponse) { assert ThreadPool.assertCurrentThreadPool(ThreadPool.Names.SEARCH_COORDINATION); // too expensive to run this on a transport worker if (fieldCapsResponse.getIndexResponses().isEmpty()) { @@ -93,8 +94,9 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp // TODO flattened is simpler - could we get away with that? String[] names = fieldsCaps.keySet().toArray(new String[0]); Arrays.sort(names); + Set forbiddenFields = new HashSet<>(); Map rootFields = new HashMap<>(); - for (String name : names) { + name: for (String name : names) { Map fields = rootFields; String fullName = name; boolean isAlias = false; @@ -105,6 +107,9 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp break; } String parent = name.substring(0, nextDot); + if (forbiddenFields.contains(parent)) { + continue name; + } EsField obj = fields.get(parent); if (obj == null) { obj = new EsField(parent, OBJECT, new HashMap<>(), false, true); @@ -116,9 +121,16 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp fields = obj.getProperties(); name = name.substring(nextDot + 1); } + + List caps = fieldsCaps.get(fullName); + if (allNested(caps)) { + forbiddenFields.add(name); + continue; + } // TODO we're careful to make isAlias match IndexResolver - but do we use it? + EsField field = firstUnsupportedParent == null - ? createField(fieldCapsResponse, name, fullName, fieldsCaps.get(fullName), isAlias) + ? createField(fieldCapsResponse, name, fullName, caps, isAlias) : new UnsupportedEsField( fullName, firstUnsupportedParent.getOriginalType(), @@ -144,6 +156,15 @@ public IndexResolution mergedMappings(String indexPattern, FieldCapabilitiesResp return IndexResolution.valid(new EsIndex(indexPattern, rootFields, concreteIndices)); } + private boolean allNested(List caps) { + for (IndexFieldCapabilities cap : caps) { + if (false == cap.type().equalsIgnoreCase("nested")) { + return false; + } + } + return true; + } + private static Map> collectFieldCaps(FieldCapabilitiesResponse fieldCapsResponse) { Set seenHashes = new HashSet<>(); Map> fieldsCaps = new HashMap<>(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java index 224fe92cdbf25..b9feded3ceb9d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/AbstractFunctionTestCase.java @@ -175,7 +175,7 @@ public static Literal randomLiteral(DataType type) { throw new UncheckedIOException(e); } } - case UNSUPPORTED, OBJECT, NESTED, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( + case UNSUPPORTED, OBJECT, DOC_DATA_TYPE, TSID_DATA_TYPE, PARTIAL_AGG -> throw new IllegalArgumentException( "can't make random values for [" + type.typeName() + "]" ); }, type); @@ -481,7 +481,7 @@ public static Stream validFunctionParameters() { */ return false; } - if (t == DataType.OBJECT || t == DataType.NESTED) { + if (t == DataType.OBJECT) { // Object and nested fields aren't supported by any functions yet return false; } diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml index c34666bb12b02..e67a76e27ce15 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/40_unsupported_types.yml @@ -390,3 +390,99 @@ unsupported with sort: - match: { values.0.26: xy } - match: { values.0.27: "foo bar" } - match: { values.0.28: 3 } + +--- +nested declared inline: + - do: + bulk: + index: test + refresh: true + body: + - { "index": { } } + - { + "find_me": 1, + "nested": { + "foo": 1, + "bar": "bar", + "baz": 1.9 + } + } + + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test | WHERE find_me == 1 | KEEP n*' + + # The `nested` field is not visible, nor are any of it's subfields. + - match: { columns: [{name: name, type: keyword}] } + - match: { values: [[null]] } + +--- +nested declared in mapping: + - do: + indices.create: + index: test_nested + body: + settings: + number_of_shards: 5 + mappings: + properties: + name: + type: keyword + nested: + type: nested + properties: + foo: + type: keyword + bar: + type: keyword + + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test_nested' + + # The `nested` field is not visible, nor are any of it's subfields. + - match: { columns: [{name: name, type: keyword}] } + +--- +double nested declared in mapping: + - do: + indices.create: + index: test_nested + body: + settings: + number_of_shards: 5 + mappings: + properties: + name: + type: keyword + nested: + type: nested + properties: + bort: + type: keyword + nested: + type: nested + properties: + foo: + type: keyword + bar: + type: keyword + + - do: + allowed_warnings_regex: + - "Field \\[.*\\] cannot be retrieved, it is unsupported or not indexed; returning null" + - "No limit defined, adding default limit of \\[.*\\]" + esql.query: + body: + query: 'FROM test_nested' + + # The `nested` field is not visible, nor are any of it's subfields. + - match: { columns: [{name: name, type: keyword}] }