diff --git a/docs/changelog/143399.yaml b/docs/changelog/143399.yaml new file mode 100644 index 0000000000000..ecefe9ae07b1a --- /dev/null +++ b/docs/changelog/143399.yaml @@ -0,0 +1,7 @@ +area: ES|QL +issues: + - 142968 + - 142959 +pr: 143399 +summary: Fix KQL/QSTR with unmapped fields in NULLIFY mode +type: bug diff --git a/server/src/main/resources/transport/definitions/referable/esql_missing_es_field.csv b/server/src/main/resources/transport/definitions/referable/esql_missing_es_field.csv new file mode 100644 index 0000000000000..67b958b9af9f6 --- /dev/null +++ b/server/src/main/resources/transport/definitions/referable/esql_missing_es_field.csv @@ -0,0 +1 @@ +9305000 diff --git a/server/src/main/resources/transport/upper_bounds/9.4.csv b/server/src/main/resources/transport/upper_bounds/9.4.csv index 3d08c5fa7867e..d0a7ad10a0f41 100644 --- a/server/src/main/resources/transport/upper_bounds/9.4.csv +++ b/server/src/main/resources/transport/upper_bounds/9.4.csv @@ -1 +1 @@ -inference_azure_openai_task_settings_headers,9304000 +esql_missing_es_field,9305000 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec index 4466350f617ca..cc2511d870281 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec @@ -482,3 +482,47 @@ ROW x = 1 x:integer |does_not_exist:null |y:keyword | language_name:keyword 1 |null |null |null ; + +kqlWithUnmappedField +required_capability: optional_fields_nullify_tech_preview +required_capability: kql_function +required_capability: fix_unmapped_fields_in_esrelation + +// Reproducer for https://github.com/elastic/elasticsearch/issues/142968 +// KQL function should work when unmapped fields are referenced later in the query +SET unmapped_fields="nullify"\; +FROM books +| WHERE KQL("author: Faulkner") +| EVAL x = does_not_exist_field + 1 +| KEEP book_no, author, does_not_exist_field, x +| SORT book_no +| LIMIT 3 +; + +book_no:keyword | author:text | does_not_exist_field:null | x:integer +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | null | null +2713 | William Faulkner | null | null +2847 | Colleen Faulkner | null | null +; + +qstrWithUnmappedField +required_capability: optional_fields_nullify_tech_preview +required_capability: qstr_function +required_capability: fix_unmapped_fields_in_esrelation + +// Reproducer for https://github.com/elastic/elasticsearch/issues/142959 +// QSTR function should work when unmapped fields are referenced later in the query +SET unmapped_fields="nullify"\; +FROM books +| WHERE QSTR("author: Faulkner") +| EVAL x = does_not_exist_field + 1 +| KEEP book_no, author, does_not_exist_field, x +| SORT book_no +| LIMIT 3 +; + +book_no:keyword | author:text | does_not_exist_field:null | x:integer +2378 | [Carol Faulkner, Holly Byers Ochoa, Lucretia Mott] | null | null +2713 | William Faulkner | null | null +2847 | Colleen Faulkner | null | null +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 04c69eb54264a..16a256fe8aa02 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -2204,6 +2204,13 @@ public enum Cap { */ FIRST_LAST_AGG_ON_INTS, + /** + * Fix for KQL/QSTR functions failing when used with unmapped fields in NULLIFY mode. + * Unmapped fields are now added directly to EsRelation output with NULL type instead of using Eval nodes. + * https://github.com/elastic/elasticsearch/issues/142968 + */ + FIX_UNMAPPED_FIELDS_IN_ESRELATION, + // Last capability should still have a comma for fewer merge conflicts when adding new ones :) // This comment prevents the semicolon from being on the previous capability when Spotless formats the file. ; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java index 004bcd33447c7..113351d7e9d6b 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/rules/ResolveUnmapped.java @@ -24,6 +24,9 @@ import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedPattern; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; +import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.MissingEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; @@ -97,19 +100,53 @@ private static LogicalPlan resolve(LogicalPlan plan, boolean load) { } /** - * The method introduces {@code EVAL missing_field = NULL}-equivalent into the plan, on top of the source, for every attribute in - * {@code unresolved}. It also "patches" the introduced attributes through the plan, where needed (like through Fork/UntionAll). + * This method introduces null-typed field attributes for every attribute in {@code unresolved}, within the {@link EsRelation}s + * in the plan. The fields are added with {@link DataType#NULL} type, which causes {@code ReplaceFieldWithConstantOrNull} + * (in the local logical optimizer) to replace them with {@code Literal.NULL}. + *

+ * For non-EsRelation sources (Row, LocalRelation), it falls back to inserting Eval nodes with null assignments. + *

+ * It also "patches" the introduced attributes through the plan, where needed (like through Fork/UnionAll). */ private static LogicalPlan nullify(LogicalPlan plan, Set unresolved) { - // insert an Eval on top of every LeafPlan, if there's a UnaryPlan atop it - var transformed = plan.transformUp( - n -> n instanceof UnaryPlan unary && unary.child() instanceof LeafPlan, + // For EsRelation sources: add null-typed fields to the relation's output + var transformed = plan.transformUp(EsRelation.class, esr -> { + if (esr.indexMode() == IndexMode.LOOKUP) { + return esr; + } + List fieldsToNullify = fieldsToNullify(unresolved, Expressions.names(esr.output())); + return fieldsToNullify.isEmpty() ? esr : esr.withAttributes(combine(esr.output(), fieldsToNullify)); + }); + + // For non-EsRelation sources (Row, LocalRelation): insert Eval nodes with null assignments + // This handles cases like: ROW x = 1 | EVAL y = unmapped_field + transformed = transformed.transformUp( + n -> n instanceof UnaryPlan unary && unary.child() instanceof LeafPlan leaf && leaf instanceof EsRelation == false, p -> evalUnresolvedAtopUnary((UnaryPlan) p, nullAliases(unresolved)) ); - // insert an Eval on top of those LeafPlan that are children of n-ary plans (could happen with UnionAll) return transformed.transformUp( n -> n instanceof UnaryPlan == false && n instanceof LeafPlan == false, - nAry -> evalUnresolvedAtopNary(nAry, nullAliases(unresolved)) + nAry -> evalUnresolvedAtopNaryNonEsRelation(nAry, nullAliases(unresolved)) + ); + } + + private static List fieldsToNullify(Set unresolved, List exclude) { + List nullified = new ArrayList<>(unresolved.size()); + for (var ua : unresolved) { + if (exclude.contains(ua.name()) == false) { + nullified.add(nullifyField(ua)); + } + } + return nullified; + } + + private static FieldAttribute nullifyField(Attribute attribute) { + return new FieldAttribute( + attribute.source(), + null, + attribute.qualifier(), + attribute.name(), + new MissingEsField(attribute.name(), DataType.NULL, Map.of(), false, EsField.TimeSeriesFieldType.NONE) ); } @@ -222,6 +259,24 @@ private static LogicalPlan evalUnresolvedAtopNary(LogicalPlan nAry, List return changed ? nAry.replaceChildren(newChildren) : nAry; } + /** + * Inserts an Eval atop each child of the given {@code nAry}, if the child is a non-EsRelation LeafPlan. + * EsRelation sources are handled separately by adding fields to their output. + */ + private static LogicalPlan evalUnresolvedAtopNaryNonEsRelation(LogicalPlan nAry, List nullAliases) { + List newChildren = new ArrayList<>(nAry.children().size()); + boolean changed = false; + for (var child : nAry.children()) { + if (child instanceof LeafPlan source && source instanceof EsRelation == false) { + assertSourceType(source); + child = new Eval(source.source(), source, nullAliases); + changed = true; + } + newChildren.add(child); + } + return changed ? nAry.replaceChildren(newChildren) : nAry; + } + /** * Inserts an Eval atop the given {@code unaryAtopSource}, if this isn't an Eval already. Otherwise it merges the nullAliases into it. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java index ff60217317fb6..ec4e3fd45296d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/DateEsField.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; @@ -51,7 +52,7 @@ public void writeContent(StreamOutput out) throws IOException { writeTimeSeriesFieldType(out); } - public String getWriteableName() { + public String getWriteableName(TransportVersion transportVersion) { return "DateEsField"; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java index 7489877ba2afa..5d93fcbdc03c3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/EsField.java @@ -105,6 +105,7 @@ public static TimeSeriesFieldType fromIndexFieldCapabilities(IndexFieldCapabilit Map.entry("EsField", EsField::new), Map.entry("InvalidMappedField", InvalidMappedField::new), Map.entry("KeywordEsField", KeywordEsField::new), + Map.entry("MissingEsField", MissingEsField::new), Map.entry("MultiTypeEsField", MultiTypeEsField::new), Map.entry("PotentiallyUnmappedKeywordEsField", PotentiallyUnmappedKeywordEsField::new), Map.entry("TextEsField", TextEsField::new), @@ -167,7 +168,7 @@ public static A readFrom(StreamInput in) throws IOException @Override public void writeTo(StreamOutput out) throws IOException { - if (((PlanStreamOutput) out).writeEsFieldCacheHeader(this)) { + if (((PlanStreamOutput) out).writeEsFieldCacheHeader(this, out.getTransportVersion())) { writeContent(out); } } @@ -201,7 +202,7 @@ protected static TimeSeriesFieldType readTimeSeriesFieldType(StreamInput in) thr /** * This needs to be overridden by subclasses for specific serialization */ - public String getWriteableName() { + public String getWriteableName(TransportVersion transportVersion) { return "EsField"; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java index cf5ea2858cfa2..8223d83b9b6df 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/InvalidMappedField.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.QlIllegalArgumentException; @@ -81,7 +82,7 @@ public void writeContent(StreamOutput out) throws IOException { writeTimeSeriesFieldType(out); } - public String getWriteableName() { + public String getWriteableName(TransportVersion transportVersion) { return "InvalidMappedField"; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java index 46222da291801..6cb778fc40131 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/KeywordEsField.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; @@ -76,7 +77,7 @@ public void writeContent(StreamOutput out) throws IOException { writeTimeSeriesFieldType(out); } - public String getWriteableName() { + public String getWriteableName(TransportVersion transportVersion) { return "KeywordEsField"; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MissingEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MissingEsField.java new file mode 100644 index 0000000000000..020f08ef2f487 --- /dev/null +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MissingEsField.java @@ -0,0 +1,54 @@ +/* + * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one + * or more contributor license agreements. Licensed under the Elastic License + * 2.0; you may not use this file except in compliance with the Elastic License + * 2.0. + */ + +package org.elasticsearch.xpack.esql.core.type; + +import org.elasticsearch.TransportVersion; +import org.elasticsearch.common.io.stream.StreamInput; + +import java.io.IOException; +import java.util.Map; + +public class MissingEsField extends EsField { + + private static final TransportVersion ESQL_MISSING_ES_FIELD = TransportVersion.fromName("esql_missing_es_field"); + + public MissingEsField( + String name, + DataType esDataType, + Map properties, + boolean aggregatable, + TimeSeriesFieldType timeSeriesFieldType + ) { + super(name, esDataType, properties, aggregatable, timeSeriesFieldType); + } + + public MissingEsField( + String name, + DataType esDataType, + Map properties, + boolean aggregatable, + boolean isAlias, + TimeSeriesFieldType timeSeriesFieldType + ) { + super(name, esDataType, properties, aggregatable, isAlias, timeSeriesFieldType); + } + + public MissingEsField(StreamInput in) throws IOException { + super(in); + } + + public String getWriteableName(TransportVersion transportVersion) { + if (transportVersion.supports(ESQL_MISSING_ES_FIELD)) { + return "MissingEsField"; + } + // This was introduced because of https://github.com/elastic/elasticsearch/issues/142968 + // Things should still work fine with nodes that don't have MissingEsField, + // so for BWC we just fall back to EsField, which is the closest thing to MissingEsField that older nodes will understand. + return "EsField"; + } +} diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java index b44c608cb92ae..d1a54a6b9680d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/MultiTypeEsField.java @@ -7,6 +7,7 @@ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.xpack.esql.core.expression.Expression; @@ -62,7 +63,7 @@ public void writeContent(StreamOutput out) throws IOException { writeTimeSeriesFieldType(out); } - public String getWriteableName() { + public String getWriteableName(TransportVersion transportVersion) { return "MultiTypeEsField"; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/PotentiallyUnmappedKeywordEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/PotentiallyUnmappedKeywordEsField.java index 679e77ad2c636..414d98e17f8ce 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/PotentiallyUnmappedKeywordEsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/PotentiallyUnmappedKeywordEsField.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import java.io.IOException; @@ -25,7 +26,7 @@ public PotentiallyUnmappedKeywordEsField(StreamInput in) throws IOException { super(in); } - public String getWriteableName() { + public String getWriteableName(TransportVersion transportVersion) { return "PotentiallyUnmappedKeywordEsField"; } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java index 3b0137d678e45..2900b0623ef2d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/TextEsField.java @@ -6,6 +6,7 @@ */ package org.elasticsearch.xpack.esql.core.type; +import org.elasticsearch.TransportVersion; import org.elasticsearch.common.io.stream.StreamInput; import org.elasticsearch.common.io.stream.StreamOutput; import org.elasticsearch.core.Tuple; @@ -54,7 +55,7 @@ public void writeContent(StreamOutput out) throws IOException { writeTimeSeriesFieldType(out); } - public String getWriteableName() { + public String getWriteableName(TransportVersion transportVersion) { return "TextEsField"; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java index eadaec2d43e63..8f4761e01899a 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/core/type/UnsupportedEsField.java @@ -80,7 +80,7 @@ public void writeContent(StreamOutput out) throws IOException { writeTimeSeriesFieldType(out); } - public String getWriteableName() { + public String getWriteableName(TransportVersion transportVersion) { return "UnsupportedEsField"; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java index 7b83795d23d61..9a8b817d16067 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/io/stream/PlanStreamOutput.java @@ -193,9 +193,11 @@ private int cacheAttribute(Attribute attr) { * Writes a cache header for an {@link org.elasticsearch.xpack.esql.core.type.EsField} and caches it if it is not already in the cache. * In that case, the field will have to serialize itself into this stream immediately after this method call. * @param field The EsField to serialize + * @param transportVersion The transport version to use for serialization, + * needed to get the correct field name in case of versioned fields. * @return true if the attribute needs to serialize itself, false otherwise (ie. if already cached) */ - public boolean writeEsFieldCacheHeader(EsField field) throws IOException { + public boolean writeEsFieldCacheHeader(EsField field, TransportVersion transportVersion) throws IOException { Integer cacheId = esFieldIdFromCache(field); if (cacheId != null) { writeZLong(cacheId); @@ -204,7 +206,7 @@ public boolean writeEsFieldCacheHeader(EsField field) throws IOException { cacheId = cacheEsField(field); writeZLong(-1 - cacheId); - writeCachedString(field.getWriteableName()); + writeCachedString(field.getWriteableName(transportVersion)); return true; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index d7b0cc348d227..9c580624a8d8d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -74,8 +74,7 @@ public class AnalyzerUnmappedTests extends ESTestCase { /* * Limit[1000[INTEGER],false,false] * \_Project[[does_not_exist_field{r}#16]] - * \_Eval[[null[NULL] AS does_not_exist_field#16]] - * \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge..] + * \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge.., does_not_exist_field{f}#16] */ public void testKeep() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -90,22 +89,16 @@ public void testKeep() { assertThat(project.projections(), hasSize(1)); assertThat(Expressions.name(project.projections().getFirst()), is("does_not_exist_field")); - var eval = as(project.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist_field")); - var literal = as(alias.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(project.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Project[[does_not_exist_field{r}#18]] - * \_Eval[[null[NULL] AS does_not_exist_field#17]] - * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..] + * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge.., does_not_exist_field{f}#17] */ public void testKeepRepeated() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -120,15 +113,10 @@ public void testKeepRepeated() { assertThat(project.projections(), hasSize(1)); assertThat(Expressions.name(project.projections().getFirst()), is("does_not_exist_field")); - var eval = as(project.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist_field")); - var literal = as(alias.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(project.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } public void testFailKeepAndNonMatchingStar() { @@ -144,8 +132,7 @@ public void testFailKeepAndNonMatchingStar() { /* * Limit[1000[INTEGER],false,false] * \_Project[[emp_no{f}#6, does_not_exist_field{r}#18]] - * \_Eval[[null[NULL] AS does_not_exist_field#18]] - * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..] + * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge.., does_not_exist_field{f}#18] */ public void testKeepAndMatchingStar() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -160,23 +147,17 @@ public void testKeepAndMatchingStar() { assertThat(project.projections(), hasSize(2)); assertThat(Expressions.names(project.projections()), is(List.of("emp_no", "does_not_exist_field"))); - var eval = as(project.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist_field")); - var literal = as(alias.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(project.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Project[[does_not_exist_field1{r}#20, does_not_exist_field2{r}#22]] * \_Eval[[TOINTEGER(does_not_exist_field1{r}#20) + 42[INTEGER] AS x#5]] - * \_Eval[[null[NULL] AS does_not_exist_field1#20, null[NULL] AS does_not_exist_field2#22]] - * \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..] + * \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, .., does_not_exist_field1{f}#20, does_not_exist_field2{f}#22] */ public void testEvalAndKeep() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -198,11 +179,11 @@ public void testEvalAndKeep() { assertThat(aliasX.name(), is("x")); assertThat(Expressions.name(aliasX.child()), is("does_not_exist_field1::INTEGER + 42")); - var eval2 = as(eval1.child(), Eval.class); - assertThat(Expressions.names(eval2.fields()), is(List.of("does_not_exist_field1", "does_not_exist_field2"))); - - var relation = as(eval2.child(), EsRelation.class); + var relation = as(eval1.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> a.name().startsWith("does_not_exist_field")).toList(); + assertThat(dneAttrs, hasSize(2)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } public void testFailKeepAndMatchingAndNonMatchingStar() { @@ -232,8 +213,7 @@ public void testFailAfterKeep() { * \_Eval[[emp_no{f}#11 + 1[INTEGER] AS x#6]] * \_Project[[_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, gender{f}#13, hire_date{f}#18, job{f}#19, job.raw{f}#20, * languages{f}#14, last_name{f}#15, long_noidx{f}#21, salary{f}#16, does_not_exist_field{r}#22]] - * \_Eval[[null[NULL] AS does_not_exist_field#22]] - * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, .., does_not_exist_field{f}#22] */ public void testEvalAfterKeepStar() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -279,17 +259,11 @@ public void testEvalAfterKeepStar() { ) ); - // The child is Eval introducing does_not_exist_field as null - var evalNull = as(esqlProject.child(), Eval.class); - assertThat(evalNull.fields(), hasSize(1)); - var alias = as(evalNull.fields().get(0), Alias.class); - assertThat(alias.name(), is("does_not_exist_field")); - var lit = as(alias.child(), Literal.class); - assertThat(lit.dataType(), is(DataType.NULL)); - - // The child is EsRelation - var relation = as(evalNull.child(), EsRelation.class); + // The child is EsRelation with does_not_exist_field in output + var relation = as(esqlProject.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* @@ -298,8 +272,7 @@ public void testEvalAfterKeepStar() { * \_Eval[[emp_no{f}#12 + 1[INTEGER] AS x#7]] * \_Project[[emp_no{f}#12, _meta_field{f}#18, first_name{f}#13, gender{f}#14, hire_date{f}#19, job{f}#20, job.raw{f}#21, * languages{f}#15, last_name{f}#16, long_noidx{f}#22, salary{f}#17, emp_does_not_exist_field{r}#23]] - * \_Eval[[null[NULL] AS emp_does_not_exist_field#23]] - * \_EsRelation[test][_meta_field{f}#18, emp_no{f}#12, first_name{f}#13, ..] + * \_EsRelation[test][_meta_field{f}#18, emp_no{f}#12, first_name{f}#13, .., emp_does_not_exist_field{f}#23] */ public void testEvalAfterMatchingKeepWithWildcard() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -345,25 +318,18 @@ public void testEvalAfterMatchingKeepWithWildcard() { ) ); - // The child is Eval introducing emp_does_not_exist_field as null - var evalNull = as(esqlProject.child(), Eval.class); - assertThat(evalNull.fields(), hasSize(1)); - var alias = as(evalNull.fields().get(0), Alias.class); - assertThat(alias.name(), is("emp_does_not_exist_field")); - var lit = as(alias.child(), Literal.class); - assertThat(lit.dataType(), is(DataType.NULL)); - - // The child is EsRelation - var relation = as(evalNull.child(), EsRelation.class); + // The child is EsRelation with emp_does_not_exist_field in output + var relation = as(esqlProject.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("emp_does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Project[[_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, gender{f}#8, hire_date{f}#13, job{f}#14, job.raw{f}#15, languages{f}#9, * last_name{f}#10, long_noidx{f}#16, salary{f}#11]] - * \_Eval[[null[NULL] AS does_not_exist_field#17, null[NULL] AS neither_this#18]] - * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..] + * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge.., does_not_exist_field{f}#17, ..] */ public void testDrop() { var extraField = randomFrom("", "does_not_exist_field", "neither_this"); @@ -398,13 +364,14 @@ public void testDrop() { ) ); - var eval = as(project.child(), Eval.class); - var expectedNames = hasExtraField && extraField.equals("does_not_exist_field") == false + var relation = as(project.child(), EsRelation.class); + assertThat(relation.indexPattern(), is("test")); + var expectedDneNames = hasExtraField && extraField.equals("does_not_exist_field") == false ? List.of("does_not_exist_field", extraField) : List.of("does_not_exist_field"); - assertThat(Expressions.names(eval.fields()), is(expectedNames)); - var relation = as(eval.child(), EsRelation.class); - assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> expectedDneNames.contains(a.name())).toList(); + assertThat(dneAttrs, hasSize(expectedDneNames.size())); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } public void testFailDropWithNonMatchingStar() { @@ -421,8 +388,7 @@ public void testFailDropWithNonMatchingStar() { * Limit[1000[INTEGER],false,false] * \_Project[[_meta_field{f}#12, first_name{f}#7, gender{f}#8, hire_date{f}#13, job{f}#14, job.raw{f}#15, languages{f}#9, * last_name{f}#10, long_noidx{f}#16, salary{f}#11]] - * \_Eval[[null[NULL] AS does_not_exist_field#18]] - * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..] + * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge.., does_not_exist_field{f}#18] */ public void testDropWithMatchingStar() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -453,10 +419,10 @@ public void testDropWithMatchingStar() { ) ); - var eval = as(project.child(), Eval.class); - assertThat(Expressions.names(eval.fields()), is(List.of("does_not_exist_field"))); - var relation = as(eval.child(), EsRelation.class); + var relation = as(project.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } public void testFailDropWithMatchingAndNonMatchingStar() { @@ -486,8 +452,7 @@ public void testFailEvalAfterDrop() { * \_Project[[_meta_field{f}#16, emp_no{f}#10 AS employee_number#8, first_name{f}#11, gender{f}#12, hire_date{f}#17, job{f}#18, * job.raw{f}#19, languages{f}#13, last_name{f}#14, long_noidx{f}#20, salary{f}#15, * now_it_does#12 AS does_not_exist_field{r}#21]] - * \_Eval[[null[NULL] AS does_not_exist_field#21]] - * \_EsRelation[test][_meta_field{f}#16, emp_no{f}#10, first_name{f}#11, ..] + * \_EsRelation[test][_meta_field{f}#16, emp_no{f}#10, first_name{f}#11, .., does_not_exist_field{f}#21] */ public void testRename() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -520,15 +485,10 @@ public void testRename() { ) ); - var eval = as(project.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist_field")); - var literal = as(alias.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(project.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* @@ -536,8 +496,7 @@ public void testRename() { * \_Project[[_meta_field{f}#19, emp_no{f}#13 AS employee_number#11, first_name{f}#14, gender{f}#15, hire_date{f}#20, * job{f}#21, job.raw{f}#22, languages{f}#16, last_name{f}#17, long_noidx{f}#23, salary{f}#18, * neither_does_this{r}#25 AS now_it_does#8]] - * \_Eval[[null[NULL] AS does_not_exist_field#24, null[NULL] AS neither_does_this#25]] - * \_EsRelation[test][_meta_field{f}#19, emp_no{f}#13, first_name{f}#14, ..] + * \_EsRelation[test][_meta_field{f}#19, emp_no{f}#13, first_name{f}#14, .., does_not_exist_field{f}#24, neither_does_this{f}#25] */ public void testRenameShadowed() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -570,19 +529,14 @@ public void testRenameShadowed() { ) ); - var eval = as(project.child(), Eval.class); - assertThat(eval.fields(), hasSize(2)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist_field")); - var literal = as(alias.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - alias = as(eval.fields().getLast(), Alias.class); - assertThat(alias.name(), is("neither_does_this")); - literal = as(alias.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(project.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output() + .stream() + .filter(a -> a.name().startsWith("does_not_exist") || a.name().equals("neither_does_this")) + .toList(); + assertThat(dneAttrs, hasSize(2)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } /* @@ -590,8 +544,7 @@ public void testRenameShadowed() { * \_Eval[[does_not_exist{r}#21 + 1[INTEGER] AS x#8]] * \_Project[[_meta_field{f}#16, emp_no{f}#10 AS employee_number#5, first_name{f}#11, gender{f}#12, hire_date{f}#17, job{f}#18, * job.raw{f}#19, languages{f}#13, last_name{f}#14, long_noidx{f}#20, salary{f}#15, does_not_exist{r}#21]] - * \_Eval[[null[NULL] AS does_not_exist#21]] - * \_EsRelation[test][_meta_field{f}#16, emp_no{f}#10, first_name{f}#11, ..] + * \_EsRelation[test][_meta_field{f}#16, emp_no{f}#10, first_name{f}#11, .., does_not_exist{f}#21] */ public void testEvalAfterRename() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -648,22 +601,16 @@ public void testEvalAfterRename() { ) ); - var evalNull = as(esqlProject.child(), Eval.class); - assertThat(evalNull.fields(), hasSize(1)); - var alias = as(evalNull.fields().get(0), Alias.class); - assertThat(alias.name(), is("does_not_exist")); - var lit = as(alias.child(), Literal.class); - assertThat(lit.dataType(), is(DataType.NULL)); - - var relation = as(evalNull.child(), EsRelation.class); + var relation = as(esqlProject.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Eval[[does_not_exist_field{r}#18 + 1[INTEGER] AS x#5]] - * \_Eval[[null[NULL] AS does_not_exist_field#18]] - * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge.., does_not_exist_field{f}#18] */ public void testEval() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -680,23 +627,18 @@ public void testEval() { assertThat(aliasX.name(), is("x")); assertThat(Expressions.name(aliasX.child()), is("does_not_exist_field + 1")); - var innerEval = as(outerEval.child(), Eval.class); - assertThat(innerEval.fields(), hasSize(1)); - var aliasField = as(innerEval.fields().getFirst(), Alias.class); - assertThat(aliasField.name(), is("does_not_exist_field")); - var literal = as(aliasField.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(innerEval.child(), EsRelation.class); + var relation = as(outerEval.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Eval[[b{r}#25 + c{r}#27 AS y#12]] * \_Eval[[a{r}#4 + b{r}#25 AS x#8]] - * \_Eval[[1[INTEGER] AS a#4, null[NULL] AS b#25, null[NULL] AS c#27]] - * \_EsRelation[test][_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, ..] + * \_Eval[[1[INTEGER] AS a#4]] + * \_EsRelation[test][_meta_field{f}#20, emp_no{f}#14, first_name{f}#15, .., b{f}#25, c{f}#27] */ public void testMultipleEvaled() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -718,24 +660,18 @@ public void testMultipleEvaled() { assertThat(evalX.fields().get(0).name(), is("x")); var evalABC = as(evalX.child(), Eval.class); - assertThat(Expressions.names(evalABC.fields()), is(List.of("a", "b", "c"))); - for (var field : evalABC.fields()) { - var alias = as(field, Alias.class); - var literal = as(alias.child(), Literal.class); - if (alias.name().equals("a") == false) { - assertThat(literal.dataType(), is(DataType.NULL)); - assertThat(literal.value(), is(nullValue())); - } - } + assertThat(Expressions.names(evalABC.fields()), is(List.of("a"))); var relation = as(evalABC.child(), EsRelation.class); + var dneAttrs = relation.output().stream().filter(a -> a.name().equals("b") || a.name().equals("c")).toList(); + assertThat(dneAttrs, hasSize(2)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } /* * Limit[1000[INTEGER],false,false] * \_Eval[[TOLONG(does_not_exist_field{r}#18) AS x#5]] - * \_Eval[[null[NULL] AS does_not_exist_field#18]] - * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge.., does_not_exist_field{f}#18] */ public void testCasting() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -752,22 +688,16 @@ public void testCasting() { assertThat(aliasX.name(), is("x")); assertThat(Expressions.name(aliasX.child()), is("does_not_exist_field::LONG")); - var innerEval = as(outerEval.child(), Eval.class); - assertThat(innerEval.fields(), hasSize(1)); - var aliasField = as(innerEval.fields().getFirst(), Alias.class); - assertThat(aliasField.name(), is("does_not_exist_field")); - var literal = as(aliasField.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(innerEval.child(), EsRelation.class); + var relation = as(outerEval.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Eval[[TOLONG(does_not_exist_field{r}#17) AS does_not_exist_field::LONG#4]] - * \_Eval[[null[NULL] AS does_not_exist_field#17]] - * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..] + * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge.., does_not_exist_field{f}#17] */ public void testCastingNoAliasing() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -782,8 +712,7 @@ public void testCastingNoAliasing() { * Limit[1000[INTEGER],false,false] * \_Eval[[42[INTEGER] AS does_not_exist_field#7]] * \_Eval[[does_not_exist_field{r}#20 + 1[INTEGER] AS x#5]] - * \_Eval[[null[NULL] AS does_not_exist_field#20]] - * \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..] + * \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g.., does_not_exist_field{f}#20] */ public void testShadowingAfterEval() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -807,23 +736,17 @@ public void testShadowingAfterEval() { assertThat(aliasX.name(), is("x")); assertThat(Expressions.name(aliasX.child()), is("does_not_exist_field + 1")); - var innerEval = as(middleEval.child(), Eval.class); - assertThat(innerEval.fields(), hasSize(1)); - var aliasField = as(innerEval.fields().getFirst(), Alias.class); - assertThat(aliasField.name(), is("does_not_exist_field")); - var literal = as(aliasField.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(innerEval.child(), EsRelation.class); + var relation = as(middleEval.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Eval[[42[INTEGER] AS does_not_exist_field#5]] * \_Project[[does_not_exist_field{r}#18]] - * \_Eval[[null[NULL] AS does_not_exist_field#18]] - * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge.., does_not_exist_field{f}#18] */ public void testShadowingAfterKeep() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -845,15 +768,10 @@ public void testShadowingAfterKeep() { assertThat(project.projections(), hasSize(1)); assertThat(Expressions.name(project.projections().getFirst()), is("does_not_exist_field")); - var innerEval = as(project.child(), Eval.class); - assertThat(innerEval.fields(), hasSize(1)); - var aliasField = as(innerEval.fields().getFirst(), Alias.class); - assertThat(aliasField.name(), is("does_not_exist_field")); - var literal = as(aliasField.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(innerEval.child(), EsRelation.class); + var relation = as(project.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } public void testFailDropThenKeep() { @@ -917,8 +835,7 @@ public void testFailStatsThenEval() { /* * Limit[1000[INTEGER],false,false] * \_Aggregate[[],[COUNT(does_not_exist_field{r}#18,true[BOOLEAN],PT0S[TIME_DURATION]) AS cnt#5]] - * \_Eval[[null[NULL] AS does_not_exist_field#18]] - * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge.., does_not_exist_field{f}#18] */ public void testStatsAgg() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -936,22 +853,16 @@ public void testStatsAgg() { assertThat(alias.name(), is("cnt")); assertThat(Expressions.name(alias.child()), is("COUNT(does_not_exist_field)")); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var aliasField = as(eval.fields().getFirst(), Alias.class); - assertThat(aliasField.name(), is("does_not_exist_field")); - var literal = as(aliasField.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Aggregate[[does_not_exist_field{r}#16],[does_not_exist_field{r}#16]] - * \_Eval[[null[NULL] AS does_not_exist_field#16]] - * \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge..] + * \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge.., does_not_exist_field{f}#16] */ public void testStatsGroup() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -966,23 +877,17 @@ public void testStatsGroup() { assertThat(agg.groupings(), hasSize(1)); assertThat(Expressions.name(agg.groupings().getFirst()), is("does_not_exist_field")); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var aliasField = as(eval.fields().getFirst(), Alias.class); - assertThat(aliasField.name(), is("does_not_exist_field")); - var literal = as(aliasField.child(), Literal.class); - assertThat(literal.dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist_field")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Aggregate[[does_not_exist2{r}#19],[SUM(does_not_exist1{r}#20,true[BOOLEAN],PT0S[TIME_DURATION],compensated[KEYWORD]) AS s * #6, does_not_exist2{r}#19]] - * \_Eval[[null[NULL] AS does_not_exist2#19, null[NULL] AS does_not_exist1#20]] - * \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] + * \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge.., does_not_exist1{f}#20, does_not_exist2{f}#19] */ public void testStatsAggAndGroup() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1001,25 +906,18 @@ public void testStatsAggAndGroup() { assertThat(alias.name(), is("s")); assertThat(Expressions.name(alias.child()), is("SUM(does_not_exist1)")); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(2)); - var alias2 = as(eval.fields().getFirst(), Alias.class); - assertThat(alias2.name(), is("does_not_exist2")); - assertThat(as(alias2.child(), Literal.class).dataType(), is(DataType.NULL)); - var alias1 = as(eval.fields().getLast(), Alias.class); - assertThat(alias1.name(), is("does_not_exist1")); - assertThat(as(alias1.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> a.name().startsWith("does_not_exist")).toList(); + assertThat(dneAttrs, hasSize(2)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } /* * Limit[1000[INTEGER],false,false] * \_Aggregate[[does_not_exist2{r}#24 AS d2#5, emp_no{f}#13],[SUM(does_not_exist1{r}#25,true[BOOLEAN],PT0S[TIME_DURATION], * compensated[KEYWORD]) + d2{r}#5 AS s#10, d2{r}#5, emp_no{f}#13]] - * \_Eval[[null[NULL] AS does_not_exist2#24, null[NULL] AS does_not_exist1#25, null[NULL] AS d2#26]] - * \_EsRelation[test][_meta_field{f}#19, emp_no{f}#13, first_name{f}#14, ..] + * \_EsRelation[test][_meta_field{f}#19, emp_no{f}#13, first_name{f}#14, .., does_not_exist1{f}#25, does_not_exist2{f}#24] */ public void testStatsAggAndAliasedGroup() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1044,29 +942,19 @@ public void testStatsAggAndAliasedGroup() { assertThat(alias.name(), is("s")); assertThat(Expressions.name(alias.child()), is("SUM(does_not_exist1) + d2")); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(3)); - var alias2 = as(eval.fields().get(0), Alias.class); - assertThat(alias2.name(), is("does_not_exist2")); - assertThat(as(alias2.child(), Literal.class).dataType(), is(DataType.NULL)); - var alias1 = as(eval.fields().get(1), Alias.class); - assertThat(alias1.name(), is("does_not_exist1")); - assertThat(as(alias1.child(), Literal.class).dataType(), is(DataType.NULL)); - var alias0 = as(eval.fields().get(2), Alias.class); - assertThat(alias0.name(), is("d2")); - assertThat(as(alias0.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> a.name().startsWith("does_not_exist")).toList(); + assertThat(dneAttrs, hasSize(2)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } /* * Limit[1000[INTEGER],false,false] * \_Aggregate[[does_not_exist2{r}#29 + does_not_exist3{r}#30 AS s0#6, emp_no{f}#18 AS s1#9],[SUM(does_not_exist1{r}#31,true[B * OOLEAN],PT0S[TIME_DURATION],compensated[KEYWORD]) + s0{r}#6 + s1{r}#9 AS sum#14, s0{r}#6, s1{r}#9]] - * \_Eval[[null[NULL] AS does_not_exist2#29, null[NULL] AS does_not_exist3#30, null[NULL] AS does_not_exist1#31, - * null[NULL] AS s0#32]] - * \_EsRelation[test][_meta_field{f}#24, emp_no{f}#18, first_name{f}#19, ..] + * \_EsRelation[test][_meta_field{f}#24, emp_no{f}#18, first_name{f}#19, .., does_not_exist1{f}#31, does_not_exist2{f}#29, + * does_not_exist3{f}#30] */ public void testStatsAggAndAliasedGroupWithExpression() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1088,21 +976,18 @@ public void testStatsAggAndAliasedGroupWithExpression() { assertThat(alias.name(), is("sum")); assertThat(Expressions.name(alias.child()), is("SUM(does_not_exist1) + s0 + s1")); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(4)); - assertThat(Expressions.names(eval.fields()), is(List.of("does_not_exist2", "does_not_exist3", "does_not_exist1", "s0"))); - eval.fields().forEach(a -> assertThat(as(as(a, Alias.class).child(), Literal.class).dataType(), is(DataType.NULL))); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> a.name().startsWith("does_not_exist")).toList(); + assertThat(dneAttrs, hasSize(3)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } /* * Limit[1000[INTEGER],false,false] * \_Aggregate[[does_not_exist2{r}#22, emp_no{f}#11],[SUM(does_not_exist1{r}#23,true[BOOLEAN],PT0S[TIME_DURATION], * compensated[KEYWORD]) AS s#7, COUNT(*[KEYWORD],true[BOOLEAN],PT0S[TIME_DURATION]) AS c#9, does_not_exist2{r}#22, emp_no{f}#11]] - * \_Eval[[null[NULL] AS does_not_exist2#22, null[NULL] AS does_not_exist1#23]] - * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, .., does_not_exist1{f}#23, does_not_exist2{f}#22] */ public void testStatsMixed() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1120,13 +1005,11 @@ public void testStatsMixed() { assertThat(agg.aggregates(), hasSize(4)); // includes grouping keys assertThat(Expressions.names(agg.aggregates()), is(List.of("s", "c", "does_not_exist2", "emp_no"))); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(2)); - assertThat(Expressions.names(eval.fields()), is(List.of("does_not_exist2", "does_not_exist1"))); - eval.fields().forEach(a -> assertThat(as(as(a, Alias.class).child(), Literal.class).dataType(), is(DataType.NULL))); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> a.name().startsWith("does_not_exist")).toList(); + assertThat(dneAttrs, hasSize(2)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } /* @@ -1134,8 +1017,7 @@ public void testStatsMixed() { * \_InlineStats[] * \_Aggregate[[does_not_exist2{r}#22, emp_no{f}#11],[SUM(does_not_exist1{r}#23,true[BOOLEAN],PT0S[TIME_DURATION],compensated[ * KEYWORD]) AS s#5, COUNT(*[KEYWORD],true[BOOLEAN],PT0S[TIME_DURATION]) AS c#7, does_not_exist2{r}#22, emp_no{f}#11]] - * \_Eval[[null[NULL] AS does_not_exist2#22, null[NULL] AS does_not_exist1#23]] - * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + * \_EsRelation[test][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, .., does_not_exist1{f}#23, does_not_exist2{f}#22] */ public void testInlineStatsMixed() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1154,13 +1036,11 @@ public void testInlineStatsMixed() { assertThat(agg.aggregates(), hasSize(4)); // includes grouping keys assertThat(Expressions.names(agg.aggregates()), is(List.of("s", "c", "does_not_exist2", "emp_no"))); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(2)); - assertThat(Expressions.names(eval.fields()), is(List.of("does_not_exist2", "does_not_exist1"))); - eval.fields().forEach(a -> assertThat(as(as(a, Alias.class).child(), Literal.class).dataType(), is(DataType.NULL))); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> a.name().startsWith("does_not_exist")).toList(); + assertThat(dneAttrs, hasSize(2)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } /* @@ -1168,8 +1048,8 @@ public void testInlineStatsMixed() { * \_Aggregate[[does_not_exist3{r}#24, emp_no{f}#13, does_not_exist2{r}#25],[SUM(does_not_exist1{r}#26,true[BOOLEAN], * PT0S[TIME_DURATION],compensated[KEYWORD]) + does_not_exist2{r}#25 AS s#9, COUNT(*[KEYWORD],true[BOOLEAN], * PT0S[TIME_DURATION]) AS c#11, does_not_exist3{r}#24, emp_no{f}#13, does_not_exist2{r}#25]] - * \_Eval[[null[NULL] AS does_not_exist3#24, null[NULL] AS does_not_exist2#25]] - * \_EsRelation[test][_meta_field{f}#19, emp_no{f}#13, first_name{f}#14, ..] + * \_EsRelation[test][_meta_field{f}#19, emp_no{f}#13, first_name{f}#14, .., does_not_exist1{f}#26, does_not_exist2{f}#25, + * does_not_exist3{f}#24] */ public void testStatsMixedAndExpressions() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1187,20 +1067,17 @@ public void testStatsMixedAndExpressions() { assertThat(agg.aggregates(), hasSize(5)); // includes grouping keys assertThat(Expressions.names(agg.aggregates()), is(List.of("s", "c", "does_not_exist3", "emp_no", "does_not_exist2"))); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(3)); - assertThat(Expressions.names(eval.fields()), is(List.of("does_not_exist3", "does_not_exist2", "does_not_exist1"))); - eval.fields().forEach(a -> assertThat(as(as(a, Alias.class).child(), Literal.class).dataType(), is(DataType.NULL))); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttrs = relation.output().stream().filter(a -> a.name().startsWith("does_not_exist")).toList(); + assertThat(dneAttrs, hasSize(3)); + dneAttrs.forEach(a -> assertThat(a.dataType(), is(DataType.NULL))); } /* * Limit[1000[INTEGER],false,false] * \_Filter[TOLONG(does_not_exist{r}#33) > 0[INTEGER]] - * \_Eval[[null[NULL] AS does_not_exist#33]] - * \_EsRelation[test][_meta_field{f}#28, emp_no{f}#22, first_name{f}#23, ..] + * \_EsRelation[test][_meta_field{f}#28, emp_no{f}#22, first_name{f}#23, .., does_not_exist{f}#33] */ public void testWhere() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1214,21 +1091,16 @@ public void testWhere() { var filter = as(limit.child(), Filter.class); assertThat(Expressions.name(filter.condition()), is("does_not_exist::LONG > 0")); - var eval = as(filter.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist")); - assertThat(as(alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(filter.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Filter[TOLONG(does_not_exist{r}#195) > 0[INTEGER] OR emp_no{f}#184 > 0[INTEGER]] - * \_Eval[[null[NULL] AS does_not_exist#195]] - * \_EsRelation[test][_meta_field{f}#190, emp_no{f}#184, first_name{f}#18..] + * \_EsRelation[test][_meta_field{f}#190, emp_no{f}#184, first_name{f}#18.., does_not_exist{f}#195] */ public void testWhereConjunction() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1242,22 +1114,17 @@ public void testWhereConjunction() { var filter = as(limit.child(), Filter.class); assertThat(Expressions.name(filter.condition()), is("does_not_exist::LONG > 0 OR emp_no > 0")); - var eval = as(filter.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist")); - assertThat(as(alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(filter.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); + var dneAttr = relation.output().stream().filter(a -> a.name().equals("does_not_exist")).findFirst().orElseThrow(); + assertThat(dneAttr.dataType(), is(DataType.NULL)); } /* * Limit[1000[INTEGER],false,false] * \_Filter[TOLONG(does_not_exist1{r}#491) > 0[INTEGER] OR emp_no{f}#480 > 0[INTEGER] * AND TOLONG(does_not_exist2{r}#492) < 100[INTEGER]] - * \_Eval[[null[NULL] AS does_not_exist1#491, null[NULL] AS does_not_exist2#492]] - * \_EsRelation[test][_meta_field{f}#486, emp_no{f}#480, first_name{f}#48..] + * \_EsRelation[test][_meta_field{f}#486, emp_no{f}#480, first_name{f}#48.., does_not_exist1{f}#491, does_not_exist2{f}#492] */ public void testWhereConjunctionMultipleFields() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1271,16 +1138,7 @@ public void testWhereConjunctionMultipleFields() { var filter = as(limit.child(), Filter.class); assertThat(Expressions.name(filter.condition()), is("does_not_exist1::LONG > 0 OR emp_no > 0 AND does_not_exist2::LONG < 100")); - var eval = as(filter.child(), Eval.class); - assertThat(eval.fields(), hasSize(2)); - var alias1 = as(eval.fields().get(0), Alias.class); - assertThat(alias1.name(), is("does_not_exist1")); - assertThat(as(alias1.child(), Literal.class).dataType(), is(DataType.NULL)); - var alias2 = as(eval.fields().get(1), Alias.class); - assertThat(alias2.name(), is("does_not_exist2")); - assertThat(as(alias2.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(filter.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } @@ -1288,8 +1146,7 @@ public void testWhereConjunctionMultipleFields() { * Limit[1000[INTEGER],false,false] * \_Aggregate[[],[FilteredExpression[COUNT(*[KEYWORD],true[BOOLEAN],PT0S[TIME_DURATION]), * TOLONG(does_not_exist1{r}#94) > 0[INTEGER]] AS c#81]] - * \_Eval[[null[NULL] AS does_not_exist1#94]] - * \_EsRelation[test][_meta_field{f}#89, emp_no{f}#83, first_name{f}#84, ..] + * \_EsRelation[test][_meta_field{f}#89, emp_no{f}#83, first_name{f}#84, .., does_not_exist1{f}#94] */ public void testAggsFiltering() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1311,13 +1168,7 @@ public void testAggsFiltering() { var tolong = as(greaterThan.left(), ToLong.class); assertThat(Expressions.name(tolong.field()), is("does_not_exist1")); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias1 = as(eval.fields().getFirst(), Alias.class); - assertThat(alias1.name(), is("does_not_exist1")); - assertThat(as(alias1.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } @@ -1327,8 +1178,8 @@ public void testAggsFiltering() { * TOLONG(does_not_exist1{r}#620) > 0[INTEGER] OR emp_no{f}#609 > 0[INTEGER] * OR TOLONG(does_not_exist2{r}#621) < 100[INTEGER]] AS c1#602, * FilteredExpression[COUNT(*[KEYWORD],true[BOOLEAN],PT0S[TIME_DURATION]),ISNULL(does_not_exist3{r}#622)] AS c2#607]] - * \_Eval[[null[NULL] AS does_not_exist1#620, null[NULL] AS does_not_exist2#621, null[NULL] AS does_not_exist3#622]] - * \_EsRelation[test][_meta_field{f}#615, emp_no{f}#609, first_name{f}#61..] + * \_EsRelation[test][_meta_field{f}#615, emp_no{f}#609, first_name{f}#61.., does_not_exist1{f}#620, does_not_exist2{f}#621, + * does_not_exist3{f}#622] */ public void testAggsFilteringMultipleFields() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1355,27 +1206,14 @@ public void testAggsFilteringMultipleFields() { assertThat(alias2.name(), is("c2")); assertThat(Expressions.name(alias2.child()), is("c2 = COUNT(*) WHERE does_not_exist3 IS NULL")); - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(3)); - var aliasDne1 = as(eval.fields().get(0), Alias.class); - assertThat(aliasDne1.name(), is("does_not_exist1")); - assertThat(as(aliasDne1.child(), Literal.class).dataType(), is(DataType.NULL)); - var aliasDne2 = as(eval.fields().get(1), Alias.class); - assertThat(aliasDne2.name(), is("does_not_exist2")); - assertThat(as(aliasDne2.child(), Literal.class).dataType(), is(DataType.NULL)); - var aliasDne3 = as(eval.fields().get(2), Alias.class); - assertThat(aliasDne3.name(), is("does_not_exist3")); - assertThat(as(aliasDne3.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } /* * Limit[1000[INTEGER],false,false] * \_OrderBy[[Order[does_not_exist{r}#16,ASC,LAST]]] - * \_Eval[[null[NULL] AS does_not_exist#16]] - * \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge..] + * \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge.., does_not_exist{f}#16] */ public void testSort() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1387,26 +1225,17 @@ public void testSort() { var limit = as(plan, Limit.class); assertThat(limit.limit().fold(FoldContext.small()), is(1000)); - // OrderBy over the Eval-produced alias + // OrderBy over the relation alias var orderBy = as(limit.child(), OrderBy.class); - // Eval introduces does_not_exist as NULL - var eval = as(orderBy.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist")); - assertThat(as(alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - // Underlying relation - var relation = as(eval.child(), EsRelation.class); + var relation = as(orderBy.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } /* * Limit[1000[INTEGER],false,false] * \_OrderBy[[Order[TOLONG(does_not_exist{r}#485) + 1[INTEGER],ASC,LAST]]] - * \_Eval[[null[NULL] AS does_not_exist#485]] - * \_EsRelation[test][_meta_field{f}#480, emp_no{f}#474, first_name{f}#47..] + * \_EsRelation[test][_meta_field{f}#480, emp_no{f}#474, first_name{f}#47.., does_not_exist{f}#485] */ public void testSortExpression() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1421,13 +1250,7 @@ public void testSortExpression() { assertThat(orderBy.order(), hasSize(1)); assertThat(Expressions.name(orderBy.order().getFirst().child()), is("does_not_exist::LONG + 1")); - var eval = as(orderBy.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist")); - assertThat(as(alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(orderBy.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } @@ -1435,8 +1258,7 @@ public void testSortExpression() { * Limit[1000[INTEGER],false,false] * \_OrderBy[[Order[TOLONG(does_not_exist{r}#370) + 1[INTEGER],ASC,LAST], Order[does_not_exist2{r}#371,DESC,FIRST], * Order[emp_no{f}#359,ASC,LAST]]] - * \_Eval[[null[NULL] AS does_not_exist1#370, null[NULL] AS does_not_exist2#371]] - * \_EsRelation[test][_meta_field{f}#365, emp_no{f}#359, first_name{f}#36..] + * \_EsRelation[test][_meta_field{f}#365, emp_no{f}#359, first_name{f}#36.., does_not_exist1{f}#370, does_not_exist2{f}#371] */ public void testSortExpressionMultipleFields() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1453,24 +1275,14 @@ public void testSortExpressionMultipleFields() { assertThat(Expressions.name(orderBy.order().get(1).child()), is("does_not_exist2")); assertThat(Expressions.name(orderBy.order().get(2).child()), is("emp_no")); - var eval = as(orderBy.child(), Eval.class); - assertThat(eval.fields(), hasSize(2)); - var alias1 = as(eval.fields().get(0), Alias.class); - assertThat(alias1.name(), is("does_not_exist1")); - assertThat(as(alias1.child(), Literal.class).dataType(), is(DataType.NULL)); - var alias2 = as(eval.fields().get(1), Alias.class); - assertThat(alias2.name(), is("does_not_exist2")); - assertThat(as(alias2.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(orderBy.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } /* * Limit[1000[INTEGER],false,false] * \_MvExpand[does_not_exist{r}#17,does_not_exist{r}#20] - * \_Eval[[null[NULL] AS does_not_exist#17]] - * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge..] + * \_EsRelation[test][_meta_field{f}#12, emp_no{f}#6, first_name{f}#7, ge.., does_not_exist{f}#17] */ public void testMvExpand() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -1484,21 +1296,14 @@ public void testMvExpand() { var mvExpand = as(limit.child(), MvExpand.class); assertThat(Expressions.name(mvExpand.expanded()), is("does_not_exist")); - var eval = as(mvExpand.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist")); - assertThat(as(alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(mvExpand.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } /* * Limit[1000[INTEGER],false,false] * \_Filter[TOLONG(does_not_exist{r}#566) > 1[INTEGER]] - * \_Eval[[null[NULL] AS does_not_exist#566]] - * \_EsRelation[languages][language_code{f}#564, language_name{f}#565] + * \_EsRelation[languages][language_code{f}#564, language_name{f}#565, does_not_exist{f}#566] */ public void testSubqueryOnly() { assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); @@ -1517,13 +1322,7 @@ public void testSubqueryOnly() { var toLong = as(gt.left(), ToLong.class); assertThat(Expressions.name(toLong.field()), is("does_not_exist")); - var eval = as(filter.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().getFirst(), Alias.class); - assertThat(alias.name(), is("does_not_exist")); - assertThat(as(alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(eval.child(), EsRelation.class); + var relation = as(filter.child(), EsRelation.class); assertThat(relation.indexPattern(), is("languages")); } @@ -1536,15 +1335,13 @@ public void testSubqueryOnly() { * | \_Eval[[null[DATETIME] AS @timestamp#16, null[IP] AS client_ip#17, null[LONG] AS event_duration#18, null[KEYWORD] AS message#19]] * | \_Subquery[] * | \_Filter[TOLONG(does_not_exist1{r}#12) > 1[INTEGER]] - * | \_Eval[[null[NULL] AS does_not_exist1#12]] - * | \_EsRelation[languages][language_code{f}#6, language_name{f}#7] + * | \_EsRelation[languages][language_code{f}#6, language_name{f}#7, does_not_exist1{f}#12] * \_Project[[language_code{r}#20, language_name{r}#21, does_not_exist1{r}#14, @timestamp{f}#8, client_ip{f}#9, event_duration{f}#10, * message{f}#11]] * \_Eval[[null[INTEGER] AS language_code#20, null[KEYWORD] AS language_name#21]] * \_Subquery[] * \_Filter[TODOUBLE(does_not_exist1{r}#14) > 10.0[DOUBLE]] - * \_Eval[[null[NULL] AS does_not_exist1#14]] - * \_EsRelation[sample_data][@timestamp{f}#8, client_ip{f}#9, event_duration{f}#..] + * \_EsRelation[sample_data][@timestamp{f}#8, client_ip{f}#9, event_duration{f}#.., does_not_exist1{f}#14] */ public void testDoubleSubqueryOnly() { assumeTrue( @@ -1590,13 +1387,7 @@ public void testDoubleSubqueryOnly() { var leftToLong = as(leftGt.left(), ToLong.class); assertThat(Expressions.name(leftToLong.field()), is("does_not_exist1")); - var leftSubEval = as(leftSubFilter.child(), Eval.class); - assertThat(leftSubEval.fields(), hasSize(1)); - var leftDoesNotExist = as(leftSubEval.fields().getFirst(), Alias.class); - assertThat(leftDoesNotExist.name(), is("does_not_exist1")); - assertThat(as(leftDoesNotExist.child(), Literal.class).dataType(), is(DataType.NULL)); - - var leftRel = as(leftSubEval.child(), EsRelation.class); + var leftRel = as(leftSubFilter.child(), EsRelation.class); assertThat(leftRel.indexPattern(), is("languages")); // Right branch: sample_data @@ -1617,13 +1408,7 @@ public void testDoubleSubqueryOnly() { var rightToDouble = as(rightGt.left(), ToDouble.class); assertThat(Expressions.name(rightToDouble.field()), is("does_not_exist1")); - var rightSubEval = as(rightSubFilter.child(), Eval.class); - assertThat(rightSubEval.fields(), hasSize(1)); - var rightDoesNotExist = as(rightSubEval.fields().getFirst(), Alias.class); - assertThat(rightDoesNotExist.name(), is("does_not_exist1")); - assertThat(as(rightDoesNotExist.child(), Literal.class).dataType(), is(DataType.NULL)); - - var rightRel = as(rightSubEval.child(), EsRelation.class); + var rightRel = as(rightSubFilter.child(), EsRelation.class); assertThat(rightRel.indexPattern(), is("sample_data")); } @@ -1641,16 +1426,14 @@ public void testDoubleSubqueryOnly() { * null[KEYWORD] AS message#20]] * | \_Subquery[] * | \_Filter[TOLONG(does_not_exist1{r}#13) > 1[INTEGER]] - * | \_Eval[[null[NULL] AS does_not_exist1#13, null[NULL] AS does_not_exist2#30]] - * | \_EsRelation[languages][language_code{f}#7, language_name{f}#8] + * | \_EsRelation[languages][language_code{f}#7, language_name{f}#8, does_not_exist1{f}#13, does_not_exist2{f}#30] * \_Project[[language_code{r}#21, language_name{r}#22, does_not_exist1{r}#15, @timestamp{f}#9, client_ip{f}#10, * event_duration{f}#11, message{f}#12, does_not_exist2{r}#31, $$does_not_exist2$converted_to$long{r$}#43]] * \_Eval[[TOLONG(does_not_exist2{r}#31) AS $$does_not_exist2$converted_to$long#43]] * \_Eval[[null[INTEGER] AS language_code#21, null[KEYWORD] AS language_name#22]] * \_Subquery[] * \_Filter[TODOUBLE(does_not_exist1{r}#15) > 10.0[DOUBLE]] - * \_Eval[[null[NULL] AS does_not_exist1#15, null[NULL] AS does_not_exist2#31]] - * \_EsRelation[sample_data][@timestamp{f}#9, client_ip{f}#10, event_duration{f}..] + * \_EsRelation[sample_data][@timestamp{f}#9, .., does_not_exist1{f}#15, does_not_exist2{f}#31] */ public void testDoubleSubqueryOnlyWithTopFilterAndNoMain() { assumeTrue( @@ -1755,18 +1538,7 @@ public void testDoubleSubqueryOnlyWithTopFilterAndNoMain() { var leftToLong = as(leftGt.left(), ToLong.class); assertThat(Expressions.name(leftToLong.field()), is("does_not_exist1")); - var leftSubEval = as(leftSubFilter.child(), Eval.class); - assertThat(leftSubEval.fields(), hasSize(2)); - var leftDoesNotExist1 = as(leftSubEval.fields().get(0), Alias.class); - assertThat(leftDoesNotExist1.name(), is("does_not_exist1")); - assertThat(as(leftDoesNotExist1.child(), Literal.class).dataType(), is(DataType.NULL)); - assertThat(leftDoesNotExist1.id(), is(unionAllLeftProjectAttribute_does_not_exist1.id())); // same IDs withing the branch - var leftDoesNotExist2 = as(leftSubEval.fields().get(1), Alias.class); - assertThat(leftDoesNotExist2.name(), is("does_not_exist2")); - assertThat(as(leftDoesNotExist2.child(), Literal.class).dataType(), is(DataType.NULL)); - assertThat(leftDoesNotExist2.id(), is(unionAllLeftProjectAttribute_does_not_exist2.id())); // same IDs withing the branch - - var leftRel = as(leftSubEval.child(), EsRelation.class); + var leftRel = as(leftSubFilter.child(), EsRelation.class); assertThat(leftRel.indexPattern(), is("languages")); // Right branch: sample_data @@ -1802,18 +1574,7 @@ public void testDoubleSubqueryOnlyWithTopFilterAndNoMain() { var rightToDouble = as(rightGt.left(), ToDouble.class); assertThat(Expressions.name(rightToDouble.field()), is("does_not_exist1")); - var rightSubEval = as(rightSubFilter.child(), Eval.class); - assertThat(rightSubEval.fields(), hasSize(2)); - var rightDoesNotExist1 = as(rightSubEval.fields().get(0), Alias.class); - assertThat(rightDoesNotExist1.name(), is("does_not_exist1")); - assertThat(as(rightDoesNotExist1.child(), Literal.class).dataType(), is(DataType.NULL)); - assertThat(rightDoesNotExist1.id(), is(unionAllRightProjectAttribute_does_not_exist1.id())); // same IDs withing the branch - var rightDoesNotExist2 = as(rightSubEval.fields().get(1), Alias.class); - assertThat(rightDoesNotExist2.name(), is("does_not_exist2")); - assertThat(as(rightDoesNotExist2.child(), Literal.class).dataType(), is(DataType.NULL)); - assertThat(rightDoesNotExist2.id(), is(unionAllRightProjectAttribute_does_not_exist2.id())); // same IDs withing the branch - - var rightRel = as(rightSubEval.child(), EsRelation.class); + var rightRel = as(rightSubFilter.child(), EsRelation.class); assertThat(rightRel.indexPattern(), is("sample_data")); } @@ -1830,9 +1591,8 @@ public void testDoubleSubqueryOnlyWithTopFilterAndNoMain() { * languages{f}#10, last_name{f}#11, long_noidx{f}#17, salary{f}#12, language_code{r}#22, language_name{r}#23, * does_not_exist1{r}#24, does_not_exist2{r}#50, $$does_not_exist2$converted_to$long{r$}#69]] * | \_Eval[[TOLONG(does_not_exist2{r}#50) AS $$does_not_exist2$converted_to$long#69]] - * | \_Eval[[null[INTEGER] AS language_code#22, null[KEYWORD] AS language_name#23, null[NULL] AS does_not_exist1#24, - * null[NULL] AS does_not_exist2#50]] - * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * | \_Eval[[null[INTEGER] AS language_code#22, null[KEYWORD] AS language_name#23]] + * | \_EsRelation[test][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge.., does_not_exist1{f}#24, does_not_exist2{f}#50] * \_Project[[_meta_field{r}#25, emp_no{r}#26, first_name{r}#27, gender{r}#28, hire_date{r}#29, job{r}#30, job.raw{r}#31, * languages{r}#32, last_name{r}#33, long_noidx{r}#34, salary{r}#35, language_code{f}#18, language_name{f}#19, * does_not_exist1{r}#20, does_not_exist2{r}#51, $$does_not_exist2$converted_to$long{r$}#70]] @@ -1843,8 +1603,7 @@ public void testDoubleSubqueryOnlyWithTopFilterAndNoMain() { * null[INTEGER] AS salary#35]] * \_Subquery[] * \_Filter[TOLONG(does_not_exist1{r}#20) > 1[INTEGER]] - * \_Eval[[null[NULL] AS does_not_exist1#20, null[NULL] AS does_not_exist2#51]] - * \_EsRelation[languages][language_code{f}#18, language_name{f}#19] + * \_EsRelation[languages][language_code{f}#18, language_name{f}#19, does_not_exist1{f}#20, does_not_exist2{f}#51] */ public void testSubqueryAndMainQuery() { assumeTrue( @@ -1916,14 +1675,11 @@ public void testSubqueryAndMainQuery() { var leftLangName = as(leftEvalEval.fields().get(1), Alias.class); assertThat(leftLangName.name(), is("language_name")); assertThat(as(leftLangName.child(), Literal.class).dataType(), is(DataType.KEYWORD)); - var leftDne1 = as(leftEvalEval.fields().get(2), Alias.class); - assertThat(leftDne1.name(), is("does_not_exist1")); - assertThat(as(leftDne1.child(), Literal.class).dataType(), is(DataType.NULL)); var leftRel = as(leftEvalEval.child(), EsRelation.class); assertThat(leftRel.indexPattern(), is("test")); - // Right branch: Project + Eval many nulls, Subquery -> Filter -> Eval -> EsRelation[languages] + // Right branch: Project + Eval many nulls, Subquery -> Filter -> EsRelation[languages] var rightProject = as(union.children().get(1), Project.class); var rightEval = as(rightProject.child(), Eval.class); assertThat(Expressions.names(rightEval.fields()), is(List.of("$$does_not_exist2$converted_to$long"))); @@ -1953,10 +1709,7 @@ public void testSubqueryAndMainQuery() { var rightToLongOnDne1 = as(rightGt.left(), ToLong.class); assertThat(Expressions.name(rightToLongOnDne1.field()), is("does_not_exist1")); - var rightSubEval = as(rightSubFilter.child(), Eval.class); - assertThat(Expressions.names(rightSubEval.fields()), is(List.of("does_not_exist1", "does_not_exist2"))); - - var rightRel = as(rightSubEval.child(), EsRelation.class); + var rightRel = as(rightSubFilter.child(), EsRelation.class); assertThat(rightRel.indexPattern(), is("languages")); } @@ -1966,8 +1719,7 @@ public void testSubqueryAndMainQuery() { * \_Project[[emp_no{f}#11, emp_no_foo{r}#22, emp_no_plus{r}#6]] * \_Filter[emp_no{f}#11 < 10003[INTEGER]] * \_Eval[[TOLONG(emp_no_foo{r}#22) + 1[INTEGER] AS emp_no_plus#6]] - * \_Eval[[null[NULL] AS emp_no_foo#22]] - * \_EsRelation[employees][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + * \_EsRelation[employees][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, .., emp_no_foo{f}#22] */ public void testSubqueryMix() { assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); @@ -2002,13 +1754,7 @@ public void testSubqueryMix() { assertThat(aliasPlus.name(), is("emp_no_plus")); assertThat(Expressions.name(aliasPlus.child()), is("emp_no_foo::LONG + 1")); - var evalFoo = as(evalPlus.child(), Eval.class); - assertThat(evalFoo.fields(), hasSize(1)); - var aliasFoo = as(evalFoo.fields().getFirst(), Alias.class); - assertThat(aliasFoo.name(), is("emp_no_foo")); - assertThat(as(aliasFoo.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(evalFoo.child(), EsRelation.class); + var relation = as(evalPlus.child(), EsRelation.class); assertThat(relation.indexPattern(), is("employees")); } @@ -2019,8 +1765,7 @@ public void testSubqueryMix() { * long_noidx{f}#21, salary{f}#16, emp_no_foo{r}#22, emp_no_plus{r}#6]] * \_Filter[emp_no{f}#11 < 10003[INTEGER]] * \_Eval[[TOLONG(emp_no_foo{r}#22) + 1[INTEGER] AS emp_no_plus#6]] - * \_Eval[[null[NULL] AS emp_no_foo#22]] - * \_EsRelation[employees][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, ..] + * \_EsRelation[employees][_meta_field{f}#17, emp_no{f}#11, first_name{f}#12, .., emp_no_foo{f}#22] */ public void testSubqueryMixWithDropPattern() { assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); @@ -2072,13 +1817,7 @@ public void testSubqueryMixWithDropPattern() { assertThat(aliasPlus.name(), is("emp_no_plus")); assertThat(Expressions.name(aliasPlus.child()), is("emp_no_foo::LONG + 1")); - var evalFoo = as(evalPlus.child(), Eval.class); - assertThat(evalFoo.fields(), hasSize(1)); - var aliasFoo = as(evalFoo.fields().getFirst(), Alias.class); - assertThat(aliasFoo.name(), is("emp_no_foo")); - assertThat(as(aliasFoo.child(), Literal.class).dataType(), is(DataType.NULL)); - - var relation = as(evalFoo.child(), EsRelation.class); + var relation = as(evalPlus.child(), EsRelation.class); assertThat(relation.indexPattern(), is("employees")); } @@ -2114,15 +1853,8 @@ public void testSubqueryAfterUnionAllOfStats() { assertThat(Expressions.name(agg.groupings().get(0)), is("does_not_exist")); assertThat(agg.aggregates(), hasSize(2)); // c and does_not_exist - // Eval introduces does_not_exist as NULL - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var alias = as(eval.fields().get(0), Alias.class); - assertThat(alias.name(), is("does_not_exist")); - assertThat(as(alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - // Underlying relation - var relation = as(eval.child(), EsRelation.class); + // Underlying relation (unmapped does_not_exist is in EsRelation output) + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("employees")); } @@ -2134,7 +1866,7 @@ public void testSubqueryAfterUnionAllOfStats() { * |_Project[[_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, gender{f}#9, hire_date{f}#14, job{f}#15, job.raw{f}#16, * languages{f}#10, last_name{f}#11, long_noidx{f}#17, salary{f}#12, c{r}#29, does_not_exist{r}#53]] * | \_Eval[[null[LONG] AS c#29, null[NULL] AS does_not_exist#53]] - * | \_EsRelation[employees][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge..] + * | \_EsRelation[employees][_meta_field{f}#13, emp_no{f}#7, first_name{f}#8, ge.., does_not_exist{f}#53] * \_Project[[_meta_field{r}#30, emp_no{r}#31, first_name{r}#32, gender{r}#33, hire_date{r}#34, job{r}#35, job.raw{r}#36, * languages{r}#37, last_name{r}#38, long_noidx{r}#39, salary{r}#40, c{r}#4, does_not_exist{r}#70]] * \_Eval[[null[NULL] AS does_not_exist#70]] @@ -2146,8 +1878,7 @@ public void testSubqueryAfterUnionAllOfStats() { * null[INTEGER] AS salary#40]] * \_Subquery[] * \_Aggregate[[],[COUNT(*[KEYWORD],true[BOOLEAN],PT0S[TIME_DURATION]) AS c#4]] - * \_Eval[[null[NULL] AS does_not_exist#54]] - * \_EsRelation[employees][_meta_field{f}#24, emp_no{f}#18, first_name{f}#19, ..] + * \_EsRelation[employees][_meta_field{f}#24, emp_no{f}#18, first_name{f}#19, .., does_not_exist{f}#54] */ public void testSubqueryAfterUnionAllOfStatsAndMain() { assumeTrue("Requires subquery in FROM command support", EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND.isEnabled()); @@ -2217,17 +1948,10 @@ public void testSubqueryAfterUnionAllOfStatsAndMain() { var leftProjectAttribute_does_not_exist = leftProject.output().get(12); assertThat(leftProjectAttribute_does_not_exist, not(unionAllAttribute_does_not_exist)); // ID is refreshed var leftEval = as(leftProject.child(), Eval.class); - // c and does_not_exist are introduced as nulls - assertThat(leftEval.fields(), hasSize(2)); - for (var alias : leftEval.fields()) { - var a = as(alias, Alias.class); - var lit = as(a.child(), Literal.class); - } + // c is introduced as null (does_not_exist is in EsRelation output) + assertThat(leftEval.fields(), hasSize(1)); assertThat(leftEval.fields().get(0).name(), is("c")); assertThat(leftEval.fields().get(0).dataType(), is(DataType.LONG)); - assertThat(leftEval.fields().get(1).name(), is("does_not_exist")); - assertThat(leftEval.fields().get(1).dataType(), is(DataType.NULL)); - assertThat(leftEval.fields().get(1).id(), is(leftProjectAttribute_does_not_exist.id())); // same ID within the branch var leftRelation = as(leftEval.child(), EsRelation.class); assertThat(leftRelation.indexPattern(), is("employees")); @@ -2316,18 +2040,7 @@ public void testSubqueryAfterUnionAllOfStatsAndMain() { var rightAgg = as(rightSubquery.child(), Aggregate.class); assertThat(rightAgg.aggregates(), hasSize(1)); assertThat(Expressions.name(rightAgg.aggregates().get(0)), is("c")); - var rightEval3 = as(rightAgg.child(), Eval.class); - // does_not_exist is introduced as null for the stats subquery - assertThat(rightEval3.fields(), hasSize(1)); - { - var a = as(rightEval3.fields().get(0), Alias.class); - var lit = as(a.child(), Literal.class); - assertThat(lit.dataType(), is(DataType.NULL)); - assertThat(a.name(), is("does_not_exist")); - } - // the upper Eval is generated by ResolveUnmapped#patchFork(), this one by ResolvedUnmapped#evalUnresolvedAtopUnary - assertThat(rightEval3.fields().get(0).id(), not(rightProjectAttribute_does_not_exist.id())); - var rightRelation2 = as(rightEval3.child(), EsRelation.class); + var rightRelation2 = as(rightAgg.child(), EsRelation.class); assertThat(rightRelation2.indexPattern(), is("employees")); } @@ -2362,9 +2075,8 @@ public void testFailAfterUnionAllOfStats() { * $$does_not_exist2$converted_to$long{r$}#92]] * | \_Eval[[TOLONG(does_not_exist2{r}#71) AS $$does_not_exist2$converted_to$long#92]] * | \_Eval[[TOLONG(does_not_exist1{r}#30) AS $$does_not_exist1$converted_to$long#67]] - * | \_Eval[[null[INTEGER] AS language_code#28, null[KEYWORD] AS language_name#29, null[NULL] AS does_not_exist1#30, - * null[NULL] AS does_not_exist2#71]] - * | \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g..] + * | \_Eval[[null[INTEGER] AS language_code#28, null[KEYWORD] AS language_name#29]] + * | \_EsRelation[test][_meta_field{f}#15, emp_no{f}#9, first_name{f}#10, g.., does_not_exist1{f}#30, does_not_exist2{f}#71] * |_Project[[_meta_field{r}#31, emp_no{r}#32, first_name{r}#33, gender{r}#34, hire_date{r}#35, job{r}#36, job.raw{r}#37, * languages{r}#38, last_name{r}#39, long_noidx{r}#40, salary{r}#41, language_code{f}#20, language_name{f}#21, * does_not_exist1{r}#24, $$does_not_exist1$converted_to$long{r$}#68, does_not_exist2{r}#72, @@ -2377,8 +2089,7 @@ public void testFailAfterUnionAllOfStats() { * null[INTEGER] AS salary#41]] * | \_Subquery[] * | \_Filter[TOLONG(does_not_exist1{r}#24) > 1[INTEGER]] - * | \_Eval[[null[NULL] AS does_not_exist1#24, null[NULL] AS does_not_exist2#72]] - * | \_EsRelation[languages][language_code{f}#20, language_name{f}#21] + * | \_EsRelation[languages][language_code{f}#20, language_name{f}#21, does_not_exist1{f}#24, does_not_exist2{f}#72] * \_Project[[_meta_field{r}#42, emp_no{r}#43, first_name{r}#44, gender{r}#45, hire_date{r}#46, job{r}#47, job.raw{r}#48, * languages{r}#49, last_name{r}#50, long_noidx{r}#51, salary{r}#52, language_code{f}#22, language_name{f}#23, * does_not_exist1{r}#26, $$does_not_exist1$converted_to$long{r$}#69, does_not_exist2{r}#73, @@ -2391,8 +2102,7 @@ public void testFailAfterUnionAllOfStats() { * null[INTEGER] AS salary#52]] * \_Subquery[] * \_Filter[TOLONG(does_not_exist1{r}#26) > 2[INTEGER]] - * \_Eval[[null[NULL] AS does_not_exist1#26, null[NULL] AS does_not_exist2#73]] - * \_EsRelation[languages][language_code{f}#22, language_name{f}#23] + * \_EsRelation[languages][language_code{f}#22, language_name{f}#23, does_not_exist1{f}#26, does_not_exist2{f}#73] */ public void testSubquerysWithMainAndSameOptional() { assumeTrue( @@ -2452,10 +2162,8 @@ public void testSubquerysWithMainAndSameOptional() { var b1EvalConvert = as(b1EvalToLong.child(), Eval.class); assertThat(Expressions.names(b1EvalConvert.fields()), is(List.of("$$does_not_exist1$converted_to$long"))); var b1EvalNulls = as(b1EvalConvert.child(), Eval.class); - assertThat( - Expressions.names(b1EvalNulls.fields()), - is(List.of("language_code", "language_name", "does_not_exist1", "does_not_exist2")) - ); + // Unmapped does_not_exist1/2 are in EsRelation output; Eval adds language_code/name for union alignment + assertThat(Expressions.names(b1EvalNulls.fields()), hasItems("language_code", "language_name")); var b1Rel = as(b1EvalNulls.child(), EsRelation.class); assertThat(b1Rel.indexPattern(), is("test")); @@ -2479,9 +2187,7 @@ public void testSubquerysWithMainAndSameOptional() { var b2Gt = as(b2Filter.condition(), GreaterThan.class); var b2GtToLong = as(b2Gt.left(), ToLong.class); assertThat(Expressions.name(b2GtToLong.field()), is("does_not_exist1")); - var b2SubEval = as(b2Filter.child(), Eval.class); - assertThat(Expressions.names(b2SubEval.fields()), is(List.of("does_not_exist1", "does_not_exist2"))); - var b2Rel = as(b2SubEval.child(), EsRelation.class); + var b2Rel = as(b2Filter.child(), EsRelation.class); assertThat(b2Rel.indexPattern(), is("languages")); // Branch 3: Subquery[languages] with Filter TOLONG(does_not_exist1) > 2, wrapped by Project nulls + Eval(TOLONG dne1) @@ -2502,9 +2208,7 @@ public void testSubquerysWithMainAndSameOptional() { var b3Gt = as(b3Filter.condition(), GreaterThan.class); var b3GtToLong = as(b3Gt.left(), ToLong.class); assertThat(Expressions.name(b3GtToLong.field()), is("does_not_exist1")); - var b3SubEval = as(b3Filter.child(), Eval.class); - assertThat(Expressions.names(b3SubEval.fields()), is(List.of("does_not_exist1", "does_not_exist2"))); - var b3Rel = as(b3SubEval.child(), EsRelation.class); + var b3Rel = as(b3Filter.child(), EsRelation.class); assertThat(b3Rel.indexPattern(), is("languages")); } @@ -2791,8 +2495,7 @@ public void testSubquerysWithMainAndStatsOnly() { * | \_Filter[emp_no{f}#29 > 3[INTEGER]] * | \_Eval[[ISNULL(does_not_exist2{r}#68) AS does_not_exist2 IS NULL#6]] * | \_Filter[first_name{f}#30 == Chris[KEYWORD] AND TOLONG(does_not_exist1{r}#62) > 5[INTEGER]] - * | \_Eval[[null[NULL] AS does_not_exist1#62, null[NULL] AS does_not_exist2#68, null[NULL] AS does_not_exist3#74]] - * | \_EsRelation[test][_meta_field{f}#35, emp_no{f}#29, first_name{f}#30, ..] + * | \_EsRelation[test][.., does_not_exist1{f}#62, does_not_exist2{f}#68, does_not_exist3{f}#74] * |_Limit[1000[INTEGER],false,false] * | \_Project[[_meta_field{f}#46, emp_no{f}#40, first_name{f}#41, gender{f}#42, hire_date{f}#47, job{f}#48, job.raw{f}#49, * languages{f}#43, last_name{f}#44, long_noidx{f}#50, salary{f}#45, does_not_exist1{r}#64, does_not_exist2{r}#70, @@ -2803,8 +2506,7 @@ public void testSubquerysWithMainAndStatsOnly() { * | \_Filter[emp_no{f}#40 > 2[INTEGER]] * | \_Eval[[ISNULL(does_not_exist2{r}#70) AS does_not_exist2 IS NULL#6]] * | \_Filter[first_name{f}#41 == Chris[KEYWORD] AND TOLONG(does_not_exist1{r}#64) > 5[INTEGER]] - * | \_Eval[[null[NULL] AS does_not_exist1#64, null[NULL] AS does_not_exist2#70, null[NULL] AS does_not_exist4#76]] - * | \_EsRelation[test][_meta_field{f}#46, emp_no{f}#40, first_name{f}#41, ..] + * | \_EsRelation[test][.., does_not_exist1{f}#64, does_not_exist2{f}#70, does_not_exist4{f}#76] * \_Limit[1000[INTEGER],false,false] * \_Project[[_meta_field{r}#87, emp_no{r}#88, first_name{r}#89, gender{r}#90, hire_date{r}#91, job{r}#92, job.raw{r}#93, * languages{r}#94, last_name{r}#95, long_noidx{r}#96, salary{r}#97, does_not_exist1{r}#98, does_not_exist2{r}#99, @@ -2823,8 +2525,7 @@ public void testSubquerysWithMainAndStatsOnly() { * parser=org.elasticsearch.dissect.DissectParser@4ba4d16b],[d{r}#22, e{r}#23, f{r}#24]] * \_Eval[[ISNULL(does_not_exist2{r}#72) AS does_not_exist2 IS NULL#6]] * \_Filter[first_name{f}#52 == Chris[KEYWORD] AND TOLONG(does_not_exist1{r}#66) > 5[INTEGER]] - * \_Eval[[null[NULL] AS does_not_exist1#66, null[NULL] AS does_not_exist2#72, null[NULL] AS does_not_exist5#78]] - * \_EsRelation[test][_meta_field{f}#57, emp_no{f}#51, first_name{f}#52, ..] + * \_EsRelation[test][.., does_not_exist1{f}#66, does_not_exist2{f}#72, does_not_exist5{f}#78] */ public void testForkBranchesWithDifferentSchemas() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -2881,18 +2582,7 @@ public void testForkBranchesWithDifferentSchemas() { assertThat(Expressions.name(b0RightToLong.field()), is("does_not_exist1")); assertThat(as(b0RightGt.right(), Literal.class).value(), is(5)); - // Chain of Evals adding dne1/dne2/dne3 NULLs - var b0EvalDne1 = as(b0Filter.child(), Eval.class); - var b0EvalDne1Alias = as(b0EvalDne1.fields().get(0), Alias.class); - assertThat(b0EvalDne1Alias.name(), is("does_not_exist1")); - assertThat(as(b0EvalDne1Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - var b0EvalDne2Alias = as(b0EvalDne1.fields().get(1), Alias.class); - assertThat(b0EvalDne2Alias.name(), is("does_not_exist2")); - assertThat(as(b0EvalDne2Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - var b0EvalDne3Alias = as(b0EvalDne1.fields().get(2), Alias.class); - assertThat(b0EvalDne3Alias.name(), is("does_not_exist3")); - assertThat(as(b0EvalDne3Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - var b0Rel = as(b0EvalDne1.child(), EsRelation.class); + var b0Rel = as(b0Filter.child(), EsRelation.class); assertThat(b0Rel.indexPattern(), is("test")); // Branch 1 @@ -2932,19 +2622,7 @@ public void testForkBranchesWithDifferentSchemas() { assertThat(Expressions.name(b1RightToLong.field()), is("does_not_exist1")); assertThat(as(b1RightGt.right(), Literal.class).value(), is(5)); - // Chain of Evals adding dne1/dne2/dne4 NULLs - var b1EvalDne1 = as(b1Filter.child(), Eval.class); - var b1EvalDne1Alias = as(b1EvalDne1.fields().get(0), Alias.class); - assertThat(b1EvalDne1Alias.name(), is("does_not_exist1")); - assertThat(as(b1EvalDne1Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - var b1EvalDne2Alias = as(b1EvalDne1.fields().get(1), Alias.class); - assertThat(b1EvalDne2Alias.name(), is("does_not_exist2")); - assertThat(as(b1EvalDne2Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - var b1EvalDne3Alias = as(b1EvalDne1.fields().get(2), Alias.class); - assertThat(b1EvalDne3Alias.name(), is("does_not_exist4")); - assertThat(as(b1EvalDne3Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var b1Rel = as(b1EvalDne1.child(), EsRelation.class); + var b1Rel = as(b1Filter.child(), EsRelation.class); assertThat(b1Rel.indexPattern(), is("test")); // Branch 2 @@ -2986,10 +2664,9 @@ public void testForkBranchesWithDifferentSchemas() { as(filtered.delegate(), Max.class); var feCondGT = as(filtered.filter(), GreaterThan.class); var feCondGTAdd = as(feCondGT.right(), Add.class); - // Right side of Add must be ToDouble(does_not_exist5) + // Right side of Add must be ToDouble(does_not_exist5) - now a FieldAttribute from EsRelation output var dne5Convert = as(feCondGTAdd.right(), ConvertFunction.class); - var dne5Ref = as(dne5Convert.field(), ReferenceAttribute.class); - assertThat(dne5Ref.name(), is("does_not_exist5")); + assertThat(Expressions.name(dne5Convert.field()), is("does_not_exist5")); var dissect = as(b2Agg.child(), Dissect.class); var evalDne2IsNull = as(dissect.child(), Eval.class); @@ -3002,18 +2679,7 @@ public void testForkBranchesWithDifferentSchemas() { assertThat(Expressions.name(rightToLong.field()), is("does_not_exist1")); assertThat(as(rightGt.right(), Literal.class).value(), is(5)); - var evalDne1 = as(filter.child(), Eval.class); - var dne1Alias = as(evalDne1.fields().get(0), Alias.class); - assertThat(dne1Alias.name(), is("does_not_exist1")); - assertThat(as(dne1Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - var dne2Alias = as(evalDne1.fields().get(1), Alias.class); - assertThat(dne2Alias.name(), is("does_not_exist2")); - assertThat(as(dne2Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - var dne3Alias = as(evalDne1.fields().get(2), Alias.class); - assertThat(dne3Alias.name(), is("does_not_exist5")); - assertThat(as(dne3Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var rel = as(evalDne1.child(), EsRelation.class); + var rel = as(filter.child(), EsRelation.class); assertThat(rel.indexPattern(), is("test")); } @@ -3106,8 +2772,7 @@ public void testFailAfterForkOfStats() { * \_InlineStats[] * \_Aggregate[[does_not_exist2{r}#19],[SUM(does_not_exist1{r}#20,true[BOOLEAN],PT0S[TIME_DURATION],compensated[KEYWORD]) AS c#5, * does_not_exist2{r}#19]] - * \_Eval[[null[NULL] AS does_not_exist2#19, null[NULL] AS does_not_exist1#20]] - * \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] + * \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge.., does_not_exist1{f}#20, does_not_exist2{f}#19] */ public void testInlineStats() { var plan = analyzeStatement(setUnmappedNullify(""" @@ -3125,28 +2790,15 @@ public void testInlineStats() { // Grouping by does_not_exist2 and SUM over does_not_exist1 assertThat(agg.groupings(), hasSize(1)); - var groupRef = as(agg.groupings().getFirst(), ReferenceAttribute.class); - assertThat(groupRef.name(), is("does_not_exist2")); + assertThat(Expressions.name(agg.groupings().getFirst()), is("does_not_exist2")); assertThat(agg.aggregates(), hasSize(2)); var cAlias = as(agg.aggregates().getFirst(), Alias.class); assertThat(cAlias.name(), is("c")); as(cAlias.child(), Sum.class); - // Upstream Eval introduces does_not_exist2 and does_not_exist1 as NULL - var eval = as(agg.child(), Eval.class); - assertThat(eval.fields(), hasSize(2)); - - var dne2Alias = as(eval.fields().get(0), Alias.class); - assertThat(dne2Alias.name(), is("does_not_exist2")); - assertThat(as(dne2Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var dne1Alias = as(eval.fields().get(1), Alias.class); - assertThat(dne1Alias.name(), is("does_not_exist1")); - assertThat(as(dne1Alias.child(), Literal.class).dataType(), is(DataType.NULL)); - // Underlying relation - var relation = as(eval.child(), EsRelation.class); + var relation = as(agg.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } @@ -3154,8 +2806,7 @@ public void testInlineStats() { * Limit[1000[INTEGER],false,false] * \_LookupJoin[LEFT,[language_code{r}#5],[language_code{f}#19],false,null] * |_Eval[[TOINTEGER(does_not_exist{r}#21) AS language_code#5]] - * | \_Eval[[null[NULL] AS does_not_exist#21]] - * | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] + * | \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge.., does_not_exist{f}#21] * \_EsRelation[languages_lookup][LOOKUP][language_code{f}#19, language_name{f}#20] */ public void testLookupJoin() { @@ -3174,20 +2825,14 @@ public void testLookupJoin() { var lj = as(limit.child(), LookupJoin.class); assertThat(lj.config().type(), is(JoinTypes.LEFT)); - // Left child: EVAL language_code = TOINTEGER(does_not_exist), with upstream NULL alias for does_not_exist + // Left child: EVAL language_code = TOINTEGER(does_not_exist) var leftEval = as(lj.left(), Eval.class); assertThat(leftEval.fields(), hasSize(1)); var langCodeAlias = as(leftEval.fields().getFirst(), Alias.class); assertThat(langCodeAlias.name(), is("language_code")); as(langCodeAlias.child(), ToInteger.class); - var upstreamEval = as(leftEval.child(), Eval.class); - assertThat(upstreamEval.fields(), hasSize(1)); - var dneAlias = as(upstreamEval.fields().getFirst(), Alias.class); - assertThat(dneAlias.name(), is("does_not_exist")); - assertThat(as(dneAlias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var leftRel = as(upstreamEval.child(), EsRelation.class); + var leftRel = as(leftEval.child(), EsRelation.class); assertThat(leftRel.indexPattern(), is("test")); // Right lookup table @@ -3200,8 +2845,7 @@ public void testLookupJoin() { * \_Enrich[ANY,languages[KEYWORD],x{r}#5,{"match":{"indices":[],"match_field":"language_code", * "enrich_fields":["language_name"]}},{=languages_idx},[language_name{r}#21]] * \_Eval[[TOSTRING(does_not_exist{r}#22) AS x#5]] - * \_Eval[[null[NULL] AS does_not_exist#22]] - * \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge..] + * \_EsRelation[test][_meta_field{f}#14, emp_no{f}#8, first_name{f}#9, ge.., does_not_exist{f}#22] */ public void testEnrich() { String query = """ @@ -3220,28 +2864,21 @@ public void testEnrich() { assertThat(enrich.matchField().name(), is("x")); assertThat(Expressions.names(enrich.enrichFields()), contains("language_name")); - // Left child: EVAL x = TOSTRING(does_not_exist), with upstream NULL alias for does_not_exist + // Left child: EVAL x = TOSTRING(does_not_exist) var leftEval = as(enrich.child(), Eval.class); assertThat(leftEval.fields(), hasSize(1)); var xAlias = as(leftEval.fields().getFirst(), Alias.class); assertThat(xAlias.name(), is("x")); as(xAlias.child(), ToString.class); - var upstreamEval = as(leftEval.child(), Eval.class); - assertThat(upstreamEval.fields(), hasSize(1)); - var dneAlias = as(upstreamEval.fields().getFirst(), Alias.class); - assertThat(dneAlias.name(), is("does_not_exist")); - assertThat(as(dneAlias.child(), Literal.class).dataType(), is(DataType.NULL)); - - var leftRel = as(upstreamEval.child(), EsRelation.class); + var leftRel = as(leftEval.child(), EsRelation.class); assertThat(leftRel.indexPattern(), is("test")); } /* * Limit[1000[INTEGER],false,false] * \_Filter[KNN(does_not_exist{r}#16,TODENSEVECTOR([0, 1, 2][INTEGER]))] - * \_Eval[[null[NULL] AS does_not_exist#16]] - * \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge..] + * \_EsRelation[test][_meta_field{f}#11, emp_no{f}#5, first_name{f}#6, ge.., does_not_exist{f}#16] */ public void testSemanticText() { String query = """ @@ -3258,15 +2895,8 @@ public void testSemanticText() { var filter = as(limit.child(), Filter.class); assertNotNull(filter.condition()); // KNN(does_not_exist, TODENSEVECTOR([...])) - // Upstream Eval introduces does_not_exist as NULL - var eval = as(filter.child(), Eval.class); - assertThat(eval.fields(), hasSize(1)); - var dneAlias = as(eval.fields().getFirst(), Alias.class); - assertThat(dneAlias.name(), is("does_not_exist")); - assertThat(as(dneAlias.child(), Literal.class).dataType(), is(DataType.NULL)); - // Underlying relation - var relation = as(eval.child(), EsRelation.class); + var relation = as(filter.child(), EsRelation.class); assertThat(relation.indexPattern(), is("test")); } @@ -3454,6 +3084,45 @@ public void testChangedTimestmapFieldWithRate() { """), "3:13: [rate(network.total_cost)] " + UnresolvedTimestamp.UNRESOLVED_SUFFIX); } + /** + * Reproducer for https://github.com/elastic/elasticsearch/issues/142968 + * KQL (and QSTR) functions should be allowed in WHERE immediately after FROM, + * even when an unmapped field is referenced later in the query. + */ + public void testKqlWithUnmappedFieldInEval() { + // This should NOT throw a verification exception. + // The KQL function is correctly placed in a WHERE directly after FROM. + var plan = analyzeStatement(setUnmappedNullify(""" + FROM test + | WHERE kql("first_name: test") + | EVAL x = does_not_exist_field + 1 + """)); + var limit = as(plan, Limit.class); + var eval = as(limit.child(), Eval.class); + var filter = as(eval.child(), Filter.class); + // Unmapped field is in EsRelation output; Filter sits directly above EsRelation + var relation = as(filter.child(), EsRelation.class); + assertThat(relation.indexPattern(), is("test")); + } + + /** + * Reproducer for https://github.com/elastic/elasticsearch/issues/142959 + * QSTR functions should be allowed after SORT, even when an unmapped field is used later. + */ + public void testQstrAfterSortWithUnmappedField() { + var plan = analyzeStatement(setUnmappedNullify(""" + FROM test + | SORT first_name + | WHERE qstr("first_name: test") + | EVAL x = does_not_exist_field + 1 + """)); + var limit = as(plan, Limit.class); + var eval = as(limit.child(), Eval.class); + var filter = as(eval.child(), Filter.class); + var orderBy = as(filter.child(), OrderBy.class); + assertThat(orderBy, not(nullValue())); + } + private void verificationFailure(String statement, String expectedFailure) { var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); assertThat(e.getMessage(), containsString(expectedFailure)); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java index 9795d2946d787..6d93d90d7fee9 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalLogicalPlanOptimizerTests.java @@ -18,6 +18,7 @@ import org.elasticsearch.xpack.esql.VerificationException; import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; +import org.elasticsearch.xpack.esql.analysis.UnmappedResolution; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeSet; @@ -506,6 +507,98 @@ public void testMissingFieldInFilterNoProjection() { ); } + /** + * Test that fields with DataType.NULL from NULLIFY mode are replaced with constant null literals. + * When unmapped_fields="nullify", the analyzer adds fields with DataType.NULL to EsRelation. + * The local optimizer's ReplaceFieldWithConstantOrNull rule should replace these with Literal.NULL. + * + * Expects: + * Project[[does_not_exist_field{r}#X]] + * \_Eval[[null[NULL] AS does_not_exist_field]] + * \_Limit[1000[INTEGER],false] + * \_EsRelation[test][...] + */ + public void testNullifyModeFieldReplacedWithNull() { + assumeTrue("Requires FIX_UNMAPPED_FIELDS_IN_ESRELATION", EsqlCapabilities.Cap.FIX_UNMAPPED_FIELDS_IN_ESRELATION.isEnabled()); + + var plan = planWithNullify("from test | keep does_not_exist_field"); + var testStats = statsForMissingField("does_not_exist_field"); + var localPlan = localPlan(plan, testStats); + + var project = as(localPlan, Project.class); + var projections = project.projections(); + assertThat(Expressions.names(projections), contains("does_not_exist_field")); + as(projections.get(0), ReferenceAttribute.class); + var eval = as(project.child(), Eval.class); + assertThat(Expressions.names(eval.fields()), contains("does_not_exist_field")); + var alias = as(eval.fields().get(0), Alias.class); + var literal = as(alias.child(), Literal.class); + assertThat(literal.value(), is(nullValue())); + assertThat(literal.dataType(), is(DataType.NULL)); + + var limit = as(eval.child(), Limit.class); + var source = as(limit.child(), EsRelation.class); + } + + /** + * Test that fields with DataType.NULL from NULLIFY mode used in EVAL are correctly replaced. + * + * Expects: + * Project[[x{r}#X]] + * \_Eval[[null[INTEGER] AS x]] + * \_Limit[1000[INTEGER],false] + * \_EsRelation[test][...] + */ + public void testNullifyModeFieldInEvalReplacedWithNull() { + assumeTrue("Requires FIX_UNMAPPED_FIELDS_IN_ESRELATION", EsqlCapabilities.Cap.FIX_UNMAPPED_FIELDS_IN_ESRELATION.isEnabled()); + + var plan = planWithNullify(""" + from test + | eval x = does_not_exist_field + 1 + | keep x + """); + + var testStats = statsForMissingField("does_not_exist_field"); + var localPlan = localPlan(plan, testStats); + + var project = as(localPlan, Project.class); + assertThat(Expressions.names(project.projections()), contains("x")); + var eval = as(project.child(), Eval.class); + assertThat(Expressions.names(eval.fields()), contains("x")); + + var alias = as(eval.fields().get(0), Alias.class); + var literal = as(alias.child(), Literal.class); + assertThat(literal.value(), is(nullValue())); + assertThat(literal.dataType(), is(INTEGER)); + + var limit = as(eval.child(), Limit.class); + var source = as(limit.child(), EsRelation.class); + } + + /** + * Test that multiple fields with DataType.NULL are all replaced. + */ + public void testNullifyModeMultipleFieldsReplacedWithNull() { + assumeTrue("Requires FIX_UNMAPPED_FIELDS_IN_ESRELATION", EsqlCapabilities.Cap.FIX_UNMAPPED_FIELDS_IN_ESRELATION.isEnabled()); + + var plan = planWithNullify(""" + from test + | eval x = field_a + field_b + | keep x + """); + + var testStats = statsForMissingField("field_a", "field_b"); + var localPlan = localPlan(plan, testStats); + + var project = as(localPlan, Project.class); + assertThat(Expressions.names(project.projections()), contains("x")); + var eval = as(project.child(), Eval.class); + + var alias = as(eval.fields().get(0), Alias.class); + var literal = as(alias.child(), Literal.class); + assertThat(literal.value(), is(nullValue())); + } + public void testSparseDocument() throws Exception { var query = """ from large @@ -2394,6 +2487,26 @@ public static EsRelation relation() { return EsqlTestUtils.relation(randomFrom(IndexMode.values())); } + private static Analyzer analyzerWithNullifyMode() { + EsIndex test = EsIndexGenerator.esIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); + return new Analyzer( + testAnalyzerContext( + EsqlTestUtils.TEST_CFG, + new EsqlFunctionRegistry(), + indexResolutions(test), + Map.of(), + emptyPolicyResolution(), + emptyInferenceResolution(), + UnmappedResolution.NULLIFY + ), + TEST_VERIFIER + ); + } + + private LogicalPlan planWithNullify(String query) { + return plan(query, analyzerWithNullifyMode()); + } + // Tests for project metadata field optimization (ReplaceFieldWithConstantOrNull) /** diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java index 8f9ac14c1c80d..fa93782ab28b7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/TestPhysicalOperationProviders.java @@ -290,6 +290,10 @@ private BiFunction getBlockExtraction(DriverCo case BlockResultSuccess success -> success.block; }; } + // For NULL-typed fields (unmapped fields with NULLIFY mode), return nulls + if (fa.dataType() == DataType.NULL) { + return (doc, copier) -> getNullsBlock(doc); + } } return (indexDoc, blockCopier) -> switch (extractBlockForSingleDoc(indexDoc, attribute.name(), blockCopier)) { case BlockResultMissing missing -> throw new EsqlIllegalArgumentException( diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterConjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterConjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected index 8f0caf6cc39a3..2c72ad909cbc1 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterConjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterConjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected @@ -1 +1 @@ -LocalSourceExec[[message{f}#0, does_not_exist{r}#1],EMPTY] \ No newline at end of file +LocalSourceExec[[message{f}#0, does_not_exist{f}#1],EMPTY] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterDisjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterDisjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected index e43a2df780fed..a089b04b205a2 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterDisjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterDisjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected @@ -1,10 +1,9 @@ -ProjectExec[[message{f}#0, does_not_exist{r}#1]] -\_EvalExec[[null[NULL] AS does_not_exist#1]] - \_LimitExec[1000[INTEGER],null] - \_ExchangeExec[[message{f}#0],false] - \_ProjectExec[[message{f}#0]] - \_FieldExtractExec[message{f}#0]<[],[]> - \_EsQueryExec[sample_data], indexMode[standard], [_doc{f}#2], limit[1000], sort[] estimatedRowSize[54] queryBuilderAndTags [[QueryBuilderAndTags[query={ +ProjectExec[[message{f}#0, does_not_exist{f}#1]] +\_LimitExec[1000[INTEGER],null] + \_ExchangeExec[[message{f}#0, does_not_exist{f}#1],false] + \_ProjectExec[[message{f}#0, does_not_exist{f}#1]] + \_FieldExtractExec[message{f}#0, does_not_exist{f}#1]<[],[]> + \_EsQueryExec[sample_data], indexMode[standard], [_doc{f}#2], limit[1000], sort[] estimatedRowSize[54] queryBuilderAndTags [[QueryBuilderAndTags[query={ "esql_single_value" : { "field" : "message", "next" : { diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterNoPushdownWithUnmapped/nullify/local_physical_optimization.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterNoPushdownWithUnmapped/nullify/local_physical_optimization.expected index 8f0caf6cc39a3..2c72ad909cbc1 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterNoPushdownWithUnmapped/nullify/local_physical_optimization.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testFilterNoPushdownWithUnmapped/nullify/local_physical_optimization.expected @@ -1 +1 @@ -LocalSourceExec[[message{f}#0, does_not_exist{r}#1],EMPTY] \ No newline at end of file +LocalSourceExec[[message{f}#0, does_not_exist{f}#1],EMPTY] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testSortConjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testSortConjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected index 3dac4b94f8e4b..809b5887c2b9b 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testSortConjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testSortConjunctionPushableAndNonPushable/nullify/local_physical_optimization.expected @@ -1,7 +1,6 @@ -ProjectExec[[message{f}#0, does_not_exist{r}#1]] -\_TopNExec[[Order[message{f}#0,ASC,LAST], Order[does_not_exist{r}#1,ASC,LAST]],5[INTEGER],null] - \_ExchangeExec[[message{f}#0, does_not_exist{r}#1],false] - \_ProjectExec[[message{f}#0, does_not_exist{r}#1]] - \_FieldExtractExec[message{f}#0]<[],[]> - \_EvalExec[[null[NULL] AS does_not_exist#1]] - \_EsQueryExec[sample_data], indexMode[standard], [_doc{f}#2], limit[5], sort[[FieldSort[field=message{f}#0, direction=ASC, nulls=LAST]]] estimatedRowSize[66] queryBuilderAndTags [[QueryBuilderAndTags[query=null, tags=[]]]] \ No newline at end of file +ProjectExec[[message{f}#0, does_not_exist{f}#1]] +\_TopNExec[[Order[message{f}#0,ASC,LAST], Order[does_not_exist{f}#1,ASC,LAST]],5[INTEGER],null] + \_ExchangeExec[[message{f}#0, does_not_exist{f}#1],false] + \_ProjectExec[[message{f}#0, does_not_exist{f}#1]] + \_FieldExtractExec[message{f}#0, does_not_exist{f}#1]<[],[]> + \_EsQueryExec[sample_data], indexMode[standard], [_doc{f}#2], limit[5], sort[[FieldSort[field=message{f}#0, direction=ASC, nulls=LAST], FieldSort[field=does_not_exist{f}#1, direction=ASC, nulls=LAST]]] estimatedRowSize[66] queryBuilderAndTags [[QueryBuilderAndTags[query=null, tags=[]]]] \ No newline at end of file diff --git a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testSortNoPushdownWithUnmapped/nullify/local_physical_optimization.expected b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testSortNoPushdownWithUnmapped/nullify/local_physical_optimization.expected index ea4e1b4e5134b..f25d59335eaf0 100644 --- a/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testSortNoPushdownWithUnmapped/nullify/local_physical_optimization.expected +++ b/x-pack/plugin/esql/src/test/resources/org/elasticsearch/xpack/esql/optimizer/golden_tests/PushdownGoldenTests/testSortNoPushdownWithUnmapped/nullify/local_physical_optimization.expected @@ -1,8 +1,6 @@ -ProjectExec[[message{f}#0, does_not_exist{r}#1]] -\_TopNExec[[Order[does_not_exist{r}#1,ASC,LAST]],5[INTEGER],null] - \_ExchangeExec[[message{f}#0, does_not_exist{r}#1],false] - \_ProjectExec[[message{f}#0, does_not_exist{r}#1]] - \_FieldExtractExec[message{f}#0]<[],[]> - \_TopNExec[[Order[does_not_exist{r}#1,ASC,LAST]],5[INTEGER],70] - \_EvalExec[[null[NULL] AS does_not_exist#1]] - \_EsQueryExec[sample_data], indexMode[standard], [_doc{f}#2], limit[], sort[] estimatedRowSize[4] queryBuilderAndTags [[QueryBuilderAndTags[query=null, tags=[]]]] \ No newline at end of file +ProjectExec[[message{f}#0, does_not_exist{f}#1]] +\_TopNExec[[Order[does_not_exist{f}#1,ASC,LAST]],5[INTEGER],null] + \_ExchangeExec[[message{f}#0, does_not_exist{f}#1],false] + \_ProjectExec[[message{f}#0, does_not_exist{f}#1]] + \_FieldExtractExec[message{f}#0, does_not_exist{f}#1]<[],[]> + \_EsQueryExec[sample_data], indexMode[standard], [_doc{f}#2], limit[5], sort[[FieldSort[field=does_not_exist{f}#1, direction=ASC, nulls=LAST]]] estimatedRowSize[66] queryBuilderAndTags [[QueryBuilderAndTags[query=null, tags=[]]]] \ No newline at end of file