From 5501a1e92501b44db0739dd33a69758c33779ebc Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Fri, 13 Mar 2026 19:21:57 +0200 Subject: [PATCH 1/7] Forbid SET nullify and load for implicit @timestamp field usage --- .../xpack/esql/analysis/Verifier.java | 44 +++- .../esql/analysis/AnalyzerUnmappedTests.java | 210 ++++++++++++++++++ 2 files changed, 252 insertions(+), 2 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 5dd111f726147..623f5b1f083c1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -20,11 +20,13 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; +import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; import org.elasticsearch.xpack.esql.core.expression.function.Function; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.tree.Node; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.util.Holder; +import org.elasticsearch.xpack.esql.expression.function.TimestampAware; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.predicate.operator.arithmetic.Neg; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals; @@ -68,6 +70,8 @@ */ public class Verifier { + static final String UNMAPPED_TIMESTAMP_SUFFIX = "; the [unmapped_fields] setting does not apply to the implicit @timestamp reference"; + /** * Extra plan verification checks defined in plugins. */ @@ -97,8 +101,13 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics, UnmappedReso assert partialMetrics != null; Failures failures = new Failures(); + boolean unmappedTimestampHandled = false; + if (unmappedResolution != UnmappedResolution.FAIL) { + unmappedTimestampHandled = checkUnmappedTimestamp(plan, failures); + } + // quick verification for unresolved attributes - checkUnresolvedAttributes(plan, failures); + checkUnresolvedAttributes(plan, failures, unmappedTimestampHandled); ConfigurationAware.verifyNoMarkerConfiguration(plan, failures); @@ -149,7 +158,7 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics, UnmappedReso return failures.failures(); } - private static void checkUnresolvedAttributes(LogicalPlan plan, Failures failures) { + private static void checkUnresolvedAttributes(LogicalPlan plan, Failures failures, boolean skipUnresolvedTimestamp) { plan.forEachUp(p -> { // if the children are unresolved, so will this node; counting it will only add noise if (p.childrenResolved() == false) { @@ -182,6 +191,9 @@ else if (p.resolved()) { return; } + if (skipUnresolvedTimestamp && ae instanceof UnresolvedTimestamp) { + return; + } if (ae instanceof Unresolvable u) { failures.add(fail(ae, u.unresolvedMessage())); } @@ -354,6 +366,34 @@ private static void checkLimitBy(LogicalPlan plan, Failures failures) { } } + /** + * {@link TimestampAware} functions implicitly reference {@code @timestamp} and require the field to be present in the index mapping. + * The {@code unmapped_fields} setting does not apply to the implicit {@code @timestamp} reference. + * Only emits the specific message when {@code @timestamp} is truly absent from all source index mappings; + * if the field was present but dropped/renamed by the query, the generic unresolved-attribute message is more appropriate. + * See https://github.com/elastic/elasticsearch/issues/142127 + */ + private static boolean checkUnmappedTimestamp(LogicalPlan plan, Failures failures) { + boolean timestampInIndex = plan.collect(EsRelation.class, r -> r.indexMode() != IndexMode.LOOKUP).stream() + .anyMatch(r -> r.output().stream().anyMatch(a -> MetadataAttribute.TIMESTAMP_FIELD.equals(a.name()))); + if (timestampInIndex) { + return false; + } + plan.forEachDown(p -> { + if (p instanceof TimestampAware ta && ta.timestamp() instanceof UnresolvedTimestamp) { + failures.add(fail(p, "[{}] " + UnresolvedTimestamp.UNRESOLVED_SUFFIX + UNMAPPED_TIMESTAMP_SUFFIX, p.sourceText())); + } + p.forEachExpression(Expression.class, e -> { + if (e instanceof TimestampAware ta && ta.timestamp() instanceof UnresolvedTimestamp) { + failures.add( + fail(e, "[{}] " + UnresolvedTimestamp.UNRESOLVED_SUFFIX + UNMAPPED_TIMESTAMP_SUFFIX, e.sourceText()) + ); + } + }); + }); + return true; + } + /** * {@code unmapped_fields="load"} does not yet support branching commands (FORK, LOOKUP JOIN, subqueries/views). * See https://github.com/elastic/elasticsearch/issues/142033 diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 0baf1902eb8af..0c876e63b6920 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -3927,6 +3927,216 @@ public void testNullifyModeAllowsFork() { assertThat(fork.children(), hasSize(2)); } + private static final String UNMAPPED_TIMESTAMP_SUFFIX = UnresolvedTimestamp.UNRESOLVED_SUFFIX + Verifier.UNMAPPED_TIMESTAMP_SUFFIX; + + public void testTbucketWithUnmappedTimestamp() { + unmappedTimestampFailure( + "FROM test | STATS c = COUNT(*) BY tbucket(1 hour)", + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTrangeWithUnmappedTimestamp() { + unmappedTimestampFailure("FROM test | WHERE trange(1 hour)", "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX); + } + + public void testTbucketAndTrangeWithUnmappedTimestamp() { + unmappedTimestampFailure( + "FROM test | WHERE trange(1 hour) | STATS c = COUNT(*) BY tbucket(1 hour)", + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX, + "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testRateWithUnmappedTimestamp() { + unmappedTimestampFailure("FROM test | STATS rate(salary)", "[rate(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + } + + public void testIrateWithUnmappedTimestamp() { + unmappedTimestampFailure("FROM test | STATS irate(salary)", "[irate(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + } + + public void testDeltaWithUnmappedTimestamp() { + unmappedTimestampFailure("FROM test | STATS delta(salary)", "[delta(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + } + + public void testIdeltaWithUnmappedTimestamp() { + unmappedTimestampFailure("FROM test | STATS idelta(salary)", "[idelta(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + } + + public void testIncreaseWithUnmappedTimestamp() { + unmappedTimestampFailure("FROM test | STATS increase(salary)", "[increase(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + } + + public void testDerivWithUnmappedTimestamp() { + unmappedTimestampFailure("FROM test | STATS deriv(salary)", "[deriv(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + } + + public void testFirstOverTimeWithUnmappedTimestamp() { + unmappedTimestampFailure( + "FROM test | STATS first_over_time(salary)", + "[first_over_time(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testLastOverTimeWithUnmappedTimestamp() { + unmappedTimestampFailure( + "FROM test | STATS last_over_time(salary)", + "[last_over_time(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testRateAndTbucketWithUnmappedTimestamp() { + unmappedTimestampFailure( + "FROM test | STATS rate(salary) BY tbucket(1 hour)", + "[rate(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX, + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTbucketWithUnmappedTimestampAfterWhere() { + unmappedTimestampFailure( + "FROM test | WHERE emp_no > 10 | STATS c = COUNT(*) BY tbucket(1 hour)", + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTbucketWithUnmappedTimestampAfterEval() { + unmappedTimestampFailure( + "FROM test | EVAL x = salary + 1 | STATS c = COUNT(*) BY tbucket(1 hour)", + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTbucketWithUnmappedTimestampMultipleGroupings() { + unmappedTimestampFailure( + "FROM test | STATS c = COUNT(*) BY tbucket(1 hour), emp_no", + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTbucketWithUnmappedTimestampAfterRename() { + unmappedTimestampFailure( + "FROM test | RENAME emp_no AS e | STATS c = COUNT(*) BY tbucket(1 hour)", + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTbucketWithUnmappedTimestampAfterDrop() { + unmappedTimestampFailure( + "FROM test | DROP emp_no | STATS c = COUNT(*) BY tbucket(1 hour)", + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTrangeWithUnmappedTimestampCompoundWhere() { + unmappedTimestampFailure( + "FROM test | WHERE trange(1 hour) AND emp_no > 10", + "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTrangeWithUnmappedTimestampAfterEval() { + unmappedTimestampFailure( + "FROM test | EVAL x = salary + 1 | WHERE trange(1 hour)", + "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTbucketWithUnmappedTimestampInInlineStats() { + unmappedTimestampFailure( + "FROM test | INLINE STATS c = COUNT(*) BY tbucket(1 hour)", + "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + ); + } + + public void testTbucketWithUnmappedTimestampWithFork() { + var query = "FROM test | FORK (STATS c = COUNT(*) BY tbucket(1 hour)) (STATS d = COUNT(*) BY emp_no)"; + for (var statement : List.of(setUnmappedNullify(query), setUnmappedLoad(query))) { + var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); + assertThat(e.getMessage(), containsString("[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX)); + assertThat(e.getMessage(), not(containsString("FORK is not supported"))); + } + } + + public void testTbucketWithUnmappedTimestampWithLookupJoin() { + var query = """ + FROM test + | EVAL language_code = languages + | LOOKUP JOIN languages_lookup ON language_code + | STATS c = COUNT(*) BY tbucket(1 hour) + """; + for (var statement : List.of(setUnmappedNullify(query), setUnmappedLoad(query))) { + var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); + assertThat(e.getMessage(), containsString("[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX)); + assertThat(e.getMessage(), not(containsString("LOOKUP JOIN is not supported"))); + } + } + + public void testTbucketWithTimestampPresent() { + for (var statement : List.of( + setUnmappedNullify("FROM sample_data | STATS c = COUNT(*) BY tbucket(1 hour)"), + setUnmappedLoad("FROM sample_data | STATS c = COUNT(*) BY tbucket(1 hour)") + )) { + var plan = analyzeStatement(statement); + var limit = as(plan, Limit.class); + var aggregate = as(limit.child(), Aggregate.class); + var relation = as(aggregate.child(), EsRelation.class); + assertThat(relation.indexPattern(), is("sample_data")); + assertTimestampInOutput(relation); + } + } + + public void testTrangeWithTimestampPresent() { + for (var statement : List.of( + setUnmappedNullify("FROM sample_data | WHERE trange(1 hour)"), + setUnmappedLoad("FROM sample_data | WHERE trange(1 hour)") + )) { + var plan = analyzeStatement(statement); + var limit = as(plan, Limit.class); + var filter = as(limit.child(), Filter.class); + var relation = as(filter.child(), EsRelation.class); + assertThat(relation.indexPattern(), is("sample_data")); + assertTimestampInOutput(relation); + } + } + + public void testTbucketTimestampPresentButDroppedNullify() { + var e = expectThrows( + VerificationException.class, + () -> analyzeStatement(setUnmappedNullify("FROM sample_data | DROP @timestamp | STATS c = COUNT(*) BY tbucket(1 hour)")) + ); + assertThat(e.getMessage(), containsString(UnresolvedTimestamp.UNRESOLVED_SUFFIX)); + assertThat(e.getMessage(), not(containsString(Verifier.UNMAPPED_TIMESTAMP_SUFFIX))); + } + + public void testTbucketTimestampPresentButRenamedNullify() { + var e = expectThrows( + VerificationException.class, + () -> analyzeStatement( + setUnmappedNullify("FROM sample_data | RENAME @timestamp AS ts | STATS c = COUNT(*) BY tbucket(1 hour)") + ) + ); + assertThat(e.getMessage(), containsString(UnresolvedTimestamp.UNRESOLVED_SUFFIX)); + assertThat(e.getMessage(), not(containsString(Verifier.UNMAPPED_TIMESTAMP_SUFFIX))); + } + + private static void assertTimestampInOutput(EsRelation relation) { + assertTrue( + "@timestamp field should be present in the EsRelation output", + relation.output().stream().anyMatch(a -> MetadataAttribute.TIMESTAMP_FIELD.equals(a.name())) + ); + } + + private void unmappedTimestampFailure(String query, String... expectedFailures) { + for (var statement : List.of(setUnmappedNullify(query), setUnmappedLoad(query))) { + var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); + for (String expected : expectedFailures) { + assertThat(e.getMessage(), containsString(expected)); + } + } + } + private void verificationFailure(String statement, String expectedFailure) { var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); assertThat(e.getMessage(), containsString(expectedFailure)); From fdfb1308d00cd8873cbbe11aa896bcfb0674391e Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 17 Mar 2026 11:13:20 +0200 Subject: [PATCH 2/7] spotless --- .../xpack/esql/analysis/Verifier.java | 7 ++--- .../esql/analysis/AnalyzerUnmappedTests.java | 31 +++++-------------- 2 files changed, 11 insertions(+), 27 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index d9b5824d6ba24..e2620e0f51b3d 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -373,7 +373,8 @@ private static void checkLimitBy(LogicalPlan plan, Failures failures) { * See https://github.com/elastic/elasticsearch/issues/142127 */ private static boolean checkUnmappedTimestamp(LogicalPlan plan, Failures failures) { - boolean timestampInIndex = plan.collect(EsRelation.class, r -> r.indexMode() != IndexMode.LOOKUP).stream() + boolean timestampInIndex = plan.collect(EsRelation.class, r -> r.indexMode() != IndexMode.LOOKUP) + .stream() .anyMatch(r -> r.output().stream().anyMatch(a -> MetadataAttribute.TIMESTAMP_FIELD.equals(a.name()))); if (timestampInIndex) { return false; @@ -384,9 +385,7 @@ private static boolean checkUnmappedTimestamp(LogicalPlan plan, Failures failure } p.forEachExpression(Expression.class, e -> { if (e instanceof TimestampAware ta && ta.timestamp() instanceof UnresolvedTimestamp) { - failures.add( - fail(e, "[{}] " + UnresolvedTimestamp.UNRESOLVED_SUFFIX + UNMAPPED_TIMESTAMP_SUFFIX, e.sourceText()) - ); + failures.add(fail(e, "[{}] " + UnresolvedTimestamp.UNRESOLVED_SUFFIX + UNMAPPED_TIMESTAMP_SUFFIX, e.sourceText())); } }); }); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 3933b8b41d85c..aeb7cafb6957f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -14,7 +14,9 @@ import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; +import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Limit; import org.elasticsearch.xpack.esql.plan.logical.Project; @@ -520,10 +522,7 @@ public void testLoadModeDisallowsSubqueryAndFork() { private static final String UNMAPPED_TIMESTAMP_SUFFIX = UnresolvedTimestamp.UNRESOLVED_SUFFIX + Verifier.UNMAPPED_TIMESTAMP_SUFFIX; public void testTbucketWithUnmappedTimestamp() { - unmappedTimestampFailure( - "FROM test | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX - ); + unmappedTimestampFailure("FROM test | STATS c = COUNT(*) BY tbucket(1 hour)", "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX); } public void testTrangeWithUnmappedTimestamp() { @@ -563,17 +562,11 @@ public void testDerivWithUnmappedTimestamp() { } public void testFirstOverTimeWithUnmappedTimestamp() { - unmappedTimestampFailure( - "FROM test | STATS first_over_time(salary)", - "[first_over_time(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX - ); + unmappedTimestampFailure("FROM test | STATS first_over_time(salary)", "[first_over_time(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); } public void testLastOverTimeWithUnmappedTimestamp() { - unmappedTimestampFailure( - "FROM test | STATS last_over_time(salary)", - "[last_over_time(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX - ); + unmappedTimestampFailure("FROM test | STATS last_over_time(salary)", "[last_over_time(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); } public void testRateAndTbucketWithUnmappedTimestamp() { @@ -620,17 +613,11 @@ public void testTbucketWithUnmappedTimestampAfterDrop() { } public void testTrangeWithUnmappedTimestampCompoundWhere() { - unmappedTimestampFailure( - "FROM test | WHERE trange(1 hour) AND emp_no > 10", - "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX - ); + unmappedTimestampFailure("FROM test | WHERE trange(1 hour) AND emp_no > 10", "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX); } public void testTrangeWithUnmappedTimestampAfterEval() { - unmappedTimestampFailure( - "FROM test | EVAL x = salary + 1 | WHERE trange(1 hour)", - "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX - ); + unmappedTimestampFailure("FROM test | EVAL x = salary + 1 | WHERE trange(1 hour)", "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX); } public void testTbucketWithUnmappedTimestampInInlineStats() { @@ -703,9 +690,7 @@ public void testTbucketTimestampPresentButDroppedNullify() { public void testTbucketTimestampPresentButRenamedNullify() { var e = expectThrows( VerificationException.class, - () -> analyzeStatement( - setUnmappedNullify("FROM sample_data | RENAME @timestamp AS ts | STATS c = COUNT(*) BY tbucket(1 hour)") - ) + () -> analyzeStatement(setUnmappedNullify("FROM sample_data | RENAME @timestamp AS ts | STATS c = COUNT(*) BY tbucket(1 hour)")) ); assertThat(e.getMessage(), containsString(UnresolvedTimestamp.UNRESOLVED_SUFFIX)); assertThat(e.getMessage(), not(containsString(Verifier.UNMAPPED_TIMESTAMP_SUFFIX))); From db88f407339c08c6e1dfe99b481b9e400a379724 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Tue, 17 Mar 2026 16:14:05 +0200 Subject: [PATCH 3/7] Simplify --- .../xpack/esql/analysis/Verifier.java | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index e2620e0f51b3d..7aad7a3853523 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -17,6 +17,7 @@ import org.elasticsearch.xpack.esql.common.Failures; import org.elasticsearch.xpack.esql.core.capabilities.Unresolvable; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TypeResolutions; @@ -373,10 +374,7 @@ private static void checkLimitBy(LogicalPlan plan, Failures failures) { * See https://github.com/elastic/elasticsearch/issues/142127 */ private static boolean checkUnmappedTimestamp(LogicalPlan plan, Failures failures) { - boolean timestampInIndex = plan.collect(EsRelation.class, r -> r.indexMode() != IndexMode.LOOKUP) - .stream() - .anyMatch(r -> r.output().stream().anyMatch(a -> MetadataAttribute.TIMESTAMP_FIELD.equals(a.name()))); - if (timestampInIndex) { + if (plan.anyMatch(p -> p instanceof EsRelation r && r.indexMode() != IndexMode.LOOKUP && hasTimestamp(r))) { return false; } plan.forEachDown(p -> { @@ -392,6 +390,15 @@ private static boolean checkUnmappedTimestamp(LogicalPlan plan, Failures failure return true; } + private static boolean hasTimestamp(EsRelation relation) { + for (Attribute attr : relation.output()) { + if (MetadataAttribute.TIMESTAMP_FIELD.equals(attr.name())) { + return true; + } + } + return false; + } + /** * {@code unmapped_fields="load"} does not yet support branching commands (FORK, LOOKUP JOIN, subqueries/views). * See https://github.com/elastic/elasticsearch/issues/142033 From e4186c451b8cc15f742ac32c8e9809b93d056eaf Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 18 Mar 2026 15:31:07 +0200 Subject: [PATCH 4/7] Address reviews --- .../xpack/esql/analysis/Verifier.java | 8 +-- .../esql/analysis/AnalyzerUnmappedTests.java | 62 +++++++++---------- 2 files changed, 31 insertions(+), 39 deletions(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 7aad7a3853523..6a3c9ea6896cc 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -101,11 +101,7 @@ public Verifier(Metrics metrics, XPackLicenseState licenseState, List verify(LogicalPlan plan, BitSet partialMetrics, UnmappedResolution unmappedResolution) { assert partialMetrics != null; Failures failures = new Failures(); - - boolean unmappedTimestampHandled = false; - if (unmappedResolution != UnmappedResolution.FAIL) { - unmappedTimestampHandled = checkUnmappedTimestamp(plan, failures); - } + boolean unmappedTimestampHandled = unmappedResolution == UnmappedResolution.FAIL || isTimestampUnmappedInAllIndices(plan, failures); // quick verification for unresolved attributes checkUnresolvedAttributes(plan, failures, unmappedTimestampHandled); @@ -373,7 +369,7 @@ private static void checkLimitBy(LogicalPlan plan, Failures failures) { * if the field was present but dropped/renamed by the query, the generic unresolved-attribute message is more appropriate. * See https://github.com/elastic/elasticsearch/issues/142127 */ - private static boolean checkUnmappedTimestamp(LogicalPlan plan, Failures failures) { + private static boolean isTimestampUnmappedInAllIndices(LogicalPlan plan, Failures failures) { if (plan.anyMatch(p -> p instanceof EsRelation r && r.indexMode() != IndexMode.LOOKUP && hasTimestamp(r))) { return false; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index aeb7cafb6957f..57be202e46a4b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -522,108 +522,108 @@ public void testLoadModeDisallowsSubqueryAndFork() { private static final String UNMAPPED_TIMESTAMP_SUFFIX = UnresolvedTimestamp.UNRESOLVED_SUFFIX + Verifier.UNMAPPED_TIMESTAMP_SUFFIX; public void testTbucketWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS c = COUNT(*) BY tbucket(1 hour)", "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS c = COUNT(*) BY tbucket(1 hour)", "[tbucket(1 hour)] "); } public void testTrangeWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | WHERE trange(1 hour)", "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | WHERE trange(1 hour)", "[trange(1 hour)] "); } public void testTbucketAndTrangeWithUnmappedTimestamp() { unmappedTimestampFailure( "FROM test | WHERE trange(1 hour) | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX, - "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + "[tbucket(1 hour)] ", + "[trange(1 hour)] " ); } public void testRateWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS rate(salary)", "[rate(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS rate(salary)", "[rate(salary)] "); } public void testIrateWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS irate(salary)", "[irate(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS irate(salary)", "[irate(salary)] "); } public void testDeltaWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS delta(salary)", "[delta(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS delta(salary)", "[delta(salary)] "); } public void testIdeltaWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS idelta(salary)", "[idelta(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS idelta(salary)", "[idelta(salary)] "); } public void testIncreaseWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS increase(salary)", "[increase(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS increase(salary)", "[increase(salary)] "); } public void testDerivWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS deriv(salary)", "[deriv(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS deriv(salary)", "[deriv(salary)] "); } public void testFirstOverTimeWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS first_over_time(salary)", "[first_over_time(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS first_over_time(salary)", "[first_over_time(salary)] "); } public void testLastOverTimeWithUnmappedTimestamp() { - unmappedTimestampFailure("FROM test | STATS last_over_time(salary)", "[last_over_time(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | STATS last_over_time(salary)", "[last_over_time(salary)] "); } public void testRateAndTbucketWithUnmappedTimestamp() { unmappedTimestampFailure( "FROM test | STATS rate(salary) BY tbucket(1 hour)", - "[rate(salary)] " + UNMAPPED_TIMESTAMP_SUFFIX, - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + "[rate(salary)] ", + "[tbucket(1 hour)] " ); } public void testTbucketWithUnmappedTimestampAfterWhere() { unmappedTimestampFailure( "FROM test | WHERE emp_no > 10 | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + "[tbucket(1 hour)] " ); } public void testTbucketWithUnmappedTimestampAfterEval() { unmappedTimestampFailure( "FROM test | EVAL x = salary + 1 | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + "[tbucket(1 hour)] " ); } public void testTbucketWithUnmappedTimestampMultipleGroupings() { unmappedTimestampFailure( "FROM test | STATS c = COUNT(*) BY tbucket(1 hour), emp_no", - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + "[tbucket(1 hour)] " ); } public void testTbucketWithUnmappedTimestampAfterRename() { unmappedTimestampFailure( "FROM test | RENAME emp_no AS e | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + "[tbucket(1 hour)] " ); } public void testTbucketWithUnmappedTimestampAfterDrop() { unmappedTimestampFailure( "FROM test | DROP emp_no | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + "[tbucket(1 hour)] " ); } public void testTrangeWithUnmappedTimestampCompoundWhere() { - unmappedTimestampFailure("FROM test | WHERE trange(1 hour) AND emp_no > 10", "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | WHERE trange(1 hour) AND emp_no > 10", "[trange(1 hour)] "); } public void testTrangeWithUnmappedTimestampAfterEval() { - unmappedTimestampFailure("FROM test | EVAL x = salary + 1 | WHERE trange(1 hour)", "[trange(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX); + unmappedTimestampFailure("FROM test | EVAL x = salary + 1 | WHERE trange(1 hour)", "[trange(1 hour)] "); } public void testTbucketWithUnmappedTimestampInInlineStats() { unmappedTimestampFailure( "FROM test | INLINE STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX + "[tbucket(1 hour)] " ); } @@ -631,7 +631,7 @@ public void testTbucketWithUnmappedTimestampWithFork() { var query = "FROM test | FORK (STATS c = COUNT(*) BY tbucket(1 hour)) (STATS d = COUNT(*) BY emp_no)"; for (var statement : List.of(setUnmappedNullify(query), setUnmappedLoad(query))) { var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); - assertThat(e.getMessage(), containsString("[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX)); + assertThat(e.getMessage(), containsString("[tbucket(1 hour)] ")); assertThat(e.getMessage(), not(containsString("FORK is not supported"))); } } @@ -645,16 +645,14 @@ public void testTbucketWithUnmappedTimestampWithLookupJoin() { """; for (var statement : List.of(setUnmappedNullify(query), setUnmappedLoad(query))) { var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); - assertThat(e.getMessage(), containsString("[tbucket(1 hour)] " + UNMAPPED_TIMESTAMP_SUFFIX)); + assertThat(e.getMessage(), containsString("[tbucket(1 hour)] ")); assertThat(e.getMessage(), not(containsString("LOOKUP JOIN is not supported"))); } } public void testTbucketWithTimestampPresent() { - for (var statement : List.of( - setUnmappedNullify("FROM sample_data | STATS c = COUNT(*) BY tbucket(1 hour)"), - setUnmappedLoad("FROM sample_data | STATS c = COUNT(*) BY tbucket(1 hour)") - )) { + var query = "FROM sample_data | STATS c = COUNT(*) BY tbucket(1 hour)"; + for (var statement : List.of(setUnmappedNullify(query), setUnmappedLoad(query))) { var plan = analyzeStatement(statement); var limit = as(plan, Limit.class); var aggregate = as(limit.child(), Aggregate.class); @@ -665,10 +663,8 @@ public void testTbucketWithTimestampPresent() { } public void testTrangeWithTimestampPresent() { - for (var statement : List.of( - setUnmappedNullify("FROM sample_data | WHERE trange(1 hour)"), - setUnmappedLoad("FROM sample_data | WHERE trange(1 hour)") - )) { + var query = "FROM sample_data | WHERE trange(1 hour)"; + for (var statement : List.of(setUnmappedNullify(query), setUnmappedLoad(query))) { var plan = analyzeStatement(statement); var limit = as(plan, Limit.class); var filter = as(limit.child(), Filter.class); @@ -707,7 +703,7 @@ private void unmappedTimestampFailure(String query, String... expectedFailures) for (var statement : List.of(setUnmappedNullify(query), setUnmappedLoad(query))) { var e = expectThrows(VerificationException.class, () -> analyzeStatement(statement)); for (String expected : expectedFailures) { - assertThat(e.getMessage(), containsString(expected)); + assertThat(e.getMessage(), containsString(expected + UNMAPPED_TIMESTAMP_SUFFIX)); } } } From a50889f26644f724e0e6602bd368a84c61a43600 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 18 Mar 2026 15:46:39 +0200 Subject: [PATCH 5/7] clean up --- .../xpack/esql/analysis/AnalyzerUnmappedTests.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index a557f60527b78..98554488d8a91 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -15,8 +15,8 @@ import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.UnresolvedTimestamp; import org.elasticsearch.xpack.esql.core.tree.Source; -import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.IndexPattern; +import org.elasticsearch.xpack.esql.plan.logical.Aggregate; import org.elasticsearch.xpack.esql.plan.logical.EsRelation; import org.elasticsearch.xpack.esql.plan.logical.Filter; import org.elasticsearch.xpack.esql.plan.logical.Limit; From 441cf43c8126c581c6f4d972b7ecd650588f9ec4 Mon Sep 17 00:00:00 2001 From: elasticsearchmachine Date: Wed, 18 Mar 2026 13:55:41 +0000 Subject: [PATCH 6/7] [CI] Auto commit changes from spotless --- .../esql/analysis/AnalyzerUnmappedTests.java | 36 ++++--------------- 1 file changed, 7 insertions(+), 29 deletions(-) diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index 98554488d8a91..a4519389666f7 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -581,46 +581,27 @@ public void testLastOverTimeWithUnmappedTimestamp() { } public void testRateAndTbucketWithUnmappedTimestamp() { - unmappedTimestampFailure( - "FROM test | STATS rate(salary) BY tbucket(1 hour)", - "[rate(salary)] ", - "[tbucket(1 hour)] " - ); + unmappedTimestampFailure("FROM test | STATS rate(salary) BY tbucket(1 hour)", "[rate(salary)] ", "[tbucket(1 hour)] "); } public void testTbucketWithUnmappedTimestampAfterWhere() { - unmappedTimestampFailure( - "FROM test | WHERE emp_no > 10 | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " - ); + unmappedTimestampFailure("FROM test | WHERE emp_no > 10 | STATS c = COUNT(*) BY tbucket(1 hour)", "[tbucket(1 hour)] "); } public void testTbucketWithUnmappedTimestampAfterEval() { - unmappedTimestampFailure( - "FROM test | EVAL x = salary + 1 | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " - ); + unmappedTimestampFailure("FROM test | EVAL x = salary + 1 | STATS c = COUNT(*) BY tbucket(1 hour)", "[tbucket(1 hour)] "); } public void testTbucketWithUnmappedTimestampMultipleGroupings() { - unmappedTimestampFailure( - "FROM test | STATS c = COUNT(*) BY tbucket(1 hour), emp_no", - "[tbucket(1 hour)] " - ); + unmappedTimestampFailure("FROM test | STATS c = COUNT(*) BY tbucket(1 hour), emp_no", "[tbucket(1 hour)] "); } public void testTbucketWithUnmappedTimestampAfterRename() { - unmappedTimestampFailure( - "FROM test | RENAME emp_no AS e | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " - ); + unmappedTimestampFailure("FROM test | RENAME emp_no AS e | STATS c = COUNT(*) BY tbucket(1 hour)", "[tbucket(1 hour)] "); } public void testTbucketWithUnmappedTimestampAfterDrop() { - unmappedTimestampFailure( - "FROM test | DROP emp_no | STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " - ); + unmappedTimestampFailure("FROM test | DROP emp_no | STATS c = COUNT(*) BY tbucket(1 hour)", "[tbucket(1 hour)] "); } public void testTrangeWithUnmappedTimestampCompoundWhere() { @@ -632,10 +613,7 @@ public void testTrangeWithUnmappedTimestampAfterEval() { } public void testTbucketWithUnmappedTimestampInInlineStats() { - unmappedTimestampFailure( - "FROM test | INLINE STATS c = COUNT(*) BY tbucket(1 hour)", - "[tbucket(1 hour)] " - ); + unmappedTimestampFailure("FROM test | INLINE STATS c = COUNT(*) BY tbucket(1 hour)", "[tbucket(1 hour)] "); } public void testTbucketWithUnmappedTimestampWithFork() { From c45fddac4365b84d6e6c9ea491cf10ca05736003 Mon Sep 17 00:00:00 2001 From: Andrei Stefan Date: Wed, 18 Mar 2026 18:12:56 +0200 Subject: [PATCH 7/7] Fix the proposed change from the review --- .../java/org/elasticsearch/xpack/esql/analysis/Verifier.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java index 7d5d7338a981b..5f72b09611df9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java @@ -109,7 +109,7 @@ Collection verify(LogicalPlan plan, BitSet partialMetrics) { Collection verify(LogicalPlan plan, BitSet partialMetrics, UnmappedResolution unmappedResolution) { assert partialMetrics != null; Failures failures = new Failures(); - boolean unmappedTimestampHandled = unmappedResolution == UnmappedResolution.FAIL || isTimestampUnmappedInAllIndices(plan, failures); + boolean unmappedTimestampHandled = unmappedResolution != UnmappedResolution.FAIL && isTimestampUnmappedInAllIndices(plan, failures); // quick verification for unresolved attributes checkUnresolvedAttributes(plan, failures, unmappedTimestampHandled);