From 7fcb1d6a93cd68b40f22fc093e67dc92d4ea38f0 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 9 May 2025 10:54:22 +0200 Subject: [PATCH 01/14] WIP Start supporting DATE_NANOS in LOOKUP JOIN --- .../compute/operator/lookup/QueryList.java | 16 ++++++++++++++++ .../xpack/esql/action/LookupJoinTypesIT.java | 16 +++++++++++++++- .../xpack/esql/enrich/AbstractLookupService.java | 1 + .../xpack/esql/plan/logical/join/Join.java | 2 -- 4 files changed, 32 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index d5e7656c637b4..a3af5450ebd0b 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -207,6 +207,22 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution ); } + /** + * Returns a list of term queries for the given field and the input block of + * {@code date_nanos} field values. + */ + public static QueryList dateNanosTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) { + return new TermQueryList( + field, + searchExecutionContext, + block, + null, + field instanceof RangeFieldMapper.RangeFieldType rangeFieldType + ? offset -> rangeFieldType.dateTimeFormatter().formatNanos(block.getLong(offset)) + : block::getLong + ); + } + /** * Returns a list of geo_shape queries for the given field and the input block. */ diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index 52c41e4056a8e..3e7a8b7544655 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -40,6 +40,7 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.BYTE; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE; import static org.elasticsearch.xpack.esql.core.type.DataType.DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.FLOAT; @@ -199,7 +200,20 @@ protected Collection> nodePlugins() { } // Tests for all types where left and right are the same type - DataType[] supported = { BOOLEAN, LONG, INTEGER, DOUBLE, SHORT, BYTE, FLOAT, HALF_FLOAT, DATETIME, IP, KEYWORD, SCALED_FLOAT }; + DataType[] supported = { + BOOLEAN, + LONG, + INTEGER, + DOUBLE, + SHORT, + BYTE, + FLOAT, + HALF_FLOAT, + DATETIME, + DATE_NANOS, + IP, + KEYWORD, + SCALED_FLOAT }; { Collection existing = testConfigurations.values(); TestConfigs configs = testConfigurations.computeIfAbsent("same", TestConfigs::new); diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java index 42d03f4e1b161..3834bdd3bbbc1 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/enrich/AbstractLookupService.java @@ -205,6 +205,7 @@ protected static QueryList termQueryList( return switch (inputDataType) { case IP -> QueryList.ipTermQueryList(field, searchExecutionContext, (BytesRefBlock) block); case DATETIME -> QueryList.dateTermQueryList(field, searchExecutionContext, (LongBlock) block); + case DATE_NANOS -> QueryList.dateNanosTermQueryList(field, searchExecutionContext, (LongBlock) block); case null, default -> QueryList.rawTermQueryList(field, searchExecutionContext, block); }; } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java index 8e887d1e92c25..cda9db32ba584 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/Join.java @@ -37,7 +37,6 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_INTEGER; import static org.elasticsearch.xpack.esql.core.type.DataType.COUNTER_LONG; -import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_PERIOD; import static org.elasticsearch.xpack.esql.core.type.DataType.DOC_DATA_TYPE; import static org.elasticsearch.xpack.esql.core.type.DataType.GEO_POINT; @@ -70,7 +69,6 @@ public class Join extends BinaryPlan implements PostAnalysisVerificationAware, S COUNTER_LONG, COUNTER_INTEGER, COUNTER_DOUBLE, - DATE_NANOS, OBJECT, SOURCE, DATE_PERIOD, From 68aaa6dafee7c2ea7b64a1454ad16aa92a5e171c Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 9 May 2025 11:19:21 +0200 Subject: [PATCH 02/14] Update docs/changelog/127962.yaml --- docs/changelog/127962.yaml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 docs/changelog/127962.yaml diff --git a/docs/changelog/127962.yaml b/docs/changelog/127962.yaml new file mode 100644 index 0000000000000..e4fa755a17812 --- /dev/null +++ b/docs/changelog/127962.yaml @@ -0,0 +1,6 @@ +pr: 127962 +summary: Supporti DATE_NANOS in LOOKUP JOIN +area: ES|QL +type: bug +issues: + - 127249 From 9c1383329266156cc213aac0d97570d8ee7c8202 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 9 May 2025 11:20:02 +0200 Subject: [PATCH 03/14] Update docs/changelog/127962.yaml --- docs/changelog/127962.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/changelog/127962.yaml b/docs/changelog/127962.yaml index e4fa755a17812..f96b79a69d5b3 100644 --- a/docs/changelog/127962.yaml +++ b/docs/changelog/127962.yaml @@ -1,5 +1,5 @@ pr: 127962 -summary: Supporti DATE_NANOS in LOOKUP JOIN +summary: Support DATE_NANOS in LOOKUP JOIN area: ES|QL type: bug issues: From 657f1c045fefdf53fd5d6d4e0e7c12fd050bc1f7 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 20 May 2025 22:04:42 +0200 Subject: [PATCH 04/14] Possible solution with passing tests But it seems we should not need to divide by 1M at this point and rather deal with this deeper in the stack, where there is ns specific support. --- .../compute/operator/lookup/QueryList.java | 2 +- .../xpack/esql/CsvTestsDataLoader.java | 3 +++ .../src/main/resources/lookup-join.csv-spec | 23 +++++++++++++++++++ 3 files changed, 27 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index a3af5450ebd0b..131569c132007 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -219,7 +219,7 @@ public static QueryList dateNanosTermQueryList(MappedFieldType field, SearchExec null, field instanceof RangeFieldMapper.RangeFieldType rangeFieldType ? offset -> rangeFieldType.dateTimeFormatter().formatNanos(block.getLong(offset)) - : block::getLong + : i -> block.getLong(i) / 1000_000 ); } diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 2abe77fe08c89..b133784a99ca8 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -83,6 +83,8 @@ public class CsvTestsDataLoader { private static final TestDataset SAMPLE_DATA_TS_NANOS = SAMPLE_DATA.withIndex("sample_data_ts_nanos") .withData("sample_data_ts_nanos.csv") .withTypeMapping(Map.of("@timestamp", "date_nanos")); + private static final TestDataset SAMPLE_DATA_TS_NANOS_LOOKUP = SAMPLE_DATA_TS_NANOS.withIndex("sample_data_ts_nanos_lookup") + .withSetting("lookup-settings.json"); private static final TestDataset MISSING_IP_SAMPLE_DATA = new TestDataset("missing_ip_sample_data"); private static final TestDataset SAMPLE_DATA_PARTIAL_MAPPING = new TestDataset("partial_mapping_sample_data"); private static final TestDataset SAMPLE_DATA_NO_MAPPING = new TestDataset( @@ -162,6 +164,7 @@ public class CsvTestsDataLoader { Map.entry(SAMPLE_DATA_STR.indexName, SAMPLE_DATA_STR), Map.entry(SAMPLE_DATA_TS_LONG.indexName, SAMPLE_DATA_TS_LONG), Map.entry(SAMPLE_DATA_TS_NANOS.indexName, SAMPLE_DATA_TS_NANOS), + Map.entry(SAMPLE_DATA_TS_NANOS_LOOKUP.indexName, SAMPLE_DATA_TS_NANOS_LOOKUP), Map.entry(MISSING_IP_SAMPLE_DATA.indexName, MISSING_IP_SAMPLE_DATA), Map.entry(CLIENT_IPS.indexName, CLIENT_IPS), Map.entry(CLIENT_IPS_LOOKUP.indexName, CLIENT_IPS_LOOKUP), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index f36d42de96c77..0b5469363f6a6 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1742,3 +1742,26 @@ max:long 3450233 8268153 ; + +############################################### +# LOOKUP JOIN on date_nanos field +############################################### + +joinDateNanos +required_capability: join_lookup_v12 +required_capability: date_nanos_type + +FROM sample_data_ts_nanos +| LOOKUP JOIN sample_data_ts_nanos_lookup ON @timestamp +| KEEP @timestamp, client_ip, event_duration, message +; + +@timestamp:date_nanos | client_ip:ip | event_duration:long | message:keyword +2023-10-23T13:55:01.543123456Z | 172.21.3.15 | 1756467 | Connected to 10.1.0.1 +2023-10-23T13:53:55.832123456Z | 172.21.3.15 | 5033755 | Connection error +2023-10-23T13:52:55.015123456Z | 172.21.3.15 | 8268153 | Connection error +2023-10-23T13:51:54.732123456Z | 172.21.3.15 | 725448 | Connection error +2023-10-23T13:33:34.937123456Z | 172.21.0.5 | 1232382 | Disconnected +2023-10-23T12:27:28.948123456Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 +2023-10-23T12:15:03.360123456Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 +; From 23bf41ab4dacd75cf3fed8fe2f9855b8d9cbc98f Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Thu, 22 May 2025 10:54:50 +0200 Subject: [PATCH 05/14] Use lookup-prefix index name to avoid clash with union-types tests --- .../src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java | 2 +- .../java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java | 4 ++-- .../qa/testFixtures/src/main/resources/lookup-join.csv-spec | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java index 3f8478fe713a3..e910a121979c2 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvAssert.java @@ -280,7 +280,7 @@ private static void dataFailure( fail(description + System.lineSeparator() + describeFailures(dataFailures) + actual + expected); } - private static final int MAX_ROWS = 25; + private static final int MAX_ROWS = 50; private static String pipeTable( String description, diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index b133784a99ca8..335d54b1ae900 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -83,7 +83,7 @@ public class CsvTestsDataLoader { private static final TestDataset SAMPLE_DATA_TS_NANOS = SAMPLE_DATA.withIndex("sample_data_ts_nanos") .withData("sample_data_ts_nanos.csv") .withTypeMapping(Map.of("@timestamp", "date_nanos")); - private static final TestDataset SAMPLE_DATA_TS_NANOS_LOOKUP = SAMPLE_DATA_TS_NANOS.withIndex("sample_data_ts_nanos_lookup") + private static final TestDataset LOOKUP_SAMPLE_DATA_TS_NANOS = SAMPLE_DATA_TS_NANOS.withIndex("lookup_sample_data_ts_nanos") .withSetting("lookup-settings.json"); private static final TestDataset MISSING_IP_SAMPLE_DATA = new TestDataset("missing_ip_sample_data"); private static final TestDataset SAMPLE_DATA_PARTIAL_MAPPING = new TestDataset("partial_mapping_sample_data"); @@ -164,7 +164,7 @@ public class CsvTestsDataLoader { Map.entry(SAMPLE_DATA_STR.indexName, SAMPLE_DATA_STR), Map.entry(SAMPLE_DATA_TS_LONG.indexName, SAMPLE_DATA_TS_LONG), Map.entry(SAMPLE_DATA_TS_NANOS.indexName, SAMPLE_DATA_TS_NANOS), - Map.entry(SAMPLE_DATA_TS_NANOS_LOOKUP.indexName, SAMPLE_DATA_TS_NANOS_LOOKUP), + Map.entry(LOOKUP_SAMPLE_DATA_TS_NANOS.indexName, LOOKUP_SAMPLE_DATA_TS_NANOS), Map.entry(MISSING_IP_SAMPLE_DATA.indexName, MISSING_IP_SAMPLE_DATA), Map.entry(CLIENT_IPS.indexName, CLIENT_IPS), Map.entry(CLIENT_IPS_LOOKUP.indexName, CLIENT_IPS_LOOKUP), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 0b5469363f6a6..96805e26577cf 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -1752,7 +1752,7 @@ required_capability: join_lookup_v12 required_capability: date_nanos_type FROM sample_data_ts_nanos -| LOOKUP JOIN sample_data_ts_nanos_lookup ON @timestamp +| LOOKUP JOIN lookup_sample_data_ts_nanos ON @timestamp | KEEP @timestamp, client_ip, event_duration, message ; From b9a713469bf553cd317a11a8eecd83b55185ee98 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 28 May 2025 00:11:08 +0200 Subject: [PATCH 06/14] More correct date_nanos support by minimizing date rendering and parsing --- .../compute/operator/lookup/QueryList.java | 79 ++++++++++++++++++- 1 file changed, 75 insertions(+), 4 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index 131569c132007..facc9b51a7848 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -11,10 +11,12 @@ import org.apache.lucene.geo.GeoEncodingUtils; import org.apache.lucene.search.BooleanClause; import org.apache.lucene.search.BooleanQuery; +import org.apache.lucene.search.ConstantScoreQuery; import org.apache.lucene.search.MatchAllDocsQuery; import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.ShapeRelation; +import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; @@ -37,9 +39,14 @@ import java.io.IOException; import java.io.UncheckedIOException; +import java.time.Instant; +import java.time.ZoneId; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.function.IntFunction; +import java.util.function.LongSupplier; /** * Generates a list of Lucene queries based on the input block. @@ -212,14 +219,14 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution * {@code date_nanos} field values. */ public static QueryList dateNanosTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) { - return new TermQueryList( + return new DateNanosQueryList( field, searchExecutionContext, block, null, field instanceof RangeFieldMapper.RangeFieldType rangeFieldType ? offset -> rangeFieldType.dateTimeFormatter().formatNanos(block.getLong(offset)) - : i -> block.getLong(i) / 1000_000 + : block::getLong ); } @@ -231,7 +238,7 @@ public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecution } private static class TermQueryList extends QueryList { - private final IntFunction blockValueReader; + protected final IntFunction blockValueReader; private TermQueryList( MappedFieldType field, @@ -259,7 +266,7 @@ public TermQueryList onlySingleValues(Warnings warnings, String multiValueWarnin Query doGetQuery(int position, int firstValueIndex, int valueCount) { return switch (valueCount) { case 0 -> null; - case 1 -> field.termQuery(blockValueReader.apply(firstValueIndex), searchExecutionContext); + case 1 -> termQuery(blockValueReader.apply(firstValueIndex)); default -> { final List terms = new ArrayList<>(valueCount); for (int i = 0; i < valueCount; i++) { @@ -270,6 +277,70 @@ Query doGetQuery(int position, int firstValueIndex, int valueCount) { } }; } + + protected Query termQuery(Object value) { + return field.termQuery(value, searchExecutionContext); + } + } + + private static class DateNanosQueryList extends QueryList implements DateMathParser { + protected final IntFunction blockValueReader; + + private DateNanosQueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + Block block, + OnlySingleValueParams onlySingleValueParams, + IntFunction blockValueReader + ) { + super(field, searchExecutionContext, block, onlySingleValueParams); + this.blockValueReader = blockValueReader; + } + + @Override + public DateNanosQueryList onlySingleValues(Warnings warnings, String multiValueWarningMessage) { + return new DateNanosQueryList( + field, + searchExecutionContext, + block, + new OnlySingleValueParams(warnings, multiValueWarningMessage), + blockValueReader + ); + } + + @Override + Query doGetQuery(int position, int firstValueIndex, int valueCount) { + return switch (valueCount) { + case 0 -> null; + case 1 -> termQuery(blockValueReader.apply(firstValueIndex)); + default -> { + final Set terms = new HashSet<>(valueCount); + BooleanQuery.Builder builder = new BooleanQuery.Builder(); + for (int i = 0; i < valueCount; i++) { + final Object value = blockValueReader.apply(firstValueIndex + i); + if (terms.contains(value)) { + continue; // Skip duplicates + } + terms.add(value); + builder.add(termQuery(value), BooleanClause.Occur.SHOULD); + } + yield new ConstantScoreQuery(builder.build()); + } + }; + } + + private Query termQuery(Object value) { + return field.rangeQuery(value, value, true, true, ShapeRelation.INTERSECTS, null, this, searchExecutionContext); + } + + @Override + public Instant parse(String text, LongSupplier now, boolean roundUpProperty, ZoneId tz) { + // TODO: It is awkward that we convert the Long to a String to an Instant and then back to a Long. + // See DateFieldMapper.parseToLong, line 822. Too much unnecessary conversion going on. + // This might make sense in the QueryDSL where the value could come from any query, but here it comes from an ES|QL LongBlock + long nanos = Long.parseLong(text); + return Instant.ofEpochSecond(nanos / 1_000_000_000, nanos % 1_000_000_000); + } } private static class GeoShapeQueryList extends QueryList { From bb81b8a9328f735baf5264579bd22d5a2a370343 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 28 May 2025 17:09:24 +0200 Subject: [PATCH 07/14] Improved solution, rather have date parsing entirely skipped in DateFieldType --- .../index/mapper/DateFieldMapper.java | 45 ++++++++++ .../compute/operator/lookup/QueryList.java | 84 +++++++++---------- 2 files changed, 83 insertions(+), 46 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3511c8dc19321..cd651f9ec152f 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -821,6 +821,51 @@ public static long parseToLong( return resolution.convert(dateParser.parse(BytesRefs.toString(value), now, roundUp, zone)); } + /** + * When the date value is already fully parsed and available as a long, use this method to skip parsing. + */ + public Query equalityQuery(Long value, @Nullable SearchExecutionContext context) { + return rangeQuery(value, value, true, true, context); + } + + /** + * When the date value is already fully parsed and available as a long, use this method to skip parsing. + */ + public Query rangeQuery( + Long lowerTerm, + Long upperTerm, + boolean includeLower, + boolean includeUpper, + SearchExecutionContext context + ) { + failIfNotIndexedNorDocValuesFallback(context); + long l, u; + if (lowerTerm == null) { + l = Long.MIN_VALUE; + } else { + l = (includeLower == false) ? lowerTerm + 1 : lowerTerm; + } + if (upperTerm == null) { + u = Long.MAX_VALUE; + } else { + u = (includeUpper == false) ? upperTerm - 1 : upperTerm; + } + Query query; + if (isIndexed()) { + query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + } + } else { + query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + } + if (hasDocValues() && context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } + return query; + } + @Override public Query distanceFeatureQuery(Object origin, String pivot, SearchExecutionContext context) { failIfNotIndexedNorDocValuesFallback(context); diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index facc9b51a7848..1644d25db933a 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -16,7 +16,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.util.BytesRef; import org.elasticsearch.common.geo.ShapeRelation; -import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.compute.data.Block; import org.elasticsearch.compute.data.BooleanBlock; import org.elasticsearch.compute.data.BytesRefBlock; @@ -32,6 +31,7 @@ import org.elasticsearch.geometry.Point; import org.elasticsearch.geometry.utils.GeometryValidator; import org.elasticsearch.geometry.utils.WellKnownBinary; +import org.elasticsearch.index.mapper.DateFieldMapper; import org.elasticsearch.index.mapper.GeoShapeQueryable; import org.elasticsearch.index.mapper.MappedFieldType; import org.elasticsearch.index.mapper.RangeFieldMapper; @@ -39,14 +39,11 @@ import java.io.IOException; import java.io.UncheckedIOException; -import java.time.Instant; -import java.time.ZoneId; import java.util.ArrayList; import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.function.IntFunction; -import java.util.function.LongSupplier; /** * Generates a list of Lucene queries based on the input block. @@ -219,15 +216,7 @@ public static QueryList dateTermQueryList(MappedFieldType field, SearchExecution * {@code date_nanos} field values. */ public static QueryList dateNanosTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) { - return new DateNanosQueryList( - field, - searchExecutionContext, - block, - null, - field instanceof RangeFieldMapper.RangeFieldType rangeFieldType - ? offset -> rangeFieldType.dateTimeFormatter().formatNanos(block.getLong(offset)) - : block::getLong - ); + return new DateNanosQueryList(field, searchExecutionContext, block, null); } /** @@ -266,7 +255,7 @@ public TermQueryList onlySingleValues(Warnings warnings, String multiValueWarnin Query doGetQuery(int position, int firstValueIndex, int valueCount) { return switch (valueCount) { case 0 -> null; - case 1 -> termQuery(blockValueReader.apply(firstValueIndex)); + case 1 -> field.termQuery(blockValueReader.apply(firstValueIndex), searchExecutionContext); default -> { final List terms = new ArrayList<>(valueCount); for (int i = 0; i < valueCount; i++) { @@ -277,24 +266,40 @@ Query doGetQuery(int position, int firstValueIndex, int valueCount) { } }; } - - protected Query termQuery(Object value) { - return field.termQuery(value, searchExecutionContext); - } } - private static class DateNanosQueryList extends QueryList implements DateMathParser { - protected final IntFunction blockValueReader; + private static class DateNanosQueryList extends QueryList { + protected final IntFunction blockValueReader; + private final DateFieldMapper.DateFieldType dateFieldType; private DateNanosQueryList( MappedFieldType field, SearchExecutionContext searchExecutionContext, - Block block, - OnlySingleValueParams onlySingleValueParams, - IntFunction blockValueReader + LongBlock block, + OnlySingleValueParams onlySingleValueParams ) { super(field, searchExecutionContext, block, onlySingleValueParams); - this.blockValueReader = blockValueReader; + if (field instanceof RangeFieldMapper.RangeFieldType rangeFieldType) { + // TODO: do this validation earlier + throw new IllegalArgumentException( + "DateNanosQueryList does not support range fields [" + rangeFieldType + "]: " + field.name() + ); + } + this.blockValueReader = block::getLong; + if (field instanceof DateFieldMapper.DateFieldType dateFieldType) { + // Validate that the field is a date_nanos field + // TODO: Consider allowing date_nanos to match normal datetime fields + if (dateFieldType.resolution() != DateFieldMapper.Resolution.NANOSECONDS) { + throw new IllegalArgumentException( + "DateNanosQueryList only supports date_nanos fields, but got: " + field.typeName() + " for field: " + field.name() + ); + } + this.dateFieldType = dateFieldType; + } else { + throw new IllegalArgumentException( + "DateNanosQueryList only supports date_nanos fields, but got: " + field.typeName() + " for field: " + field.name() + ); + } } @Override @@ -302,9 +307,8 @@ public DateNanosQueryList onlySingleValues(Warnings warnings, String multiValueW return new DateNanosQueryList( field, searchExecutionContext, - block, - new OnlySingleValueParams(warnings, multiValueWarningMessage), - blockValueReader + (LongBlock) block, + new OnlySingleValueParams(warnings, multiValueWarningMessage) ); } @@ -312,35 +316,23 @@ public DateNanosQueryList onlySingleValues(Warnings warnings, String multiValueW Query doGetQuery(int position, int firstValueIndex, int valueCount) { return switch (valueCount) { case 0 -> null; - case 1 -> termQuery(blockValueReader.apply(firstValueIndex)); + case 1 -> dateFieldType.equalityQuery(blockValueReader.apply(firstValueIndex), searchExecutionContext); default -> { - final Set terms = new HashSet<>(valueCount); + // The following code is a slight simplification of the DateFieldMapper.termsQuery method + final Set values = new HashSet<>(valueCount); BooleanQuery.Builder builder = new BooleanQuery.Builder(); for (int i = 0; i < valueCount; i++) { - final Object value = blockValueReader.apply(firstValueIndex + i); - if (terms.contains(value)) { + final Long value = blockValueReader.apply(firstValueIndex + i); + if (values.contains(value)) { continue; // Skip duplicates } - terms.add(value); - builder.add(termQuery(value), BooleanClause.Occur.SHOULD); + values.add(value); + builder.add(dateFieldType.equalityQuery(value, searchExecutionContext), BooleanClause.Occur.SHOULD); } yield new ConstantScoreQuery(builder.build()); } }; } - - private Query termQuery(Object value) { - return field.rangeQuery(value, value, true, true, ShapeRelation.INTERSECTS, null, this, searchExecutionContext); - } - - @Override - public Instant parse(String text, LongSupplier now, boolean roundUpProperty, ZoneId tz) { - // TODO: It is awkward that we convert the Long to a String to an Instant and then back to a Long. - // See DateFieldMapper.parseToLong, line 822. Too much unnecessary conversion going on. - // This might make sense in the QueryDSL where the value could come from any query, but here it comes from an ES|QL LongBlock - long nanos = Long.parseLong(text); - return Instant.ofEpochSecond(nanos / 1_000_000_000, nanos % 1_000_000_000); - } } private static class GeoShapeQueryList extends QueryList { From 71b056f50e61c92f929f2e6246133514aab04a25 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Fri, 30 May 2025 19:09:54 +0200 Subject: [PATCH 08/14] Add EsqlCapabilities to block csv-spec tests in mixed clusters --- .../qa/testFixtures/src/main/resources/lookup-join.csv-spec | 2 +- .../elasticsearch/xpack/esql/action/EsqlCapabilities.java | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index ff91c93aab338..83be4d12efddb 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -4591,7 +4591,7 @@ emp_no:integer | language_code:integer | language_name:keyword joinDateNanos required_capability: join_lookup_v12 -required_capability: date_nanos_type +required_capability: date_nanos_lookup_join FROM sample_data_ts_nanos | LOOKUP JOIN lookup_sample_data_ts_nanos ON @timestamp diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 565a6519a831e..acc025a122936 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -586,6 +586,12 @@ public enum Cap { * e.g. {@code WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")} */ FIX_DATE_NANOS_MIXED_RANGE_PUSHDOWN_BUG(), + + /** + * Support for date nanos in lookup join. Done in #127962 + */ + DATE_NANOS_LOOKUP_JOIN, + /** * DATE_PARSE supports reading timezones */ From f21115801b250575b0e48aa2cd79d63ebb09b0bc Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Sun, 1 Jun 2025 23:03:55 +0200 Subject: [PATCH 09/14] Make csv-spec test multi-node and serverless safe with a SORT --- .../esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec | 1 + 1 file changed, 1 insertion(+) diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec index 83be4d12efddb..fc9932e330295 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/lookup-join.csv-spec @@ -4596,6 +4596,7 @@ required_capability: date_nanos_lookup_join FROM sample_data_ts_nanos | LOOKUP JOIN lookup_sample_data_ts_nanos ON @timestamp | KEEP @timestamp, client_ip, event_duration, message +| SORT @timestamp DESC ; @timestamp:date_nanos | client_ip:ip | event_duration:long | message:keyword From fd01818abbcb97134aec51f31d3ca49e40726276 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 2 Jun 2025 18:51:56 +0200 Subject: [PATCH 10/14] After merging with main, fixed compile error --- .../java/org/elasticsearch/index/mapper/DateFieldMapper.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3732ccc2c1cfc..3f1196ee64ac0 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -861,7 +861,7 @@ public Query rangeQuery( query = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); } if (hasDocValues() && context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + query = new XIndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); } return query; } From e5cdf7c0c35f72dd55c2b1a8bc18916a3b16b217 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Mon, 2 Jun 2025 18:52:11 +0200 Subject: [PATCH 11/14] Reverted temporary change --- .../org/elasticsearch/compute/operator/lookup/QueryList.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index 1644d25db933a..0c7a3c985c675 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -227,7 +227,7 @@ public static QueryList geoShapeQueryList(MappedFieldType field, SearchExecution } private static class TermQueryList extends QueryList { - protected final IntFunction blockValueReader; + private final IntFunction blockValueReader; private TermQueryList( MappedFieldType field, From 31d0375cc1e8a18e3a1ee9132e22628c9ef3605f Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 3 Jun 2025 15:26:23 +0200 Subject: [PATCH 12/14] Add tests asserting datetime/date_nanos cannot be mixed in joins But this might not be that hard to add support for. We need: * commonType to work (us wider range type DATETIME) * refactor QueryList.dateQuery to new approach used for QueryList.dateNanos --- .../xpack/esql/action/LookupJoinTypesIT.java | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java index eab1c4e89bce2..4070e11120ab9 100644 --- a/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java +++ b/x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/LookupJoinTypesIT.java @@ -171,6 +171,19 @@ protected Collection> nodePlugins() { } } + // Tests for mixed-date/time types + var dateTypes = List.of(DATETIME, DATE_NANOS); + { + TestConfigs configs = testConfigurations.computeIfAbsent("mixed-temporal", TestConfigs::new); + for (DataType mainType : dateTypes) { + for (DataType lookupType : dateTypes) { + if (mainType != lookupType) { + configs.addFails(mainType, lookupType); + } + } + } + } + // Tests for all unsupported types DataType[] unsupported = Join.UNSUPPORTED_TYPES; { @@ -285,6 +298,10 @@ public void testLookupJoinMixedNumerical() { testLookupJoinTypes("mixed-numerical"); } + public void testLookupJoinMixedTemporal() { + testLookupJoinTypes("mixed-temporal"); + } + public void testLookupJoinSame() { testLookupJoinTypes("same"); } From f478122083b537928d5c8cda3d990b3b71835f8f Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Tue, 3 Jun 2025 15:32:56 +0200 Subject: [PATCH 13/14] Fix after merging main --- .../compute/operator/lookup/QueryList.java | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java index 058a1a21b2f2a..56b857ad90a82 100644 --- a/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java +++ b/x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/operator/lookup/QueryList.java @@ -246,8 +246,13 @@ public static QueryList dateTermQueryList( * Returns a list of term queries for the given field and the input block of * {@code date_nanos} field values. */ - public static QueryList dateNanosTermQueryList(MappedFieldType field, SearchExecutionContext searchExecutionContext, LongBlock block) { - return new DateNanosQueryList(field, searchExecutionContext, block, null); + public static QueryList dateNanosTermQueryList( + MappedFieldType field, + SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, + LongBlock block + ) { + return new DateNanosQueryList(field, searchExecutionContext, aliasFilter, block, null); } /** @@ -313,10 +318,11 @@ private static class DateNanosQueryList extends QueryList { private DateNanosQueryList( MappedFieldType field, SearchExecutionContext searchExecutionContext, + AliasFilter aliasFilter, LongBlock block, OnlySingleValueParams onlySingleValueParams ) { - super(field, searchExecutionContext, block, onlySingleValueParams); + super(field, searchExecutionContext, aliasFilter, block, onlySingleValueParams); if (field instanceof RangeFieldMapper.RangeFieldType rangeFieldType) { // TODO: do this validation earlier throw new IllegalArgumentException( @@ -345,6 +351,7 @@ public DateNanosQueryList onlySingleValues(Warnings warnings, String multiValueW return new DateNanosQueryList( field, searchExecutionContext, + aliasFilter, (LongBlock) block, new OnlySingleValueParams(warnings, multiValueWarningMessage) ); From 7de8ee773fca9ce29d2d69c4bc0c981aeef32049 Mon Sep 17 00:00:00 2001 From: Craig Taverner Date: Wed, 4 Jun 2025 20:12:46 +0200 Subject: [PATCH 14/14] Add unit tests for new DateFieldType methods --- .../index/mapper/DateFieldMapper.java | 15 +- .../index/mapper/DateFieldTypeTests.java | 294 ++++++++++++++---- 2 files changed, 251 insertions(+), 58 deletions(-) diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 3f1196ee64ac0..e79b1937cd8f3 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -589,7 +589,7 @@ public DateFieldType(String name) { ); } - public DateFieldType(String name, boolean isIndexed) { + public DateFieldType(String name, boolean isIndexed, Resolution resolution) { this( name, isIndexed, @@ -599,13 +599,17 @@ public DateFieldType(String name, boolean isIndexed) { false, false, DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, + resolution, null, null, Collections.emptyMap() ); } + public DateFieldType(String name, boolean isIndexed) { + this(name, isIndexed, Resolution.MILLISECONDS); + } + public DateFieldType(String name, DateFormatter dateFormatter) { this(name, true, true, false, true, false, false, dateFormatter, Resolution.MILLISECONDS, null, null, Collections.emptyMap()); } @@ -822,14 +826,17 @@ public static long parseToLong( } /** - * When the date value is already fully parsed and available as a long, use this method to skip parsing. + * Similar to the {@link DateFieldType#termQuery} method, but works on dates that are already parsed to a long + * in the same precision as the field mapper. */ public Query equalityQuery(Long value, @Nullable SearchExecutionContext context) { return rangeQuery(value, value, true, true, context); } /** - * When the date value is already fully parsed and available as a long, use this method to skip parsing. + * Similar to the existing + * {@link DateFieldType#rangeQuery(Object, Object, boolean, boolean, ShapeRelation, ZoneId, DateMathParser, SearchExecutionContext)} + * method, but works on dates that are already parsed to a long in the same precision as the field mapper. */ public Query rangeQuery( Long lowerTerm, diff --git a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java index 66c324563d330..5decadedf02cb 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/DateFieldTypeTests.java @@ -250,13 +250,11 @@ public void testValueForSearch() { assertEquals(date, ft.valueForDisplay(instant)); } + /** + * If the term field is a string of date-time format with exact seconds (no sub-seconds), any data within a 1second range will match. + */ public void testTermQuery() { - Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); - SearchExecutionContext context = SearchExecutionContextHelper.createSimple( - new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - parserConfig(), - writableRegistry() - ); + SearchExecutionContext context = prepareIndexForTermQuery(); MappedFieldType ft = new DateFieldType("field"); String date = "2015-10-12T14:10:55"; long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); @@ -270,45 +268,99 @@ public void testTermQuery() { expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant + 999); assertEquals(expected, ft.termQuery(date, context)); - MappedFieldType unsearchable = new DateFieldType( - "field", - false, - false, - false, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, - Resolution.MILLISECONDS, - null, - null, - Collections.emptyMap() + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(date, context)); + } + + /** + * If the term field is a string of date-time format with sub-seconds, only data with exact ms precision will match. + */ + public void testTermQuerySubseconds() { + SearchExecutionContext context = prepareIndexForTermQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55.01"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) ); - IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> unsearchable.termQuery(date, context)); - assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + assertEquals(expected, ft.termQuery(date, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.termQuery(date, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(date, context)); } - public void testRangeQuery() throws IOException { - Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); - SearchExecutionContext context = new SearchExecutionContext( - 0, - 0, - new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), - null, - null, - null, - MappingLookup.EMPTY, - null, - null, - parserConfig(), - writableRegistry(), - null, - null, - () -> nowInMillis, - null, - null, - () -> true, - null, - Collections.emptyMap(), - MapperMetrics.NOOP + /** + * If the term field is a string of the long value (ms since epoch), only data with exact ms precision will match. + */ + public void testTermQueryMillis() { + SearchExecutionContext context = prepareIndexForTermQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) ); + assertEquals(expected, ft.termQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.termQuery(instant, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.termQuery(instant, context)); + } + + /** + * This query has similar behaviour to passing a String containing a long to termQuery, only data with exact ms precision will match. + */ + public void testEqualityQuery() { + SearchExecutionContext context = prepareIndexForTermQuery(); + DateFieldType ft = new DateFieldType("field"); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) + ); + assertEquals(expected, ft.equalityQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.equalityQuery(instant, context)); + + assertIndexUnsearchable(Resolution.MILLISECONDS, (unsearchable) -> unsearchable.equalityQuery(instant, context)); + } + + /** + * This query supports passing a ns value, and only data with exact ns precision will match. + */ + public void testEqualityNanosQuery() { + SearchExecutionContext context = prepareIndexForTermQuery(); + DateFieldType ft = new DateFieldType("field", Resolution.NANOSECONDS); + String date = "2015-10-12T14:10:55"; + long instant = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date)).toInstant().toEpochMilli() * 1000000L; + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant, instant), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant) + ); + assertEquals(expected, ft.equalityQuery(instant, context)); + + ft = new DateFieldType("field", false); + expected = SortedNumericDocValuesField.newSlowRangeQuery("field", instant, instant); + assertEquals(expected, ft.equalityQuery(instant, context)); + + assertIndexUnsearchable(Resolution.NANOSECONDS, (unsearchable) -> unsearchable.equalityQuery(instant, context)); + } + + /** + * If the term fields are strings of date-time format with exact seconds (no sub-seconds), + * the second field will be rounded up to the next second. + */ + public void testRangeQuery() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); MappedFieldType ft = new DateFieldType("field"); String date1 = "2015-10-12T14:10:55"; String date2 = "2016-04-28T11:33:52"; @@ -340,22 +392,105 @@ public void testRangeQuery() throws IOException { expected2 = new DateRangeIncludingNowQuery(SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)); assertEquals(expected2, ft2.rangeQuery("now", instant2, true, true, null, null, null, context)); - MappedFieldType unsearchable = new DateFieldType( - "field", - false, - false, - false, - DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + assertIndexUnsearchable( Resolution.MILLISECONDS, - null, - null, - Collections.emptyMap() + (unsearchable) -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) ); - IllegalArgumentException e = expectThrows( - IllegalArgumentException.class, - () -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) + } + + /** + * If the term fields are strings of date-time format with sub-seconds, + * the lower and upper values will be matched inclusively to the ms. + */ + public void testRangeQuerySubseconds() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + MappedFieldType ft = new DateFieldType("field"); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader()))); + + MappedFieldType ft2 = new DateFieldType("field", false); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals( + expected2, + ft2.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader())) + ); + + instant1 = nowInMillis; + instant2 = instant1 + 100; + expected = new DateRangeIncludingNowQuery( + new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ) + ); + assertEquals(expected, ft.rangeQuery("now", instant2, true, true, null, null, null, context)); + + expected2 = new DateRangeIncludingNowQuery(SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2)); + assertEquals(expected2, ft2.rangeQuery("now", instant2, true, true, null, null, null, context)); + + assertIndexUnsearchable( + Resolution.MILLISECONDS, + (unsearchable) -> unsearchable.rangeQuery(date1, date2, true, true, null, null, null, context) ); - assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + } + + /** + * If the term fields are strings of long ms, the lower and upper values will be matched inclusively to the ms. + */ + public void testRangeQueryMillis() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + DateFieldType ft = new DateFieldType("field"); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli(); + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli(); + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + DateFieldType ft2 = new DateFieldType("field", false); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals(expected2, ft2.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + assertIndexUnsearchable( + Resolution.MILLISECONDS, + (unsearchable) -> unsearchable.rangeQuery(instant1, instant2, true, true, context) + ); + } + + /** + * If the term fields are strings of long ns, the lower and upper values will be matched inclusively to the ns. + */ + public void testRangeQueryNanos() throws IOException { + SearchExecutionContext context = prepareIndexForRangeQuery(); + DateFieldType ft = new DateFieldType("field", Resolution.NANOSECONDS); + String date1 = "2015-10-12T14:10:55.01"; + String date2 = "2016-04-28T11:33:52.01"; + long instant1 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date1)).toInstant().toEpochMilli() * 1000000L; + long instant2 = DateFormatters.from(DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER.parse(date2)).toInstant().toEpochMilli() * 1000000L; + Query expected = new IndexOrDocValuesQuery( + LongPoint.newRangeQuery("field", instant1, instant2), + SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2) + ); + assertEquals(expected, ft.rangeQuery(instant1, instant2, true, true, context).rewrite(newSearcher(new MultiReader()))); + + DateFieldType ft2 = new DateFieldType("field", false, Resolution.NANOSECONDS); + Query expected2 = SortedNumericDocValuesField.newSlowRangeQuery("field", instant1, instant2); + assertEquals( + expected2, + ft2.rangeQuery(date1, date2, true, true, null, null, null, context).rewrite(newSearcher(new MultiReader())) + ); + + assertIndexUnsearchable(Resolution.NANOSECONDS, (unsearchable) -> unsearchable.rangeQuery(instant1, instant2, true, true, context)); } public void testRangeQueryWithIndexSort() { @@ -462,4 +597,55 @@ public void testParseSourceValueNanos() throws IOException { MappedFieldType nullValueMapper = fieldType(Resolution.NANOSECONDS, "strict_date_time||epoch_millis", nullValueDate); assertEquals(List.of(nullValueDate), fetchSourceValue(nullValueMapper, null)); } + + private SearchExecutionContext prepareIndexForTermQuery() { + Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); + return SearchExecutionContextHelper.createSimple( + new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), + parserConfig(), + writableRegistry() + ); + } + + private SearchExecutionContext prepareIndexForRangeQuery() { + Settings indexSettings = indexSettings(IndexVersion.current(), 1, 1).build(); + return new SearchExecutionContext( + 0, + 0, + new IndexSettings(IndexMetadata.builder("foo").settings(indexSettings).build(), indexSettings), + null, + null, + null, + MappingLookup.EMPTY, + null, + null, + parserConfig(), + writableRegistry(), + null, + null, + () -> nowInMillis, + null, + null, + () -> true, + null, + Collections.emptyMap(), + MapperMetrics.NOOP + ); + } + + private void assertIndexUnsearchable(Resolution resolution, ThrowingConsumer runnable) { + DateFieldType unsearchable = new DateFieldType( + "field", + false, + false, + false, + DateFieldMapper.DEFAULT_DATE_TIME_FORMATTER, + resolution, + null, + null, + Collections.emptyMap() + ); + IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> runnable.accept(unsearchable)); + assertEquals("Cannot search on field [field] since it is not indexed nor has doc values.", e.getMessage()); + } }