diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml index 98c8bdff66fc6..36ac1f0c7548f 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/search/90_search_after.yml @@ -1,4 +1,4 @@ -setup: +"search with search_after parameter": - do: indices.create: index: test @@ -24,9 +24,6 @@ setup: indices.refresh: index: test ---- -"search with search_after parameter": - - do: search: rest_total_hits_as_int: true @@ -94,3 +91,130 @@ setup: - match: {hits.total: 3} - length: {hits.hits: 0 } + +--- +"date": + - do: + indices.create: + index: test + body: + mappings: + properties: + timestamp: + type: date + format: yyyy-MM-dd HH:mm:ss.SSS + - do: + bulk: + refresh: true + index: test + body: | + {"index":{}} + {"timestamp":"2019-10-21 00:30:04.828"} + {"index":{}} + {"timestamp":"2019-10-21 08:30:04.828"} + + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828" } + - match: {hits.hits.0.sort: [1571646604828] } + + # search_after with the sort + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + search_after: [1571646604828] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" } + - match: {hits.hits.0.sort: [1571617804828] } + + # search_after with the formatted date + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + search_after: ["2019-10-21 08:30:04.828"] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828" } + - match: {hits.hits.0.sort: [1571617804828] } + +--- +"date_nanos": + - skip: + version: " - 7.99.99" + reason: fixed in 8.0.0 to be backported to 7.10.0 + + - do: + indices.create: + index: test + body: + mappings: + properties: + timestamp: + type: date_nanos + format: yyyy-MM-dd HH:mm:ss.SSSSSS + - do: + bulk: + refresh: true + index: test + body: | + {"index":{}} + {"timestamp":"2019-10-21 00:30:04.828740"} + {"index":{}} + {"timestamp":"2019-10-21 08:30:04.828733"} + + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 08:30:04.828733" } + - match: {hits.hits.0.sort: [1571646604828733000] } + + # search_after with the sort + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + search_after: [1571646604828733000] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" } + - match: {hits.hits.0.sort: [1571617804828740000] } + + # search_after with the formatted date + - do: + search: + index: test + body: + size: 1 + sort: [{ timestamp: desc }] + search_after: ["2019-10-21 08:30:04.828733"] + - match: {hits.total.value: 2 } + - length: {hits.hits: 1 } + - match: {hits.hits.0._index: test } + - match: {hits.hits.0._source.timestamp: "2019-10-21 00:30:04.828740" } + - match: {hits.hits.0.sort: [1571617804828740000] } + diff --git a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java index 84df5be537f0a..82549c0527777 100644 --- a/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java +++ b/server/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericIndexFieldData.java @@ -105,7 +105,7 @@ protected boolean sortRequiresCustomComparator() { @Override protected XFieldComparatorSource dateComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { if (numericType == NumericType.DATE_NANOSECONDS) { - // converts date values to nanosecond resolution + // converts date_nanos values to millisecond resolution return new LongValuesComparatorSource(this, missingValue, sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toMilliSeconds)); } @@ -115,7 +115,7 @@ protected XFieldComparatorSource dateComparatorSource(Object missingValue, Multi @Override protected XFieldComparatorSource dateNanosComparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) { if (numericType == NumericType.DATE) { - // converts date_nanos values to millisecond resolution + // converts date values to nanosecond resolution return new LongValuesComparatorSource(this, missingValue, sortMode, nested, dvs -> convertNumeric(dvs, DateUtils::toNanoSeconds)); } diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index fa4f11d9a0c45..c9d6869eabf94 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -440,6 +440,7 @@ public DocValueFormat docValueFormat(@Nullable String format, ZoneId timeZone) { } // the resolution here is always set to milliseconds, as aggregations use this formatter mainly and those are always in // milliseconds. The only special case here is docvalue fields, which are handled somewhere else + // TODO maybe aggs should force millis because lots so of other places want nanos? return new DocValueFormat.DateTime(dateTimeFormatter, timeZone, Resolution.MILLISECONDS); } } diff --git a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java index 5b45cd660697a..02551250dee7a 100644 --- a/server/src/main/java/org/elasticsearch/search/DocValueFormat.java +++ b/server/src/main/java/org/elasticsearch/search/DocValueFormat.java @@ -253,6 +253,11 @@ public long parseLong(String value, boolean roundUp, LongSupplier now) { public double parseDouble(String value, boolean roundUp, LongSupplier now) { return parseLong(value, roundUp, now); } + + @Override + public String toString() { + return "DocValueFormat.DateTime(" + formatter + ", " + timeZone + ", " + resolution + ")"; + } } DocValueFormat GEOHASH = new DocValueFormat() { diff --git a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java index ca6cd77fc4f62..353e0ddb36d2f 100644 --- a/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java +++ b/server/src/main/java/org/elasticsearch/search/sort/FieldSortBuilder.java @@ -338,6 +338,7 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { throw new QueryShardException(context, "we only support AVG, MEDIAN and SUM on number based fields"); } final SortField field; + boolean isNanosecond = false; if (numericType != null) { if (fieldData instanceof IndexNumericFieldData == false) { throw new QueryShardException(context, @@ -348,8 +349,15 @@ public SortFieldAndFormat build(QueryShardContext context) throws IOException { field = numericFieldData.sortField(resolvedType, missing, localSortMode(), nested, reverse); } else { field = fieldData.sortField(missing, localSortMode(), nested, reverse); + if (fieldData instanceof IndexNumericFieldData) { + isNanosecond = ((IndexNumericFieldData) fieldData).getNumericType() == NumericType.DATE_NANOSECONDS; + } + } + DocValueFormat format = fieldType.docValueFormat(null, null); + if (isNanosecond) { + format = DocValueFormat.withNanosecondResolution(format); } - return new SortFieldAndFormat(field, fieldType.docValueFormat(null, null)); + return new SortFieldAndFormat(field, format); } public boolean canRewriteToMatchNone() { diff --git a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java index 00e5a4a5e33a6..89be4d563be2e 100644 --- a/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/search/sort/FieldSortBuilderTests.java @@ -79,7 +79,7 @@ public class FieldSortBuilderTests extends AbstractSortTestCase