From 033cd0eb2e9dad8d842bbd8f347d5f864b2acfbd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 14 Oct 2020 18:12:25 +0200 Subject: [PATCH 01/13] Fix range quers on date fields for number inputs Currently, if you write a date range query with numeric 'to' or 'from' bounds, they can be interpreted as years if no format is provided. We use "strict_date_optional_time||epoch_millis" in this case that can interpret inputs like 1000 as the year 1000 for example. We should change this to always interpret and parse numbers in this case with the second option "epoch_millis" if no other formatter was provided. Closes #63680 --- .../search/simple/SimpleSearchIT.java | 17 ++++++++++++++ .../index/mapper/DateFieldMapper.java | 22 +++++++++++++++---- .../index/query/IdsQueryBuilderTests.java | 18 ++++++++++++++- 3 files changed, 52 insertions(+), 5 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 6a163543e3073..66ea90330dcbb 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -198,6 +198,7 @@ public void testSimpleDateRange() throws Exception { createIndex("test"); client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); client().prepareIndex("test").setId("2").setSource("field", "2010-01-06T02:00").get(); + client().prepareIndex("test").setId("3").setSource("field", "1967-01-01T00:00").get(); ensureGreen(); refresh(); SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d") @@ -223,6 +224,22 @@ public void testSimpleDateRange() throws Exception { searchResponse = client().prepareSearch("test").setQuery( QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")).get(); assertHitCount(searchResponse, 2L); + + // a string value of "1000" should be parsed as the year 1000 and return all three docs + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt("1000")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 3L); + + // a numeric value of 1000 should be parsed as 1000 millis since epoch and return only docs after 1970 + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt(1000)) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + assertEquals("1", searchResponse.getHits().getHits()[0].getId()); + assertEquals("2", searchResponse.getHits().getHits()[1].getId()); } public void testSimpleTerminateAfterCount() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 10f918149cb5d..37d705e28dd1e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -73,6 +73,7 @@ public final class DateFieldMapper extends ParametrizedFieldMapper { public static final String CONTENT_TYPE = "date"; public static final String DATE_NANOS_CONTENT_TYPE = "date_nanos"; public static final DateFormatter DEFAULT_DATE_TIME_FORMATTER = DateFormatter.forPattern("strict_date_optional_time||epoch_millis"); + private static final DateMathParser EPOCH_MILLIS_PARSER = DateFormatter.forPattern("epoch_millis").toDateMathParser(); public enum Resolution { MILLISECONDS(CONTENT_TYPE, NumericType.DATE) { @@ -349,9 +350,17 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } - DateMathParser parser = forcedDateParser == null - ? dateMathParser - : forcedDateParser; + DateMathParser parser; + if (forcedDateParser == null) { + if (lowerTerm instanceof Integer || upperTerm instanceof Integer) { + // force epoch_millis + parser = EPOCH_MILLIS_PARSER; + } else { + parser = dateMathParser; + } + } else { + parser = forcedDateParser; + } return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { Query query = LongPoint.newRangeQuery(name(), l, u); if (hasDocValues()) { @@ -443,7 +452,12 @@ public Relation isFieldWithinQuery(IndexReader reader, Object from, Object to, boolean includeLower, boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { if (dateParser == null) { - dateParser = this.dateMathParser; + if (from instanceof Integer || to instanceof Integer) { + // force epoch_millis + dateParser = EPOCH_MILLIS_PARSER; + } else { + dateParser = this.dateMathParser; + } } long fromInclusive = Long.MIN_VALUE; diff --git a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java index 7185210caadb8..01a67451d54e1 100644 --- a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java @@ -24,6 +24,8 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; import org.elasticsearch.common.ParsingException; +import org.elasticsearch.common.time.DateFormatter; +import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractQueryTestCase; @@ -97,7 +99,7 @@ public void testFromJson() throws IOException { protected QueryBuilder parseQuery(XContentParser parser) throws IOException { QueryBuilder query = super.parseQuery(parser); assertThat(query, instanceOf(IdsQueryBuilder.class)); - return (IdsQueryBuilder) query; + return query; } @Override @@ -109,4 +111,18 @@ public void testMustRewrite() throws IOException { () -> queryBuilder.toQuery(context)); assertEquals("Rewrite first", e.getMessage()); } + + public void testFoo() { + DateMathParser dateMathParser = DateFormatter.forPattern("strict_date_optional_time||epoch_millis").toDateMathParser(); + System.out.println(dateMathParser.parse("10", null).toEpochMilli()); + System.out.println(dateMathParser.parse("100", null).toEpochMilli()); + System.out.println(dateMathParser.parse("999", null).toEpochMilli()); + System.out.println(dateMathParser.parse("1000", null).toEpochMilli()); + System.out.println(dateMathParser.parse("1500", null).toEpochMilli()); + System.out.println(dateMathParser.parse("1968", null).toEpochMilli()); + System.out.println(dateMathParser.parse("1969", null).toEpochMilli()); + System.out.println(dateMathParser.parse("1970", null).toEpochMilli()); + System.out.println(dateMathParser.parse("1990", null).toEpochMilli()); + System.out.println(dateMathParser.parse("10000", null).toEpochMilli()); + } } From f233e474971c167691486b764890e07f0f399f86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Wed, 14 Oct 2020 21:38:22 +0200 Subject: [PATCH 02/13] Iter --- .../search/simple/SimpleSearchIT.java | 62 +++++++++---------- .../index/mapper/DateFieldMapper.java | 46 +++++++------- .../index/query/IdsQueryBuilderTests.java | 18 +----- 3 files changed, 57 insertions(+), 69 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 66ea90330dcbb..244152988c01e 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -201,39 +201,39 @@ public void testSimpleDateRange() throws Exception { client().prepareIndex("test").setId("3").setSource("field", "1967-01-01T00:00").get(); ensureGreen(); refresh(); - SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d") - .lte("2010-01-04||+2d/d")).get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 2L); - - searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00") - .lte("2010-01-06T02:00")).get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 2L); - - searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00") - .lt("2010-01-06T02:00")).get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 1L); - - searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gt("2010-01-05T02:00") - .lt("2010-01-06T02:00")).get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 0L); - - searchResponse = client().prepareSearch("test").setQuery( - QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")).get(); - assertHitCount(searchResponse, 2L); - - // a string value of "1000" should be parsed as the year 1000 and return all three docs - searchResponse = client().prepareSearch("test") - .setQuery(QueryBuilders.rangeQuery("field").gt("1000")) - .get(); - assertNoFailures(searchResponse); - assertHitCount(searchResponse, 3L); +// SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d") +// .lte("2010-01-04||+2d/d")).get(); +// assertNoFailures(searchResponse); +// assertHitCount(searchResponse, 2L); +// +// searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00") +// .lte("2010-01-06T02:00")).get(); +// assertNoFailures(searchResponse); +// assertHitCount(searchResponse, 2L); +// +// searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00") +// .lt("2010-01-06T02:00")).get(); +// assertNoFailures(searchResponse); +// assertHitCount(searchResponse, 1L); +// +// searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gt("2010-01-05T02:00") +// .lt("2010-01-06T02:00")).get(); +// assertNoFailures(searchResponse); +// assertHitCount(searchResponse, 0L); +// +// searchResponse = client().prepareSearch("test").setQuery( +// QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")).get(); +// assertHitCount(searchResponse, 2L); +// +// // a string value of "1000" should be parsed as the year 1000 and return all three docs +// searchResponse = client().prepareSearch("test") +// .setQuery(QueryBuilders.rangeQuery("field").gt("1000")) +// .get(); +// assertNoFailures(searchResponse); +// assertHitCount(searchResponse, 3L); // a numeric value of 1000 should be parsed as 1000 millis since epoch and return only docs after 1970 - searchResponse = client().prepareSearch("test") + SearchResponse searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gt(1000)) .get(); assertNoFailures(searchResponse); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 37d705e28dd1e..cc03f4cf6fea1 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -350,17 +350,7 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } - DateMathParser parser; - if (forcedDateParser == null) { - if (lowerTerm instanceof Integer || upperTerm instanceof Integer) { - // force epoch_millis - parser = EPOCH_MILLIS_PARSER; - } else { - parser = dateMathParser; - } - } else { - parser = forcedDateParser; - } + DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { Query query = LongPoint.newRangeQuery(name(), l, u); if (hasDocValues()) { @@ -391,7 +381,12 @@ public static Query dateRangeQuery( if (lowerTerm == null) { l = Long.MIN_VALUE; } else { - l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution); + if (lowerTerm instanceof Number && parser == null) { + // force epoch_millis + l = ((Number) lowerTerm).longValue(); + } else { + l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution); + } if (includeLower == false) { ++l; } @@ -399,7 +394,12 @@ public static Query dateRangeQuery( if (upperTerm == null) { u = Long.MAX_VALUE; } else { - u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution); + if (lowerTerm instanceof Number && parser == null) { + // force epoch_millis + u = ((Number) lowerTerm).longValue(); + } else { + u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution); + } if (includeUpper == false) { --u; } @@ -451,18 +451,18 @@ public Query distanceFeatureQuery(Object origin, String pivot, float boost, Quer public Relation isFieldWithinQuery(IndexReader reader, Object from, Object to, boolean includeLower, boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { + DateMathParser forcedDateParser = dateParser; if (dateParser == null) { - if (from instanceof Integer || to instanceof Integer) { - // force epoch_millis - dateParser = EPOCH_MILLIS_PARSER; - } else { - dateParser = this.dateMathParser; - } + forcedDateParser = this.dateMathParser; } long fromInclusive = Long.MIN_VALUE; if (from != null) { - fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis, resolution); + if (dateParser == null && from instanceof Number) { + fromInclusive = ((Number) from).longValue(); + } else { + fromInclusive = parseToLong(from, !includeLower, timeZone, forcedDateParser, context::nowInMillis, resolution); + } if (includeLower == false) { if (fromInclusive == Long.MAX_VALUE) { return Relation.DISJOINT; @@ -473,7 +473,11 @@ public Relation isFieldWithinQuery(IndexReader reader, long toInclusive = Long.MAX_VALUE; if (to != null) { - toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis, resolution); + if (dateParser == null && to instanceof Number) { + toInclusive = ((Number) to).longValue(); + } else { + toInclusive = parseToLong(to, includeUpper, timeZone, forcedDateParser, context::nowInMillis, resolution); + } if (includeUpper == false) { if (toInclusive == Long.MIN_VALUE) { return Relation.DISJOINT; diff --git a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java index 01a67451d54e1..7185210caadb8 100644 --- a/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java +++ b/server/src/test/java/org/elasticsearch/index/query/IdsQueryBuilderTests.java @@ -24,8 +24,6 @@ import org.apache.lucene.search.Query; import org.apache.lucene.search.TermInSetQuery; import org.elasticsearch.common.ParsingException; -import org.elasticsearch.common.time.DateFormatter; -import org.elasticsearch.common.time.DateMathParser; import org.elasticsearch.common.xcontent.XContentParser; import org.elasticsearch.test.AbstractQueryTestCase; @@ -99,7 +97,7 @@ public void testFromJson() throws IOException { protected QueryBuilder parseQuery(XContentParser parser) throws IOException { QueryBuilder query = super.parseQuery(parser); assertThat(query, instanceOf(IdsQueryBuilder.class)); - return query; + return (IdsQueryBuilder) query; } @Override @@ -111,18 +109,4 @@ public void testMustRewrite() throws IOException { () -> queryBuilder.toQuery(context)); assertEquals("Rewrite first", e.getMessage()); } - - public void testFoo() { - DateMathParser dateMathParser = DateFormatter.forPattern("strict_date_optional_time||epoch_millis").toDateMathParser(); - System.out.println(dateMathParser.parse("10", null).toEpochMilli()); - System.out.println(dateMathParser.parse("100", null).toEpochMilli()); - System.out.println(dateMathParser.parse("999", null).toEpochMilli()); - System.out.println(dateMathParser.parse("1000", null).toEpochMilli()); - System.out.println(dateMathParser.parse("1500", null).toEpochMilli()); - System.out.println(dateMathParser.parse("1968", null).toEpochMilli()); - System.out.println(dateMathParser.parse("1969", null).toEpochMilli()); - System.out.println(dateMathParser.parse("1970", null).toEpochMilli()); - System.out.println(dateMathParser.parse("1990", null).toEpochMilli()); - System.out.println(dateMathParser.parse("10000", null).toEpochMilli()); - } } From 18a5da5f90d97f7e1499e268138b72b0df023bc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 15 Oct 2020 12:29:32 +0200 Subject: [PATCH 03/13] iter --- .../search/simple/SimpleSearchIT.java | 69 ++++++++++--------- .../index/mapper/DateFieldMapper.java | 48 ++++++++----- .../mapper/DateScriptFieldType.java | 11 +-- 3 files changed, 73 insertions(+), 55 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 244152988c01e..99124bcc3369a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -54,6 +54,8 @@ import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertNoFailures; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.equalTo; +import static org.hamcrest.Matchers.is; +import static org.hamcrest.Matchers.oneOf; public class SimpleSearchIT extends ESIntegTestCase { @@ -201,45 +203,46 @@ public void testSimpleDateRange() throws Exception { client().prepareIndex("test").setId("3").setSource("field", "1967-01-01T00:00").get(); ensureGreen(); refresh(); -// SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d") -// .lte("2010-01-04||+2d/d")).get(); -// assertNoFailures(searchResponse); -// assertHitCount(searchResponse, 2L); -// -// searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00") -// .lte("2010-01-06T02:00")).get(); -// assertNoFailures(searchResponse); -// assertHitCount(searchResponse, 2L); -// -// searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00") -// .lt("2010-01-06T02:00")).get(); -// assertNoFailures(searchResponse); -// assertHitCount(searchResponse, 1L); -// -// searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gt("2010-01-05T02:00") -// .lt("2010-01-06T02:00")).get(); -// assertNoFailures(searchResponse); -// assertHitCount(searchResponse, 0L); -// -// searchResponse = client().prepareSearch("test").setQuery( -// QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")).get(); -// assertHitCount(searchResponse, 2L); -// -// // a string value of "1000" should be parsed as the year 1000 and return all three docs -// searchResponse = client().prepareSearch("test") -// .setQuery(QueryBuilders.rangeQuery("field").gt("1000")) -// .get(); -// assertNoFailures(searchResponse); -// assertHitCount(searchResponse, 3L); + SearchResponse searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-03||+2d") + .lte("2010-01-04||+2d/d")).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00") + .lte("2010-01-06T02:00")).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 2L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gte("2010-01-05T02:00") + .lt("2010-01-06T02:00")).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 1L); + + searchResponse = client().prepareSearch("test").setQuery(QueryBuilders.rangeQuery("field").gt("2010-01-05T02:00") + .lt("2010-01-06T02:00")).get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 0L); + + searchResponse = client().prepareSearch("test").setQuery( + QueryBuilders.queryStringQuery("field:[2010-01-03||+2d TO 2010-01-04||+2d/d]")).get(); + assertHitCount(searchResponse, 2L); + + // a string value of "1000" should be parsed as the year 1000 and return all three docs + searchResponse = client().prepareSearch("test") + .setQuery(QueryBuilders.rangeQuery("field").gt("1000")) + .get(); + assertNoFailures(searchResponse); + assertHitCount(searchResponse, 3L); // a numeric value of 1000 should be parsed as 1000 millis since epoch and return only docs after 1970 - SearchResponse searchResponse = client().prepareSearch("test") + searchResponse = client().prepareSearch("test") .setQuery(QueryBuilders.rangeQuery("field").gt(1000)) .get(); assertNoFailures(searchResponse); assertHitCount(searchResponse, 2L); - assertEquals("1", searchResponse.getHits().getHits()[0].getId()); - assertEquals("2", searchResponse.getHits().getHits()[1].getId()); + String[] expectedIds = new String[] {"1", "2"}; + assertThat(searchResponse.getHits().getHits()[0].getId(), is(oneOf(expectedIds))); + assertThat(searchResponse.getHits().getHits()[1].getId(), is(oneOf(expectedIds))); } public void testSimpleTerminateAfterCount() throws Exception { diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index cc03f4cf6fea1..517ea07a5df5e 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -350,19 +350,29 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } - DateMathParser parser = forcedDateParser == null ? dateMathParser : forcedDateParser; - return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { - Query query = LongPoint.newRangeQuery(name(), l, u); - if (hasDocValues()) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - - if (context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + return dateRangeQuery( + lowerTerm, + upperTerm, + includeLower, + includeUpper, + timeZone, + forcedDateParser, + this.dateMathParser, + context, + resolution, + (l, u) -> { + Query query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); + } } + return query; } - return query; - }); + ); } public static Query dateRangeQuery( @@ -371,19 +381,21 @@ public static Query dateRangeQuery( boolean includeLower, boolean includeUpper, @Nullable ZoneId timeZone, - DateMathParser parser, + DateMathParser forcedDateParser, + DateMathParser fallbackParser, QueryShardContext context, Resolution resolution, BiFunction builder ) { return handleNow(context, nowSupplier -> { long l, u; + DateMathParser parser = forcedDateParser == null ? fallbackParser : forcedDateParser; if (lowerTerm == null) { l = Long.MIN_VALUE; } else { - if (lowerTerm instanceof Number && parser == null) { + if (lowerTerm instanceof Number && forcedDateParser == null) { // force epoch_millis - l = ((Number) lowerTerm).longValue(); + l = resolution.convert(Instant.ofEpochMilli(((Number) lowerTerm).longValue())); } else { l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution); } @@ -394,9 +406,9 @@ public static Query dateRangeQuery( if (upperTerm == null) { u = Long.MAX_VALUE; } else { - if (lowerTerm instanceof Number && parser == null) { + if (upperTerm instanceof Number && forcedDateParser == null) { // force epoch_millis - u = ((Number) lowerTerm).longValue(); + u = resolution.convert(Instant.ofEpochMilli(((Number) upperTerm).longValue())); } else { u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution); } @@ -459,7 +471,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long fromInclusive = Long.MIN_VALUE; if (from != null) { if (dateParser == null && from instanceof Number) { - fromInclusive = ((Number) from).longValue(); + fromInclusive = resolution.convert(Instant.ofEpochMilli(((Number) from).longValue())); } else { fromInclusive = parseToLong(from, !includeLower, timeZone, forcedDateParser, context::nowInMillis, resolution); } @@ -474,7 +486,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long toInclusive = Long.MAX_VALUE; if (to != null) { if (dateParser == null && to instanceof Number) { - toInclusive = ((Number) to).longValue(); + toInclusive = resolution.convert(Instant.ofEpochMilli(((Number) to).longValue())); } else { toInclusive = parseToLong(to, includeUpper, timeZone, forcedDateParser, context::nowInMillis, resolution); } diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java index c69148fe43751..969774493f758 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java @@ -8,6 +8,7 @@ import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; + import org.apache.lucene.search.Query; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.search.Queries; @@ -37,6 +38,7 @@ public class DateScriptFieldType extends AbstractScriptFieldType { private final DateFormatter dateTimeFormatter; + private final DateMathParser dateMathParser; DateScriptFieldType( String name, @@ -47,6 +49,7 @@ public class DateScriptFieldType extends AbstractScriptFieldType scriptFactory.newFactory(n, params, ctx, dateTimeFormatter), meta); this.dateTimeFormatter = dateTimeFormatter; + this.dateMathParser = dateTimeFormatter.toDateMathParser(); } @Override @@ -88,7 +91,7 @@ public Query distanceFeatureQuery(Object origin, String pivot, float boost, Quer origin, true, null, - dateTimeFormatter.toDateMathParser(), + dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS ); @@ -120,7 +123,6 @@ public Query rangeQuery( @Nullable DateMathParser parser, QueryShardContext context ) { - parser = parser == null ? dateTimeFormatter.toDateMathParser() : parser; checkAllowExpensiveQueries(context); return DateFieldType.dateRangeQuery( lowerTerm, @@ -129,6 +131,7 @@ public Query rangeQuery( includeUpper, timeZone, parser, + dateMathParser, context, DateFieldMapper.Resolution.MILLISECONDS, (l, u) -> new LongScriptFieldRangeQuery(script, leafFactory(context)::newInstance, name(), l, u) @@ -142,7 +145,7 @@ public Query termQuery(Object value, QueryShardContext context) { value, false, null, - dateTimeFormatter.toDateMathParser(), + dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS ); @@ -164,7 +167,7 @@ public Query termsQuery(List values, QueryShardContext context) { value, false, null, - dateTimeFormatter.toDateMathParser(), + dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS ) From aad02706f3b8ea3fe6e005dbb47cac94f29eb561 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 15 Oct 2020 17:36:54 +0200 Subject: [PATCH 04/13] spotless --- .../mapper/DateScriptFieldType.java | 29 ++----------------- 1 file changed, 3 insertions(+), 26 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java index 969774493f758..fb29b84b44d62 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java @@ -87,14 +87,7 @@ public DateScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) { checkAllowExpensiveQueries(context); return DateFieldType.handleNow(context, now -> { - long originLong = DateFieldType.parseToLong( - origin, - true, - null, - dateMathParser, - now, - DateFieldMapper.Resolution.MILLISECONDS - ); + long originLong = DateFieldType.parseToLong(origin, true, null, dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS); TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot"); return new LongScriptFieldDistanceFeatureQuery( script, @@ -141,14 +134,7 @@ public Query rangeQuery( @Override public Query termQuery(Object value, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { - long l = DateFieldType.parseToLong( - value, - false, - null, - dateMathParser, - now, - DateFieldMapper.Resolution.MILLISECONDS - ); + long l = DateFieldType.parseToLong(value, false, null, dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS); checkAllowExpensiveQueries(context); return new LongScriptFieldTermQuery(script, leafFactory(context)::newInstance, name(), l); }); @@ -162,16 +148,7 @@ public Query termsQuery(List values, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { LongSet terms = new LongHashSet(values.size()); for (Object value : values) { - terms.add( - DateFieldType.parseToLong( - value, - false, - null, - dateMathParser, - now, - DateFieldMapper.Resolution.MILLISECONDS - ) - ); + terms.add(DateFieldType.parseToLong(value, false, null, dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS)); } checkAllowExpensiveQueries(context); return new LongScriptFieldTermsQuery(script, leafFactory(context)::newInstance, name(), terms); From 84582e9ccc17c8d4b11a66a4502c4750120fcb17 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 15 Oct 2020 18:51:14 +0200 Subject: [PATCH 05/13] Go back --- .../search/simple/SimpleSearchIT.java | 3 + .../index/mapper/DateFieldMapper.java | 80 ++++++++----------- 2 files changed, 35 insertions(+), 48 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index 99124bcc3369a..ac648f79e3cdd 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -19,6 +19,8 @@ package org.elasticsearch.search.simple; +import com.carrotsearch.randomizedtesting.annotations.Repeat; + import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -196,6 +198,7 @@ public void testSimpleId() { assertHitCount(searchResponse, 1L); } + @Repeat(iterations = 100) public void testSimpleDateRange() throws Exception { createIndex("test"); client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); diff --git a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java index 517ea07a5df5e..af8d870ae3181 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java @@ -350,29 +350,29 @@ public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support DISJOINT ranges"); } - return dateRangeQuery( - lowerTerm, - upperTerm, - includeLower, - includeUpper, - timeZone, - forcedDateParser, - this.dateMathParser, - context, - resolution, - (l, u) -> { - Query query = LongPoint.newRangeQuery(name(), l, u); - if (hasDocValues()) { - Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); - query = new IndexOrDocValuesQuery(query, dvQuery); - - if (context.indexSortedOnField(name())) { - query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); - } + DateMathParser parser; + if (forcedDateParser == null) { + if (lowerTerm instanceof Number || upperTerm instanceof Number) { + // force epoch_millis + parser = EPOCH_MILLIS_PARSER; + } else { + parser = dateMathParser; + } + } else { + parser = forcedDateParser; + } + return dateRangeQuery(lowerTerm, upperTerm, includeLower, includeUpper, timeZone, parser, context, resolution, (l, u) -> { + Query query = LongPoint.newRangeQuery(name(), l, u); + if (hasDocValues()) { + Query dvQuery = SortedNumericDocValuesField.newSlowRangeQuery(name(), l, u); + query = new IndexOrDocValuesQuery(query, dvQuery); + + if (context.indexSortedOnField(name())) { + query = new IndexSortSortedNumericDocValuesRangeQuery(name(), l, u, query); } - return query; } - ); + return query; + }); } public static Query dateRangeQuery( @@ -381,24 +381,17 @@ public static Query dateRangeQuery( boolean includeLower, boolean includeUpper, @Nullable ZoneId timeZone, - DateMathParser forcedDateParser, - DateMathParser fallbackParser, + DateMathParser parser, QueryShardContext context, Resolution resolution, BiFunction builder ) { return handleNow(context, nowSupplier -> { long l, u; - DateMathParser parser = forcedDateParser == null ? fallbackParser : forcedDateParser; if (lowerTerm == null) { l = Long.MIN_VALUE; } else { - if (lowerTerm instanceof Number && forcedDateParser == null) { - // force epoch_millis - l = resolution.convert(Instant.ofEpochMilli(((Number) lowerTerm).longValue())); - } else { - l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution); - } + l = parseToLong(lowerTerm, !includeLower, timeZone, parser, nowSupplier, resolution); if (includeLower == false) { ++l; } @@ -406,12 +399,7 @@ public static Query dateRangeQuery( if (upperTerm == null) { u = Long.MAX_VALUE; } else { - if (upperTerm instanceof Number && forcedDateParser == null) { - // force epoch_millis - u = resolution.convert(Instant.ofEpochMilli(((Number) upperTerm).longValue())); - } else { - u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution); - } + u = parseToLong(upperTerm, includeUpper, timeZone, parser, nowSupplier, resolution); if (includeUpper == false) { --u; } @@ -463,18 +451,18 @@ public Query distanceFeatureQuery(Object origin, String pivot, float boost, Quer public Relation isFieldWithinQuery(IndexReader reader, Object from, Object to, boolean includeLower, boolean includeUpper, ZoneId timeZone, DateMathParser dateParser, QueryRewriteContext context) throws IOException { - DateMathParser forcedDateParser = dateParser; if (dateParser == null) { - forcedDateParser = this.dateMathParser; + if (from instanceof Number || to instanceof Number) { + // force epoch_millis + dateParser = EPOCH_MILLIS_PARSER; + } else { + dateParser = this.dateMathParser; + } } long fromInclusive = Long.MIN_VALUE; if (from != null) { - if (dateParser == null && from instanceof Number) { - fromInclusive = resolution.convert(Instant.ofEpochMilli(((Number) from).longValue())); - } else { - fromInclusive = parseToLong(from, !includeLower, timeZone, forcedDateParser, context::nowInMillis, resolution); - } + fromInclusive = parseToLong(from, !includeLower, timeZone, dateParser, context::nowInMillis, resolution); if (includeLower == false) { if (fromInclusive == Long.MAX_VALUE) { return Relation.DISJOINT; @@ -485,11 +473,7 @@ public Relation isFieldWithinQuery(IndexReader reader, long toInclusive = Long.MAX_VALUE; if (to != null) { - if (dateParser == null && to instanceof Number) { - toInclusive = resolution.convert(Instant.ofEpochMilli(((Number) to).longValue())); - } else { - toInclusive = parseToLong(to, includeUpper, timeZone, forcedDateParser, context::nowInMillis, resolution); - } + toInclusive = parseToLong(to, includeUpper, timeZone, dateParser, context::nowInMillis, resolution); if (includeUpper == false) { if (toInclusive == Long.MIN_VALUE) { return Relation.DISJOINT; From d2f74c98e48e0a50c0fc0d322841c3cfa5e11a4b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Thu, 15 Oct 2020 18:56:39 +0200 Subject: [PATCH 06/13] fixing yaml test --- .../resources/rest-api-spec/test/field_caps/30_filter.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml index da692dbe8e850..31e8661c9e472 100644 --- a/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml +++ b/rest-api-spec/src/main/resources/rest-api-spec/test/field_caps/30_filter.yml @@ -81,7 +81,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { gte: 2010 }}}} + body: { index_filter: { range: { timestamp: { gte: "2010" }}}} - match: {indices: ["test-1", "test-2", "test-3"]} - length: {fields.field1: 3} @@ -90,7 +90,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { gte: 2019 } } } } + body: { index_filter: { range: { timestamp: { gte: "2019" } } } } - match: {indices: ["test-2", "test-3"]} - length: {fields.field1: 2} @@ -106,7 +106,7 @@ setup: field_caps: index: test-* fields: "*" - body: { index_filter: { range: { timestamp: { lt: 2019 } } } } + body: { index_filter: { range: { timestamp: { lt: "2019" } } } } - match: {indices: ["test-1"]} - length: {fields.field1: 1} From 41f0c9bc5d81743fc7518547e6f1baf7774282ed Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 16 Oct 2020 12:07:37 +0200 Subject: [PATCH 07/13] Fix compile problem --- .../mapper/DateScriptFieldType.java | 32 ++++++++++++++++--- 1 file changed, 27 insertions(+), 5 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java index fb29b84b44d62..7930c58d77f60 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java @@ -8,7 +8,6 @@ import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; - import org.apache.lucene.search.Query; import org.elasticsearch.common.Nullable; import org.elasticsearch.common.lucene.search.Queries; @@ -87,7 +86,14 @@ public DateScriptFieldData.Builder fielddataBuilder(String fullyQualifiedIndexNa public Query distanceFeatureQuery(Object origin, String pivot, float boost, QueryShardContext context) { checkAllowExpensiveQueries(context); return DateFieldType.handleNow(context, now -> { - long originLong = DateFieldType.parseToLong(origin, true, null, dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS); + long originLong = DateFieldType.parseToLong( + origin, + true, + null, + this.dateMathParser, + now, + DateFieldMapper.Resolution.MILLISECONDS + ); TimeValue pivotTime = TimeValue.parseTimeValue(pivot, "distance_feature.pivot"); return new LongScriptFieldDistanceFeatureQuery( script, @@ -116,6 +122,7 @@ public Query rangeQuery( @Nullable DateMathParser parser, QueryShardContext context ) { + parser = parser == null ? this.dateMathParser : parser; checkAllowExpensiveQueries(context); return DateFieldType.dateRangeQuery( lowerTerm, @@ -124,7 +131,6 @@ public Query rangeQuery( includeUpper, timeZone, parser, - dateMathParser, context, DateFieldMapper.Resolution.MILLISECONDS, (l, u) -> new LongScriptFieldRangeQuery(script, leafFactory(context)::newInstance, name(), l, u) @@ -134,7 +140,14 @@ public Query rangeQuery( @Override public Query termQuery(Object value, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { - long l = DateFieldType.parseToLong(value, false, null, dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS); + long l = DateFieldType.parseToLong( + value, + false, + null, + this.dateMathParser, + now, + DateFieldMapper.Resolution.MILLISECONDS + ); checkAllowExpensiveQueries(context); return new LongScriptFieldTermQuery(script, leafFactory(context)::newInstance, name(), l); }); @@ -148,7 +161,16 @@ public Query termsQuery(List values, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { LongSet terms = new LongHashSet(values.size()); for (Object value : values) { - terms.add(DateFieldType.parseToLong(value, false, null, dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS)); + terms.add( + DateFieldType.parseToLong( + value, + false, + null, + this.dateMathParser, + now, + DateFieldMapper.Resolution.MILLISECONDS + ) + ); } checkAllowExpensiveQueries(context); return new LongScriptFieldTermsQuery(script, leafFactory(context)::newInstance, name(), terms); From 15c9c8242f6a5745f5776d20369e0a5ae76154f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 16 Oct 2020 12:26:26 +0200 Subject: [PATCH 08/13] reapplying yaml fix --- .../rest-api-spec/test/multi_cluster/30_field_caps.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml index cfecd009bd1ef..5d7182cf9be9a 100644 --- a/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml +++ b/qa/multi-cluster-search/src/test/resources/rest-api-spec/test/multi_cluster/30_field_caps.yml @@ -106,7 +106,7 @@ field_caps: index: 'field_caps_index_4,my_remote_cluster:field_*' fields: [number] - body: { index_filter: { range: { created_at: { lt: 2018 } } } } + body: { index_filter: { range: { created_at: { lt: "2018" } } } } - match: {indices: ["field_caps_index_4","my_remote_cluster:field_caps_index_1"]} - length: {fields.number: 1} @@ -117,7 +117,7 @@ field_caps: index: 'field_caps_index_4,my_remote_cluster:field_*' fields: [number] - body: { index_filter: { range: { created_at: { gt: 2019 } } } } + body: { index_filter: { range: { created_at: { gt: "2019" } } } } - match: {indices: ["my_remote_cluster:field_caps_index_3"]} - length: {fields.number: 1} From 94ceac8e89b9cd7abf45a225b4146150e59d8ed3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Fri, 16 Oct 2020 12:32:13 +0200 Subject: [PATCH 09/13] spotless again --- .../mapper/DateScriptFieldType.java | 20 ++----------------- 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java index 7930c58d77f60..f4c021b424e9c 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java @@ -140,14 +140,7 @@ public Query rangeQuery( @Override public Query termQuery(Object value, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { - long l = DateFieldType.parseToLong( - value, - false, - null, - this.dateMathParser, - now, - DateFieldMapper.Resolution.MILLISECONDS - ); + long l = DateFieldType.parseToLong(value, false, null, this.dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS); checkAllowExpensiveQueries(context); return new LongScriptFieldTermQuery(script, leafFactory(context)::newInstance, name(), l); }); @@ -161,16 +154,7 @@ public Query termsQuery(List values, QueryShardContext context) { return DateFieldType.handleNow(context, now -> { LongSet terms = new LongHashSet(values.size()); for (Object value : values) { - terms.add( - DateFieldType.parseToLong( - value, - false, - null, - this.dateMathParser, - now, - DateFieldMapper.Resolution.MILLISECONDS - ) - ); + terms.add(DateFieldType.parseToLong(value, false, null, this.dateMathParser, now, DateFieldMapper.Resolution.MILLISECONDS)); } checkAllowExpensiveQueries(context); return new LongScriptFieldTermsQuery(script, leafFactory(context)::newInstance, name(), terms); From 463b1b2337a56fac4f4f66030db5b5783916e52a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 20 Oct 2020 14:24:26 +0200 Subject: [PATCH 10/13] remove repeat --- .../java/org/elasticsearch/search/simple/SimpleSearchIT.java | 3 --- 1 file changed, 3 deletions(-) diff --git a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java index ac648f79e3cdd..99124bcc3369a 100644 --- a/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java +++ b/server/src/internalClusterTest/java/org/elasticsearch/search/simple/SimpleSearchIT.java @@ -19,8 +19,6 @@ package org.elasticsearch.search.simple; -import com.carrotsearch.randomizedtesting.annotations.Repeat; - import org.elasticsearch.action.index.IndexRequestBuilder; import org.elasticsearch.action.search.SearchPhaseExecutionException; import org.elasticsearch.action.search.SearchRequestBuilder; @@ -198,7 +196,6 @@ public void testSimpleId() { assertHitCount(searchResponse, 1L); } - @Repeat(iterations = 100) public void testSimpleDateRange() throws Exception { createIndex("test"); client().prepareIndex("test").setId("1").setSource("field", "2010-01-05T02:00").get(); From e0044a7354eebbde9648e1d903282c3bd6af5009 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 1 Dec 2020 15:13:15 +0100 Subject: [PATCH 11/13] Add docs and breaking changes entry --- .../migration/migrate_8_0/search.asciidoc | 20 ++++++++++++++++++- docs/reference/query-dsl/range-query.asciidoc | 8 ++++++++ 2 files changed, 27 insertions(+), 1 deletion(-) diff --git a/docs/reference/migration/migrate_8_0/search.asciidoc b/docs/reference/migration/migrate_8_0/search.asciidoc index ce58ca6d245e6..0608e3b99527e 100644 --- a/docs/reference/migration/migrate_8_0/search.asciidoc +++ b/docs/reference/migration/migrate_8_0/search.asciidoc @@ -142,4 +142,22 @@ an error already. We now also reject `-1` as an invalid value. Change any use of `-1` as `from` parameter in request body or url parameters by either setting it to `0` or omitting it entirely. Requests containing negative values will return an error. -==== \ No newline at end of file +==== + +.Range queries on date fields treat numeric values alwas as milliseconds-since-epoch. +[%collapsible] +==== +*Details* + +Range queries on date fields used to misinterpret small numbers (e.g. four digits like 1000) +as a year when no additional format was set, but would interpret other numeric values as +milliseconds since epoch. We now treat all numeric values in absence of a specific `format` +parameter as milliseconds since epoch. If you want to query for years instead, with a missing +`format` you now need to quote the input value (e.g. "1984"). + +*Impact* + +If you query date fields without a specified `format`, check if the values in your queries are +actually meant to be milliseconds-since-epoch and use a numeric value in this case. If not, use +a string value which gets parsed by either the date format set on the field in the mappings or +by `strict_date_optional_time` by default. + +==== diff --git a/docs/reference/query-dsl/range-query.asciidoc b/docs/reference/query-dsl/range-query.asciidoc index 32184875aa0b6..7c078fc4067ca 100644 --- a/docs/reference/query-dsl/range-query.asciidoc +++ b/docs/reference/query-dsl/range-query.asciidoc @@ -183,6 +183,14 @@ to `2099-12-01T23:59:59.999_999_999Z`. This date uses the provided year (`2099`) and month (`12`) but uses the default day (`01`), hour (`23`), minute (`59`), second (`59`), and nanosecond (`999_999_999`). +[[numeric-date]] +====== Numeric date range value + +When no date format is specified and the range query is targeting a date field, numeric +values are interpreted representing milliseconds-since-the-epoch. If you want the value +to represent a year, e.g. 2020, you need to pass it as a String value (e.g. "2020") that +will be parsed according to the default format or the set format. + [[range-query-date-math-rounding]] ====== Date math and rounding {es} rounds <> values in parameters as follows: From 95e6e954b76f467efde8dc5328ee615e2e2e8c39 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 1 Dec 2020 16:51:49 +0100 Subject: [PATCH 12/13] Adding to breaking changes section in release notes --- docs/reference/release-notes/8.0.0-alpha1.asciidoc | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/docs/reference/release-notes/8.0.0-alpha1.asciidoc b/docs/reference/release-notes/8.0.0-alpha1.asciidoc index 9851c32c709e2..3a2823b222132 100644 --- a/docs/reference/release-notes/8.0.0-alpha1.asciidoc +++ b/docs/reference/release-notes/8.0.0-alpha1.asciidoc @@ -29,5 +29,6 @@ by using appropriate thresholds. If for instance we want to simulate `index.inde all we need to do is to set `index.indexing.slowlog.threshold.index.debug` and `index.indexing.slowlog.threshold.index.trace` to `-1` {es-pull}57591[#57591] - +Search:: +* Consistent treatment of numeric values for range query on date fields without `format` {es-pull}[#63692] From 5439e6c7900c835a066aa58687e9c524908d73f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Christoph=20B=C3=BCscher?= Date: Tue, 1 Dec 2020 17:47:50 +0100 Subject: [PATCH 13/13] init dateMathParser in DateScriptFieldType --- .../xpack/runtimefields/mapper/DateScriptFieldType.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java index 1fd8bda9fc89f..c54ad030cb9da 100644 --- a/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java +++ b/x-pack/plugin/runtime-fields/src/main/java/org/elasticsearch/xpack/runtimefields/mapper/DateScriptFieldType.java @@ -8,6 +8,7 @@ import com.carrotsearch.hppc.LongHashSet; import com.carrotsearch.hppc.LongSet; + import org.apache.lucene.search.Query; import org.elasticsearch.common.CheckedBiConsumer; import org.elasticsearch.common.Nullable; @@ -96,6 +97,7 @@ protected AbstractScriptFieldType buildFieldType() { private DateScriptFieldType(String name, DateFieldScript.Factory scriptFactory, DateFormatter dateTimeFormatter, Builder builder) { super(name, (n, params, ctx) -> scriptFactory.newFactory(n, params, ctx, dateTimeFormatter), builder); this.dateTimeFormatter = dateTimeFormatter; + this.dateMathParser = dateTimeFormatter.toDateMathParser(); } DateScriptFieldType(