Skip to content

Commit 636cf55

Browse files
albertzaharovitsalbertzaharovits
committed
Epoch millis and second formats parse float implicitly (Closes #14641) (#26119)
`epoch_millis` and `epoch_second` date formats truncate float values, as numbers or as strings. The `coerce` parameter is not defined for `date` field type and this is not changing. See PR #26119 Closes #14641
1 parent 4cfcb27 commit 636cf55

File tree

6 files changed

+84
-34
lines changed

6 files changed

+84
-34
lines changed

‎core/src/main/java/org/elasticsearch/common/joda/Joda.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,7 @@
4242

4343
import java.io.IOException;
4444
import java.io.Writer;
45+
import java.math.BigDecimal;
4546
import java.util.Locale;
4647

4748
public class Joda {
@@ -331,7 +332,8 @@ public int estimateParsedLength() {
331332
@Override
332333
public int parseInto(DateTimeParserBucket bucket, String text, int position) {
333334
boolean isPositive = text.startsWith("-") == false;
334-
boolean isTooLong = text.length() > estimateParsedLength();
335+
int firstDotIndex = text.indexOf('.');
336+
boolean isTooLong = (firstDotIndex == -1 ? text.length() : firstDotIndex) > estimateParsedLength();
335337

336338
if (bucket.getZone() != DateTimeZone.UTC) {
337339
String format = hasMilliSecondPrecision ? "epoch_millis" : "epoch_second";
@@ -342,7 +344,7 @@ public int parseInto(DateTimeParserBucket bucket, String text, int position) {
342344

343345
int factor = hasMilliSecondPrecision ? 1 : 1000;
344346
try {
345-
long millis = Long.valueOf(text) * factor;
347+
long millis = new BigDecimal(text).longValue() * factor;
346348
DateTime dt = new DateTime(millis, DateTimeZone.UTC);
347349
bucket.saveField(DateTimeFieldType.year(), dt.getYear());
348350
bucket.saveField(DateTimeFieldType.monthOfYear(), dt.getMonthOfYear());

‎core/src/test/java/org/elasticsearch/deps/joda/SimpleJodaTests.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -260,6 +260,10 @@ public void testThatEpochsCanBeParsed() {
260260
} else {
261261
assertThat(dateTime.getMillisOfSecond(), is(0));
262262
}
263+
264+
// test floats get truncated
265+
String epochFloatValue = String.format(Locale.US, "%d.%d", dateTime.getMillis() / (parseMilliSeconds ? 1L : 1000L), randomNonNegativeLong());
266+
assertThat(formatter.parser().parseDateTime(epochFloatValue).getMillis(), is(dateTime.getMillis()));
263267
}
264268

265269
public void testThatNegativeEpochsCanBeParsed() {
@@ -281,16 +285,26 @@ public void testThatNegativeEpochsCanBeParsed() {
281285
assertThat(dateTime.getSecondOfMinute(), is(20));
282286
}
283287

288+
// test floats get truncated
289+
String epochFloatValue = String.format(Locale.US, "%d.%d", dateTime.getMillis() / (parseMilliSeconds ? 1L : 1000L), randomNonNegativeLong());
290+
assertThat(formatter.parser().parseDateTime(epochFloatValue).getMillis(), is(dateTime.getMillis()));
291+
284292
// every negative epoch must be parsed, no matter if exact the size or bigger
285293
if (parseMilliSeconds) {
286294
formatter.parser().parseDateTime("-100000000");
287295
formatter.parser().parseDateTime("-999999999999");
288296
formatter.parser().parseDateTime("-1234567890123");
289297
formatter.parser().parseDateTime("-1234567890123456789");
298+
299+
formatter.parser().parseDateTime("-1234567890123.9999");
300+
formatter.parser().parseDateTime("-1234567890123456789.9999");
290301
} else {
291302
formatter.parser().parseDateTime("-100000000");
292303
formatter.parser().parseDateTime("-1234567890");
293304
formatter.parser().parseDateTime("-1234567890123456");
305+
306+
formatter.parser().parseDateTime("-1234567890.9999");
307+
formatter.parser().parseDateTime("-1234567890123456.9999");
294308
}
295309
}
296310

‎core/src/test/java/org/elasticsearch/index/mapper/DateFieldMapperTests.java‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535

3636
import java.io.IOException;
3737
import java.util.Collection;
38+
import java.util.Locale;
3839

3940
import static org.hamcrest.Matchers.containsString;
4041
import static org.hamcrest.Matchers.notNullValue;
@@ -214,6 +215,32 @@ public void testChangeFormat() throws IOException {
214215
assertEquals(1457654400000L, pointField.numericValue().longValue());
215216
}
216217

218+
public void testFloatEpochFormat() throws IOException {
219+
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
220+
.startObject("properties").startObject("field").field("type", "date")
221+
.field("format", "epoch_millis").endObject().endObject()
222+
.endObject().endObject().string();
223+
224+
DocumentMapper mapper = parser.parse("type", new CompressedXContent(mapping));
225+
226+
assertEquals(mapping, mapper.mappingSource().toString());
227+
228+
double epochFloatMillisFromEpoch = (randomDouble() * 2 - 1) * 1000000;
229+
String epochFloatValue = String.format(Locale.US, "%f", epochFloatMillisFromEpoch);
230+
231+
ParsedDocument doc = mapper.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
232+
.startObject()
233+
.field("field", epochFloatValue)
234+
.endObject()
235+
.bytes(),
236+
XContentType.JSON));
237+
238+
IndexableField[] fields = doc.rootDoc().getFields("field");
239+
assertEquals(2, fields.length);
240+
IndexableField pointField = fields[0];
241+
assertEquals((long)epochFloatMillisFromEpoch, pointField.numericValue().longValue());
242+
}
243+
217244
public void testChangeLocale() throws IOException {
218245
String mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
219246
.startObject("properties").startObject("field").field("type", "date").field("locale", "fr").endObject().endObject()

‎core/src/test/java/org/elasticsearch/index/mapper/RangeFieldMapperTests.java‎

Lines changed: 18 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -245,24 +245,24 @@ public void doTestCoerce(String type) throws IOException {
245245
IndexableField pointField = fields[1];
246246
assertEquals(2, pointField.fieldType().pointDimensionCount());
247247

248-
mapping = XContentFactory.jsonBuilder().startObject().startObject("type")
249-
.startObject("properties").startObject("field").field("type", type).field("coerce", false).endObject().endObject()
250-
.endObject().endObject();
251-
DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping.string()));
252-
253-
assertEquals(mapping.string(), mapper2.mappingSource().toString());
254-
255-
ThrowingRunnable runnable = () -> mapper2.parse(SourceToParse.source("test", "type", "1", XContentFactory.jsonBuilder()
256-
.startObject()
257-
.startObject("field")
258-
.field(getFromField(), "5.2")
259-
.field(getToField(), "10")
260-
.endObject()
261-
.endObject().bytes(),
262-
XContentType.JSON));
263-
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
264-
assertThat(e.getCause().getMessage(), anyOf(containsString("passed as String"),
265-
containsString("failed to parse date"), containsString("is not an IP string literal")));
248+
// date_range ignores the coerce parameter and epoch_millis date format truncates floats (see issue: #14641)
249+
if (type.equals("date_range") == false) {
250+
251+
mapping = XContentFactory.jsonBuilder().startObject().startObject("type").startObject("properties").startObject("field")
252+
.field("type", type).field("coerce", false).endObject().endObject().endObject().endObject();
253+
DocumentMapper mapper2 = parser.parse("type", new CompressedXContent(mapping.string()));
254+
255+
assertEquals(mapping.string(), mapper2.mappingSource().toString());
256+
257+
ThrowingRunnable runnable = () -> mapper2
258+
.parse(SourceToParse.source(
259+
"test", "type", "1", XContentFactory.jsonBuilder().startObject().startObject("field")
260+
.field(getFromField(), "5.2").field(getToField(), "10").endObject().endObject().bytes(),
261+
XContentType.JSON));
262+
MapperParsingException e = expectThrows(MapperParsingException.class, runnable);
263+
assertThat(e.getCause().getMessage(), anyOf(containsString("passed as String"), containsString("failed to parse date"),
264+
containsString("is not an IP string literal")));
265+
}
266266
}
267267

268268
@Override

‎core/src/test/java/org/elasticsearch/search/aggregations/bucket/DateRangeIT.java‎

Lines changed: 14 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1014,17 +1014,20 @@ public void testRangeWithFormatNumericValue() throws Exception {
10141014
assertBucket(buckets.get(0), 2L, "1000-3000", 1000000L, 3000000L);
10151015
assertBucket(buckets.get(1), 1L, "3000-4000", 3000000L, 4000000L);
10161016

1017-
// however, e-notation should and fractional parts provided as string
1018-
// should be parsed and error if not compatible
1019-
Exception e = expectThrows(Exception.class, () -> client().prepareSearch(indexName).setSize(0)
1020-
.addAggregation(dateRange("date_range").field("date").addRange("1.0e3", "3.0e3").addRange("3.0e3", "4.0e3")).get());
1021-
assertThat(e.getCause(), instanceOf(ElasticsearchParseException.class));
1022-
assertEquals("failed to parse date field [1.0e3] with format [epoch_second]", e.getCause().getMessage());
1023-
1024-
e = expectThrows(Exception.class, () -> client().prepareSearch(indexName).setSize(0)
1025-
.addAggregation(dateRange("date_range").field("date").addRange("1000.123", "3000.8").addRange("3000.8", "4000.3")).get());
1026-
assertThat(e.getCause(), instanceOf(ElasticsearchParseException.class));
1027-
assertEquals("failed to parse date field [1000.123] with format [epoch_second]", e.getCause().getMessage());
1017+
// also e-notation and floats provided as string also be truncated (see: #14641)
1018+
searchResponse = client().prepareSearch(indexName).setSize(0)
1019+
.addAggregation(dateRange("date_range").field("date").addRange("1.0e3", "3.0e3").addRange("3.0e3", "4.0e3")).get();
1020+
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L));
1021+
buckets = checkBuckets(searchResponse.getAggregations().get("date_range"), "date_range", 2);
1022+
assertBucket(buckets.get(0), 2L, "1000-3000", 1000000L, 3000000L);
1023+
assertBucket(buckets.get(1), 1L, "3000-4000", 3000000L, 4000000L);
1024+
1025+
searchResponse = client().prepareSearch(indexName).setSize(0)
1026+
.addAggregation(dateRange("date_range").field("date").addRange("1000.123", "3000.8").addRange("3000.8", "4000.3")).get();
1027+
assertThat(searchResponse.getHits().getTotalHits(), equalTo(3L));
1028+
buckets = checkBuckets(searchResponse.getAggregations().get("date_range"), "date_range", 2);
1029+
assertBucket(buckets.get(0), 2L, "1000-3000", 1000000L, 3000000L);
1030+
assertBucket(buckets.get(1), 1L, "3000-4000", 3000000L, 4000000L);
10281031

10291032
// using different format should work when to/from is compatible with
10301033
// format in aggregation

‎core/src/test/java/org/elasticsearch/search/query/SearchQueryIT.java‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1699,11 +1699,15 @@ public void testDateProvidedAsNumber() throws ExecutionException, InterruptedExc
16991699
assertAcked(client().admin().indices().preparePutMapping("test").setType("type").setSource("field", "type=date,format=epoch_millis").get());
17001700
indexRandom(true, client().prepareIndex("test", "type", "1").setSource("field", -1000000000001L),
17011701
client().prepareIndex("test", "type", "2").setSource("field", -1000000000000L),
1702-
client().prepareIndex("test", "type", "3").setSource("field", -999999999999L));
1702+
client().prepareIndex("test", "type", "3").setSource("field", -999999999999L),
1703+
client().prepareIndex("test", "type", "4").setSource("field", -1000000000001.0123456789),
1704+
client().prepareIndex("test", "type", "5").setSource("field", -1000000000000.0123456789),
1705+
client().prepareIndex("test", "type", "6").setSource("field", -999999999999.0123456789));
17031706

17041707

1705-
assertHitCount(client().prepareSearch("test").setSize(0).setQuery(rangeQuery("field").lte(-1000000000000L)).get(), 2);
1706-
assertHitCount(client().prepareSearch("test").setSize(0).setQuery(rangeQuery("field").lte(-999999999999L)).get(), 3);
1708+
assertHitCount(client().prepareSearch("test").setSize(0).setQuery(rangeQuery("field").lte(-1000000000000L)).get(), 4);
1709+
assertHitCount(client().prepareSearch("test").setSize(0).setQuery(rangeQuery("field").lte(-999999999999L)).get(), 6);
1710+
17071711
}
17081712

17091713
public void testRangeQueryWithTimeZone() throws Exception {

0 commit comments

Comments
 (0)