-
Notifications
You must be signed in to change notification settings - Fork 25.6k
Reject out of range numbers for float, double and half_float #25826
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Changes from 10 commits
2d0e5c1
c0fff6f
6e0a6ea
7d4f315
d1ebd6f
1358fed
44983d8
d072e69
ecf3424
b5d231d
187982a
d236158
7423c4d
8e88c9d
11ce7c8
3902911
656cf63
cd38455
b578b5a
3c666a9
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -162,12 +162,25 @@ public enum NumberType { | |
| HALF_FLOAT("half_float", NumericType.HALF_FLOAT) { | ||
| @Override | ||
| Float parse(Object value, boolean coerce) { | ||
| return (Float) FLOAT.parse(value, false); | ||
| final Float result; | ||
|
|
||
| if (value instanceof Number) { | ||
| result = ((Number) value).floatValue(); | ||
| } else { | ||
| if (value instanceof BytesRef) { | ||
| value = ((BytesRef) value).utf8ToString(); | ||
| } | ||
| result = Float.parseFloat(value.toString()); | ||
| } | ||
| validateParsed(result); | ||
| return result; | ||
| } | ||
|
|
||
| @Override | ||
| Float parse(XContentParser parser, boolean coerce) throws IOException { | ||
| return parser.floatValue(coerce); | ||
| Float parsed = parser.floatValue(coerce); | ||
| validateParsed(parsed); | ||
| return parsed; | ||
| } | ||
|
|
||
| @Override | ||
|
|
@@ -231,22 +244,39 @@ public List<Field> createFields(String name, Number value, | |
| } | ||
| return fields; | ||
| } | ||
|
|
||
| private void validateParsed(Float value) { | ||
|
||
| if ( | ||
| value.isNaN() || value.isInfinite() | ||
| || value > 65504 | ||
|
||
| || !Float.isFinite(HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(value))) | ||
|
||
| ) { | ||
| throw new IllegalArgumentException("[half_float] supports only finite values, but got [" + value + "]"); | ||
| } | ||
| } | ||
| }, | ||
| FLOAT("float", NumericType.FLOAT) { | ||
| @Override | ||
| Float parse(Object value, boolean coerce) { | ||
| final Float result; | ||
|
||
|
|
||
| if (value instanceof Number) { | ||
| return ((Number) value).floatValue(); | ||
| } | ||
| if (value instanceof BytesRef) { | ||
| value = ((BytesRef) value).utf8ToString(); | ||
| result = ((Number) value).floatValue(); | ||
| } else { | ||
| if (value instanceof BytesRef) { | ||
| value = ((BytesRef) value).utf8ToString(); | ||
| } | ||
| result = Float.parseFloat(value.toString()); | ||
| } | ||
| return Float.parseFloat(value.toString()); | ||
| validateParsed(result); | ||
| return result; | ||
| } | ||
|
|
||
| @Override | ||
| Float parse(XContentParser parser, boolean coerce) throws IOException { | ||
| return parser.floatValue(coerce); | ||
| Float parsed = parser.floatValue(coerce); | ||
| validateParsed(parsed); | ||
| return parsed; | ||
| } | ||
|
|
||
| @Override | ||
|
|
@@ -308,22 +338,35 @@ public List<Field> createFields(String name, Number value, | |
| } | ||
| return fields; | ||
| } | ||
|
|
||
| private void validateParsed(Float value) { | ||
|
||
| if (value.isInfinite() || value.isNaN()) { | ||
| throw new IllegalArgumentException("[float] supports only finite values, but got [" + value + "]"); | ||
| } | ||
| } | ||
| }, | ||
| DOUBLE("double", NumericType.DOUBLE) { | ||
| @Override | ||
| Double parse(Object value, boolean coerce) { | ||
| final Double result; | ||
|
|
||
| if (value instanceof Number) { | ||
| return ((Number) value).doubleValue(); | ||
| } | ||
| if (value instanceof BytesRef) { | ||
| value = ((BytesRef) value).utf8ToString(); | ||
| result = ((Number) value).doubleValue(); | ||
| } else { | ||
| if (value instanceof BytesRef) { | ||
| value = ((BytesRef) value).utf8ToString(); | ||
| } | ||
| result = Double.parseDouble(value.toString()); | ||
| } | ||
| return Double.parseDouble(value.toString()); | ||
| validateParsed(result); | ||
| return result; | ||
| } | ||
|
|
||
| @Override | ||
| Double parse(XContentParser parser, boolean coerce) throws IOException { | ||
| return parser.doubleValue(coerce); | ||
| Double parsed = parser.doubleValue(coerce); | ||
| validateParsed(parsed); | ||
| return parsed; | ||
| } | ||
|
|
||
| @Override | ||
|
|
@@ -385,6 +428,12 @@ public List<Field> createFields(String name, Number value, | |
| } | ||
| return fields; | ||
| } | ||
|
|
||
| private void validateParsed(Double value) { | ||
|
||
| if (value.isInfinite() || value.isNaN()) { | ||
| throw new IllegalArgumentException("[double] supports only finite values, but got [" + value + "]"); | ||
| } | ||
| } | ||
| }, | ||
| BYTE("byte", NumericType.BYTE) { | ||
| @Override | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -21,11 +21,13 @@ | |
|
|
||
| import org.apache.lucene.index.DocValuesType; | ||
| import org.apache.lucene.index.IndexableField; | ||
| import org.elasticsearch.common.bytes.BytesReference; | ||
| import org.elasticsearch.common.compress.CompressedXContent; | ||
| import org.elasticsearch.common.xcontent.XContentFactory; | ||
| import org.elasticsearch.common.xcontent.XContentType; | ||
|
|
||
| import java.io.IOException; | ||
| import java.util.List; | ||
| import java.util.Arrays; | ||
| import java.util.HashSet; | ||
|
|
||
|
|
@@ -314,4 +316,76 @@ public void testEmptyName() throws IOException { | |
| assertThat(e.getMessage(), containsString("name cannot be empty string")); | ||
| } | ||
| } | ||
|
|
||
| public void testOutOfRangeValues() throws IOException { | ||
| final List<Triple<String, Object, String>> inputs = Arrays.asList( | ||
| Triple.of("byte", "128", "is out of range for a byte"), | ||
| Triple.of("short", "32768", "is out of range for a short"), | ||
| Triple.of("integer", "2147483648", "For input string"), | ||
| Triple.of("long", "92233720368547758080", "For input string"), | ||
|
|
||
| Triple.of("half_float", "65504.1", "[half_float] supports only finite values"), | ||
| Triple.of("float", "3.4028235E39", "[float] supports only finite values"), | ||
| Triple.of("double", "1.7976931348623157E309", "[double] supports only finite values"), | ||
|
|
||
| Triple.of("half_float", Float.NaN, "[half_float] supports only finite values"), | ||
| Triple.of("float", Float.NaN, "[float] supports only finite values"), | ||
| Triple.of("double", Double.NaN, "[double] supports only finite values"), | ||
|
|
||
| Triple.of("half_float", Float.POSITIVE_INFINITY, "[half_float] supports only finite values"), | ||
| Triple.of("float", Float.POSITIVE_INFINITY, "[float] supports only finite values"), | ||
| Triple.of("double", Double.POSITIVE_INFINITY, "[double] supports only finite values") | ||
| ); | ||
|
|
||
| for(Triple<String, Object, String> item: inputs) { | ||
| try { | ||
| parseRequest(item.type, createIndexRequest(item.value)); | ||
| fail("Mapper parsing exception expected for [" + item.type + "] with value [" + item.value + "]"); | ||
| } catch (MapperParsingException e) { | ||
| assertThat("Incorrect error message for [" + item.type + "] with value [" + item.value + "]", | ||
| e.getCause().getMessage(), containsString(item.message)); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| private static class Triple<K,V,M> { | ||
|
||
|
|
||
| final K type; | ||
| final V value; | ||
| final M message; | ||
|
|
||
| static <K,V,M> Triple<K,V,M> of(K t, V v, M m) { | ||
| return new Triple<>(t, v, m); | ||
| } | ||
|
|
||
| Triple(K t, V v, M m) { | ||
| type = t; | ||
| value = v; | ||
| message = m; | ||
| } | ||
| } | ||
|
|
||
| private void parseRequest(String type, BytesReference content) throws IOException { | ||
| createDocumentMapper(type).parse(SourceToParse.source("test", "type", "1", content, XContentType.JSON)); | ||
| } | ||
|
|
||
| private DocumentMapper createDocumentMapper(String type) throws IOException { | ||
| String mapping = XContentFactory.jsonBuilder() | ||
| .startObject() | ||
| .startObject("type") | ||
| .startObject("properties") | ||
| .startObject("field") | ||
| .field("type", type) | ||
| .endObject() | ||
| .endObject() | ||
| .endObject() | ||
| .endObject() | ||
| .string(); | ||
|
|
||
| return parser.parse("type", new CompressedXContent(mapping)); | ||
| } | ||
|
|
||
| private BytesReference createIndexRequest(Object value) throws IOException { | ||
| return XContentFactory.jsonBuilder().startObject().field("field", value).endObject().bytes(); | ||
| } | ||
| } | ||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -43,9 +43,14 @@ | |
| import org.junit.Before; | ||
|
|
||
| import java.io.IOException; | ||
| import java.math.BigDecimal; | ||
| import java.math.BigInteger; | ||
| import java.util.Arrays; | ||
| import java.util.List; | ||
| import java.util.function.Supplier; | ||
|
|
||
| import static org.hamcrest.Matchers.containsString; | ||
|
|
||
| public class NumberFieldTypeTests extends FieldTypeTestCase { | ||
|
|
||
| NumberType type; | ||
|
|
@@ -266,8 +271,8 @@ public void testHalfFloatRange() throws IOException { | |
| IndexSearcher searcher = newSearcher(reader); | ||
| final int numQueries = 1000; | ||
| for (int i = 0; i < numQueries; ++i) { | ||
| float l = (randomFloat() * 2 - 1) * 70000; | ||
| float u = (randomFloat() * 2 - 1) * 70000; | ||
| float l = (randomFloat() * 2 - 1) * 65504; | ||
| float u = (randomFloat() * 2 - 1) * 65504; | ||
| boolean includeLower = randomBoolean(); | ||
| boolean includeUpper = randomBoolean(); | ||
| Query floatQ = NumberFieldMapper.NumberType.FLOAT.rangeQuery("float", l, u, includeLower, includeUpper, false); | ||
|
|
@@ -348,4 +353,61 @@ public void doTestDocValueRangeQueries(NumberType type, Supplier<Number> valueSu | |
| reader.close(); | ||
| dir.close(); | ||
| } | ||
|
|
||
| public void testParseOutOfRangeValues() throws IOException { | ||
| final List<Triple<NumberType, Object, String>> inputs = Arrays.asList( | ||
| Triple.of(NumberType.BYTE, "128", "Value out of range"), | ||
| Triple.of(NumberType.SHORT, "32768", "Value out of range"), | ||
| Triple.of(NumberType.INTEGER, "2147483648", "For input string"), | ||
| Triple.of(NumberType.LONG, "9223372036854775808", "For input string"), | ||
|
|
||
| Triple.of(NumberType.BYTE, 128, "is out of range for a byte"), | ||
| Triple.of(NumberType.SHORT, 32768, "is out of range for a short"), | ||
| Triple.of(NumberType.INTEGER, 2147483648L, "is out of range for an integer"), | ||
| Triple.of(NumberType.LONG, new BigInteger("92233720368547758080"), " is out of range for a long"), | ||
|
|
||
| Triple.of(NumberType.HALF_FLOAT, "65504.1", "[half_float] supports only finite values"), | ||
| Triple.of(NumberType.FLOAT, "3.4028235E39", "[float] supports only finite values"), | ||
| Triple.of(NumberType.DOUBLE, "1.7976931348623157E309", "[double] supports only finite values"), | ||
|
|
||
| Triple.of(NumberType.HALF_FLOAT, 65504.1, "[half_float] supports only finite values"), | ||
| Triple.of(NumberType.FLOAT, 3.4028235E39, "[float] supports only finite values"), | ||
| Triple.of(NumberType.DOUBLE, new BigDecimal("1.7976931348623157E309"), "[double] supports only finite values"), | ||
|
|
||
| Triple.of(NumberType.HALF_FLOAT, Float.NaN, "[half_float] supports only finite values"), | ||
| Triple.of(NumberType.FLOAT, Float.NaN, "[float] supports only finite values"), | ||
| Triple.of(NumberType.DOUBLE, Double.NaN, "[double] supports only finite values"), | ||
|
|
||
| Triple.of(NumberType.HALF_FLOAT, Float.POSITIVE_INFINITY, "[half_float] supports only finite values"), | ||
| Triple.of(NumberType.FLOAT, Float.POSITIVE_INFINITY, "[float] supports only finite values"), | ||
| Triple.of(NumberType.DOUBLE, Double.POSITIVE_INFINITY, "[double] supports only finite values") | ||
| ); | ||
|
|
||
| for (Triple<NumberType, Object, String> item: inputs) { | ||
| try { | ||
| item.type.parse(item.value, false); | ||
| fail("Parsing exception expected for [" + item.type + "] with value [" + item.value + "]"); | ||
| } catch (IllegalArgumentException e) { | ||
| assertThat("Incorrect error message for [" + item.type + "] with value [" + item.value + "]", | ||
| e.getMessage(), containsString(item.message)); | ||
| } | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. could you use
Contributor
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Despite expectThrows is more concise and readable, I think we should keep try/fail/catch/assert because
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. fair enough |
||
| } | ||
| } | ||
|
|
||
| private static class Triple<K,V,M> { | ||
|
|
||
| final K type; | ||
| final V value; | ||
| final M message; | ||
|
|
||
| static <K,V,M> Triple<K,V,M> of(K t, V v, M m) { | ||
| return new Triple<>(t, v, m); | ||
| } | ||
|
|
||
| Triple(K t, V v, M m) { | ||
| type = t; | ||
| value = v; | ||
| message = m; | ||
| } | ||
| } | ||
| } | ||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
let's store it as a float in order to delay boxing as much as possible?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
agreed, I will fix in half_float, float and double.