From 35fac68ed28833b2d5d5040deea36e37bae6b01d Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 5 Jan 2016 14:48:08 -0500 Subject: [PATCH 1/4] Checkpoint --- .../fielddata/IndexFieldDataService.java | 5 + .../fielddata/IndexNumericFieldData.java | 16 + .../plain/DocValuesIndexFieldData.java | 2 +- .../plain/SortedNumericDVIndexFieldData.java | 94 ++++- .../index/mapper/MapperBuilders.java | 5 + .../mapper/fixed/FixedPointFieldMapper.java | 390 ++++++++++++++++++ .../elasticsearch/indices/IndicesModule.java | 2 + 7 files changed, 511 insertions(+), 3 deletions(-) create mode 100644 core/src/main/java/org/elasticsearch/index/mapper/fixed/FixedPointFieldMapper.java diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java index 8ac0bda2f0ba3..3201f9d35d5b0 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexFieldDataService.java @@ -89,6 +89,7 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo buildersByTypeBuilder.put(IndexFieldMapper.NAME, new IndexIndexFieldData.Builder()); buildersByTypeBuilder.put("binary", new DisabledIndexFieldData.Builder()); buildersByTypeBuilder.put(BooleanFieldMapper.CONTENT_TYPE, MISSING_DOC_VALUES_BUILDER); + buildersByTypeBuilder.put("fixed", MISSING_DOC_VALUES_BUILDER); buildersByType = unmodifiableMap(buildersByTypeBuilder); @@ -103,6 +104,7 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo .put("geo_point", new AbstractGeoPointDVIndexFieldData.Builder()) .put("binary", new BytesBinaryDVIndexFieldData.Builder()) .put(BooleanFieldMapper.CONTENT_TYPE, new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BOOLEAN)) + .put("fixed", new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FIXED)) .immutableMap(); buildersByTypeAndFormat = MapBuilder., IndexFieldData.Builder>newMapBuilder() @@ -138,6 +140,9 @@ public class IndexFieldDataService extends AbstractIndexComponent implements Clo .put(Tuple.tuple(BooleanFieldMapper.CONTENT_TYPE, DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.BOOLEAN)) .put(Tuple.tuple(BooleanFieldMapper.CONTENT_TYPE, DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + .put(Tuple.tuple("fixed", DOC_VALUES_FORMAT), new DocValuesIndexFieldData.Builder().numericType(IndexNumericFieldData.NumericType.FIXED)) + .put(Tuple.tuple("fixed", DISABLED_FORMAT), new DisabledIndexFieldData.Builder()) + .immutableMap(); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java index 44d90d9dd1325..0008357e146ab 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java @@ -150,6 +150,22 @@ public void toIndexForm(Number number, BytesRefBuilder bytes) { public Number toNumber(BytesRef indexForm) { return NumericUtils.sortableLongToDouble(NumericUtils.prefixCodedToLong(indexForm)); } + }, + FIXED(64, false, SortField.Type.LONG, Long.MIN_VALUE, Long.MAX_VALUE) { + @Override + public long toLong(BytesRef indexForm) { + return NumericUtils.prefixCodedToLong(indexForm); + } + + @Override + public void toIndexForm(Number number, BytesRefBuilder bytes) { + NumericUtils.longToPrefixCodedBytes(number.longValue(), 0, bytes); + } + + @Override + public Number toNumber(BytesRef indexForm) { + return NumericUtils.prefixCodedToLong(indexForm); + } }; private final int requiredBits; diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java index 27db531a2186f..420d94cee8fcb 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/DocValuesIndexFieldData.java @@ -102,7 +102,7 @@ public IndexFieldData build(IndexSettings indexSettings, MappedFieldType fiel assert numericType == null; return new BinaryDVIndexFieldData(indexSettings.getIndex(), fieldName, fieldType.fieldDataType()); } else if (numericType != null) { - return new SortedNumericDVIndexFieldData(indexSettings.getIndex(), fieldName, numericType, fieldType.fieldDataType()); + return new SortedNumericDVIndexFieldData(indexSettings.getIndex(), fieldName, numericType, fieldType); } else { return new SortedSetDVOrdinalsIndexFieldData(indexSettings, cache, fieldName, breakerService, fieldType.fieldDataType()); } diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java index 3e0dffa89ac7b..0ba33d005e007 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java @@ -38,6 +38,8 @@ import org.elasticsearch.index.fielddata.fieldcomparator.DoubleValuesComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.FloatValuesComparatorSource; import org.elasticsearch.index.fielddata.fieldcomparator.LongValuesComparatorSource; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.fixed.FixedPointFieldMapper; import org.elasticsearch.search.MultiValueMode; import java.io.IOException; @@ -50,13 +52,15 @@ */ public class SortedNumericDVIndexFieldData extends DocValuesIndexFieldData implements IndexNumericFieldData { private final NumericType numericType; + private final MappedFieldType fieldType; - public SortedNumericDVIndexFieldData(Index index, String fieldNames, NumericType numericType, FieldDataType fieldDataType) { - super(index, fieldNames, fieldDataType); + public SortedNumericDVIndexFieldData(Index index, String fieldNames, NumericType numericType, MappedFieldType fieldType) { + super(index, fieldNames, fieldType.fieldDataType()); if (numericType == null) { throw new IllegalArgumentException("numericType must be non-null"); } this.numericType = numericType; + this.fieldType = fieldType; } @Override @@ -66,6 +70,8 @@ public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource c return new FloatValuesComparatorSource(this, missingValue, sortMode, nested); case DOUBLE: return new DoubleValuesComparatorSource(this, missingValue, sortMode, nested); + case FIXED: + return new LongValuesComparatorSource(this, missingValue, sortMode, nested); default: assert !numericType.isFloatingPoint(); return new LongValuesComparatorSource(this, missingValue, sortMode, nested); @@ -92,6 +98,9 @@ public AtomicNumericFieldData load(LeafReaderContext context) { return new SortedNumericFloatFieldData(reader, field); case DOUBLE: return new SortedNumericDoubleFieldData(reader, field); + case FIXED: + long decimalFactor = ((FixedPointFieldMapper.FixedPointFieldType)fieldType).getDecimalFactor(); + return new SortedNumericFixedPointFieldData(reader, field, decimalFactor); default: return new SortedNumericLongFieldData(reader, field); } @@ -264,4 +273,85 @@ public Collection getChildResources() { return Collections.emptyList(); } } + + /* + static final class SortedNumericFixedPointFieldData extends AtomicLongFieldData { + final LeafReader reader; + final String field; + + SortedNumericFixedPointFieldData(LeafReader reader, String field) { + super(0L); + this.reader = reader; + this.field = field; + } + + @Override + public SortedNumericDocValues getLongValues() { + try { + return DocValues.getSortedNumeric(reader, field); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + } + */ + + static final class SortedNumericFixedPointFieldData extends AtomicDoubleFieldData { + final LeafReader reader; + final String field; + final long decimalFactor; + + SortedNumericFixedPointFieldData(LeafReader reader, String field, long decimalFactor) { + super(0L); + this.reader = reader; + this.field = field; + this.decimalFactor = decimalFactor; + } + + @Override + public SortedNumericDoubleValues getDoubleValues() { + try { + SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field); + return new SortedNumericFixedPointDocValues(raw, decimalFactor); + } catch (IOException e) { + throw new IllegalStateException("Cannot load doc values", e); + } + } + + @Override + public Collection getChildResources() { + return Collections.emptyList(); + } + + public static class SortedNumericFixedPointDocValues extends SortedNumericDoubleValues { + private final long decimalFactor; + private final SortedNumericDocValues dv; + + protected SortedNumericFixedPointDocValues(SortedNumericDocValues dv, long decimalFactor) { + super(); + this.decimalFactor = decimalFactor; + this.dv = dv; + } + + @Override + public void setDocument(int doc) { + dv.setDocument(doc); + } + + @Override + public double valueAt(int index) { + return ((double)dv.valueAt(index)) / decimalFactor; + } + + @Override + public int count() { + return dv.count(); + } + } + } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java b/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java index 9ea9e99f01be8..631b37fbd145a 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/MapperBuilders.java @@ -31,6 +31,7 @@ import org.elasticsearch.index.mapper.core.ShortFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.core.TokenCountFieldMapper; +import org.elasticsearch.index.mapper.fixed.FixedPointFieldMapper; import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.ip.IpFieldMapper; import org.elasticsearch.index.mapper.object.ObjectMapper; @@ -107,4 +108,8 @@ public static GeoShapeFieldMapper.Builder geoShapeField(String name) { public static CompletionFieldMapper.Builder completionField(String name) { return new CompletionFieldMapper.Builder(name); } + + public static FixedPointFieldMapper.Builder fixedPointField(String name) { + return new FixedPointFieldMapper.Builder(name); + } } diff --git a/core/src/main/java/org/elasticsearch/index/mapper/fixed/FixedPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/fixed/FixedPointFieldMapper.java new file mode 100644 index 0000000000000..cf5afd78228f9 --- /dev/null +++ b/core/src/main/java/org/elasticsearch/index/mapper/fixed/FixedPointFieldMapper.java @@ -0,0 +1,390 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.fixed; + +import org.apache.lucene.analysis.Analyzer; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.IndexOptions; +import org.apache.lucene.index.Terms; +import org.apache.lucene.search.NumericRangeQuery; +import org.apache.lucene.search.Query; +import org.apache.lucene.util.BytesRef; +import org.apache.lucene.util.BytesRefBuilder; +import org.apache.lucene.util.NumericUtils; +import org.elasticsearch.Version; +import org.elasticsearch.action.fieldstats.FieldStats; +import org.elasticsearch.common.Explicit; +import org.elasticsearch.common.Numbers; +import org.elasticsearch.common.Strings; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.unit.Fuzziness; +import org.elasticsearch.common.xcontent.XContentBuilder; +import org.elasticsearch.common.xcontent.XContentParser; +import org.elasticsearch.index.analysis.NamedAnalyzer; +import org.elasticsearch.index.analysis.NumericLongAnalyzer; +import org.elasticsearch.index.mapper.MappedFieldType; +import org.elasticsearch.index.mapper.Mapper; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext; +import org.elasticsearch.index.mapper.core.NumberFieldMapper; + +import java.io.IOException; +import java.util.Iterator; +import java.util.List; +import java.util.Map; + +import static org.elasticsearch.common.xcontent.support.XContentMapValues.nodeLongValue; +import static org.elasticsearch.index.mapper.MapperBuilders.fixedPointField; +import static org.elasticsearch.index.mapper.MapperBuilders.longField; +import static org.elasticsearch.index.mapper.core.TypeParsers.parseNumberField; + +/** + * + */ +public class FixedPointFieldMapper extends NumberFieldMapper { + + public static final String CONTENT_TYPE = "fixed"; + + public static class Defaults extends NumberFieldMapper.Defaults { + public static final MappedFieldType FIELD_TYPE = new FixedPointFieldType(); + public static final long DECIMAL_PLACES = 2; + public static final long DECIMAL_FACTOR = 100; + + static { + FIELD_TYPE.freeze(); + } + } + + public static class Builder extends NumberFieldMapper.Builder { + + public Builder(String name) { + super(name, Defaults.FIELD_TYPE, Defaults.PRECISION_STEP_64_BIT); + builder = this; + } + + public Builder nullValue(long nullValue) { + this.fieldType.setNullValue(nullValue); + return this; + } + + public Builder decimalPlaces(long decimalPlaces) { + fieldType().setDecimalPlaces(decimalPlaces); + return this; + } + + @Override + public FixedPointFieldType fieldType() { + return (FixedPointFieldType)fieldType; + } + + @Override + public FixedPointFieldMapper build(BuilderContext context) { + setupFieldType(context); + FixedPointFieldMapper fieldMapper = new FixedPointFieldMapper(name, fieldType, defaultFieldType, + ignoreMalformed(context), coerce(context), context.indexSettings(), multiFieldsBuilder.build(this, context), copyTo); + return (FixedPointFieldMapper) fieldMapper.includeInAll(includeInAll); + } + + @Override + protected NamedAnalyzer makeNumberAnalyzer(int precisionStep) { + return NumericLongAnalyzer.buildNamedAnalyzer(precisionStep); + } + + @Override + protected int maxPrecisionStep() { + return 64; + } + } + + public static class TypeParser implements Mapper.TypeParser { + @Override + public Mapper.Builder parse(String name, Map node, ParserContext parserContext) throws MapperParsingException { + FixedPointFieldMapper.Builder builder = fixedPointField(name); + parseNumberField(builder, name, node, parserContext); + for (Iterator> iterator = node.entrySet().iterator(); iterator.hasNext();) { + Map.Entry entry = iterator.next(); + String propName = Strings.toUnderscoreCase(entry.getKey()); + Object propNode = entry.getValue(); + if (propName.equals("null_value")) { + if (propNode == null) { + throw new MapperParsingException("Property [null_value] cannot be null."); + } + builder.nullValue(nodeLongValue(propNode)); + iterator.remove(); + } else if (propName.equals("decimal_places")) { + if (propNode == null) { + throw new MapperParsingException("Property [decimal_places] cannot be null."); + } + long decimalPlaces = nodeLongValue(propNode); + if (decimalPlaces <= 0) { + throw new MapperParsingException("Property [decimal_places] must be greater than zero."); + } + if (decimalPlaces >= 19) { + // would cause an overflow in decimalFactor later + // + // Long.MAX_VALUE == 9223372036854775807 + // 10^19 == 10000000000000000000 + throw new MapperParsingException("Property [decimal_places] must be smaller than 19."); + } + + builder.decimalPlaces((byte)decimalPlaces); + iterator.remove(); + } + } + return builder; + } + } + + public static class FixedPointFieldType extends NumberFieldType { + private long decimalPlaces = Defaults.DECIMAL_PLACES; + private long decimalFactor = Defaults.DECIMAL_FACTOR; + + public FixedPointFieldType() { + super(NumericType.LONG); + } + + protected FixedPointFieldType(FixedPointFieldType ref) { + super(ref); + } + + public void setDecimalPlaces(long decimalPlaces) { + this.decimalPlaces = decimalPlaces; + this.decimalFactor = (long) Math.pow(10.0, decimalPlaces); + } + + public long getDecimalPlaces() { + return this.decimalPlaces; + } + + public long getDecimalFactor() { + return this.decimalFactor; + } + + @Override + public NumberFieldType clone() { + return new FixedPointFieldType(this); + } + + @Override + public String typeName() { + return CONTENT_TYPE; + } + + @Override + public Double nullValue() { + return (Double)super.nullValue(); + } + + @Override + public Long value(Object value) { + if (value == null) { + return null; + } + if (value instanceof Number) { + return ((Number) value).longValue(); + } + if (value instanceof BytesRef) { + return Numbers.bytesToLong((BytesRef) value); + } + return Long.parseLong(value.toString()); + } + + @Override + public BytesRef indexedValueForSearch(Object value) { + long longValue = (long)(parseDoubleValue(value) * decimalFactor); + BytesRefBuilder bytesRef = new BytesRefBuilder(); + NumericUtils.longToPrefixCoded(longValue, 0, bytesRef); // 0 because of exact match + return bytesRef.get(); + } + + @Override + public Query rangeQuery(Object lowerTerm, Object upperTerm, boolean includeLower, boolean includeUpper) { + return NumericRangeQuery.newLongRange(name(), numericPrecisionStep(), + lowerTerm == null ? null : (long) (parseDoubleValue(lowerTerm) * decimalFactor), + upperTerm == null ? null : (long) (parseDoubleValue(upperTerm) * decimalFactor), + includeLower, includeUpper); + } + + @Override + public Query fuzzyQuery(Object value, Fuzziness fuzziness, int prefixLength, int maxExpansions, boolean transpositions) { + final long iValue = (long)(parseDoubleValue(value) * decimalFactor); + final long iSim = fuzziness.asLong(); + return NumericRangeQuery.newLongRange(name(), numericPrecisionStep(), + iValue - iSim, + iValue + iSim, + true, true); + } + + @Override + public FieldStats stats(Terms terms, int maxDoc) throws IOException { + long minValue = NumericUtils.getMinLong(terms); + long maxValue = NumericUtils.getMaxLong(terms); + return new FieldStats.Long( + maxDoc, terms.getDocCount(), terms.getSumDocFreq(), terms.getSumTotalTermFreq(), minValue, maxValue + ); + } + } + + protected FixedPointFieldMapper(String simpleName, MappedFieldType fieldType, MappedFieldType defaultFieldType, + Explicit ignoreMalformed, Explicit coerce, + Settings indexSettings, MultiFields multiFields, CopyTo copyTo) { + super(simpleName, fieldType, defaultFieldType, ignoreMalformed, coerce, indexSettings, multiFields, copyTo); + } + + @Override + public FixedPointFieldType fieldType() { + return (FixedPointFieldType) super.fieldType(); + } + + @Override + protected boolean customBoost() { + return true; + } + + @Override + protected void innerParseCreateField(ParseContext context, List fields) throws IOException { + double value; + float boost = fieldType().boost(); + if (context.externalValueSet()) { + Object externalValue = context.externalValue(); + if (externalValue == null) { + if (fieldType().nullValue() == null) { + return; + } + value = fieldType().nullValue(); + } else if (externalValue instanceof String) { + String sExternalValue = (String) externalValue; + if (sExternalValue.length() == 0) { + if (fieldType().nullValue() == null) { + return; + } + value = fieldType().nullValue(); + } else { + value = Double.parseDouble(sExternalValue); + } + } else { + value = ((Number) externalValue).doubleValue(); + } + if (context.includeInAll(includeInAll, this)) { + context.allEntries().addText(fieldType().name(), Double.toString(value), boost); + } + } else { + XContentParser parser = context.parser(); + if (parser.currentToken() == XContentParser.Token.VALUE_NULL || + (parser.currentToken() == XContentParser.Token.VALUE_STRING && parser.textLength() == 0)) { + if (fieldType().nullValue() == null) { + return; + } + value = fieldType().nullValue(); + if (fieldType().nullValueAsString() != null && (context.includeInAll(includeInAll, this))) { + context.allEntries().addText(fieldType().name(), fieldType().nullValueAsString(), boost); + } + } else if (parser.currentToken() == XContentParser.Token.START_OBJECT + && Version.indexCreated(context.indexSettings()).before(Version.V_3_0_0)) { + XContentParser.Token token; + String currentFieldName = null; + Double objValue = fieldType().nullValue(); + while ((token = parser.nextToken()) != XContentParser.Token.END_OBJECT) { + if (token == XContentParser.Token.FIELD_NAME) { + currentFieldName = parser.currentName(); + } else { + if ("value".equals(currentFieldName) || "_value".equals(currentFieldName)) { + if (parser.currentToken() != XContentParser.Token.VALUE_NULL) { + objValue = parser.doubleValue(coerce.value()); + } + } else if ("boost".equals(currentFieldName) || "_boost".equals(currentFieldName)) { + boost = parser.floatValue(); + } else { + throw new IllegalArgumentException("unknown property [" + currentFieldName + "]"); + } + } + } + if (objValue == null) { + // no value + return; + } + value = objValue; + } else { + value = parser.doubleValue(coerce.value()); + if (context.includeInAll(includeInAll, this)) { + context.allEntries().addText(fieldType().name(), parser.text(), boost); + } + } + } + final long scaledValue = (long)(value * (double)fieldType().decimalFactor); + if (fieldType().indexOptions() != IndexOptions.NONE || fieldType().stored()) { + CustomFixedPointNumericField field = new CustomFixedPointNumericField(scaledValue, fieldType().decimalFactor, fieldType()); + field.setBoost(boost); + fields.add(field); + } + if (fieldType().hasDocValues()) { + addDocValue(context, fields, scaledValue); + } + } + + @Override + protected String contentType() { + return CONTENT_TYPE; + } + + @Override + protected void doXContentBody(XContentBuilder builder, boolean includeDefaults, Params params) throws IOException { + super.doXContentBody(builder, includeDefaults, params); + + if (includeDefaults || fieldType().numericPrecisionStep() != Defaults.PRECISION_STEP_64_BIT) { + builder.field("precision_step", fieldType().numericPrecisionStep()); + } + if (includeDefaults || fieldType().nullValue() != null) { + builder.field("null_value", fieldType().nullValue()); + } + if (includeInAll != null) { + builder.field("include_in_all", includeInAll); + } else if (includeDefaults) { + builder.field("include_in_all", false); + } + builder.field("decimal_places", fieldType().decimalPlaces); + } + + public static class CustomFixedPointNumericField extends CustomNumericField { + + private final long number; + private final long decimalFactor; + + public CustomFixedPointNumericField(long number, long decimalFactor, MappedFieldType fieldType) { + super(number, fieldType); + this.number = number; + this.decimalFactor = decimalFactor; + } + + @Override + public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) throws IOException { + if (fieldType().indexOptions() != IndexOptions.NONE) { + return getCachedStream().setLongValue(number); + } + return null; + } + + @Override + public String numericAsString() { + return Double.toString(number / decimalFactor); + } + } +} diff --git a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java index faf7f73dc2307..f6141794843e6 100644 --- a/core/src/main/java/org/elasticsearch/indices/IndicesModule.java +++ b/core/src/main/java/org/elasticsearch/indices/IndicesModule.java @@ -40,6 +40,7 @@ import org.elasticsearch.index.mapper.core.ShortFieldMapper; import org.elasticsearch.index.mapper.core.StringFieldMapper; import org.elasticsearch.index.mapper.core.TokenCountFieldMapper; +import org.elasticsearch.index.mapper.fixed.FixedPointFieldMapper; import org.elasticsearch.index.mapper.geo.GeoPointFieldMapper; import org.elasticsearch.index.mapper.geo.GeoShapeFieldMapper; import org.elasticsearch.index.mapper.internal.AllFieldMapper; @@ -211,6 +212,7 @@ private void registerBuiltInMappers() { registerMapper(ObjectMapper.NESTED_CONTENT_TYPE, new ObjectMapper.TypeParser()); registerMapper(CompletionFieldMapper.CONTENT_TYPE, new CompletionFieldMapper.TypeParser()); registerMapper(GeoPointFieldMapper.CONTENT_TYPE, new GeoPointFieldMapper.TypeParser()); + registerMapper(FixedPointFieldMapper.CONTENT_TYPE, new FixedPointFieldMapper.TypeParser()); registerMapper(PercolatorFieldMapper.CONTENT_TYPE, new PercolatorFieldMapper.TypeParser()); if (ShapesAvailability.JTS_AVAILABLE && ShapesAvailability.SPATIAL4J_AVAILABLE) { From 114028e1ec52a8269588f2a5888f2fe036922c4f Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 12 Jan 2016 14:29:43 -0500 Subject: [PATCH 2/4] Add tests --- .../mapper/fixed/FixedPointFieldMapper.java | 11 +- .../index/mapper/fixed/FixedPointTests.java | 460 ++++++++++++++++++ 2 files changed, 470 insertions(+), 1 deletion(-) create mode 100644 core/src/test/java/org/elasticsearch/index/mapper/fixed/FixedPointTests.java diff --git a/core/src/main/java/org/elasticsearch/index/mapper/fixed/FixedPointFieldMapper.java b/core/src/main/java/org/elasticsearch/index/mapper/fixed/FixedPointFieldMapper.java index cf5afd78228f9..bb56ade84acfe 100644 --- a/core/src/main/java/org/elasticsearch/index/mapper/fixed/FixedPointFieldMapper.java +++ b/core/src/main/java/org/elasticsearch/index/mapper/fixed/FixedPointFieldMapper.java @@ -384,7 +384,16 @@ public TokenStream tokenStream(Analyzer analyzer, TokenStream previous) throws I @Override public String numericAsString() { - return Double.toString(number / decimalFactor); + return Double.toString(ToDouble()); + } + + @Override + public Number numericValue() { + return ToDouble(); + } + + private double ToDouble() { + return (double)number / (double)decimalFactor; } } } diff --git a/core/src/test/java/org/elasticsearch/index/mapper/fixed/FixedPointTests.java b/core/src/test/java/org/elasticsearch/index/mapper/fixed/FixedPointTests.java new file mode 100644 index 0000000000000..53b9a0a9423f7 --- /dev/null +++ b/core/src/test/java/org/elasticsearch/index/mapper/fixed/FixedPointTests.java @@ -0,0 +1,460 @@ +/* + * Licensed to Elasticsearch under one or more contributor + * license agreements. See the NOTICE file distributed with + * this work for additional information regarding copyright + * ownership. Elasticsearch licenses this file to you under + * the Apache License, Version 2.0 (the "License"); you may + * not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.elasticsearch.index.mapper.fixed; + +import org.apache.lucene.analysis.NumericTokenStream; +import org.apache.lucene.analysis.TokenStream; +import org.apache.lucene.document.Field; +import org.apache.lucene.index.DocValuesType; +import org.apache.lucene.index.IndexableField; +import org.elasticsearch.action.search.SearchResponse; +import org.elasticsearch.common.compress.CompressedXContent; +import org.elasticsearch.common.settings.Settings; +import org.elasticsearch.common.xcontent.XContentFactory; +import org.elasticsearch.index.IndexService; +import org.elasticsearch.index.mapper.DocumentMapper; +import org.elasticsearch.index.mapper.DocumentMapperParser; +import org.elasticsearch.index.mapper.MapperParsingException; +import org.elasticsearch.index.mapper.ParseContext.Document; +import org.elasticsearch.index.mapper.ParsedDocument; +import org.elasticsearch.index.mapper.core.NumberFieldMapper; +import org.elasticsearch.index.mapper.string.SimpleStringMappingTests; +import org.elasticsearch.test.ESSingleNodeTestCase; + +import java.io.IOException; + +import static org.elasticsearch.common.settings.Settings.settingsBuilder; +import static org.elasticsearch.index.query.QueryBuilders.rangeQuery; +import static org.elasticsearch.index.query.QueryBuilders.termQuery; +import static org.elasticsearch.test.hamcrest.DoubleMatcher.nearlyEqual; +import static org.elasticsearch.test.hamcrest.ElasticsearchAssertions.assertHitCount; +import static org.hamcrest.Matchers.instanceOf; +import static org.hamcrest.Matchers.notNullValue; +import static org.hamcrest.Matchers.nullValue; + +/** + */ +public class FixedPointTests extends ESSingleNodeTestCase { + + public void testInvalidDecimalPlaces() throws Exception { + IndexService index = createIndex("test"); + + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field1").field("type", "fixed").field("decimal_places", 0).endObject() + .endObject() + .endObject().endObject().string(); + + try { + client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get(); + fail("Should have thrown MapperParsingException[Property [decimal_places] must be greater than zero.]"); + } catch (MapperParsingException e) { + assertEquals(e.getMessage(), "Property [decimal_places] must be greater than zero."); + } + + mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field1").field("type", "fixed").field("decimal_places", 20).endObject() + .endObject() + .endObject().endObject().string(); + + try { + client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get(); + fail("Should have thrown MapperParsingException[Property [decimal_places] must be smaller than 19.]"); + } catch (MapperParsingException e) { + assertEquals(e.getMessage(), "Property [decimal_places] must be smaller than 19."); + } + + } + + public void testDecimalPlaces() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field1").field("type", "fixed").field("decimal_places", 1).endObject() + .startObject("field2").field("type", "fixed").field("decimal_places", 2).endObject() + .startObject("field3").field("type", "fixed").field("decimal_places", 3).endObject() + .startObject("field4").field("type", "fixed").endObject() // default: 2 + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + + double doubleValue = 1.2345; + + ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field1", doubleValue) + .field("field2", doubleValue) + .field("field3", doubleValue) + .field("field4", doubleValue) + .endObject() + .bytes()); + + assertTrue(nearlyEqual(1.2, doc.rootDoc().getField("field1").numericValue().doubleValue(), 0.01)); + assertTrue(nearlyEqual(1.23, doc.rootDoc().getField("field2").numericValue().doubleValue(), 0.01)); + assertTrue(nearlyEqual(1.234, doc.rootDoc().getField("field3").numericValue().doubleValue(), 0.01)); + assertTrue(nearlyEqual(1.23, doc.rootDoc().getField("field4").numericValue().doubleValue(), 0.01)); + } + + public void testIgnoreMalformedOption() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field1").field("type", "fixed").field("ignore_malformed", true).endObject() + .startObject("field2").field("type", "fixed").field("ignore_malformed", false).endObject() + .startObject("field3").field("type", "fixed").endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + + ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field1", "a") + .field("field2", "1") + .endObject() + .bytes()); + assertThat(doc.rootDoc().getField("field1"), nullValue()); + assertThat(doc.rootDoc().getField("field2"), notNullValue()); + + try { + defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field2", "a") + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(NumberFormatException.class)); + } + + // Verify that the default is false + try { + defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field3", "a") + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(NumberFormatException.class)); + } + + // Unless the global ignore_malformed option is set to true + Settings indexSettings = settingsBuilder().put("index.mapping.ignore_malformed", true).build(); + defaultMapper = createIndex("test2", indexSettings).mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field3", "a") + .endObject() + .bytes()); + assertThat(doc.rootDoc().getField("field3"), nullValue()); + + // This should still throw an exception, since field2 is specifically set to ignore_malformed=false + try { + defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("field2", "a") + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(NumberFormatException.class)); + } + } + + public void testCoerceOption() throws Exception { + + DocumentMapperParser parser = createIndex("test").mapperService().documentMapperParser(); + + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("noErrorNoCoerceField").field("type", "fixed").field("ignore_malformed", true) + .field("coerce", false).endObject() + .startObject("noErrorCoerceField").field("type", "fixed").field("ignore_malformed", true) + .field("coerce", true).endObject() + .startObject("errorDefaultCoerce").field("type", "fixed").field("ignore_malformed", false).endObject() + .startObject("errorNoCoerce").field("type", "fixed").field("ignore_malformed", false) + .field("coerce", false).endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = parser.parse("type", new CompressedXContent(mapping)); + + //Test numbers passed as strings + String invalidJsonNumberAsString="1"; + ParsedDocument doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", invalidJsonNumberAsString) + .field("noErrorCoerceField", invalidJsonNumberAsString) + .field("errorDefaultCoerce", invalidJsonNumberAsString) + .endObject() + .bytes()); + assertThat(doc.rootDoc().getField("noErrorNoCoerceField"), nullValue()); + assertThat(doc.rootDoc().getField("noErrorCoerceField"), notNullValue()); + //Default is ignore_malformed=true and coerce=true + assertThat(doc.rootDoc().getField("errorDefaultCoerce"), notNullValue()); + + //Test valid case of numbers passed as numbers + int validNumber=1; + doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", validNumber) + .field("noErrorCoerceField", validNumber) + .field("errorDefaultCoerce", validNumber) + .endObject() + .bytes()); + assertEquals(validNumber, doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue()); + assertEquals(validNumber,doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); + assertEquals(validNumber,doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); + + //Test valid case of negative numbers passed as numbers + int validNegativeNumber=-1; + doc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("noErrorNoCoerceField", validNegativeNumber) + .field("noErrorCoerceField", validNegativeNumber) + .field("errorDefaultCoerce", validNegativeNumber) + .endObject() + .bytes()); + assertEquals(validNegativeNumber, doc.rootDoc().getField("noErrorNoCoerceField").numericValue().intValue()); + assertEquals(validNegativeNumber, doc.rootDoc().getField("noErrorCoerceField").numericValue().intValue()); + assertEquals(validNegativeNumber, doc.rootDoc().getField("errorDefaultCoerce").numericValue().intValue()); + + + try { + defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("errorNoCoerce", invalidJsonNumberAsString) + .endObject() + .bytes()); + } catch (MapperParsingException e) { + assertThat(e.getCause(), instanceOf(IllegalArgumentException.class)); + } + + } + + + public void testDocValues() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("fixed") + .field("type", "fixed") + .startObject("fielddata") + .field("format", "doc_values") + .endObject() + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + + ParsedDocument parsedDoc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("fixed", "1234") + .endObject() + .bytes()); + final Document doc = parsedDoc.rootDoc(); + assertEquals(DocValuesType.SORTED_NUMERIC, SimpleStringMappingTests.docValuesType(doc, "fixed")); + } + + public void testDocValuesOnNested() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("nested") + .field("type", "nested") + .startObject("properties") + .startObject("fixed") + .field("type", "fixed") + .startObject("fielddata") + .field("format", "doc_values") + .endObject() + .endObject() + .endObject() + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper defaultMapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + + ParsedDocument parsedDoc = defaultMapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .startArray("nested") + .startObject() + .field("fixed", "1234") + .endObject() + .startObject() + .field("fixed", "0.23") + .endObject() + .startObject() + .field("fixed", "-0.12") + .endObject() + .endArray() + .endObject() + .bytes()); + for (Document doc : parsedDoc.docs()) { + if (doc == parsedDoc.rootDoc()) { + continue; + } + assertEquals(DocValuesType.SORTED_NUMERIC, SimpleStringMappingTests.docValuesType(doc, "nested.fixed")); + } + } + + /** Test default precision step for numeric types */ + public void testPrecisionStepDefaultsMapped() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("fixed") + .field("type", "fixed") + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + + ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("fixed", "34.545") + .endObject() + .bytes()); + + assertEquals(1, doc.docs().size()); + Document luceneDoc = doc.docs().get(0); + + assertPrecisionStepEquals(NumberFieldMapper.Defaults.PRECISION_STEP_64_BIT, luceneDoc.getField("fixed")); + } + + /** Test precision step set to silly explicit values */ + public void testPrecisionStepExplicit() throws Exception { + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("fixed") + .field("type", "fixed") + .field("precision_step", "2") + .endObject() + .endObject() + .endObject().endObject().string(); + + DocumentMapper mapper = createIndex("test").mapperService().documentMapperParser().parse("type", new CompressedXContent(mapping)); + + ParsedDocument doc = mapper.parse("test", "type", "1", XContentFactory.jsonBuilder() + .startObject() + .field("fixed", "34.545") + .endObject() + .bytes()); + + assertEquals(1, doc.docs().size()); + Document luceneDoc = doc.docs().get(0); + + assertPrecisionStepEquals(2, luceneDoc.getField("fixed")); + } + + /** checks precisionstep on both the fieldtype and the tokenstream */ + private static void assertPrecisionStepEquals(int expected, IndexableField field) throws IOException { + assertNotNull(field); + assertThat(field, instanceOf(Field.class)); + + // check fieldtype's precisionstep + assertEquals(expected, ((Field)field).fieldType().numericPrecisionStep()); + + // check the tokenstream actually used by the indexer + TokenStream ts = field.tokenStream(null, null); + assertThat(ts, instanceOf(NumericTokenStream.class)); + assertEquals(expected, ((NumericTokenStream) ts).getPrecisionStep()); + } + + public void testBasicSearch() throws Exception { + IndexService index = createIndex("test"); + + String mapping = XContentFactory.jsonBuilder().startObject().startObject("type") + .startObject("properties") + .startObject("field1").field("type", "fixed").field("decimal_places", 2).endObject() + .endObject() + .endObject().endObject().string(); + + client().admin().indices().preparePutMapping("test").setType("type").setSource(mapping).get(); + + client().prepareIndex("test", "type", "1") + .setSource("field1", 1.2345).setRefresh(true).execute().actionGet(); + + // Check 1 decimal places, should not match + SearchResponse search = client().prepareSearch() + .setQuery(termQuery("field1", 1.2)) + .execute().actionGet(); + + assertHitCount(search, 0l); + + // Check 2 decimal places + search = client().prepareSearch() + .setQuery(termQuery("field1", 1.23)) + .execute().actionGet(); + + assertHitCount(search, 1l); + assertEquals(search.getHits().getAt(0).getSource().get("field1"), 1.2345); + + // Check 3 decimal places -- should truncate down to 2 + search = client().prepareSearch() + .setQuery(termQuery("field1", 1.239)) + .execute().actionGet(); + + assertHitCount(search, 1l); + assertEquals(search.getHits().getAt(0).getSource().get("field1"), 1.2345); + + // Check 4 decimal places -- should truncate down to 2 + search = client().prepareSearch() + .setQuery(termQuery("field1", 1.2349)) + .execute().actionGet(); + + assertHitCount(search, 1l); + assertEquals(search.getHits().getAt(0).getSource().get("field1"), 1.2345); + + + // Check ranges + search = client().prepareSearch() + .setQuery(rangeQuery("field1").gte(1.0).lte(2.0)) + .execute().actionGet(); + + assertHitCount(search, 1l); + assertEquals(search.getHits().getAt(0).getSource().get("field1"), 1.2345); + + search = client().prepareSearch() + .setQuery(rangeQuery("field1").gte(1.23).lte(2.0)) + .execute().actionGet(); + + assertHitCount(search, 1l); + assertEquals(search.getHits().getAt(0).getSource().get("field1"), 1.2345); + + search = client().prepareSearch() + .setQuery(rangeQuery("field1").gte(1.0).lte(1.23)) + .execute().actionGet(); + + assertHitCount(search, 1l); + assertEquals(search.getHits().getAt(0).getSource().get("field1"), 1.2345); + + search = client().prepareSearch() + .setQuery(rangeQuery("field1").gte(1.2399).lte(2.0)) // extra decimal places truncated + .execute().actionGet(); + + assertHitCount(search, 1l); + assertEquals(search.getHits().getAt(0).getSource().get("field1"), 1.2345); + + search = client().prepareSearch() + .setQuery(rangeQuery("field1").gte(1.0).lte(1.2)) // not enough decimal places to match + .execute().actionGet(); + + assertHitCount(search, 0l); + } +} From 9b82ad21b8ee86ee833ce9d1ee47d990197b6ac1 Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 12 Jan 2016 15:07:23 -0500 Subject: [PATCH 3/4] Add documentation --- docs/reference/mapping/types.asciidoc | 2 +- docs/reference/mapping/types/numeric.asciidoc | 51 +++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) diff --git a/docs/reference/mapping/types.asciidoc b/docs/reference/mapping/types.asciidoc index 60d96577a4367..5b34b312b833c 100644 --- a/docs/reference/mapping/types.asciidoc +++ b/docs/reference/mapping/types.asciidoc @@ -8,7 +8,7 @@ document: === Core datatypes <>:: `string` -<>:: `long`, `integer`, `short`, `byte`, `double`, `float` +<>:: `long`, `integer`, `short`, `byte`, `double`, `float`, `fixed` <>:: `date` <>:: `boolean` <>:: `binary` diff --git a/docs/reference/mapping/types/numeric.asciidoc b/docs/reference/mapping/types/numeric.asciidoc index 77f5808e6b00c..8085054b00aeb 100644 --- a/docs/reference/mapping/types/numeric.asciidoc +++ b/docs/reference/mapping/types/numeric.asciidoc @@ -10,6 +10,7 @@ The following numeric types are supported: `byte`:: A signed 8-bit integer with a minimum value of +-128+ and a maximum value of +127+. `double`:: A double-precision 64-bit IEEE 754 floating point. `float`:: A single-precision 32-bit IEEE 754 floating point. +`fixed`:: A variable-precision representation of real numbers Below is an example of configuring a mapping with numeric fields: @@ -92,3 +93,53 @@ The following parameters are accepted by numeric types: (default). +==== fixed Type + +The `fixed` numeric type provides a variable-precision representation of real-valued quantities, +like a `float` or `double`. Precision is controlled by specifying `decimal_places`, which +controls how many significant digits should appear after the radix point ("decimal point"). + +For example, if `decimal_places` is set to `2`, the following values are all considered +equivalent: `1.23`, `1.234`, `1.23999999`. They are equivalent because only two digits are +stored after the decimal point, so all values are converted into `1.23`. + +Internally, `fixed` fields are stored as a `long` with an associated scaling factor. With +two decimal places, the value `1.23` is stored as `123` and has an associated scaling factor of +`100`. To obtain our original real-valued numeric, we simply divide the stored long by +the scaling factor: `123 / 100 == 1.23` + +Fixed point fields provide different precision/range tradeoffs compared to floating point values. +They are primarily useful when your data only contains a few significant digits (e.g. your +sensor only guarantees two sig-figs of real signal before noise takes over), or you only care +about `n` decimal places. In these cases, storing the data in a long saves considerable +on-disk space. Longs are much easier to compress in a random-access fashion compared to +floats / doubles, which are stored at their full width. + +Fixed point precision is controlled with the `decimal_places` parameter: + +[horizontal] + +`decimal_places`:: + Controls the number of significant digits that should be stored after the radix point + ("decimal point"). The value must be greater than `0` and smaller than `19`. + + +[WARNING] +==== +Increasing the size of `decimal_places` directly decreases the range of values that can be stored, +because more bits are going to the fractional portion. The range of a particular fixed point is +`[-9223372036854775807 * 10^n^, 9223372036854775807 * 10^n^]` where `n` is the number of decimal +places. + +For example, `n=18` will give you 18 decimal places of precision, but a range of roughly `-9.223` to +`9.223`. +==== + +[WARNING] +==== +Fixed point fields are similar to `Decimal` or `Currency` fields in databases, but *should not be +used as such*. Fixed point fields are cast to Doubles for use in aggregations, which means +floating point math (and all the caveats/trade-offs associated with floating point math) will be +used for calculations. Fixed-point arithmetic *is not* used. It is simply a storage optimization. +==== + From 878e1b087bc8af533ec54cc37be49d37a193ac8f Mon Sep 17 00:00:00 2001 From: Zachary Tong Date: Tue, 12 Jan 2016 15:17:00 -0500 Subject: [PATCH 4/4] Cleanup --- .../plain/SortedNumericDVIndexFieldData.java | 30 ++----------------- 1 file changed, 3 insertions(+), 27 deletions(-) diff --git a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java index 0ba33d005e007..5851bf299810b 100644 --- a/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java +++ b/core/src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java @@ -274,33 +274,6 @@ public Collection getChildResources() { } } - /* - static final class SortedNumericFixedPointFieldData extends AtomicLongFieldData { - final LeafReader reader; - final String field; - - SortedNumericFixedPointFieldData(LeafReader reader, String field) { - super(0L); - this.reader = reader; - this.field = field; - } - - @Override - public SortedNumericDocValues getLongValues() { - try { - return DocValues.getSortedNumeric(reader, field); - } catch (IOException e) { - throw new IllegalStateException("Cannot load doc values", e); - } - } - - @Override - public Collection getChildResources() { - return Collections.emptyList(); - } - } - */ - static final class SortedNumericFixedPointFieldData extends AtomicDoubleFieldData { final LeafReader reader; final String field; @@ -328,6 +301,9 @@ public Collection getChildResources() { return Collections.emptyList(); } + /** + * Thin Wrapper class which converts the long into a double based on the scaling factor + */ public static class SortedNumericFixedPointDocValues extends SortedNumericDoubleValues { private final long decimalFactor; private final SortedNumericDocValues dv;