Skip to content

Expose half-floats. #18887

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
jpountz merged 1 commit into from
Jun 16, 2016
Merged

Expose half-floats. #18887

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions core/src/main/java/org/elasticsearch/index/fielddata/IndexNumericFieldData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,6 +29,7 @@ public static enum NumericType {
SHORT(false),
INT(false),
LONG(false),
HALF_FLOAT(true),
FLOAT(true),
DOUBLE(true);

Expand Down
93 changes: 93 additions & 0 deletions .../src/main/java/org/elasticsearch/index/fielddata/plain/SortedNumericDVIndexFieldData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.index.fielddata.plain;

import org.apache.lucene.document.HalfFloatPoint;
import org.apache.lucene.index.DocValues;
import org.apache.lucene.index.DocValuesType;
import org.apache.lucene.index.LeafReader;
Expand Down Expand Up @@ -61,6 +62,7 @@ public SortedNumericDVIndexFieldData(Index index, String fieldNames, NumericType
@Override
public org.elasticsearch.index.fielddata.IndexFieldData.XFieldComparatorSource comparatorSource(Object missingValue, MultiValueMode sortMode, Nested nested) {
switch (numericType) {
case HALF_FLOAT:
case FLOAT:
return new FloatValuesComparatorSource(this, missingValue, sortMode, nested);
case DOUBLE:
Expand All @@ -87,6 +89,8 @@ public AtomicNumericFieldData load(LeafReaderContext context) {
final String field = fieldName;

switch (numericType) {
case HALF_FLOAT:
return new SortedNumericHalfFloatFieldData(reader, field);
case FLOAT:
return new SortedNumericFloatFieldData(reader, field);
case DOUBLE:
Expand Down Expand Up @@ -134,6 +138,95 @@ public Collection<Accountable> getChildResources() {
}
}

/**
* FieldData implementation for 16-bit float values.
* <p>
* Order of values within a document is consistent with
* {@link Float#compareTo(Float)}, hence the following reversible
* transformation is applied at both index and search:
* {@code bits ^ (bits >> 15) & 0x7fff}
* <p>
* Although the API is multi-valued, most codecs in Lucene specialize
* for the case where documents have at most one value. In this case
* {@link FieldData#unwrapSingleton(SortedNumericDoubleValues)} will return
* the underlying single-valued NumericDoubleValues representation, and
* {@link FieldData#unwrapSingletonBits(SortedNumericDoubleValues)} will return
* a Bits matching documents that have a real value (as opposed to missing).
*/
static final class SortedNumericHalfFloatFieldData extends AtomicDoubleFieldData {
final LeafReader reader;
final String field;

SortedNumericHalfFloatFieldData(LeafReader reader, String field) {
super(0L);
this.reader = reader;
this.field = field;
}

@Override
public SortedNumericDoubleValues getDoubleValues() {
try {
SortedNumericDocValues raw = DocValues.getSortedNumeric(reader, field);

NumericDocValues single = DocValues.unwrapSingleton(raw);
if (single != null) {
return FieldData.singleton(new SingleHalfFloatValues(single), DocValues.unwrapSingletonBits(raw));
} else {
return new MultiHalfFloatValues(raw);
}
} catch (IOException e) {
throw new IllegalStateException("Cannot load doc values", e);
}
}

@Override
public Collection<Accountable> getChildResources() {
return Collections.emptyList();
}
}

/**
* Wraps a NumericDocValues and exposes a single 16-bit float per document.
*/
static final class SingleHalfFloatValues extends NumericDoubleValues {
final NumericDocValues in;

SingleHalfFloatValues(NumericDocValues in) {
this.in = in;
}

@Override
public double get(int docID) {
return HalfFloatPoint.sortableShortToHalfFloat((short) in.get(docID));
}
}

/**
* Wraps a SortedNumericDocValues and exposes multiple 16-bit floats per document.
*/
static final class MultiHalfFloatValues extends SortedNumericDoubleValues {
final SortedNumericDocValues in;

MultiHalfFloatValues(SortedNumericDocValues in) {
this.in = in;
}

@Override
public void setDocument(int doc) {
in.setDocument(doc);
}

@Override
public double valueAt(int index) {
return HalfFloatPoint.sortableShortToHalfFloat((short) in.valueAt(index));
}

@Override
public int count() {
return in.count();
}
}

/**
* FieldData implementation for 32-bit float values.
* <p>
Expand Down
81 changes: 81 additions & 0 deletions core/src/main/java/org/elasticsearch/index/mapper/core/NumberFieldMapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import org.apache.lucene.document.DoublePoint;
import org.apache.lucene.document.Field;
import org.apache.lucene.document.FloatPoint;
import org.apache.lucene.document.HalfFloatPoint;
import org.apache.lucene.document.IntPoint;
import org.apache.lucene.document.LongPoint;
import org.apache.lucene.document.SortedNumericDocValuesField;
Expand Down Expand Up @@ -180,6 +181,86 @@ public Mapper.Builder<?,?> parse(String name, Map<String, Object> node,
}

public enum NumberType {
HALF_FLOAT("half_float", NumericType.HALF_FLOAT) {
@Override
Float parse(Object value) {
return (Float) FLOAT.parse(value);
}

@Override
Float parse(XContentParser parser, boolean coerce) throws IOException {
return parser.floatValue(coerce);
}

@Override
Query termQuery(String field, Object value) {
float v = parse(value);
return HalfFloatPoint.newExactQuery(field, v);
}

@Override
Query termsQuery(String field, List<Object> values) {
float[] v = new float[values.size()];
for (int i = 0; i < values.size(); ++i) {
v[i] = parse(values.get(i));
}
return HalfFloatPoint.newSetQuery(field, v);
}

@Override
Query rangeQuery(String field, Object lowerTerm, Object upperTerm,
boolean includeLower, boolean includeUpper) {
float l = Float.NEGATIVE_INFINITY;
float u = Float.POSITIVE_INFINITY;
if (lowerTerm != null) {
l = parse(lowerTerm);
if (includeLower) {
l = Math.nextDown(l);
}
l = HalfFloatPoint.nextUp(l);
}
if (upperTerm != null) {
u = parse(upperTerm);
if (includeUpper) {
u = Math.nextUp(u);
}
u = HalfFloatPoint.nextDown(u);
}
return HalfFloatPoint.newRangeQuery(field, l, u);
}

@Override
public List<Field> createFields(String name, Number value,
boolean indexed, boolean docValued, boolean stored) {
List<Field> fields = new ArrayList<>();
if (indexed) {
fields.add(new HalfFloatPoint(name, value.floatValue()));
}
if (docValued) {
fields.add(new SortedNumericDocValuesField(name,
HalfFloatPoint.halfFloatToSortableShort(value.floatValue())));
}
if (stored) {
fields.add(new StoredField(name, value.floatValue()));
}
return fields;
}

@Override
FieldStats.Double stats(IndexReader reader, String fieldName,
boolean isSearchable, boolean isAggregatable) throws IOException {
long size = XPointValues.size(reader, fieldName);
if (size == 0) {
return null;
}
int docCount = XPointValues.getDocCount(reader, fieldName);
byte[] min = XPointValues.getMinPackedValue(reader, fieldName);
byte[] max = XPointValues.getMaxPackedValue(reader, fieldName);
return new FieldStats.Double(reader.maxDoc(),docCount, -1L, size,
isSearchable, isAggregatable,
HalfFloatPoint.decodeDimension(min, 0), HalfFloatPoint.decodeDimension(max, 0));
}
},
FLOAT("float", NumericType.FLOAT) {
@Override
Float parse(Object value) {
Expand Down
23 changes: 18 additions & 5 deletions core/src/test/java/org/elasticsearch/fieldstats/FieldStatsIntegrationIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@

package org.elasticsearch.fieldstats;

import org.apache.lucene.document.HalfFloatPoint;
import org.apache.lucene.util.BytesRef;
import org.elasticsearch.action.ActionRequestValidationException;
import org.elasticsearch.action.fieldstats.FieldStats;
Expand Down Expand Up @@ -54,6 +55,7 @@ public void testRandom() throws Exception {
"string", "type=text",
"date", "type=date",
"double", "type=double",
"half_float", "type=half_float",
"float", "type=float",
"long", "type=long",
"integer", "type=integer",
Expand All @@ -67,6 +69,7 @@ public void testRandom() throws Exception {
"string", "type=text,index=false",
"date", "type=date,index=false",
"double", "type=double,index=false",
"half_float", "type=half_float",
"float", "type=float,index=false",
"long", "type=long,index=false",
"integer", "type=integer,index=false",
Expand All @@ -81,6 +84,7 @@ public void testRandom() throws Exception {
"string", "type=text,index=false",
"date", "type=date,index=false",
"double", "type=double,index=false",
"half_float", "type=half_float",
"float", "type=float,index=false",
"long", "type=long,index=false",
"integer", "type=integer,index=false",
Expand All @@ -97,10 +101,12 @@ public void testRandom() throws Exception {
long maxInt = Integer.MIN_VALUE;
long minLong = Long.MAX_VALUE;
long maxLong = Long.MIN_VALUE;
double minFloat = Float.MAX_VALUE;
double maxFloat = Float.MIN_VALUE;
double minDouble = Double.MAX_VALUE;
double maxDouble = Double.MIN_VALUE;
double minHalfFloat = Double.POSITIVE_INFINITY;
double maxHalfFloat = Double.NEGATIVE_INFINITY;
double minFloat = Double.POSITIVE_INFINITY;
double maxFloat = Double.NEGATIVE_INFINITY;
double minDouble = Double.POSITIVE_INFINITY;
double maxDouble = Double.NEGATIVE_INFINITY;
String minString = new String(Character.toChars(1114111));
String maxString = "0";

Expand All @@ -119,6 +125,10 @@ public void testRandom() throws Exception {
long l = randomLong();
minLong = Math.min(minLong, l);
maxLong = Math.max(maxLong, l);
float hf = randomFloat();
hf = HalfFloatPoint.sortableShortToHalfFloat(HalfFloatPoint.halfFloatToSortableShort(hf));
minHalfFloat = Math.min(minHalfFloat, hf);
maxHalfFloat = Math.max(maxHalfFloat, hf);
float f = randomFloat();
minFloat = Math.min(minFloat, f);
maxFloat = Math.max(maxFloat, f);
Expand All @@ -138,6 +148,7 @@ public void testRandom() throws Exception {
"short", s,
"integer", i,
"long", l,
"half_float", hf,
"float", f,
"double", d,
"string", str)
Expand All @@ -147,7 +158,7 @@ public void testRandom() throws Exception {

FieldStatsResponse response = client()
.prepareFieldStats()
.setFields("byte", "short", "integer", "long", "float", "double", "string").get();
.setFields("byte", "short", "integer", "long", "half_float", "float", "double", "string").get();
assertAllSuccessful(response);

for (FieldStats<?> stats : response.getAllFieldStats().values()) {
Expand All @@ -164,6 +175,8 @@ public void testRandom() throws Exception {
assertThat(response.getAllFieldStats().get("integer").getMaxValue(), equalTo(maxInt));
assertThat(response.getAllFieldStats().get("long").getMinValue(), equalTo(minLong));
assertThat(response.getAllFieldStats().get("long").getMaxValue(), equalTo(maxLong));
assertThat(response.getAllFieldStats().get("half_float").getMinValue(), equalTo(minHalfFloat));
assertThat(response.getAllFieldStats().get("half_float").getMaxValue(), equalTo(maxHalfFloat));
assertThat(response.getAllFieldStats().get("float").getMinValue(), equalTo(minFloat));
assertThat(response.getAllFieldStats().get("float").getMaxValue(), equalTo(maxFloat));
assertThat(response.getAllFieldStats().get("double").getMinValue(), equalTo(minDouble));
Expand Down
18 changes: 18 additions & 0 deletions core/src/test/java/org/elasticsearch/fieldstats/FieldStatsTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -108,6 +108,24 @@ public void testDouble() {
assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Double.toString(-1)));
}

public void testHalfFloat() {
String fieldName = "field";
createIndex("test", Settings.EMPTY, "test", fieldName, "type=half_float");
for (float value = -1; value <= 9; value++) {
client().prepareIndex("test", "test").setSource(fieldName, value).get();
}
client().admin().indices().prepareRefresh().get();

FieldStatsResponse result = client().prepareFieldStats().setFields(fieldName).get();
assertThat(result.getAllFieldStats().get(fieldName).getMaxDoc(), equalTo(11L));
assertThat(result.getAllFieldStats().get(fieldName).getDocCount(), equalTo(11L));
assertThat(result.getAllFieldStats().get(fieldName).getDensity(), equalTo(100));
assertThat(result.getAllFieldStats().get(fieldName).getMinValue(), equalTo(-1d));
assertThat(result.getAllFieldStats().get(fieldName).getMaxValue(), equalTo(9d));
assertThat(result.getAllFieldStats().get(fieldName).getMinValueAsString(), equalTo(Float.toString(-1)));
assertThat(result.getAllFieldStats().get(fieldName).getMaxValueAsString(), equalTo(Float.toString(9)));
}

public void testFloat() {
String fieldName = "field";
createIndex("test", Settings.EMPTY, "test", fieldName, "type=float");
Expand Down
Loading