diff --git a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java index 4f784852905c6..4dbaf15961aeb 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java @@ -438,6 +438,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return HalfFloatPoint.halfFloatToSortableShort(value.floatValue()); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedDoublesIndexFieldData.Builder( @@ -622,6 +627,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return NumericUtils.floatToSortableInt(value.floatValue()); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedDoublesIndexFieldData.Builder( @@ -772,6 +782,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return NumericUtils.doubleToSortableLong(value.doubleValue()); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedDoublesIndexFieldData.Builder( @@ -891,6 +906,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea INTEGER.addFields(document, name, value, indexed, docValued, stored); } + @Override + public long toSortableLong(Number value) { + return INTEGER.toSortableLong(value); + } + @Override Number valueForSearch(Number value) { return value.byteValue(); @@ -1009,6 +1029,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea INTEGER.addFields(document, name, value, indexed, docValued, stored); } + @Override + public long toSortableLong(Number value) { + return INTEGER.toSortableLong(value); + } + @Override Number valueForSearch(Number value) { return value.shortValue(); @@ -1206,6 +1231,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return value.intValue(); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedNumericIndexFieldData.Builder( @@ -1358,6 +1388,11 @@ public void addFields(LuceneDocument document, String name, Number value, boolea } } + @Override + public long toSortableLong(Number value) { + return value.longValue(); + } + @Override public IndexFieldData.Builder getFieldDataBuilder(MappedFieldType ft, ValuesSourceType valuesSourceType) { return new SortedNumericIndexFieldData.Builder( @@ -1506,6 +1541,13 @@ public abstract void addFields( boolean stored ); + /** + * For a given {@code Number}, returns the sortable long representation that will be stored in the doc values. + * @param value number to convert + * @return sortable long representation + */ + public abstract long toSortableLong(Number value); + public FieldValues compile(String fieldName, Script script, ScriptCompiler compiler) { // only implemented for long and double fields throw new IllegalArgumentException("Unknown parameter [script] for mapper [" + fieldName + "]"); @@ -2140,7 +2182,10 @@ protected void parseCreateField(DocumentParserContext context) throws IOExceptio } if (offsetsFieldName != null && context.isImmediateParentAnArray() && context.canAddIgnoredField()) { if (value != null) { - context.getOffSetContext().recordOffset(offsetsFieldName, (Comparable) value); + // We cannot simply cast value to Comparable<> because we need to also capture the potential loss of precision that occurs + // when the value is stored into the doc values. + long sortableLongValue = type.toSortableLong(value); + context.getOffSetContext().recordOffset(offsetsFieldName, sortableLongValue); } else { context.getOffSetContext().recordNull(offsetsFieldName); } diff --git a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatSyntheticSourceNativeArrayIntegrationTests.java b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatSyntheticSourceNativeArrayIntegrationTests.java index a15834f074c81..96689ba29c0b4 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatSyntheticSourceNativeArrayIntegrationTests.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/HalfFloatSyntheticSourceNativeArrayIntegrationTests.java @@ -13,8 +13,32 @@ import org.apache.lucene.sandbox.document.HalfFloatPoint; +import static org.elasticsearch.xcontent.XContentFactory.jsonBuilder; + public class HalfFloatSyntheticSourceNativeArrayIntegrationTests extends NativeArrayIntegrationTestCase { + public void testSynthesizeArray() throws Exception { + var inputArrayValues = new Float[][] { new Float[] { 0.78151345F, 0.6886488F, 0.6882413F } }; + var expectedArrayValues = new Float[inputArrayValues.length][inputArrayValues[0].length]; + for (int i = 0; i < inputArrayValues.length; i++) { + for (int j = 0; j < inputArrayValues[i].length; j++) { + expectedArrayValues[i][j] = HalfFloatPoint.sortableShortToHalfFloat( + HalfFloatPoint.halfFloatToSortableShort(inputArrayValues[i][j]) + ); + } + } + + var mapping = jsonBuilder().startObject() + .startObject("properties") + .startObject("field") + .field("type", getFieldTypeName()) + .endObject() + .endObject() + .endObject(); + + verifySyntheticArray(inputArrayValues, expectedArrayValues, mapping, "_id"); + } + @Override protected String getFieldTypeName() { return "half_float"; diff --git a/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java b/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java index 17f39be5b1a60..b02ceb31c96db 100644 --- a/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java +++ b/server/src/test/java/org/elasticsearch/index/mapper/NativeArrayIntegrationTestCase.java @@ -259,39 +259,53 @@ protected void verifySyntheticArray(Object[][] arrays) throws IOException { } protected void verifySyntheticArray(Object[][] arrays, XContentBuilder mapping, String... expectedStoredFields) throws IOException { + verifySyntheticArray(arrays, arrays, mapping, expectedStoredFields); + } + + private XContentBuilder arrayToSource(Object[] array) throws IOException { + var source = jsonBuilder().startObject(); + if (array != null) { + source.startArray("field"); + for (Object arrayValue : array) { + source.value(arrayValue); + } + source.endArray(); + } else { + source.field("field").nullValue(); + } + return source.endObject(); + } + + protected void verifySyntheticArray( + Object[][] inputArrays, + Object[][] expectedArrays, + XContentBuilder mapping, + String... expectedStoredFields + ) throws IOException { + assertThat(inputArrays.length, equalTo(expectedArrays.length)); + var indexService = createIndex( "test-index", Settings.builder().put("index.mapping.source.mode", "synthetic").put("index.mapping.synthetic_source_keep", "arrays").build(), mapping ); - for (int i = 0; i < arrays.length; i++) { - var array = arrays[i]; - + for (int i = 0; i < inputArrays.length; i++) { var indexRequest = new IndexRequest("test-index"); indexRequest.id("my-id-" + i); - var source = jsonBuilder().startObject(); - if (array != null) { - source.startArray("field"); - for (Object arrayValue : array) { - source.value(arrayValue); - } - source.endArray(); - } else { - source.field("field").nullValue(); - } - source.endObject(); - var expectedSource = Strings.toString(source); - indexRequest.source(source); + var inputSource = arrayToSource(inputArrays[i]); + indexRequest.source(inputSource); indexRequest.setRefreshPolicy(WriteRequest.RefreshPolicy.IMMEDIATE); client().index(indexRequest).actionGet(); + var expectedSource = arrayToSource(expectedArrays[i]); + var searchRequest = new SearchRequest("test-index"); searchRequest.source().query(new IdsQueryBuilder().addIds("my-id-" + i)); var searchResponse = client().search(searchRequest).actionGet(); try { var hit = searchResponse.getHits().getHits()[0]; assertThat(hit.getId(), equalTo("my-id-" + i)); - assertThat(hit.getSourceAsString(), equalTo(expectedSource)); + assertThat(hit.getSourceAsString(), equalTo(Strings.toString(expectedSource))); } finally { searchResponse.decRef(); } @@ -299,7 +313,7 @@ protected void verifySyntheticArray(Object[][] arrays, XContentBuilder mapping, try (var searcher = indexService.getShard(0).acquireSearcher(getTestName())) { var reader = searcher.getDirectoryReader(); - for (int i = 0; i < arrays.length; i++) { + for (int i = 0; i < expectedArrays.length; i++) { var document = reader.storedFields().document(i); // Verify that there is no ignored source: Set storedFieldNames = new LinkedHashSet<>(document.getFields().stream().map(IndexableField::name).toList());