diff --git a/CHANGELOG.md b/CHANGELOG.md index 55b39f474d..3006321097 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,3 +22,4 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), ### Enhancements * Add the bulkscore logic in MOS when K is greater than number of docs in segment [#3285](https://github.com/opensearch-project/k-NN/pull/3285) * Added capability to retrieve float data type vectors using doc_values [#3321](https://github.com/opensearch-project/k-NN/pull/3321) +* Add base64 binary encoding as default format for knn_vector docvalue_fields [#3324](https://github.com/opensearch-project/k-NN/pull/3324) diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java index 5c3b55e776..fae877172b 100644 --- a/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java +++ b/src/main/java/org/opensearch/knn/index/KNNVectorDVLeafFieldData.java @@ -102,16 +102,29 @@ public SortedBinaryDocValues getBytesValues() { * iterator so that multiple {@code Leaf} instances obtained from the same * {@code KNNVectorDVLeafFieldData} cannot interfere with each other's state. * - *

Return type: {@code float[]} for FLOAT vectors — - * {@link org.opensearch.core.xcontent.XContentBuilder} serializes this as a JSON numeric array. + *

Return type: Depends on the format: + *

* - * @param format the doc value format — currently unused but required by the interface + * @param format the {@link KNNVectorDocValueFormat} that determines output encoding (array or binary) * @return a leaf fetcher that yields vector values per document, or an empty fetcher * if the field has no vectors in this segment * @throws UnsupportedOperationException if the vector data type is BYTE or BINARY + * @throws IllegalArgumentException if format is not an instance of {@link KNNVectorDocValueFormat} */ @Override public DocValueFetcher.Leaf getLeafValueFetcher(final DocValueFormat format) { + if (!(format instanceof KNNVectorDocValueFormat knnFormat)) { + throw new IllegalArgumentException( + "Unsupported DocValueFormat [" + format + "] for knn_vector field '" + fieldName + "'. Expected KNNVectorDocValueFormat." + ); + } + final boolean isBinary = knnFormat.isBinary(); + if (vectorDataType == VectorDataType.BYTE || vectorDataType == VectorDataType.BINARY) { throw new UnsupportedOperationException( "docvalue_fields is not supported for [" + vectorDataType + "] vector field '" + fieldName + "'" @@ -152,6 +165,12 @@ public int docValueCount() { @Override public Object nextValue() throws IOException { + // We don't need a conditional clone here since encodeToBinary will convert the array to string. + if (isBinary) { + return KNNVectorDocValueFormat.encodeToBinary((float[]) vectorValues.getVector()); + } + // We need a conditional clone since vector returned from here will be added in a map, so we do the clone + // since vectorValues keep a single copy of vector array for all docs. return vectorValues.conditionalCloneVector(); } }; diff --git a/src/main/java/org/opensearch/knn/index/KNNVectorDocValueFormat.java b/src/main/java/org/opensearch/knn/index/KNNVectorDocValueFormat.java new file mode 100644 index 0000000000..f4f303c7fb --- /dev/null +++ b/src/main/java/org/opensearch/knn/index/KNNVectorDocValueFormat.java @@ -0,0 +1,96 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import lombok.Getter; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.core.common.io.stream.StreamOutput; +import org.opensearch.search.DocValueFormat; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Arrays; +import java.util.Base64; + +/** + * DocValueFormat for knn_vector fields. Supports two modes: + * + */ +@Getter +public enum KNNVectorDocValueFormat implements DocValueFormat { + + ARRAY_FORMAT("array", false), + BINARY_FORMAT("binary", true); + + public static final String NAME = "knn_vector"; + + private final String formatName; + private final boolean binary; + private static final Base64.Encoder BASE64_ENCODER = Base64.getEncoder(); + + KNNVectorDocValueFormat(final String formatName, boolean binary) { + this.formatName = formatName; + this.binary = binary; + } + + public static KNNVectorDocValueFormat fromStream(final StreamInput in) throws IOException { + return in.readBoolean() ? BINARY_FORMAT : ARRAY_FORMAT; + } + + /** + * Resolves the format string to the corresponding enum constant. + * Returns {@link #BINARY_FORMAT} when format is null (the default). + * + * @param format the format string from the docvalue_fields request, or null for default + * @return the matching {@link KNNVectorDocValueFormat} + * @throws IllegalArgumentException if the format string is not recognized + */ + public static KNNVectorDocValueFormat fromFormatString(final String format) { + if (format == null || BINARY_FORMAT.formatName.equals(format)) { + return BINARY_FORMAT; + } + if (ARRAY_FORMAT.formatName.equals(format)) { + return ARRAY_FORMAT; + } + throw new IllegalArgumentException( + "Unsupported knn_vector docvalue_fields format [" + + format + + "]. Supported formats are " + + Arrays.toString(KNNVectorDocValueFormat.values()) + ); + } + + @Override + public String getWriteableName() { + return NAME; + } + + @Override + public void writeTo(final StreamOutput out) throws IOException { + out.writeBoolean(binary); + } + + /** + * Encodes a float[] vector as a base64 string with little-endian byte order. + * Each float is written as 4 bytes in little-endian format (native byte order on x86/ARM), + * then the resulting byte array is base64-encoded. + */ + public static String encodeToBinary(final float[] vector) { + final ByteBuffer buffer = ByteBuffer.allocate(vector.length * Float.BYTES).order(ByteOrder.LITTLE_ENDIAN); + buffer.asFloatBuffer().put(vector); // Bulk operation optimized by the JVM + final byte[] bytes = buffer.array(); + return BASE64_ENCODER.encodeToString(bytes); + } + + @Override + public String toString() { + return "knn_vector(" + formatName + ")"; + } +} diff --git a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java index ff55dac334..fd5573b7ad 100644 --- a/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java +++ b/src/main/java/org/opensearch/knn/index/mapper/KNNVectorFieldType.java @@ -20,6 +20,7 @@ import org.opensearch.index.mapper.ValueFetcher; import org.opensearch.index.query.QueryShardContext; import org.opensearch.index.query.QueryShardException; +import org.opensearch.knn.index.KNNVectorDocValueFormat; import org.opensearch.knn.index.KNNVectorIndexFieldData; import org.opensearch.knn.index.VectorDataType; import org.opensearch.knn.index.engine.KNNEngine; @@ -29,9 +30,11 @@ import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.knn.indices.ModelDao; import org.opensearch.knn.indices.ModelMetadata; +import org.opensearch.search.DocValueFormat; import org.opensearch.search.aggregations.support.CoreValuesSourceType; import org.opensearch.search.lookup.SearchLookup; +import java.time.ZoneId; import java.util.ArrayList; import java.util.Collections; import java.util.Locale; @@ -152,6 +155,14 @@ public IndexFieldData.Builder fielddataBuilder(String fullyQualifiedIndexName, S return new KNNVectorIndexFieldData.Builder(name(), CoreValuesSourceType.BYTES, this.vectorDataType); } + @Override + public DocValueFormat docValueFormat(final String format, final ZoneId timeZone) { + if (timeZone != null) { + throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] does not support custom time zones"); + } + return KNNVectorDocValueFormat.fromFormatString(format); + } + @Override public Object valueForDisplay(Object value) { return deserializeStoredVector((BytesRef) value, vectorDataType); diff --git a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java index a2632cfbb0..b167fc510e 100644 --- a/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java +++ b/src/main/java/org/opensearch/knn/plugin/KNNPlugin.java @@ -37,6 +37,7 @@ import org.opensearch.index.shard.IndexSettingProvider; import org.opensearch.indices.SystemIndexDescriptor; import org.opensearch.knn.index.KNNCircuitBreaker; +import org.opensearch.knn.index.KNNVectorDocValueFormat; import org.opensearch.knn.index.KNNSettings; import org.opensearch.knn.index.codec.KNNCodecService; import org.opensearch.knn.index.codec.derivedsource.DerivedSourceIndexOperationListener; @@ -120,6 +121,7 @@ import org.opensearch.script.ScriptContext; import org.opensearch.script.ScriptEngine; import org.opensearch.script.ScriptService; +import org.opensearch.search.DocValueFormat; import org.opensearch.search.SearchExtBuilder; import org.opensearch.search.deciders.ConcurrentSearchRequestDecider; import org.opensearch.search.pipeline.SearchRequestProcessor; @@ -447,6 +449,9 @@ public List getNamedWriteables() { entries.add(new NamedWriteableRegistry.Entry(Metadata.Custom.class, ModelGraveyard.TYPE, ModelGraveyard::new)); entries.add(new NamedWriteableRegistry.Entry(NamedDiff.class, ModelGraveyard.TYPE, ModelGraveyard::readDiffFrom)); + entries.add( + new NamedWriteableRegistry.Entry(DocValueFormat.class, KNNVectorDocValueFormat.NAME, KNNVectorDocValueFormat::fromStream) + ); return entries; } diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorDVLeafFieldDataTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorDVLeafFieldDataTests.java index 4c68aa9be4..8fedeaf29e 100644 --- a/src/test/java/org/opensearch/knn/index/KNNVectorDVLeafFieldDataTests.java +++ b/src/test/java/org/opensearch/knn/index/KNNVectorDVLeafFieldDataTests.java @@ -18,10 +18,11 @@ import org.opensearch.index.fielddata.ScriptDocValues; import org.opensearch.index.mapper.DocValueFetcher; import org.opensearch.knn.KNNTestCase; -import org.opensearch.search.DocValueFormat; import org.junit.Before; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; public class KNNVectorDVLeafFieldDataTests extends KNNTestCase { @@ -101,7 +102,7 @@ public void testGetLeafValueFetcher_floatVector_returnsCorrectValues() throws IO MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT ); - DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(DocValueFormat.RAW); + DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.ARRAY_FORMAT); assertNotNull(leaf); float[][] results = new float[ALL_VECTORS.length][]; @@ -124,7 +125,7 @@ public void testGetLeafValueFetcher_advanceExact_nonExistentDoc_returnsFalse() t MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT ); - DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(DocValueFormat.RAW); + DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.ARRAY_FORMAT); boolean[] advanceResults = new boolean[ALL_VECTORS.length + 1]; int[] docValueCounts = new int[ALL_VECTORS.length + 1]; @@ -150,7 +151,7 @@ public void testGetLeafValueFetcher_docValueCount_isOne() throws IOException { MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT ); - DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(DocValueFormat.RAW); + DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.ARRAY_FORMAT); int[] docValueCounts = new int[ALL_VECTORS.length]; for (int docId = 0; docId < ALL_VECTORS.length; docId++) { @@ -195,7 +196,7 @@ public void testGetLeafValueFetcher_multipleDocuments_iteratesCorrectly() throws MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT ); - DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(DocValueFormat.RAW); + DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.ARRAY_FORMAT); float[][] expected = { vector1, vector2, vector3 }; float[][] results = new float[expected.length][]; @@ -212,6 +213,78 @@ public void testGetLeafValueFetcher_multipleDocuments_iteratesCorrectly() throws } } + public void testGetLeafValueFetcher_binaryFormat_returnsBase64String() throws IOException { + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData( + leafReaderContext.reader(), + MOCK_INDEX_FIELD_NAME, + VectorDataType.FLOAT + ); + DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.BINARY_FORMAT); + assertNotNull(leaf); + + assertTrue(leaf.advanceExact(0)); + assertEquals(1, leaf.docValueCount()); + Object value = leaf.nextValue(); + assertTrue(value instanceof String); + String base64 = (String) value; + + // Decode and verify + byte[] decoded = java.util.Base64.getDecoder().decode(base64); + assertEquals(SAMPLE_VECTOR_1.length * Float.BYTES, decoded.length); + ByteBuffer buffer = ByteBuffer.wrap(decoded).order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < SAMPLE_VECTOR_1.length; i++) { + assertEquals(SAMPLE_VECTOR_1[i], buffer.getFloat(), 0.001f); + } + } + + public void testGetLeafValueFetcher_nonKNNFormat_throwsIllegalArgument() { + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData( + leafReaderContext.reader(), + MOCK_INDEX_FIELD_NAME, + VectorDataType.FLOAT + ); + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> leafFieldData.getLeafValueFetcher(org.opensearch.search.DocValueFormat.RAW) + ); + assertTrue("Error should mention unsupported format", ex.getMessage().contains("Unsupported DocValueFormat")); + assertTrue("Error should mention the field name", ex.getMessage().contains(MOCK_INDEX_FIELD_NAME)); + } + + public void testGetLeafValueFetcher_nullFormat_throwsIllegalArgument() { + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData( + leafReaderContext.reader(), + MOCK_INDEX_FIELD_NAME, + VectorDataType.FLOAT + ); + IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> leafFieldData.getLeafValueFetcher(null)); + assertTrue("Error should mention unsupported format", ex.getMessage().contains("Unsupported DocValueFormat")); + assertTrue("Error should mention the field name", ex.getMessage().contains(MOCK_INDEX_FIELD_NAME)); + } + + public void testGetLeafValueFetcher_binaryFormat_multipleDocuments() throws IOException { + KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData( + leafReaderContext.reader(), + MOCK_INDEX_FIELD_NAME, + VectorDataType.FLOAT + ); + DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.BINARY_FORMAT); + + for (int docId = 0; docId < ALL_VECTORS.length; docId++) { + assertTrue("advanceExact should succeed for doc " + docId, leaf.advanceExact(docId)); + assertEquals("docValueCount should be 1 for doc " + docId, 1, leaf.docValueCount()); + Object value = leaf.nextValue(); + assertTrue("Binary format should produce a String for doc " + docId, value instanceof String); + + byte[] decoded = java.util.Base64.getDecoder().decode((String) value); + assertEquals("Decoded byte length mismatch for doc " + docId, ALL_VECTORS[docId].length * Float.BYTES, decoded.length); + ByteBuffer buffer = ByteBuffer.wrap(decoded).order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < ALL_VECTORS[docId].length; i++) { + assertEquals("Value mismatch at index " + i + " for doc " + docId, ALL_VECTORS[docId][i], buffer.getFloat(), 0.001f); + } + } + } + public void testGetLeafValueFetcher_byteVectorDataType_throwsUnsupportedOp() { KNNVectorDVLeafFieldData leafFieldData = new KNNVectorDVLeafFieldData( leafReaderContext.reader(), @@ -220,7 +293,7 @@ public void testGetLeafValueFetcher_byteVectorDataType_throwsUnsupportedOp() { ); UnsupportedOperationException ex = expectThrows( UnsupportedOperationException.class, - () -> leafFieldData.getLeafValueFetcher(DocValueFormat.RAW) + () -> leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.ARRAY_FORMAT) ); assertTrue(ex.getMessage().contains("docvalue_fields is not supported")); assertTrue(ex.getMessage().contains("BYTE")); @@ -234,7 +307,7 @@ public void testGetLeafValueFetcher_binaryVectorDataType_throwsUnsupportedOp() { ); UnsupportedOperationException ex = expectThrows( UnsupportedOperationException.class, - () -> leafFieldData.getLeafValueFetcher(DocValueFormat.RAW) + () -> leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.ARRAY_FORMAT) ); assertTrue(ex.getMessage().contains("docvalue_fields is not supported")); assertTrue(ex.getMessage().contains("BINARY")); @@ -261,20 +334,17 @@ public void testGetLeafValueFetcher_fieldNotInSegment_returnsEmptyLeaf() throws MOCK_INDEX_FIELD_NAME, VectorDataType.FLOAT ); - DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(DocValueFormat.RAW); - assertNotNull(leaf); + DocValueFetcher.Leaf leaf = leafFieldData.getLeafValueFetcher(KNNVectorDocValueFormat.ARRAY_FORMAT); + assertNotNull("Empty leaf should not be null", leaf); - boolean[] advanceResults = new boolean[numDocs]; - int[] docValueCounts = new int[numDocs]; + // Verify advanceExact returns false for all docs for (int docId = 0; docId < numDocs; docId++) { - advanceResults[docId] = leaf.advanceExact(docId); - docValueCounts[docId] = leaf.docValueCount(); + assertFalse("advanceExact should fail for doc " + docId + " in segment without vector field", leaf.advanceExact(docId)); + assertEquals("docValueCount should be 0 for doc " + docId, 0, leaf.docValueCount()); } - for (int docId = 0; docId < numDocs; docId++) { - assertFalse("advanceExact should fail for doc " + docId + " in segment without vector field", advanceResults[docId]); - assertEquals(0, docValueCounts[docId]); - } + // Verify nextValue returns null on the empty leaf + assertNull("Empty leaf nextValue should return null", leaf.nextValue()); } } } diff --git a/src/test/java/org/opensearch/knn/index/KNNVectorDocValueFormatTests.java b/src/test/java/org/opensearch/knn/index/KNNVectorDocValueFormatTests.java new file mode 100644 index 0000000000..645518b88b --- /dev/null +++ b/src/test/java/org/opensearch/knn/index/KNNVectorDocValueFormatTests.java @@ -0,0 +1,152 @@ +/* + * Copyright OpenSearch Contributors + * SPDX-License-Identifier: Apache-2.0 + */ + +package org.opensearch.knn.index; + +import org.opensearch.common.io.stream.BytesStreamOutput; +import org.opensearch.core.common.io.stream.StreamInput; +import org.opensearch.knn.KNNTestCase; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; +import java.util.Base64; + +public class KNNVectorDocValueFormatTests extends KNNTestCase { + + public void testIsBinary() { + assertFalse("ARRAY_FORMAT should not be binary", KNNVectorDocValueFormat.ARRAY_FORMAT.isBinary()); + assertTrue("BINARY_FORMAT should be binary", KNNVectorDocValueFormat.BINARY_FORMAT.isBinary()); + } + + public void testGetWriteableName() { + assertEquals("ARRAY_FORMAT writeable name mismatch", "knn_vector", KNNVectorDocValueFormat.ARRAY_FORMAT.getWriteableName()); + assertEquals("BINARY_FORMAT writeable name mismatch", "knn_vector", KNNVectorDocValueFormat.BINARY_FORMAT.getWriteableName()); + } + + public void testStreamRoundTrip() throws IOException { + // Array format round-trip + BytesStreamOutput arrayOut = new BytesStreamOutput(); + KNNVectorDocValueFormat.ARRAY_FORMAT.writeTo(arrayOut); + try (StreamInput arrayIn = arrayOut.bytes().streamInput()) { + KNNVectorDocValueFormat arrayDeserialized = KNNVectorDocValueFormat.fromStream(arrayIn); + assertFalse("Deserialized ARRAY_FORMAT should not be binary", arrayDeserialized.isBinary()); + assertEquals("Deserialized ARRAY_FORMAT writeable name mismatch", "knn_vector", arrayDeserialized.getWriteableName()); + assertSame( + "Deserialized ARRAY_FORMAT should be the singleton instance", + KNNVectorDocValueFormat.ARRAY_FORMAT, + arrayDeserialized + ); + } + + // Binary format round-trip + BytesStreamOutput binaryOut = new BytesStreamOutput(); + KNNVectorDocValueFormat.BINARY_FORMAT.writeTo(binaryOut); + try (StreamInput binaryIn = binaryOut.bytes().streamInput()) { + KNNVectorDocValueFormat binaryDeserialized = KNNVectorDocValueFormat.fromStream(binaryIn); + assertTrue("Deserialized BINARY_FORMAT should be binary", binaryDeserialized.isBinary()); + assertEquals("Deserialized BINARY_FORMAT writeable name mismatch", "knn_vector", binaryDeserialized.getWriteableName()); + assertSame( + "Deserialized BINARY_FORMAT should be the singleton instance", + KNNVectorDocValueFormat.BINARY_FORMAT, + binaryDeserialized + ); + } + } + + public void testFromFormatString() { + // null defaults to binary + assertSame( + "null format should return BINARY_FORMAT", + KNNVectorDocValueFormat.BINARY_FORMAT, + KNNVectorDocValueFormat.fromFormatString(null) + ); + + // explicit "binary" returns BINARY_FORMAT + assertSame( + "'binary' should return BINARY_FORMAT", + KNNVectorDocValueFormat.BINARY_FORMAT, + KNNVectorDocValueFormat.fromFormatString("binary") + ); + + // explicit "array" returns ARRAY_FORMAT + assertSame( + "'array' should return ARRAY_FORMAT", + KNNVectorDocValueFormat.ARRAY_FORMAT, + KNNVectorDocValueFormat.fromFormatString("array") + ); + + // unsupported format throws + IllegalArgumentException ex = expectThrows( + IllegalArgumentException.class, + () -> KNNVectorDocValueFormat.fromFormatString("epoch_millis") + ); + assertTrue("Error should mention unsupported format", ex.getMessage().contains("epoch_millis")); + assertTrue("Error should list supported formats", ex.getMessage().contains("array")); + assertTrue("Error should list supported formats", ex.getMessage().contains("binary")); + } + + public void testEncodeToBinary() { + // Simple vector + float[] vector = { 1.0f, 2.0f, 3.0f, 4.0f }; + String encoded = KNNVectorDocValueFormat.encodeToBinary(vector); + assertNotNull("Encoded string should not be null", encoded); + assertFalse("Encoded string should not be empty", encoded.isEmpty()); + assertDecodedVectorEquals("Simple vector", vector, encoded); + + // Single element + float[] single = { 42.5f }; + assertDecodedVectorEquals("Single element vector", single, KNNVectorDocValueFormat.encodeToBinary(single)); + + // Negative and edge values + float[] edgeCases = { -1.5f, -100.0f, 0.0f, Float.MAX_VALUE, Float.MIN_VALUE }; + assertDecodedVectorEquals("Edge case vector", edgeCases, KNNVectorDocValueFormat.encodeToBinary(edgeCases)); + + // Empty vector + float[] empty = {}; + String emptyEncoded = KNNVectorDocValueFormat.encodeToBinary(empty); + assertNotNull("Empty vector encoding should not be null", emptyEncoded); + assertEquals("Empty vector should decode to 0 bytes", 0, Base64.getDecoder().decode(emptyEncoded).length); + + // High dimension (768d) + int dimension = 768; + float[] highDim = new float[dimension]; + for (int i = 0; i < dimension; i++) { + highDim[i] = i * 0.01f; + } + assertDecodedVectorEquals("768d vector", highDim, KNNVectorDocValueFormat.encodeToBinary(highDim)); + } + + public void testEncodeToBinary_nullVector_throwsNPE() { + expectThrows(NullPointerException.class, () -> KNNVectorDocValueFormat.encodeToBinary(null)); + } + + public void testEncodeToBinary_usesLittleEndian() { + float[] vector = { 1.0f }; + byte[] decoded = Base64.getDecoder().decode(KNNVectorDocValueFormat.encodeToBinary(vector)); + + // 1.0f in IEEE 754 is 0x3F800000 + // Little-endian: 0x00, 0x00, 0x80, 0x3F + assertEquals("Byte 0 should be 0x00 (little-endian)", (byte) 0x00, decoded[0]); + assertEquals("Byte 1 should be 0x00 (little-endian)", (byte) 0x00, decoded[1]); + assertEquals("Byte 2 should be 0x80 (little-endian)", (byte) 0x80, decoded[2]); + assertEquals("Byte 3 should be 0x3F (little-endian)", (byte) 0x3F, decoded[3]); + } + + public void testToString() { + assertEquals("ARRAY_FORMAT toString mismatch", "knn_vector(array)", KNNVectorDocValueFormat.ARRAY_FORMAT.toString()); + assertEquals("BINARY_FORMAT toString mismatch", "knn_vector(binary)", KNNVectorDocValueFormat.BINARY_FORMAT.toString()); + } + + private void assertDecodedVectorEquals(String label, float[] expected, String base64Encoded) { + byte[] decoded = Base64.getDecoder().decode(base64Encoded); + assertEquals(label + ": decoded byte length mismatch", expected.length * Float.BYTES, decoded.length); + + ByteBuffer buffer = ByteBuffer.wrap(decoded).order(ByteOrder.LITTLE_ENDIAN); + for (int i = 0; i < expected.length; i++) { + assertEquals(label + ": value mismatch at index " + i, expected[i], buffer.getFloat(), 0.0f); + } + } +} diff --git a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldTypeTests.java b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldTypeTests.java index 3bafac848b..054be4efb2 100644 --- a/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldTypeTests.java +++ b/src/test/java/org/opensearch/knn/index/mapper/KNNVectorFieldTypeTests.java @@ -15,6 +15,7 @@ import org.opensearch.knn.index.engine.KNNEngine; import org.opensearch.knn.index.engine.KNNMethodContext; import org.opensearch.knn.index.engine.MethodComponentContext; +import org.opensearch.knn.index.KNNVectorDocValueFormat; import org.opensearch.knn.index.query.rescore.RescoreContext; import org.opensearch.search.DocValueFormat; @@ -145,7 +146,7 @@ public void testKNNVectorFieldType_whenNonSQOneBitEncoder_thenAlwaysUseMemoryOpt assertTrue(fieldType.isMemoryOptimizedSearchAvailable()); } - public void testDocValueFormat_nullFormatAndTimezone_returnsRaw() { + public void testDocValueFormat_nullFormat_returnsBinaryFormat() { KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); KNNVectorFieldType fieldType = new KNNVectorFieldType( FIELD_NAME, @@ -154,10 +155,36 @@ public void testDocValueFormat_nullFormatAndTimezone_returnsRaw() { getMappingConfigForMethodMapping(knnMethodContext, 3) ); DocValueFormat format = fieldType.docValueFormat(null, null); - assertSame(DocValueFormat.RAW, format); + assertSame(KNNVectorDocValueFormat.BINARY_FORMAT, format); } - public void testDocValueFormat_nonNullFormat_throwsIllegalArgument() { + public void testDocValueFormat_arrayFormat_returnsArrayFormat() { + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); + KNNVectorFieldType fieldType = new KNNVectorFieldType( + FIELD_NAME, + Collections.emptyMap(), + VectorDataType.FLOAT, + getMappingConfigForMethodMapping(knnMethodContext, 3) + ); + DocValueFormat format = fieldType.docValueFormat("array", null); + assertSame(KNNVectorDocValueFormat.ARRAY_FORMAT, format); + assertFalse(((KNNVectorDocValueFormat) format).isBinary()); + } + + public void testDocValueFormat_binaryFormat_returnsBinaryFormat() { + KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); + KNNVectorFieldType fieldType = new KNNVectorFieldType( + FIELD_NAME, + Collections.emptyMap(), + VectorDataType.FLOAT, + getMappingConfigForMethodMapping(knnMethodContext, 3) + ); + DocValueFormat format = fieldType.docValueFormat("binary", null); + assertSame(KNNVectorDocValueFormat.BINARY_FORMAT, format); + assertTrue(((KNNVectorDocValueFormat) format).isBinary()); + } + + public void testDocValueFormat_unsupportedFormat_throwsIllegalArgument() { KNNMethodContext knnMethodContext = getDefaultKNNMethodContext(); KNNVectorFieldType fieldType = new KNNVectorFieldType( FIELD_NAME, @@ -166,8 +193,8 @@ public void testDocValueFormat_nonNullFormat_throwsIllegalArgument() { getMappingConfigForMethodMapping(knnMethodContext, 3) ); IllegalArgumentException ex = expectThrows(IllegalArgumentException.class, () -> fieldType.docValueFormat("epoch_millis", null)); - assertTrue(ex.getMessage().contains(FIELD_NAME)); - assertTrue(ex.getMessage().contains("does not support custom formats")); + assertTrue(ex.getMessage().contains("epoch_millis")); + assertTrue(ex.getMessage().contains("Unsupported knn_vector docvalue_fields format")); } public void testDocValueFormat_nonNullTimezone_throwsIllegalArgument() { diff --git a/src/test/java/org/opensearch/knn/integ/DocValueFieldsIT.java b/src/test/java/org/opensearch/knn/integ/DocValueFieldsIT.java index 9273884d29..efaa6b4504 100644 --- a/src/test/java/org/opensearch/knn/integ/DocValueFieldsIT.java +++ b/src/test/java/org/opensearch/knn/integ/DocValueFieldsIT.java @@ -25,7 +25,10 @@ import org.opensearch.knn.index.KNNSettings; import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.util.ArrayList; +import java.util.Base64; import java.util.List; import java.util.Map; @@ -110,14 +113,19 @@ public void testDocValueFields_vectorValuesMatchSource() { String sourceBody = EntityUtils.toString(sourceResponse.getEntity()); List> sourceHits = parseSearchHits(sourceBody); - // Fetch all docs with docvalue_fields + // Fetch all docs with docvalue_fields (explicitly request array format) String dvQuery = XContentFactory.jsonBuilder() .startObject() .startObject("query") .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", true) .startObject("sort") .field("_id", "asc") @@ -256,7 +264,12 @@ public void testDocValueFields_highDimension_768d() { .endObject() .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .toString(); @@ -326,8 +339,14 @@ public void testDocValueFields_multipleVectorFields() { .endObject() .endObject() .startArray("docvalue_fields") - .value(VECTOR_FIELD) - .value(secondVectorField) + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .startObject() + .field("field", secondVectorField) + .field("format", "array") + .endObject() .endArray() .field("_source", false) .endObject() @@ -383,7 +402,10 @@ public void testDocValueFields_mixedWithScalarFields() { .endObject() .endObject() .startArray("docvalue_fields") - .value(VECTOR_FIELD) + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() .value(numericField) .endArray() .field("_source", false) @@ -457,7 +479,12 @@ public void testDocValueFields_emptyIndex_returnsEmptyHits() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .toString(); @@ -489,7 +516,12 @@ public void testDocValueFields_afterDocDeletion_noGhostVectors() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .toString(); @@ -524,7 +556,12 @@ public void testDocValueFields_afterForcemerge_returnsCorrectly() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .toString(); @@ -560,7 +597,12 @@ public void testDocValueFields_paginationWithFrom() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .field("from", 5) .field("size", 5) @@ -611,7 +653,12 @@ public void testDocValueFields_byteVectorField_throwsError() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .toString(); @@ -656,7 +703,12 @@ public void testDocValueFields_binaryVectorField_throwsError() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .toString(); @@ -720,14 +772,19 @@ public void testDocValueFields_knnDerivedSourceEnabled_vectorsMatchSource() { String sourceBody = EntityUtils.toString(sourceResponse.getEntity()); List> sourceHits = parseSearchHits(sourceBody); - // Fetch with docvalue_fields + // Fetch with docvalue_fields (explicitly request array format) String dvQuery = XContentFactory.jsonBuilder() .startObject() .startObject("query") .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .startObject("sort") .field("_id", "asc") @@ -812,7 +869,12 @@ public void testDocValueFields_coreDerivedSourceEnabled_vectorsMatchSource() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .startObject("sort") .field("_id", "asc") @@ -865,7 +927,249 @@ public void testDocValueFields_customFormat_throwsError() { .toString(); ResponseException ex = expectThrows(ResponseException.class, () -> searchKNNIndex(TEST_INDEX, query, 1)); - assertTrue(ex.getMessage().contains("does not support custom formats")); + assertTrue(ex.getMessage().contains("Unsupported knn_vector docvalue_fields format")); + } + + /** + * Verifies that the default format (no format specified) returns base64-encoded binary + * for both Faiss and Lucene engines, since binary is now the default doc value format. + */ + @SneakyThrows + public void testDocValueFields_defaultFormat_returnsBinaryBase64() { + for (KNNEngine engine : List.of(KNNEngine.FAISS, KNNEngine.LUCENE)) { + String indexName = TEST_INDEX + "_default_fmt_" + engine.getName(); + + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .spaceType(SpaceType.L2.getValue()) + .engine(engine.getName()) + .build(); + + String mapping = KNNJsonIndexMappingsBuilder.builder() + .fieldName(VECTOR_FIELD) + .dimension(DIMENSION) + .vectorDataType(VectorDataType.FLOAT.getValue()) + .method(method) + .build() + .getIndexMapping(); + + createKnnIndex(indexName, mapping); + addKnnDoc(indexName, "1", VECTOR_FIELD, Floats.asList(VECTOR_1).toArray()); + addKnnDoc(indexName, "2", VECTOR_FIELD, Floats.asList(VECTOR_2).toArray()); + refreshIndex(indexName); + + // No format specified — should default to binary + String query = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(VECTOR_FIELD) + .field("vector", VECTOR_1) + .field("k", 2) + .endObject() + .endObject() + .endObject() + .array("docvalue_fields", VECTOR_FIELD) + .field("_source", false) + .endObject() + .toString(); + + Response response = searchKNNIndex(indexName, query, 2); + String responseBody = EntityUtils.toString(response.getEntity()); + List> hits = parseSearchHits(responseBody); + + assertEquals("Expected 2 hits for engine " + engine.getName(), 2, hits.size()); + + float[][] expectedVectors = { VECTOR_1, VECTOR_2 }; + for (int i = 0; i < hits.size(); i++) { + List binaryValues = getBinaryDocValueField(hits.get(i), VECTOR_FIELD); + assertNotNull("Default format should return binary for engine " + engine.getName(), binaryValues); + assertEquals(1, binaryValues.size()); + + String base64Value = binaryValues.get(0); + assertNotNull(base64Value); + assertFalse(base64Value.isEmpty()); + + byte[] decoded = Base64.getDecoder().decode(base64Value); + assertEquals(DIMENSION * Float.BYTES, decoded.length); + + ByteBuffer buffer = ByteBuffer.wrap(decoded).order(ByteOrder.LITTLE_ENDIAN); + for (int j = 0; j < DIMENSION; j++) { + assertEquals( + "Mismatch at index " + j + " for engine " + engine.getName(), + expectedVectors[i][j], + buffer.getFloat(), + 0.001f + ); + } + } + + deleteKNNIndex(indexName); + } + } + + /** + * Verifies that explicitly requesting binary format returns a valid base64-encoded + * little-endian float vector for both Faiss and Lucene engines. + */ + @SneakyThrows + public void testDocValueFields_explicitBinaryFormat_returnsCorrectBase64ForBothEngines() { + for (KNNEngine engine : List.of(KNNEngine.FAISS, KNNEngine.LUCENE)) { + String indexName = TEST_INDEX + "_binary_fmt_" + engine.getName(); + + KNNJsonIndexMappingsBuilder.Method method = KNNJsonIndexMappingsBuilder.Method.builder() + .methodName(METHOD_HNSW) + .spaceType(SpaceType.L2.getValue()) + .engine(engine.getName()) + .build(); + + String mapping = KNNJsonIndexMappingsBuilder.builder() + .fieldName(VECTOR_FIELD) + .dimension(DIMENSION) + .vectorDataType(VectorDataType.FLOAT.getValue()) + .method(method) + .build() + .getIndexMapping(); + + createKnnIndex(indexName, mapping); + addKnnDoc(indexName, "1", VECTOR_FIELD, Floats.asList(VECTOR_1).toArray()); + addKnnDoc(indexName, "2", VECTOR_FIELD, Floats.asList(VECTOR_2).toArray()); + refreshIndex(indexName); + + String query = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("knn") + .startObject(VECTOR_FIELD) + .field("vector", VECTOR_1) + .field("k", 2) + .endObject() + .endObject() + .endObject() + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "binary") + .endObject() + .endArray() + .field("_source", false) + .endObject() + .toString(); + + Response response = searchKNNIndex(indexName, query, 2); + String responseBody = EntityUtils.toString(response.getEntity()); + List> hits = parseSearchHits(responseBody); + + assertEquals("Expected 2 hits for engine " + engine.getName(), 2, hits.size()); + + // KNN with VECTOR_1 as query: closest is VECTOR_1, then VECTOR_2 + float[][] expectedVectors = { VECTOR_1, VECTOR_2 }; + for (int i = 0; i < hits.size(); i++) { + Map fields = getFields(hits.get(i)); + assertNotNull("Fields should not be null for engine " + engine.getName(), fields); + List binaryValues = getBinaryDocValueField(hits.get(i), VECTOR_FIELD); + assertNotNull("Binary field should not be null for engine " + engine.getName(), binaryValues); + assertEquals(1, binaryValues.size()); + + String base64Value = binaryValues.get(0); + assertNotNull(base64Value); + assertFalse(base64Value.isEmpty()); + + byte[] decoded = Base64.getDecoder().decode(base64Value); + assertEquals(DIMENSION * Float.BYTES, decoded.length); + + ByteBuffer buffer = ByteBuffer.wrap(decoded).order(ByteOrder.LITTLE_ENDIAN); + for (int j = 0; j < DIMENSION; j++) { + assertEquals( + "Mismatch at index " + j + " for engine " + engine.getName(), + expectedVectors[i][j], + buffer.getFloat(), + 0.001f + ); + } + } + + deleteKNNIndex(indexName); + } + } + + /** + * Verifies that docvalue_fields works correctly when index.knn is false and a script_score + * query is used for search. This validates that vector retrieval via doc values does not + * depend on the KNN graph being present. + */ + @SneakyThrows + public void testDocValueFields_scriptScoreQuery_indexKnnFalse() { + String indexName = TEST_INDEX + "_script_score"; + + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject("properties") + .startObject(VECTOR_FIELD) + .field("type", "knn_vector") + .field("dimension", DIMENSION) + .endObject() + .endObject() + .endObject() + .toString(); + + Settings settings = Settings.builder().put("number_of_shards", 1).put("number_of_replicas", 0).put("index.knn", false).build(); + + createKnnIndex(indexName, settings, mapping); + addKnnDoc(indexName, "1", VECTOR_FIELD, Floats.asList(VECTOR_1).toArray()); + addKnnDoc(indexName, "2", VECTOR_FIELD, Floats.asList(VECTOR_2).toArray()); + addKnnDoc(indexName, "3", VECTOR_FIELD, Floats.asList(VECTOR_3).toArray()); + refreshIndex(indexName); + + String query = XContentFactory.jsonBuilder() + .startObject() + .startObject("query") + .startObject("script_score") + .startObject("query") + .startObject("match_all") + .endObject() + .endObject() + .startObject("script") + .field("source", "knn_score") + .field("lang", "knn") + .startObject("params") + .field("field", VECTOR_FIELD) + .field("query_value", VECTOR_1) + .field("space_type", SpaceType.L2.getValue()) + .endObject() + .endObject() + .endObject() + .endObject() + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() + .field("_source", false) + .endObject() + .toString(); + + Response response = searchKNNIndex(indexName, query, 10); + String responseBody = EntityUtils.toString(response.getEntity()); + List> hits = parseSearchHits(responseBody); + + assertEquals(3, hits.size()); + for (Map hit : hits) { + assertNull("Expected no _source in script_score response", hit.get("_source")); + List> vectorField = getDocValueField(hit, VECTOR_FIELD); + assertNotNull("Expected vector in docvalue_fields for script_score query", vectorField); + assertEquals("Expected single vector value", 1, vectorField.size()); + assertEquals("Expected correct dimension", DIMENSION, vectorField.get(0).size()); + } + + // Verify the closest vector is VECTOR_1 (doc "1" should be first since we query with VECTOR_1) + List> firstHitVector = getDocValueField(hits.get(0), VECTOR_FIELD); + for (int i = 0; i < DIMENSION; i++) { + assertEquals("Vector component mismatch at index " + i, VECTOR_1[i], firstHitVector.get(0).get(i).floatValue(), 0.001f); + } + + deleteKNNIndex(indexName); } /** @@ -917,7 +1221,12 @@ public void testDocValueFields_docsWithoutVectorField_returnsEmptyFields() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", true) .startObject("sort") .field("_id", "asc") @@ -975,7 +1284,12 @@ public void testDocValueFields_nestedField_topLevelReturnsEmpty() { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", nestedVectorField) + .startArray("docvalue_fields") + .startObject() + .field("field", nestedVectorField) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .toString(); @@ -1028,7 +1342,12 @@ public void testDocValueFields_nestedField_innerHitsReturnsVector() { .endObject() .endObject() .startObject("inner_hits") - .array("docvalue_fields", nestedVectorPath) + .startArray("docvalue_fields") + .startObject() + .field("field", nestedVectorPath) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .endObject() @@ -1122,7 +1441,12 @@ public void testDocValueFields_nestedField_innerHitsVectorsMatchSource() { .endObject() .endObject() .startObject("inner_hits") - .array("docvalue_fields", nestedVectorPath) + .startArray("docvalue_fields") + .startObject() + .field("field", nestedVectorPath) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .endObject() @@ -1207,7 +1531,12 @@ public void testDocValueFields_nestedField_expandNestedReturnsAllInnerHitVectors .endObject() .startObject("inner_hits") .field("size", 10) - .array("docvalue_fields", nestedVectorPath) + .startArray("docvalue_fields") + .startObject() + .field("field", nestedVectorPath) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .endObject() .endObject() @@ -1305,7 +1634,12 @@ private String buildDocValueFieldsQuery(float[] queryVector, boolean includeSour .endObject() .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD); + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray(); if (!includeSource) { builder.field("_source", false); } @@ -1341,6 +1675,13 @@ private List> getDocValueField(Map hit, String fiel return (List>) fields.get(fieldName); } + @SuppressWarnings("unchecked") + private List getBinaryDocValueField(Map hit, String fieldName) { + Map fields = (Map) hit.get("fields"); + if (fields == null) return null; + return (List) fields.get(fieldName); + } + @SuppressWarnings("unchecked") private Map getFields(Map hit) { return (Map) hit.get("fields"); @@ -1413,7 +1754,12 @@ private String buildSortedDocValueFieldsQuery() throws IOException { .startObject("match_all") .endObject() .endObject() - .array("docvalue_fields", VECTOR_FIELD) + .startArray("docvalue_fields") + .startObject() + .field("field", VECTOR_FIELD) + .field("format", "array") + .endObject() + .endArray() .field("_source", false) .startObject("sort") .field("_id", "asc")