Skip to content
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
### Features

### Enhancements
- [Storage perf] Stop writing the redundant binary doc values copy of the vector for jVector fields from 3.9.0 onwards. Preserve existing indices. [715] (https://github.com/opensearch-project/opensearch-jvector/pull/715)

### Bug Fixes
- Fix flaky testJVectorKnnIndex_filter_maxInnerProduct test case [681](https://github.com/opensearch-project/opensearch-jvector/pull/681)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -59,6 +59,36 @@ public static FieldType buildDocValuesFieldType(KNNEngine knnEngine) {
return field;
}

/**
* index version after which jVector fields stop writing binary doc values copy of the vector as it is unused.
* Indices created before this keep writing it; see {@link #shouldWriteBinaryDocValues(KNNEngine, Version)}.
*/
static final Version SKIP_BINARY_DOC_VALUES_FOR_JVECTOR_VERSION = Version.V_3_9_0;

/**
* Method to determine whether a vector field needs the extra binary doc values copy of the vector alongside
* the copy the {@link org.apache.lucene.codecs.KnnVectorsFormat} already writes.
* <p>
* jVector engine stores full-precision vectors inline in the graph and exposes them through
* {@code JVectorReader#getFloatVectorValues}. Every reader in this plugin — script doc values, {@code docvalue_fields},
* derived source, and merge — checks {@link org.apache.lucene.index.FieldInfo#hasVectorValues()} before falling back to
* binary doc values, and {@link KNNVectorFieldType} always reports {@code hasDocValues() == true} regardless of whether
* the physical field exists. The binary doc values copy for jvector engine is therefore unused costing one raw fp32 copy
* per document on disk, and adds to merge time.
* <p>
* Skip binary doc values storage for newly created indices after version SKIP_BINARY_DOC_VALUES_FOR_JVECTOR_VERSION.
*
* @param knnEngine KNNEngine backing the field
* @param indexCreatedVersion version the index was created on
* @return true if the binary doc values copy of the vector should be written
*/
public static boolean shouldWriteBinaryDocValues(final KNNEngine knnEngine, final Version indexCreatedVersion) {
if (knnEngine != KNNEngine.JVECTOR) {
return true;
}
return indexCreatedVersion.before(SKIP_BINARY_DOC_VALUES_FOR_JVECTOR_VERSION);
}

/**
* Creates a stored field for a byte vector
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -107,7 +107,12 @@ private LuceneFieldMapper(

this.fieldType = vectorDataType.createKnnVectorFieldType(knnMappingConfig.getDimension(), knnVectorSimilarityFunction);

if (this.hasDocValues) {
// jVector keeps the full-precision vectors inline in the graph (.data-jvector) and serves them through
// JVectorReader#getFloatVectorValues. Binary doc values copy is never read.
// Skipping it removes one raw fp32 copy per document giving indexing/merge time improvements. Do this
// for new indices after v3.9.0.0.
if (this.hasDocValues
Comment thread
reta marked this conversation as resolved.
Resolved
&& KNNVectorFieldMapperUtil.shouldWriteBinaryDocValues(resolvedKnnMethodContext.getKnnEngine(), this.indexCreatedVersion)) {
this.vectorFieldType = buildDocValuesFieldType(resolvedKnnMethodContext.getKnnEngine());
} else {
this.vectorFieldType = null;
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,8 @@

import lombok.SneakyThrows;
import lombok.extern.log4j.Log4j2;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.DocValuesType;
import org.opensearch.Version;
import org.opensearch.cluster.metadata.IndexMetadata;
import org.opensearch.common.ValidationException;
Expand All @@ -23,6 +25,7 @@
import org.opensearch.index.mapper.ParseContext;
import org.opensearch.knn.KNNTestCase;
import org.opensearch.knn.common.KNNConstants;
import org.opensearch.knn.index.DerivedKnnFloatVectorField;
import org.opensearch.knn.index.SpaceType;
import org.opensearch.knn.index.VectorDataType;
import org.opensearch.knn.index.engine.KNNEngine;
Expand Down Expand Up @@ -125,6 +128,98 @@ public void testTypeParser_build_fromKnnMethodContext() throws IOException {
);
}

/**
* jVector keeps full-precision vectors inline in the graph and serves them via
* {@code JVectorReader#getFloatVectorValues}, so the extra binary doc values copy is never read. Only indices created
* on or after the cutoff may skip it — older ones already recorded the field as BINARY and should be unchanged.
*/
public void testShouldWriteBinaryDocValues_jvectorSkipsOnlyForNewIndices() {
Version preCutoff = Version.V_3_8_0;
Version cutoff = KNNVectorFieldMapperUtil.SKIP_BINARY_DOC_VALUES_FOR_JVECTOR_VERSION;

assertFalse(KNNVectorFieldMapperUtil.shouldWriteBinaryDocValues(KNNEngine.JVECTOR, cutoff));
assertTrue(KNNVectorFieldMapperUtil.shouldWriteBinaryDocValues(KNNEngine.JVECTOR, preCutoff));
assertTrue(KNNVectorFieldMapperUtil.shouldWriteBinaryDocValues(KNNEngine.JVECTOR, Version.V_2_17_0));

// Engine scoping: every other engine keeps binary doc values regardless of version.
assertTrue(KNNVectorFieldMapperUtil.shouldWriteBinaryDocValues(KNNEngine.LUCENE, cutoff));
assertTrue(KNNVectorFieldMapperUtil.shouldWriteBinaryDocValues(KNNEngine.LUCENE, preCutoff));
}

public void testLuceneFieldMapper_jvector_newIndex_doesNotWriteBinaryDocValues() throws IOException {
KNNVectorFieldMapper mapper = buildLuceneBackedMapper(
KNNEngine.JVECTOR,
DISK_ANN,
KNNVectorFieldMapperUtil.SKIP_BINARY_DOC_VALUES_FOR_JVECTOR_VERSION
);
assertTrue(mapper instanceof LuceneFieldMapper);

List<Field> fields = mapper.getFieldsForFloatVector(new float[TEST_DIMENSION], false);

// The KnnVectorsFormat copy is still written ...
assertEquals(1, fields.size());
assertTrue(fields.get(0) instanceof DerivedKnnFloatVectorField);
// ... and the redundant fp32 copy in binary doc values is not.
assertTrue(fields.stream().noneMatch(field -> field.fieldType().docValuesType() == DocValuesType.BINARY));
}

/**
* A jVector index created before the cutoff already has the field as BINARY in its existing segments, so new segments
* must keep declaring BINARY or Lucene's FieldNumbers rejects the schema change on the next write after upgrade.
*/
public void testLuceneFieldMapper_jvector_preExistingIndex_stillWritesBinaryDocValues() throws IOException {
KNNVectorFieldMapper mapper = buildLuceneBackedMapper(KNNEngine.JVECTOR, DISK_ANN, Version.V_3_8_0);
assertTrue(mapper instanceof LuceneFieldMapper);

List<Field> fields = mapper.getFieldsForFloatVector(new float[TEST_DIMENSION], false);

assertTrue(fields.stream().anyMatch(field -> field.fieldType().docValuesType() == DocValuesType.BINARY));
}

/**
* Guards the narrow scope of the jVector change: the Lucene engine shares {@link LuceneFieldMapper} and must keep
* writing binary doc values at any index version.
*/
public void testLuceneFieldMapper_lucene_stillWritesBinaryDocValues() throws IOException {
KNNVectorFieldMapper mapper = buildLuceneBackedMapper(
KNNEngine.LUCENE,
METHOD_HNSW,
KNNVectorFieldMapperUtil.SKIP_BINARY_DOC_VALUES_FOR_JVECTOR_VERSION
);
assertTrue(mapper instanceof LuceneFieldMapper);

List<Field> fields = mapper.getFieldsForFloatVector(new float[TEST_DIMENSION], false);

assertTrue(fields.stream().anyMatch(field -> field.fieldType().docValuesType() == DocValuesType.BINARY));
}

private KNNVectorFieldMapper buildLuceneBackedMapper(
final KNNEngine knnEngine,
final String methodName,
final Version indexCreatedVersion
) throws IOException {
XContentBuilder xContentBuilder = XContentFactory.jsonBuilder()
.startObject()
.field(TYPE_FIELD_NAME, KNN_VECTOR_TYPE)
.field(DIMENSION_FIELD_NAME, TEST_DIMENSION)
.startObject(KNN_METHOD)
.field(NAME, methodName)
.field(KNN_ENGINE, knnEngine.getName())
.field(METHOD_PARAMETER_SPACE_TYPE, SpaceType.L2.getValue())
.endObject()
.endObject();

Settings settings = Settings.builder().put(settings(indexCreatedVersion).build()).put(KNN_INDEX, true).build();
// buildParserContext pins the parser context to Version.CURRENT, but the whole point here is the index created
// version, which is what KNNMethodConfigContext#getVersionCreated ends up carrying into the mapper.
KNNVectorFieldMapper.Builder builder = (KNNVectorFieldMapper.Builder) new KNNVectorFieldMapper.TypeParser().parse(
TEST_FIELD_NAME,
xContentBuilderToMap(xContentBuilder),
dobuildParserContext(TEST_INDEX_NAME, settings, indexCreatedVersion)
);
return builder.build(new Mapper.BuilderContext(settings, new ContentPath()));
}

public void testTypeParser_parse_fromKnnMethodContext_invalidDimension() throws IOException {
KNNVectorFieldMapper.TypeParser typeParser = new KNNVectorFieldMapper.TypeParser();
SpaceType spaceType = SpaceType.L2;
Expand Down
Loading