From 819a51292126927b5d2f5f56a521045a77d475a8 Mon Sep 17 00:00:00 2001 From: Takuo Kuroki Date: Sat, 20 Jun 2026 16:28:11 +0900 Subject: [PATCH] Fix copy_to functionality with vector fields #3162 --- CHANGELOG.md | 1 + .../opensearch/knn/index/OpenSearchIT.java | 225 ++++++++++++++++++ .../opensearch/knn/integ/DerivedSourceIT.java | 173 ++++++++++++++ 3 files changed, 399 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 279c912c05..8f152c424f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), * Preserve raw non-XContent `_source` fields when derived source is enabled [#3402](https://github.com/opensearch-project/k-NN/pull/3402) * Fix dimension-based oversampling not applying for 32x compression [#3455](https://github.com/opensearch-project/k-NN/pull/3455) * Fix NPE in nested kNN search when index contains documents without nested object [#3368](https://github.com/opensearch-project/k-NN/pull/3368) +* Fix copy_to functionality with vector fields [#3162](https://github.com/opensearch-project/k-NN/pull/3162) * Turn off ACORN for MOS to match default Lucene HNSW behavior [#3346](https://github.com/opensearch-project/k-NN/pull/3346) * Preserve mixed-case derived source vector field names and add backward-compatible field resolution for previously lowercased segment metadata [#3313](https://github.com/opensearch-project/k-NN/pull/3313) * Fix rescore flag not propagating over transport layer in multi-node clusters [#3343](https://github.com/opensearch-project/k-NN/pull/3343) diff --git a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java index 295de42544..9078f13f8d 100644 --- a/src/test/java/org/opensearch/knn/index/OpenSearchIT.java +++ b/src/test/java/org/opensearch/knn/index/OpenSearchIT.java @@ -23,6 +23,7 @@ import org.opensearch.knn.KNNCompressionRestTestCase; import org.opensearch.knn.KNNResult; import org.apache.hc.core5.http.io.entity.EntityUtils; +import org.opensearch.Version; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -1786,6 +1787,230 @@ public void testKNNSearchWithProfilerEnabled_Radial() throws Exception { deleteKNNIndex(INDEX_NAME); } + @SneakyThrows + public void testCopyTo_whenSearchOnTargetField_thenSuccess() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String indexName = "test_copy_to_search"; + String sourceField = "source_vector"; + String targetField1 = "target_vector_1"; + String targetField2 = "target_vector_2"; + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + buildKnnVectorFieldMapping(builder, sourceField, 2, SpaceType.L2, KNNEngine.FAISS, new String[] { targetField1, targetField2 }); + buildKnnVectorFieldMapping(builder, targetField1, 2, SpaceType.L2, KNNEngine.FAISS, null); + buildKnnVectorFieldMapping(builder, targetField2, 2, SpaceType.L2, KNNEngine.LUCENE, null); + String mapping = builder.endObject().endObject().toString(); + + createKnnIndex(indexName, mapping); + + addKnnDoc(indexName, "1", sourceField, new Float[] { 1.0f, 1.0f }); + addKnnDoc(indexName, "2", sourceField, new Float[] { 10.0f, 10.0f }); + refreshAllIndices(); + + float[] queryVector = { 1.0f, 1.0f }; + List results1 = getResults(indexName, targetField1, queryVector, 1); + assertEquals(1, results1.size()); + assertEquals("1", results1.get(0).getDocId()); + + List results2 = getResults(indexName, targetField2, queryVector, 1); + assertEquals(1, results2.size()); + assertEquals("1", results2.get(0).getDocId()); + + deleteKNNIndex(indexName); + } + + @SneakyThrows + public void testCopyTo_whenDocUpdated_thenTargetFieldReflectsUpdate() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String indexName = "test_copy_to_update"; + String sourceField = "source_vector"; + String targetField = "target_vector"; + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + buildKnnVectorFieldMapping(builder, sourceField, 2, SpaceType.L2, KNNEngine.FAISS, targetField); + buildKnnVectorFieldMapping(builder, targetField, 2, SpaceType.L2, KNNEngine.FAISS, null); + String mapping = builder.endObject().endObject().toString(); + + createKnnIndex(indexName, mapping); + + // Insert doc near [10, 10] + addKnnDoc(indexName, "1", sourceField, new Float[] { 10.0f, 10.0f }); + refreshAllIndices(); + + // Verify target field finds doc 1 near [10, 10] + float[] queryNearTen = { 10.0f, 10.0f }; + List results = getResults(indexName, targetField, queryNearTen, 1); + assertEquals(1, results.size()); + assertEquals("1", results.get(0).getDocId()); + + // Update doc to be near [1, 1] + updateKnnDoc(indexName, "1", sourceField, new Float[] { 1.0f, 1.0f }); + refreshAllIndices(); + + // Verify target field reflects the update + float[] queryNearOne = { 1.0f, 1.0f }; + results = getResults(indexName, targetField, queryNearOne, 1); + assertEquals(1, results.size()); + assertEquals("1", results.get(0).getDocId()); + + deleteKNNIndex(indexName); + } + + @SneakyThrows + public void testCopyTo_whenDocDeleted_thenTargetFieldReflectsDeletion() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String indexName = "test_copy_to_delete"; + String sourceField = "source_vector"; + String targetField = "target_vector"; + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + buildKnnVectorFieldMapping(builder, sourceField, 2, SpaceType.L2, KNNEngine.FAISS, targetField); + buildKnnVectorFieldMapping(builder, targetField, 2, SpaceType.L2, KNNEngine.FAISS, null); + String mapping = builder.endObject().endObject().toString(); + + createKnnIndex(indexName, mapping); + + addKnnDoc(indexName, "1", sourceField, new Float[] { 1.0f, 1.0f }); + addKnnDoc(indexName, "2", sourceField, new Float[] { 10.0f, 10.0f }); + refreshAllIndices(); + assertEquals(2, getDocCount(indexName)); + + deleteKnnDoc(indexName, "1"); + refreshAllIndices(); + assertEquals(1, getDocCount(indexName)); + + // Search on target field should only return doc 2 + float[] queryVector = { 1.0f, 1.0f }; + List results = getResults(indexName, targetField, queryVector, 10); + assertEquals(1, results.size()); + assertEquals("2", results.get(0).getDocId()); + + deleteKNNIndex(indexName); + } + + @SneakyThrows + public void testCopyTo_whenForceMerge_thenTargetFieldSearchable() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String indexName = "test_copy_to_merge"; + String sourceField = "source_vector"; + String targetField = "target_vector"; + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + buildKnnVectorFieldMapping(builder, sourceField, 2, SpaceType.L2, KNNEngine.FAISS, targetField); + buildKnnVectorFieldMapping(builder, targetField, 2, SpaceType.L2, KNNEngine.FAISS, null); + String mapping = builder.endObject().endObject().toString(); + + createKnnIndex(indexName, mapping); + + addKnnDoc(indexName, "1", sourceField, new Float[] { 1.0f, 1.0f }); + addKnnDoc(indexName, "2", sourceField, new Float[] { 5.0f, 5.0f }); + addKnnDoc(indexName, "3", sourceField, new Float[] { 10.0f, 10.0f }); + refreshAllIndices(); + + forceMergeKnnIndex(indexName); + + float[] queryVector = { 1.0f, 1.0f }; + List results = getResults(indexName, targetField, queryVector, 2); + assertEquals(2, results.size()); + assertEquals("1", results.get(0).getDocId()); + + deleteKNNIndex(indexName); + } + + @SneakyThrows + public void testCopyTo_whenSourceAndTargetBothSearched_thenBothReturnResults() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String indexName = "test_copy_to_both"; + String sourceField = "source_vector"; + String targetField = "target_vector"; + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + buildKnnVectorFieldMapping(builder, sourceField, 2, SpaceType.L2, KNNEngine.FAISS, targetField); + buildKnnVectorFieldMapping(builder, targetField, 2, SpaceType.L2, KNNEngine.FAISS, null); + String mapping = builder.endObject().endObject().toString(); + + createKnnIndex(indexName, mapping); + + addKnnDoc(indexName, "1", sourceField, new Float[] { 1.0f, 1.0f }); + addKnnDoc(indexName, "2", sourceField, new Float[] { 10.0f, 10.0f }); + refreshAllIndices(); + + float[] queryVector = { 1.0f, 1.0f }; + + // Both source and target should return the same nearest neighbor + List sourceResults = getResults(indexName, sourceField, queryVector, 1); + List targetResults = getResults(indexName, targetField, queryVector, 1); + + assertEquals(1, sourceResults.size()); + assertEquals(1, targetResults.size()); + assertEquals(sourceResults.get(0).getDocId(), targetResults.get(0).getDocId()); + + deleteKNNIndex(indexName); + } + + @SneakyThrows + public void testCopyTo_whenTargetFieldHasDifferentDimension_thenFail() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String indexName = "test_copy_to_dim_mismatch"; + String sourceField = "source_vector"; + String targetField = "target_vector"; + + XContentBuilder builder = XContentFactory.jsonBuilder().startObject().startObject("properties"); + buildKnnVectorFieldMapping(builder, sourceField, 2, SpaceType.L2, KNNEngine.FAISS, targetField); + buildKnnVectorFieldMapping(builder, targetField, 3, SpaceType.L2, KNNEngine.FAISS, null); + String mapping = builder.endObject().endObject().toString(); + + createKnnIndex(indexName, mapping); + + ResponseException ex = expectThrows( + ResponseException.class, + () -> addKnnDoc(indexName, "1", sourceField, new Float[] { 1.0f, 1.0f }) + ); + assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString("Vector dimension mismatch")); + + deleteKNNIndex(indexName); + } + + private static XContentBuilder buildKnnVectorFieldMapping( + XContentBuilder builder, + String fieldName, + int dimension, + SpaceType spaceType, + KNNEngine engine, + Object copyTo + ) throws IOException { + builder.startObject(fieldName) + .field("type", "knn_vector") + .field("dimension", dimension) + .startObject("method") + .field(KNNConstants.NAME, KNNConstants.METHOD_HNSW) + .field(KNNConstants.METHOD_PARAMETER_SPACE_TYPE, spaceType.getValue()) + .field(KNNConstants.KNN_ENGINE, engine.getName()) + .endObject(); + if (copyTo != null) { + builder.field("copy_to", copyTo); + } + return builder.endObject(); + } + private List getResults(final String indexName, final String fieldName, final float[] vector, final int k) throws IOException, ParseException { final Response searchResponseField = searchKNNIndex(indexName, new KNNQueryBuilder(fieldName, vector, k), k); diff --git a/src/test/java/org/opensearch/knn/integ/DerivedSourceIT.java b/src/test/java/org/opensearch/knn/integ/DerivedSourceIT.java index 7b37ed2151..1b40428f9c 100644 --- a/src/test/java/org/opensearch/knn/integ/DerivedSourceIT.java +++ b/src/test/java/org/opensearch/knn/integ/DerivedSourceIT.java @@ -9,6 +9,7 @@ import lombok.SneakyThrows; import org.apache.hc.core5.http.io.entity.EntityUtils; import org.junit.Before; +import org.opensearch.Version; import org.opensearch.client.Request; import org.opensearch.client.Response; import org.opensearch.client.ResponseException; @@ -22,6 +23,8 @@ import org.opensearch.knn.Pair; import org.opensearch.knn.common.KNNConstants; import org.opensearch.knn.index.VectorDataType; +import org.opensearch.knn.index.query.KNNQueryBuilder; +import org.opensearch.knn.KNNResult; import org.opensearch.knn.common.annotation.ExpectRemoteBuildValidation; import java.io.IOException; @@ -344,6 +347,176 @@ public void testDerivedSource_withDynamicTemplates_andBulkIndexing() { deleteKNNIndex(indexName); } + @SneakyThrows + public void testDerivedSource_withCopyTo_thenTargetFieldSearchable() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String indexName = "test-derived-copy-to"; + String sourceField = "source_vector"; + String targetField = "target_vector"; + int dimension = 3; + + Settings settings = Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 0) + .put("index.knn", true) + .put("index.knn.derived_source.enabled", true) + .build(); + + XContentBuilder mappingBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject(KNNConstants.PROPERTIES) + .startObject(sourceField) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .field("copy_to", targetField) + .endObject() + .startObject(targetField) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .endObject() + .endObject() + .endObject(); + + createKnnIndex(indexName, settings, mappingBuilder.toString()); + + addKnnDoc(indexName, "1", sourceField, new Float[] { 1.0f, 2.0f, 3.0f }); + addKnnDoc(indexName, "2", sourceField, new Float[] { 10.0f, 20.0f, 30.0f }); + refreshIndex(indexName); + + // Verify _source is correctly reconstructed with derived source + Map doc1Source = getKnnDoc(indexName, "1"); + assertNotNull("Source vector should be reconstructed from derived source", doc1Source.get(sourceField)); + List sourceVector = (List) doc1Source.get(sourceField); + assertEquals(dimension, sourceVector.size()); + assertEquals(1.0, ((Number) sourceVector.get(0)).doubleValue(), 0.01); + + // Verify search on target (copy_to) field returns correct results + float[] queryVector = { 1.0f, 2.0f, 3.0f }; + Response searchResponse = searchKNNIndex(indexName, new KNNQueryBuilder(targetField, queryVector, 1), 1); + String searchResponseBody = EntityUtils.toString(searchResponse.getEntity()); + List results = parseSearchResponse(searchResponseBody, targetField); + assertEquals(1, results.size()); + assertEquals("1", results.get(0).getDocId()); + + deleteKNNIndex(indexName); + } + + @SneakyThrows + public void testDerivedSource_withCopyTo_thenSourceMatchesNonDerived() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String sourceField = "source_vector"; + String targetField = "target_vector"; + int dimension = 3; + String derivedIndex = "test-derived-copy-to-enabled"; + String baselineIndex = "test-derived-copy-to-disabled"; + + String mapping = XContentFactory.jsonBuilder() + .startObject() + .startObject(KNNConstants.PROPERTIES) + .startObject(sourceField) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .field("copy_to", targetField) + .endObject() + .startObject(targetField) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(DIMENSION, dimension) + .endObject() + .endObject() + .endObject() + .toString(); + + createKnnIndex( + derivedIndex, + Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 0) + .put("index.knn", true) + .put("index.knn.derived_source.enabled", true) + .build(), + mapping + ); + createKnnIndex( + baselineIndex, + Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 0) + .put("index.knn", true) + .put("index.knn.derived_source.enabled", false) + .build(), + mapping + ); + + Float[][] vectors = { { 1.0f, 2.0f, 3.0f }, { 10.0f, 20.0f, 30.0f }, { 5.0f, 5.0f, 5.0f } }; + for (int i = 0; i < vectors.length; i++) { + String docId = String.valueOf(i + 1); + addKnnDoc(derivedIndex, docId, sourceField, vectors[i]); + addKnnDoc(baselineIndex, docId, sourceField, vectors[i]); + } + refreshIndex(derivedIndex); + refreshIndex(baselineIndex); + + // Verify _source matches between derived and non-derived indices + for (int i = 0; i < vectors.length; i++) { + String docId = String.valueOf(i + 1); + Map derivedDoc = getKnnDoc(derivedIndex, docId); + Map baselineDoc = getKnnDoc(baselineIndex, docId); + assertEquals("Source field _source mismatch for doc " + docId, baselineDoc.get(sourceField), derivedDoc.get(sourceField)); + } + + deleteKNNIndex(derivedIndex); + deleteKNNIndex(baselineIndex); + } + + @SneakyThrows + public void testDerivedSource_withCopyTo_whenDimensionMismatch_thenFail() { + // copy_to for knn_vector requires OpenSearch 3.6.0+ + if (Version.CURRENT.before(Version.V_3_6_0)) { + return; + } + String indexName = "test-derived-copy-to-dim-mismatch"; + String sourceField = "source_vector"; + String targetField = "target_vector"; + + Settings settings = Settings.builder() + .put("number_of_shards", 1) + .put("number_of_replicas", 0) + .put("index.knn", true) + .put("index.knn.derived_source.enabled", true) + .build(); + + XContentBuilder mappingBuilder = XContentFactory.jsonBuilder() + .startObject() + .startObject(KNNConstants.PROPERTIES) + .startObject(sourceField) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(DIMENSION, 2) + .field("copy_to", targetField) + .endObject() + .startObject(targetField) + .field(KNNConstants.TYPE, KNNConstants.TYPE_KNN_VECTOR) + .field(DIMENSION, 3) + .endObject() + .endObject() + .endObject(); + + createKnnIndex(indexName, settings, mappingBuilder.toString()); + + ResponseException ex = expectThrows( + ResponseException.class, + () -> addKnnDoc(indexName, "1", sourceField, new Float[] { 1.0f, 1.0f }) + ); + assertThat(EntityUtils.toString(ex.getResponse().getEntity()), containsString("Vector dimension mismatch")); + + deleteKNNIndex(indexName); + } + @SneakyThrows public void testDerivedSource_withMixedCaseObjectVectorField() { String indexName = getIndexName("derived-source", "mixed-case", false);