Fix Tests/CI, refactor merge to call CAGRA's merge(), implement CAGRA prefiltering

build.sh

@@ -14,6 +14,47 @@ function hasArg {
     (( NUMARGS != 0 )) && (echo " ${ARGS} " | grep -q " $1 ")
 }
 
+if hasArg --build-cuvs-from-source; then
+
+    CUR_DIR=$(pwd)
+    # Get cmake locally if not found
+    if ! cmake; then
+        echo "==> cmake not found. Downloading..."
+        CMAKE_URL="https://github.com/Kitware/CMake/releases/download/v4.1.1/cmake-4.1.1-linux-x86_64.tar.gz"
+        wget $CMAKE_URL
+        tar -xvf cmake-4.1.1-linux-x86_64.tar.gz
+        CMAKE_HOME="cmake-4.1.1-linux-x86_64"
+        export PATH="$CUR_DIR/$CMAKE_HOME/bin:$PATH"
+        echo "PATH: $PATH"
+    fi
+
+    unset LD_LIBRARY_PATH
+    CUVS_REPO="https://github.com/rapidsai/cuvs.git"
+    CUVS_DIR="cuvs"
+    BUILD_DIR=$CUVS_DIR/cpp/build
+
+    # checkout cuvs
+    if [ ! -d "$CUVS_DIR" ]; then
+        echo "==> Current working directory is: $CUR_DIR, checking out cuvs repository into $CUR_DIR..."
+        RV=$(git clone $CUVS_REPO $CUVS_DIR)
+        if [ "$RV" -ne 0 ]; then
+            echo "==> cuvs checkout failed."
+            exit "$RV"
+        fi
+    fi
+
+    # build cuvs
+    echo "==> Building cuvs"
+    if ! ./$CUVS_DIR/build.sh libcuvs; then
+        echo "==> cuvs build.sh returned non-zero."
+        exit "$RV"
+    fi
+
+    # set LD_LIBRARY_PATH variable
+    export LD_LIBRARY_PATH="$CUR_DIR/$BUILD_DIR"
+    echo "==> LD_LIBRARY_PATH is set to: $LD_LIBRARY_PATH"
+fi
+
 MAVEN_VERIFY_ARGS=()
 if ! hasArg --run-java-tests; then
   MAVEN_VERIFY_ARGS=("-DskipTests")

ci/build_java.sh

@@ -7,9 +7,14 @@ set -euo pipefail
 #       and test_java.sh no longer calls build_java.sh
 #       ref: https://github.com/rapidsai/cuvs/issues/868
 EXTRA_BUILD_ARGS=()
-if [[ "${1:-}" == "--run-java-tests" ]]; then
-  EXTRA_BUILD_ARGS+=("--run-java-tests")
-fi
+for arg in "$@"; do
+  if [[ "$arg" == "--run-java-tests" ]]; then
+    EXTRA_BUILD_ARGS+=("--run-java-tests")
+  fi
+  if [[ "$arg" == "--build-cuvs-from-source" ]]; then
+    EXTRA_BUILD_ARGS+=("--build-cuvs-from-source")
+  fi
+done
 
 # shellcheck disable=SC1091
 . /opt/conda/etc/profile.d/conda.sh

ci/test_java.sh

@@ -10,4 +10,4 @@ rapids-logger "Run Java build and tests"
 
 # TODO: switch to installing pre-built artifacts instead of rebuilding in test jobs
 #       ref: https://github.com/rapidsai/cuvs/issues/868
-ci/build_java.sh --run-java-tests
+ci/build_java.sh --run-java-tests --build-cuvs-from-source

pom.xml

@@ -69,7 +69,7 @@
       <groupId>com.nvidia.cuvs</groupId>
       <artifactId>cuvs-java</artifactId>
       <!-- Note: This is temporary measure (artifact coming from https://maven.searchscale.com/snapshots, for now) and will get updated in subsequent PRs. -->
-      <version>25.8.0-4f53f-SNAPSHOT</version>
+      <version>25.10.0-2c0e1-SNAPSHOT</version>
     </dependency>
   </dependencies>
 

src/main/java/com/nvidia/cuvs/lucene/CuVSIndex.java

@@ -15,8 +15,6 @@
  */
 package com.nvidia.cuvs.lucene;
 
-import static com.nvidia.cuvs.lucene.CuVSVectorsReader.handleThrowable;
-
 import com.nvidia.cuvs.BruteForceIndex;
 import com.nvidia.cuvs.CagraIndex;
 import com.nvidia.cuvs.HnswIndex;
@@ -103,16 +101,16 @@ public void close() throws IOException {
   private void destroyIndices() throws IOException {
     try {
       if (cagraIndex != null) {
-        cagraIndex.destroyIndex();
+        cagraIndex.close();
       }
       if (bruteforceIndex != null) {
-        bruteforceIndex.destroyIndex();
+        bruteforceIndex.close();
       }
       if (hnswIndex != null) {
-        hnswIndex.destroyIndex();
+        hnswIndex.close();
       }
     } catch (Throwable t) {
-      handleThrowable(t);
+      Utils.handleThrowable(t);
     }
   }
 }

src/main/java/com/nvidia/cuvs/lucene/CuVSKnnFloatVectorQuery.java

@@ -19,6 +19,7 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.search.KnnFloatVectorQuery;
+import org.apache.lucene.search.Query;
 import org.apache.lucene.search.TopDocs;
 import org.apache.lucene.search.knn.KnnCollectorManager;
 import org.apache.lucene.util.Bits;
@@ -29,8 +30,9 @@ public class CuVSKnnFloatVectorQuery extends KnnFloatVectorQuery {
   private final int iTopK;
   private final int searchWidth;
 
-  public CuVSKnnFloatVectorQuery(String field, float[] target, int k, int iTopK, int searchWidth) {
-    super(field, target, k);
+  public CuVSKnnFloatVectorQuery(
+      String field, float[] target, int k, Query filter, int iTopK, int searchWidth) {
+    super(field, target, k, filter);
     this.iTopK = iTopK;
     this.searchWidth = searchWidth;
   }
@@ -46,7 +48,7 @@ protected TopDocs approximateSearch(
     PerLeafCuVSKnnCollector results = new PerLeafCuVSKnnCollector(k, iTopK, searchWidth);
 
     LeafReader reader = context.reader();
-    reader.searchNearestVectors(field, this.getTargetCopy(), results, null);
+    reader.searchNearestVectors(field, this.getTargetCopy(), results, acceptDocs);
     return results.topDocs();
   }
 }

src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsFormat.java

@@ -86,6 +86,12 @@ public CuVSVectorsFormat(
   }
 
   private static CuVSResources cuVSResourcesOrNull() {
+    try {
+      System.loadLibrary(
+          "cudart"); // nocommit: this is here so as to pass CI, should goto cuvs-java
+    } catch (UnsatisfiedLinkError e) {
+      LOG.warning("Could not load CUDA runtime library: " + e.getMessage());
+    }
     try {
       resources = CuVSResources.create();
       return resources;

src/main/java/com/nvidia/cuvs/lucene/CuVSVectorsReader.java

@@ -271,7 +271,7 @@ private CuVSIndex loadCuVSIndex(FieldEntry fieldEntry) throws IOException {
         }
       }
     } catch (Throwable t) {
-      handleThrowable(t);
+      Utils.handleThrowable(t);
     }
     return new CuVSIndex(cagraIndex, bruteForceIndex, hnswIndex);
   }
@@ -367,7 +367,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits
       try {
         searchResult = cagraIndex.search(query).getResults();
       } catch (Throwable t) {
-        handleThrowable(t);
+        Utils.handleThrowable(t);
       }
       // List expected to have only one entry because of single query "target".
       assert searchResult.size() == 1;
@@ -385,7 +385,7 @@ public void search(String field, float[] target, KnnCollector knnCollector, Bits
       try {
         searchResult = bruteforceIndex.search(query).getResults();
       } catch (Throwable t) {
-        handleThrowable(t);
+        Utils.handleThrowable(t);
       }
       assert searchResult.size() == 1;
       result = searchResult.getFirst();
@@ -472,12 +472,15 @@ static void checkVersion(int versionMeta, int versionVectorData, IndexInput in)
     }
   }
 
-  static void handleThrowable(Throwable t) throws IOException {
-    switch (t) {
-      case IOException ioe -> throw ioe;
-      case Error error -> throw error;
-      case RuntimeException re -> throw re;
-      case null, default -> throw new RuntimeException("UNEXPECTED: exception type", t);
-    }
+  public FieldInfos getFieldInfos() {
+    return fieldInfos;
+  }
+
+  public IntObjectHashMap<CuVSIndex> getCuvsIndexes() {
+    return cuvsIndices;
+  }
+
+  public IntObjectHashMap<FieldEntry> getFieldEntries() {
+    return fields;
   }
 }