Adding bulkSize for benchmarking to better reflect realworld usage

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt7uBulkBenchmark.java

@@ -59,7 +59,7 @@
  * Benchmark that compares bulk scoring of various scalar quantized vector similarity function
  * implementations (scalar, lucene's panama-ized, and Elasticsearch's native) against sequential
  * and random access target vectors.
- * Run with ./gradlew -p benchmarks run --args 'VectorScorerInt7uBulkScorerBenchmark'
+ * Run with ./gradlew -p benchmarks run --args 'VectorScorerInt7uBulkBenchmark'
  */
 @Fork(value = 1, jvmArgsPrepend = { "--add-modules=jdk.incubator.vector" })
 @Warmup(iterations = 3, time = 3)
@@ -80,14 +80,22 @@ public class VectorScorerInt7uBulkBenchmark {
     @Param({ "1024" })
     public int dims;
 
-    // 128k is typically enough to not fit in L1 (core) cache for most processors;
-    // 1.5M is typically enough to not fit in L2 (core) cache;
-    // 130M is enough to not fit in L3 cache
+    // 128kb is typically enough to not fit in L1 (core) cache for most processors;
+    // 1.5Mb is typically enough to not fit in L2 (core) cache;
+    // 130Mb is enough to not fit in L3 cache
     @Param({ "128", "1500", "130000" })
     public int numVectors;
     public int numVectorsToScore;
 
-    @Param
+    // Bulk sizes to test.
+    // DiskBBQ has two bulk sizes, 16 and 32
+    // HNSW params will have the distributed ordinal bulk sizes depending on the number of connections in the graph
+    // The default is 16, maximum is 512, and the bottom layer is 2x that the configured setting, so 1024 is a maximum
+    // the MOST common case here is 32
+    @Param({ "16", "32", "64", "256", "1024" })
+    public int bulkSize;
+
+    @Param({ "SCALAR", "LUCENE", "NATIVE" })
     public VectorImplementation implementation;
 
     @Param({ "DOT_PRODUCT", "EUCLIDEAN" })
@@ -163,6 +171,7 @@ public void setScoringOrdinal(int targetOrd) throws IOException {
     private float[] scores;
     private int[] ordinals;
     private int[] ids;
+    private int[] toScore; // scratch array for bulk scoring
 
     private UpdateableRandomVectorScorer scorer;
     private RandomVectorScorer queryScorer;
@@ -213,7 +222,8 @@ void setup(VectorData vectorData) throws IOException {
         writeInt7VectorData(dir, vectorData.vectorData, vectorData.offsets);
 
         numVectorsToScore = vectorData.numVectorsToScore;
-        scores = new float[numVectorsToScore];
+        scores = new float[bulkSize];
+        toScore = new int[bulkSize]; // scratch array for ordinal slices
         ids = IntStream.range(0, numVectors).toArray();
         ordinals = vectorData.ordinals;
 
@@ -255,43 +265,67 @@ public void teardown() throws IOException {
 
     @Benchmark
     public float[] scoreMultipleSequential() throws IOException {
-        for (int v = 0; v < numVectorsToScore; v++) {
-            scores[v] = scorer.score(v);
+        int v = 0;
+        while (v < numVectorsToScore) {
+            for (int i = 0; i < bulkSize && v < numVectorsToScore; i++, v++) {
+                scores[i] = scorer.score(v);
+            }
         }
         return scores;
     }
 
     @Benchmark
     public float[] scoreMultipleRandom() throws IOException {
-        for (int v = 0; v < numVectorsToScore; v++) {
-            scores[v] = scorer.score(ordinals[v]);
+        int v = 0;
+        while (v < numVectorsToScore) {
+            for (int i = 0; i < bulkSize && v < numVectorsToScore; i++, v++) {
+                scores[i] = scorer.score(ordinals[v]);
+            }
         }
         return scores;
     }
 
     @Benchmark
     public float[] scoreQueryMultipleRandom() throws IOException {
-        for (int v = 0; v < numVectorsToScore; v++) {
-            scores[v] = queryScorer.score(ordinals[v]);
+        int v = 0;
+        while (v < numVectorsToScore) {
+            for (int i = 0; i < bulkSize && v < numVectorsToScore; i++, v++) {
+                scores[i] = queryScorer.score(ordinals[v]);
+            }
         }
         return scores;
     }
 
     @Benchmark
     public float[] scoreMultipleSequentialBulk() throws IOException {
-        scorer.bulkScore(ids, scores, ordinals.length);
+        for (int i = 0; i < numVectorsToScore; i += bulkSize) {
+            int toScoreInThisBatch = Math.min(bulkSize, numVectorsToScore - i);
+            // Copy the slice of sequential IDs to the scratch array
+            System.arraycopy(ids, i, toScore, 0, toScoreInThisBatch);
+            scorer.bulkScore(toScore, scores, toScoreInThisBatch);
+        }
         return scores;
     }
 
     @Benchmark
     public float[] scoreMultipleRandomBulk() throws IOException {
-        scorer.bulkScore(ordinals, scores, ordinals.length);
+        for (int i = 0; i < numVectorsToScore; i += bulkSize) {
+            int toScoreInThisBatch = Math.min(bulkSize, numVectorsToScore - i);
+            // Copy the slice of random ordinals to the scratch array
+            System.arraycopy(ordinals, i, toScore, 0, toScoreInThisBatch);
+            scorer.bulkScore(toScore, scores, toScoreInThisBatch);
+        }
         return scores;
     }
 
     @Benchmark
     public float[] scoreQueryMultipleRandomBulk() throws IOException {
-        queryScorer.bulkScore(ordinals, scores, ordinals.length);
+        for (int i = 0; i < numVectorsToScore; i += bulkSize) {
+            int toScoreInThisBatch = Math.min(bulkSize, numVectorsToScore - i);
+            // Copy the slice of random ordinals to the scratch array
+            System.arraycopy(ordinals, i, toScore, 0, toScoreInThisBatch);
+            queryScorer.bulkScore(toScore, scores, toScoreInThisBatch);
+        }
         return scores;
     }
 }

benchmarks/src/test/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt7uBulkBenchmarkTests.java

@@ -49,6 +49,7 @@ public void testSequential() throws Exception {
                 bench.dims = dims;
                 bench.numVectors = 1000;
                 bench.numVectorsToScore = 200;
+                bench.bulkSize = 200;
                 bench.setup(vectorData);
 
                 try {
@@ -79,6 +80,7 @@ public void testRandom() throws Exception {
                 bench.dims = dims;
                 bench.numVectors = 1000;
                 bench.numVectorsToScore = 200;
+                bench.bulkSize = 200;
                 bench.setup(vectorData);
 
                 try {
@@ -110,6 +112,7 @@ public void testQueryRandom() throws Exception {
                 bench.dims = dims;
                 bench.numVectors = 1000;
                 bench.numVectorsToScore = 200;
+                bench.bulkSize = 200;
                 bench.setup(vectorData);
 
                 try {