Updated regression log and batches to include BGE (#2338)

castorini · Jan 14, 2024 · e3134e9 · e3134e9
1 parent 53514b1
commit e3134e9
Show file tree

Hide file tree

Showing 3 changed files with 19 additions and 1 deletion.
diff --git a/docs/regressions-log.md b/docs/regressions-log.md
@@ -3,11 +3,17 @@
 The following change log details commits to regression tests that alter effectiveness and the addition of new regression tests.
 This documentation is useful for figuring why results may have changed over time.
 
+### January 13, 2024
+
++ commit [`883539`](https://github.com/castorini/anserini/commit/47876e8f7b5e65b11c19692d7c07a2e87a410533) (1/13/2024)
+
+Added HNSW regressions for the BGE-base-en-v1.5 model on the MS MARCO passage corpus: dev, DL19, and DL20; both original and int8 quantized versions. 
+
 ### December 19, 2023
 
 + commit [`883539`](https://github.com/castorini/anserini/commit/883539b119e3c4f236f4fda3dd748a3b2d4ff1a1) (12/19/2023)
 
-Upgraded to Lucene 9.9.1, added following regressions:
+Upgraded to Lucene 9.9.1, added the following HNSW regressions:
 
 + `msmarco-passage-cos-dpr-distil-hnsw-int8-onnx`
 + `msmarco-passage-cos-dpr-distil-hnsw-int8`

diff --git a/docs/regressions.md b/docs/regressions.md
@@ -54,6 +54,8 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre
 nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-cos-dpr-distil-hnsw-int8 >& logs/log.msmarco-passage-cos-dpr-distil-hnsw-int8 &
 nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-cos-dpr-distil-fw >& logs/log.msmarco-passage-cos-dpr-distil-fw &
 nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-cos-dpr-distil-lexlsh >& logs/log.msmarco-passage-cos-dpr-distil-lexlsh &
+nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-bge-base-en-v1.5-hnsw >& logs/log.msmarco-passage-bge-base-en-v1.5-hnsw &
+nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-bge-base-en-v1.5-hnsw-int8 >& logs/log.msmarco-passage-bge-base-en-v1.5-hnsw-int8 &
 nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-openai-ada2 >& logs/log.msmarco-passage-openai-ada2 &
 
 nohup python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-splade-pp-ed-onnx >& logs/log.msmarco-passage-splade-pp-ed-onnx &
@@ -88,6 +90,8 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-cos-dpr-distil-hnsw-int8 >& logs/log.dl19-passage-cos-dpr-distil-hnsw-int8 &
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-cos-dpr-distil-fw >& logs/log.dl19-passage-cos-dpr-distil-fw &
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-cos-dpr-distil-lexlsh >& logs/log.dl19-passage-cos-dpr-distil-lexlsh &
+nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-bge-base-en-v1.5-hnsw >& logs/log.dl19-passage-bge-base-en-v1.5-hnsw &
+nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-bge-base-en-v1.5-hnsw-int8 >& logs/log.dl19-passage-bge-base-en-v1.5-hnsw-int8 &
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-openai-ada2 >& logs/log.dl19-passage-openai-ada2 &
 
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl19-passage-splade-pp-ed-onnx >& logs/log.dl19-passage-splade-pp-ed-onnx &
@@ -122,6 +126,8 @@ nohup python src/main/python/run_regression.py --index --verify --search --regre
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-cos-dpr-distil-hnsw-int8 >& logs/log.dl20-passage-cos-dpr-distil-hnsw-int8 &
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-cos-dpr-distil-fw >& logs/log.dl20-passage-cos-dpr-distil-fw &
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-cos-dpr-distil-lexlsh >& logs/log.dl20-passage-cos-dpr-distil-lexlsh &
+nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-bge-base-en-v1.5-hnsw >& logs/log.dl20-passage-bge-base-en-v1.5-hnsw &
+nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-bge-base-en-v1.5-hnsw-int8 >& logs/log.dl20-passage-bge-base-en-v1.5-hnsw-int8 &
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-openai-ada2 >& logs/log.dl20-passage-openai-ada2 &
 
 nohup python src/main/python/run_regression.py --index --verify --search --regression dl20-passage-splade-pp-ed-onnx >& logs/log.dl20-passage-splade-pp-ed-onnx &

diff --git a/src/main/python/regressions-batch03.txt b/src/main/python/regressions-batch03.txt
@@ -1,3 +1,5 @@
+python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-bge-base-en-v1.5-hnsw > logs/log.msmarco-passage-bge-base-en-v1.5-hnsw 2>&1
+python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-bge-base-en-v1.5-hnsw-int8 > logs/log.msmarco-passage-bge-base-en-v1.5-hnsw-int8 2>&1
 python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-cos-dpr-distil-hnsw > logs/log.msmarco-passage-cos-dpr-distil-hnsw 2>&1
 python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-cos-dpr-distil-hnsw-int8 > logs/log.msmarco-passage-cos-dpr-distil-hnsw-int8 2>&1
 python src/main/python/run_regression.py --index --verify --search --regression msmarco-passage-openai-ada2 > logs/log.msmarco-passage-openai-ada2 2>&1
@@ -136,6 +138,8 @@ python src/main/python/run_regression.py --search --regression dl19-passage-docT
 python src/main/python/run_regression.py --search --regression dl19-passage-splade-pp-ed-onnx > logs/log.dl19-passage-splade-pp-ed-onnx 2>&1
 python src/main/python/run_regression.py --search --regression dl19-passage-splade-pp-sd-onnx > logs/log.dl19-passage-splade-pp-sd-onnx 2>&1
 
+python src/main/python/run_regression.py --search --regression dl19-passage-bge-base-en-v1.5-hnsw > logs/log.dl19-passage-bge-base-en-v1.5-hnsw 2>&1
+python src/main/python/run_regression.py --search --regression dl19-passage-bge-base-en-v1.5-hnsw-int8 > logs/log.dl19-passage-bge-base-en-v1.5-hnsw-int8 2>&1
 python src/main/python/run_regression.py --search --regression dl19-passage-cos-dpr-distil-hnsw > logs/log.dl19-passage-cos-dpr-distil-hnsw 2>&1
 python src/main/python/run_regression.py --search --regression dl19-passage-cos-dpr-distil-hnsw-int8 > logs/log.dl19-passage-cos-dpr-distil-hnsw-int8 2>&1
 python src/main/python/run_regression.py --search --regression dl19-passage-cos-dpr-distil-fw > logs/log.dl19-passage-cos-dpr-distil-fw 2>&1
@@ -171,6 +175,8 @@ python src/main/python/run_regression.py --search --regression dl20-passage-docT
 python src/main/python/run_regression.py --search --regression dl20-passage-splade-pp-ed-onnx > logs/log.dl20-passage-splade-pp-ed-onnx 2>&1
 python src/main/python/run_regression.py --search --regression dl20-passage-splade-pp-sd-onnx > logs/log.dl20-passage-splade-pp-sd-onnx 2>&1
 
+python src/main/python/run_regression.py --search --regression dl20-passage-bge-base-en-v1.5-hnsw > logs/log.dl20-passage-bge-base-en-v1.5-hnsw 2>&1
+python src/main/python/run_regression.py --search --regression dl20-passage-bge-base-en-v1.5-hnsw-int8 > logs/log.dl20-passage-bge-base-en-v1.5-hnsw-int8 2>&1
 python src/main/python/run_regression.py --search --regression dl20-passage-cos-dpr-distil-hnsw > logs/log.dl20-passage-cos-dpr-distil-hnsw 2>&1
 python src/main/python/run_regression.py --search --regression dl20-passage-cos-dpr-distil-hnsw-int8 > logs/log.dl20-passage-cos-dpr-distil-hnsw-int8 2>&1
 python src/main/python/run_regression.py --search --regression dl20-passage-cos-dpr-distil-fw > logs/log.dl20-passage-cos-dpr-distil-fw 2>&1