MB-65243: Merge facebookresearch/faiss@v1.11.0 into blevesearch/faiss@bleve

.github/actions/build_cmake/action.yml

@@ -39,11 +39,11 @@ runs:
         conda update -y -q conda
         echo "$CONDA/bin" >> $GITHUB_PATH
 
-        conda install -y -q python=3.11 cmake=3.26 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2
+        conda install -y -q python=3.11 cmake=3.30.4 make=4.2 swig=4.0 "numpy<2" scipy=1.14 pytest=7.4 gflags=2.2
 
         # install base packages for ARM64
         if [ "${{ runner.arch }}" = "ARM64" ]; then
-          conda install -y -q -c conda-forge openblas=0.3 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
+          conda install -y -q -c conda-forge openblas=0.3.29 gxx_linux-aarch64=14.2 sysroot_linux-aarch64=2.17
         fi
 
         # install base packages for X86_64
@@ -61,7 +61,7 @@ runs:
           conda install -y -q cuda-toolkit=12.4 -c "nvidia/label/cuda-12.4.0"
         # and CUDA from cuVS channel for cuVS builds
         elif [ "${{ inputs.cuvs }}" = "ON" ]; then
-          conda install -y -q libcuvs=24.12 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c conda-forge
+          conda install -y -q libcuvs=25.04 'cuda-version>=12.0,<=12.5' cuda-toolkit=12.4.1 gxx_linux-64=12.4 -c rapidsai -c rapidsai-nightly -c conda-forge
         fi
 
         # install test packages

.github/actions/build_conda/action.yml

@@ -44,7 +44,7 @@ runs:
         # Ensure starting packages are from conda-forge.
         conda list --show-channel-urls
         conda install -y -q "conda!=24.11.0"
-        conda install -y -q "conda-build!=24.11.0"
+        conda install -y -q "conda-build=25.3.1" "liblief=0.14.1"
         conda list --show-channel-urls
     - name: Enable anaconda uploads
       if: inputs.label != ''

.github/workflows/retry_build.yml

@@ -15,7 +15,10 @@ jobs:
           GH_TOKEN: ${{ github.token }}
           GH_DEBUG: api
         run: |
-          while gh run view ${{ inputs.run_id }} --json status | grep -q in_progress
+          # status can be one of "queued", "in_progress", "completed", "waiting", "requested", "pending"
+          # https://docs.github.com/en/rest/checks/runs
+          # while not completed, sleep for 10 minutes
+          while gh run view ${{ inputs.run_id }} --json status | grep -v completed
           do
             echo Workflow in progress - sleeping for 10 minutes then checking again
             sleep 10m

CHANGELOG.md

@@ -3,6 +3,87 @@ All notable changes to this project will be documented in this file.
 
 ## [Unreleased]
 
+## [1.11.0] - 2025-04-24
+
+
+Added
+- RaBitQ implementation (#4235)
+- Add RaBitQ to the swigfaiss so we can access its properties correctly in python (#4304)
+- Add date and time to the codec file path so that the file doesn't get overridden with each run (#4303)
+- Add missing header in faiss/CMakeLists.txt (#4285)
+- Implement is_spherical and normalize_L2 booleans as part of the training APIs (#4279)
+- Add normalize_l2 boolean to distributed training API
+- re-land mmap diff (#4250)
+- SearchParameters support for IndexBinaryFlat (#4055)
+- Support non-partition col and map in the embedding reader (#4229)
+- Support cosine distance for training vectors (#4227)
+- Add missing #include in code_distance-sve.h (#4219)
+- Add the support for IndexIDMap with Cagra index (#4188)
+- Add bounds checking to hnsw nb_neighbors (#4185)
+- Add sharding convenience function for IVF indexes (#4150)
+- Added support for building for MinGW, in addition to MSVC (#4145)
+
+Changed
+- Skip mmap test case in AIX. (#4275)
+- Handle insufficient driver gracefully (#4271)
+- relax input params for IndexIVFRaBitQ::get_InvertedListScanner() (#4270)
+- Allow using custom index readers and writers (#4180)
+- Upgrade to libcuvs=25.04 (#4164)
+- ignore regex (#4264)
+- Publish the C API to Conda (#4186)
+- Pass row filters to Hive Reader to filter rows (#4256)
+- Back out "test merge with internal repo" (#4244)
+- test merge with internal repo (#4242)
+- Revert D69972250: Memory-mapping and Zero-copy deserializers
+- Revert D69984379: mem mapping and zero-copy python fixes
+- mem mapping and zero-copy python fixes (#4212)
+- Memory-mapping and Zero-copy deserializers (#4199)
+- Use `nullptr` in faiss/gpu/StandardGpuResources.cpp (#4232)
+- Make static method in header inline (#4214)
+- Upgrade openblas to 0.3.29 for ARM architectures (#4203)
+- Pass `store_dataset` argument along to cuVS CAGRA (#4173)
+- Handle plain SearchParameters in HNSW searches (#4167)
+- Update INSTALL.md to remove some raft references, add missing dependency (#4176)
+- Update README.md (#4169)
+- Update CAGRA docs (#4152)
+- Expose IDSelectorBitmap in the C_API (#4158)
+
+Fixed
+- fix: algorithm of spreading vectors over shards (#4299)
+- Fix overflow of int32 in IndexNSG (#4297)
+- Fix Type Error in Conditional Logic (#4294)
+- faiss/gpu/GpuAutoTune.cpp: fix llvm-19-exposed -Wunused-but-set-variable warnings
+- Fix nightly by pinning conda-build to prevent regression in 25.3.2 (#4287)
+- Fix CQS signal. Id] 88153895 -- readability-redundant-string-init in fbcode/faiss (#4283)
+- Fix a placeholder for 'unimplemented' in mapped_io.cpp (#4268)
+- fix bug: IVFPQ of raft/cuvs does not require redundant check (#4241)
+- fix a serialization problem in RaBitQ (#4261)
+- Grammar fix in FlatIndexHNSW (#4253)
+- Fix CUDA kernel index data type in faiss/gpu/impl/DistanceUtils.cuh +10 (#4246)
+- fix `IVFPQFastScan::RangeSearch()` on the `ARM` architecture (#4247)
+- fix integer overflow issue when calculating imbalance_factor (#4245)
+- Fix bug with metric_arg in IndexHNSW (#4239)
+- Address compile errors and warnings (#4238)
+- faiss: fix non-templated hammings function (#4195)
+- Fix LLVM-19 compilation issue in faiss/AutoTune.cpp (#4220)
+- Fix cloning and reverse index factory for NSG indices (#4151)
+- Remove python_abi to fix nightly (#4217)
+- Fix IVF quantizer centroid sharding so IDs are generated (#4197)
+- Pin lief to fix nightly (#4211)
+- Fix Sapphire Rapids never loading in Python bindings (#4209)
+- Attempt to nightly fix (#4204)
+- Fix nightly by installing earlier version of lief (#4198)
+- Check for not completed
+- Fix install error when building avx512_spr variant (#4170)
+- fix: gpu tests link failure with static lib (#4137)
+- Fix the order of parameters in bench_scalar_quantizer_distance. (#4159)
+
+Deprecated
+- Remove unused exception parameter from faiss/impl/ResultHandler.h (#4243)
+- Remove unused variable (#4205)
+
+
+
 ## [1.10.0] - 2025-01-30
 
 
@@ -459,7 +540,9 @@ by conda install -c pytorch faiss-gpu cudatoolkit=10.0.
 - C bindings.
 - Extended tutorial to GPU indices.
 
-[Unreleased]: https://github.com/facebookresearch/faiss/compare/v1.9.0...HEAD
+[Unreleased]: https://github.com/facebookresearch/faiss/compare/v1.11.0...HEAD
+[1.11.0]: https://github.com/facebookresearch/faiss/compare/v1.10.0...v1.11.0
+[1.10.0]: https://github.com/facebookresearch/faiss/compare/v1.9.0...v1.10.0
 [1.9.0]: https://github.com/facebookresearch/faiss/compare/v1.8.0...v1.9.0
 [1.8.0]: https://github.com/facebookresearch/faiss/compare/v1.7.4...v1.8.0
 [1.7.4]: https://github.com/facebookresearch/faiss/compare/v1.7.3...v1.7.4

CMakeLists.txt

@@ -34,20 +34,20 @@ if(FAISS_ENABLE_GPU)
 endif()
 
 if(FAISS_ENABLE_CUVS)
-include(cmake/thirdparty/fetch_rapids.cmake)
-include(rapids-cmake)
-include(rapids-cpm)
-include(rapids-cuda)
-include(rapids-export)
-include(rapids-find)
-
-rapids_cuda_init_architectures(faiss)
-rapids_cuda_init_architectures(pyfaiss)
-rapids_cuda_init_architectures(faiss_c_library)
+  include(cmake/thirdparty/fetch_rapids.cmake)
+  include(rapids-cmake)
+  include(rapids-cpm)
+  include(rapids-cuda)
+  include(rapids-export)
+  include(rapids-find)
+
+  rapids_cuda_init_architectures(faiss)
+  rapids_cuda_init_architectures(pyfaiss)
+  rapids_cuda_init_architectures(faiss_c_library)
 endif()
 
 project(faiss
-  VERSION 1.10.0
+  VERSION 1.11.0
   DESCRIPTION "A library for efficient similarity search and clustering of dense vectors."
   HOMEPAGE_URL "https://github.com/facebookresearch/faiss"
   LANGUAGES ${FAISS_LANGUAGES})

INSTALL.md

@@ -6,26 +6,26 @@ pre-release nightly builds.
 
 - The CPU-only faiss-cpu conda package is currently available on Linux (x86-64 and aarch64), OSX (arm64 only), and Windows (x86-64)
 - faiss-gpu, containing both CPU and GPU indices, is available on Linux (x86-64 only) for CUDA 11.4 and 12.1
-- faiss-gpu-raft [^1] package containing GPU indices provided by [NVIDIA RAFT](https://github.com/rapidsai/raft/) version 24.06, is available on Linux (x86-64 only) for CUDA 11.8 and 12.4. 
+- faiss-gpu-cuvs package containing GPU indices provided by [NVIDIA cuVS](https://github.com/rapidsai/cuvs/) version 24.12, is available on Linux (x86-64 only) for CUDA 11.8 and 12.4.
 
 To install the latest stable release:
 
 ``` shell
 # CPU-only version
-$ conda install -c pytorch faiss-cpu=1.10.0
+$ conda install -c pytorch faiss-cpu=1.11.0
 
 # GPU(+CPU) version
-$ conda install -c pytorch -c nvidia faiss-gpu=1.10.0
+$ conda install -c pytorch -c nvidia faiss-gpu=1.11.0
 
-# GPU(+CPU) version with NVIDIA RAFT
-$ conda install -c pytorch -c nvidia -c rapidsai -c conda-forge faiss-gpu-raft=1.10.0
+# GPU(+CPU) version with NVIDIA cuVS
+$ conda install -c pytorch -c nvidia -c rapidsai -c conda-forge libnvjitlink faiss-gpu-cuvs=1.11.0
 
 # GPU(+CPU) version using AMD ROCm not yet available
 ```
 
 For faiss-gpu, the nvidia channel is required for CUDA, which is not published in the main anaconda channel.
 
-For faiss-gpu-raft, the rapidsai, conda-forge and nvidia channels are required.
+For faiss-gpu-cuvs, the rapidsai, conda-forge and nvidia channels are required.
 
 Nightly pre-release packages can be installed as follows:
 
@@ -34,13 +34,13 @@ Nightly pre-release packages can be installed as follows:
 $ conda install -c pytorch/label/nightly faiss-cpu
 
 # GPU(+CPU) version
-$ conda install -c pytorch/label/nightly -c nvidia faiss-gpu=1.10.0
+$ conda install -c pytorch/label/nightly -c nvidia faiss-gpu=1.11.0
 
 # GPU(+CPU) version with NVIDIA cuVS (package built with CUDA 12.4)
-conda install -c pytorch -c rapidsai -c conda-forge -c nvidia pytorch/label/nightly::faiss-gpu-cuvs 'cuda-version>=12.0,<=12.5'
+conda install -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia pytorch/label/nightly::faiss-gpu-cuvs 'cuda-version>=12.0,<=12.5'
 
 # GPU(+CPU) version with NVIDIA cuVS (package built with CUDA 11.8)
-conda install -c pytorch -c rapidsai -c conda-forge -c nvidia pytorch/label/nightly::faiss-gpu-cuvs 'cuda-version>=11.4,<=11.8'
+conda install -c pytorch -c rapidsai -c rapidsai-nightly -c conda-forge -c nvidia pytorch/label/nightly::faiss-gpu-cuvs 'cuda-version>=11.4,<=11.8'
 
 # GPU(+CPU) version using AMD ROCm not yet available
 ```
@@ -321,5 +321,3 @@ and you can run
 $ python demos/demo_auto_tune.py
 ```
 to test the GPU code.
-
-[^1]: The vector search and clustering algorithms in NVIDIA RAFT have been formally migrated to [NVIDIA cuVS](https://github.com/rapidsai/cuvs). This package is being renamed to `faiss-gpu-cuvs` in the next stable release, which will use these GPU implementations from the pre-compiled `libcuvs=24.12` binary.

README.md

@@ -80,7 +80,7 @@ For the GPU version of Faiss, please cite:
 
 ## Join the Faiss community
 
-For public discussion of Faiss or for questions, there is a Facebook group at https://www.facebook.com/groups/faissusers/
+For public discussion of Faiss or for questions, visit https://github.com/facebookresearch/faiss/discussions.
 
 We monitor the [issues page](https://github.com/facebookresearch/faiss/issues) of the repository.
 You can report bugs, ask questions, etc.

benchs/bench_fw/descriptors.py

@@ -83,8 +83,15 @@ class DatasetDescriptor:
 
     embedding_column: Optional[str] = None
 
+    # only when the embedding column is a map
+    embedding_column_key: Optional[Any] = None
+
     embedding_id_column: Optional[str] = None
 
+    # filters on the dataset where each filter is a
+    # string rep of a filter expression
+    filters: Optional[List[str]] = None
+
     # unused in open-source
     splits_distribution: Optional[List[List[bytes]]] = None
 
@@ -106,6 +113,10 @@ class DatasetDescriptor:
     # desc_name
     desc_name: Optional[str] = None
 
+    filename_suffix: Optional[str] = None
+
+    normalize_L2: bool = False
+
     def __hash__(self):
         return hash(self.get_filename())
 
@@ -129,6 +140,8 @@ def get_filename(
             ).replace("=", "_").replace("/", "_")
         if self.num_vectors is not None:
             filename += f"_{self.num_vectors}"
+        if self.filename_suffix is not None:
+            filename += f"_{self.filename_suffix}"
         filename += "."
 
         self.desc_name = filename
@@ -214,6 +227,8 @@ class CodecDescriptor(IndexBaseDescriptor):
     factory: Optional[str] = None
     construction_params: Optional[List[Dict[str, int]]] = None
     training_vectors: Optional[DatasetDescriptor] = None
+    normalize_l2: bool = False
+    is_spherical: bool = False
     FILENAME_PREFIX: str = "xt"
 
     def __post_init__(self):

benchs/bench_fw/index.py

@@ -1138,6 +1138,8 @@ def assemble(self, dry_run):
                 return None, None, ""
             logger.info(f"assemble, train {self.factory}")
             xt = self.io.get_dataset(self.training_vectors)
+            if self.training_vectors.normalize_L2:
+                faiss.normalize_L2(xt)
             _, t, _ = timer("train", lambda: codec.train(xt), once=True)
             t_aggregate += t