Upgrade FAISS to 1.7.x (#3509)

Answers #3459

Authors:
  - Victor Lafargue (@viclafargue)
  - Dante Gama Dessavre (@dantegd)

Approvers:
  - AJ Schmidt (@ajschmidt8)
  - Dante Gama Dessavre (@dantegd)

URL: #3509

conda/environments/cuml_dev_cuda10.1.yml

@@ -18,7 +18,7 @@ dependencies:
 - ucx-proc=*=gpu
 - dask-ml
 - doxygen>=1.8.20
-- libfaiss>=1.6.3
+- libfaiss>=1.7.0
 - faiss-proc=*=cuda
 - umap-learn
 - scikit-learn=0.23.1

conda/environments/cuml_dev_cuda10.2.yml

@@ -18,7 +18,7 @@ dependencies:
 - ucx-proc=*=gpu
 - dask-ml
 - doxygen>=1.8.20
-- libfaiss>=1.6.3
+- libfaiss>=1.7.0
 - faiss-proc=*=cuda
 - umap-learn
 - scikit-learn=0.23.1

conda/environments/cuml_dev_cuda11.0.yml

@@ -18,7 +18,7 @@ dependencies:
 - ucx-proc=*=gpu
 - dask-ml
 - doxygen>=1.8.20
-- libfaiss>=1.6.3
+- libfaiss>=1.7.0
 - faiss-proc=*=cuda
 - umap-learn
 - scikit-learn=0.23.1

conda/recipes/libcuml/meta.yaml

@@ -48,7 +48,7 @@ requirements:
     - treelite=1.0.0
     - faiss-proc=*=cuda
     - gtest=1.10.0
-    - libfaiss=1.6.3
+    - conda-forge::libfaiss=1.7.0
   run:
     - libcumlprims {{ minor_version }}
     - cudf {{ minor_version }}
@@ -58,7 +58,7 @@ requirements:
     - {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
     - treelite=1.0.0
     - faiss-proc=*=cuda
-    - libfaiss=1.6.3
+    - conda-forge::libfaiss=1.7.0
 
 about:
   home: http://rapids.ai/

cpp/CMakeLists.txt

@@ -307,12 +307,12 @@ message("-- Building for GPU_ARCHS = ${GPU_ARCHS}")
 
 foreach(arch ${GPU_ARCHS})
   set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode arch=compute_${arch},code=sm_${arch}")
-  set(FAISS_GPU_ARCHS "${FAISS_GPU_ARCHS} -gencode arch=compute_${arch},code=sm_${arch}")
+  set(FAISS_GPU_ARCHS "${FAISS_GPU_ARCHS} ${arch}")
 endforeach()
 
 list(GET GPU_ARCHS -1 ptx)
 set(CMAKE_CUDA_FLAGS "${CMAKE_CUDA_FLAGS} -gencode arch=compute_${ptx},code=compute_${ptx}")
-set(FAISS_GPU_ARCHS "${FAISS_GPU_ARCHS} -gencode arch=compute_${ptx},code=compute_${ptx}")
+set(FAISS_GPU_ARCHS "${FAISS_GPU_ARCHS} ${ptx}")
 
 set(CMAKE_CUDA_FLAGS
   "${CMAKE_CUDA_FLAGS} -Xcudafe --diag_suppress=unrecognized_gcc_pragma")
@@ -357,6 +357,13 @@ set(CUML_PRIVATE_LINK_LIBRARIES
   treelite::treelite_runtime
   )
 
+if(BUILD_STATIC_FAISS)
+  set(CUML_PRIVATE_LINK_LIBRARIES
+   ${CUML_PRIVATE_LINK_LIBRARIES}
+   ${BLAS_LIBRARIES}
+  )
+endif(BUILD_STATIC_FAISS)
+
 if(BUILD_CUML_STD_COMMS OR BUILD_CUML_MPI_COMMS)
 	list(APPEND CUML_INCLUDE_DIRECTORIES
 		${NCCL_INCLUDE_DIRS})

cpp/cmake/Dependencies.cmake

@@ -150,33 +150,32 @@ if(BUILD_STATIC_FAISS)
     "Path to FAISS source directory")
   ExternalProject_Add(faiss
     GIT_REPOSITORY    https://github.com/facebookresearch/faiss.git
-    GIT_TAG           a5b850dec6f1cd6c88ab467bfd5e87b0cac2e41d
+    GIT_TAG           7c2d2388a492d65fdda934c7e74ae87acaeed066
     CONFIGURE_COMMAND LIBS=-pthread
                       CPPFLAGS=-w
                       LDFLAGS=-L${CMAKE_INSTALL_PREFIX}/lib
-                              ${CMAKE_CURRENT_BINARY_DIR}/faiss/src/faiss/configure
-	                      --prefix=${CMAKE_CURRENT_BINARY_DIR}/faiss
-	                      --with-blas=${BLAS_LIBRARIES}
-	                      --with-cuda=${CUDA_TOOLKIT_ROOT_DIR}
-	                      --with-cuda-arch=${FAISS_GPU_ARCHS}
-	                      -v
+                        cmake -B build .
+                        -DCMAKE_BUILD_TYPE=Release
+                        -DBUILD_TESTING=OFF
+                        -DFAISS_ENABLE_PYTHON=OFF
+                        -DBUILD_SHARED_LIBS=OFF
+                        -DFAISS_ENABLE_GPU=ON
+                        -DCUDAToolkit_ROOT=${CUDA_TOOLKIT_ROOT_DIR}
+                        -DCUDA_ARCHITECTURES=${FAISS_GPU_ARCHS}
+                        -DBLAS_LIBRARIES=${BLAS_LIBRARIES}
     PREFIX            ${FAISS_DIR}
-    BUILD_COMMAND     make -j${PARALLEL_LEVEL} VERBOSE=1
-    BUILD_BYPRODUCTS  ${FAISS_DIR}/lib/libfaiss.a
+    BUILD_COMMAND     make -C build -j${PARALLEL_LEVEL} VERBOSE=1
+    BUILD_BYPRODUCTS  ${FAISS_DIR}/src/faiss/build/faiss/libfaiss.a
     BUILD_ALWAYS      1
-    INSTALL_COMMAND   make -s install > /dev/null
+    INSTALL_COMMAND   ""
     UPDATE_COMMAND    ""
-    BUILD_IN_SOURCE   1
-    PATCH_COMMAND     patch -p1 -N < ${CMAKE_CURRENT_SOURCE_DIR}/cmake/faiss_cuda11.patch || true)
+    BUILD_IN_SOURCE   1)
 
   ExternalProject_Get_Property(faiss install_dir)
   add_library(FAISS::FAISS STATIC IMPORTED)
   set_property(TARGET FAISS::FAISS PROPERTY
-    IMPORTED_LOCATION ${FAISS_DIR}/lib/libfaiss.a)
-  # to account for the FAISS file reorg that happened recently after the current
-  # pinned commit, just change the following line to
-  # set(FAISS_INCLUDE_DIRS "${FAISS_DIR}/src/faiss")
-  set(FAISS_INCLUDE_DIRS "${FAISS_DIR}/src")
+    IMPORTED_LOCATION ${FAISS_DIR}/src/faiss/build/faiss/libfaiss.a)
+  set(FAISS_INCLUDE_DIRS "${FAISS_DIR}/src/faiss")
 else()
   set(FAISS_INSTALL_DIR ENV{FAISS_ROOT})
   find_package(FAISS REQUIRED)

cpp/src_prims/selection/knn.cuh

@@ -302,7 +302,6 @@ void approx_knn_build_index(ML::knnIndex *index, ML::knnIndexParam *params,
   faiss::gpu::StandardGpuResources *gpu_res =
     new faiss::gpu::StandardGpuResources();
   gpu_res->noTempMemory();
-  gpu_res->setCudaMallocWarning(false);
   gpu_res->setDefaultStream(device, userStream);
   index->gpu_res = gpu_res;
   index->device = device;
@@ -455,7 +454,6 @@ void brute_force_knn(std::vector<float *> &input, std::vector<int> &sizes,
         faiss::gpu::StandardGpuResources gpu_res;
 
         gpu_res.noTempMemory();
-        gpu_res.setCudaMallocWarning(false);
         gpu_res.setDefaultStream(device, stream);
 
         faiss::gpu::GpuDistanceParams args;

python/setup.py

@@ -1,5 +1,5 @@
 #
-# Copyright (c) 2018-2020, NVIDIA CORPORATION.
+# Copyright (c) 2018-2021, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -160,6 +160,7 @@ def finalize_options(self):
             raft_include_dir,
             cuda_include_dir,
             numpy.get_include(),
+            '../cpp/build/faiss/src/faiss',
             os.path.dirname(sysconfig.get_path("include"))
         ]