Skip to content

fix raft contbuild and switch to libraft 23.12 #3116

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
algoriddle wants to merge 22 commits into from
Closed

fix raft contbuild and switch to libraft 23.12 #3116

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
22 commits
Select commit Hold shift + click to select a range
5751fb9
test build
algoriddle Oct 30, 2023
f166a9c
cuda-toolkit
algoriddle Oct 30, 2023
6ee6261
typo
algoriddle Oct 30, 2023
324f4e7
try 3
algoriddle Oct 30, 2023
3f48de4
pin to cuda 11.4
algoriddle Oct 31, 2023
3385d81
revert to cudatoolkit
algoriddle Oct 31, 2023
8d03048
take 2
algoriddle Oct 31, 2023
64b8e55
conda cuda
algoriddle Oct 31, 2023
1d050e1
-mamba
algoriddle Oct 31, 2023
7bc70c2
libcublas-dev 11.5
algoriddle Oct 31, 2023
fb756d5
cuda-toolkit
algoriddle Oct 31, 2023
ca857ef
pytorch-cuda=11
algoriddle Oct 31, 2023
fd9f184
curand
algoriddle Oct 31, 2023
470d235
missing and
algoriddle Oct 31, 2023
d307c07
cudart
algoriddle Oct 31, 2023
fca849a
back to host cuda
algoriddle Oct 31, 2023
82c207a
cuda-version=11.4 only
algoriddle Oct 31, 2023
a7cc966
disable libmamba
algoriddle Nov 1, 2023
290f8e4
workaround cudatoolkit and cuda-cudart clobbering
algoriddle Nov 1, 2023
da32b7f
cleanup
algoriddle Nov 1, 2023
af563cb
cleanup
algoriddle Nov 1, 2023
9671a57
Merge branch 'main' into raft_circleci
algoriddle Nov 1, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ executors:
environment:
CONDA_ARCH: Linux-x86_64
machine:
image: linux-cuda-11:2023.02.1
image: linux-cuda-11:default
resource_class: gpu.nvidia.medium
linux-arm64-cpu:
environment:
Expand Down Expand Up @@ -91,7 +91,7 @@ jobs:
- run:
name: Install conda build tools
command: |
conda config --set solver libmamba
# conda config --set solver libmamba
# conda config --set verbosity 3
conda update -y -q conda
conda install -y -q conda-build
Expand Down Expand Up @@ -171,7 +171,7 @@ jobs:
sudo update-alternatives --set cuda /usr/local/cuda-<<parameters.cuda>>
cd conda
conda build faiss-gpu-raft --variants '{ "cudatoolkit": "<<parameters.cuda>>", "c_compiler_version": "<<parameters.compiler_version>>", "cxx_compiler_version": "<<parameters.compiler_version>>" }' \
-c pytorch -c nvidia -c rapidsai -c conda-forge
-c pytorch -c nvidia -c rapidsai-nightly -c conda-forge
- when:
condition:
and:
Expand All @@ -186,7 +186,7 @@ jobs:
sudo update-alternatives --set cuda /usr/local/cuda-<<parameters.cuda>>
cd conda
conda build faiss-gpu-raft --variants '{ "cudatoolkit": "<<parameters.cuda>>", "c_compiler_version": "<<parameters.compiler_version>>", "cxx_compiler_version": "<<parameters.compiler_version>>" }' \
--user pytorch --label <<parameters.label>> -c pytorch -c nvidia -c rapidsai -c conda-forge
--user pytorch --label <<parameters.label>> -c pytorch -c nvidia -c rapidsai-nightly -c conda-forge

build_cmake:
parameters:
Expand Down Expand Up @@ -236,7 +236,7 @@ jobs:
- run:
name: Install libraft
command: |
conda install -y -q libraft cudatoolkit=11.4 -c rapidsai-nightly -c nvidia -c pkgs/main -c conda-forge
conda install -y -q libraft cuda-version=11.4 -c rapidsai-nightly -c nvidia -c pkgs/main -c conda-forge
- run:
name: Build all targets
no_output_timeout: 30m
Expand Down Expand Up @@ -283,7 +283,7 @@ jobs:
- run:
name: Python tests (CPU + GPU)
command: |
conda install -y -q pytorch pytorch-cuda -c pytorch -c nvidia
conda install -y -q pytorch pytorch-cuda=11 -c pytorch -c nvidia
pytest --junitxml=test-results/pytest/results.xml tests/test_*.py
pytest --junitxml=test-results/pytest/results-torch.xml tests/torch_*.py
cp tests/common_faiss_tests.py faiss/gpu/test
Expand Down
2 changes: 1 addition & 1 deletion cmake/thirdparty/fetch_rapids.cmake
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,7 +15,7 @@
# or implied. See the License for the specific language governing permissions and limitations under
# the License.
# =============================================================================
set(RAPIDS_VERSION "23.08")
set(RAPIDS_VERSION "23.12")

if(NOT EXISTS ${CMAKE_CURRENT_BINARY_DIR}/FAISS_RAPIDS.cmake)
file(DOWNLOAD https://raw.githubusercontent.com/rapidsai/rapids-cmake/branch-${RAPIDS_VERSION}/RAPIDS.cmake
Expand Down
10 changes: 6 additions & 4 deletions conda/faiss-gpu-raft/meta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -47,13 +47,13 @@ outputs:
host:
- mkl =2023 # [x86_64]
- openblas # [not x86_64]
- cudatoolkit {{ cudatoolkit }}
- libraft =23.08
- cuda-version {{ cudatoolkit }}
- libraft =23.12
run:
- mkl =2023 # [x86_64]
- openblas # [not x86_64]
- {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
- libraft =23.08
- {{ pin_compatible('cuda-version', max_pin='x') }}
- libraft =23.12
test:
requires:
- conda-build
Expand Down Expand Up @@ -90,6 +90,8 @@ outputs:
- numpy
- scipy
- pytorch
- pytorch-cuda =11.8
- cuda-version =11.8
commands:
- python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
- python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
Expand Down
3 changes: 2 additions & 1 deletion conda/faiss-gpu/meta.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,7 @@ outputs:
run:
- mkl =2023 # [x86_64]
- openblas # [not x86_64]
- {{ pin_compatible('cudatoolkit', max_pin='x.x') }}
- {{ pin_compatible('cudatoolkit', max_pin='x') }}
test:
requires:
- conda-build
Expand Down Expand Up @@ -88,6 +88,7 @@ outputs:
- numpy
- scipy
- pytorch
- pytorch-cuda =11.8
commands:
- python -X faulthandler -m unittest discover -v -s tests/ -p "test_*"
- python -X faulthandler -m unittest discover -v -s tests/ -p "torch_*"
Expand Down