From acd8fb0fe6f31275dc609a5dfab0c16dba788ba6 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Wed, 4 Jan 2023 12:00:42 -0500 Subject: [PATCH 01/10] Update builds for CUDA `11.8` and Python `3.10` This PR updates the `raft` CI workflows to build against the CUDA `11.8` / Python `3.10` [branch](https://github.com/rapidsai/shared-action-workflows/tree/cuda-118) of the `shared-action-workflows` repository. --- .github/workflows/build.yaml | 6 +++--- .github/workflows/pr.yaml | 12 ++++++------ .github/workflows/test.yaml | 4 ++-- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 620a13fe17..d7e81bcd3a 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -28,7 +28,7 @@ concurrency: jobs: cpp-build: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -37,7 +37,7 @@ jobs: python-build: needs: [cpp-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} @@ -46,7 +46,7 @@ jobs: upload-conda: needs: [cpp-build, python-build] secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-upload-packages.yaml@cuda-118 with: build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index ca2e2356c0..c2610b147d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -18,32 +18,32 @@ jobs: - conda-python-build - conda-python-tests secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/pr-builder.yaml@cuda-118 checks: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/checks.yaml@cuda-118 conda-cpp-build: needs: checks secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-matrix-build.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-build.yaml@cuda-118 with: build_type: pull-request node_type: cpu16 conda-cpp-tests: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118 with: build_type: pull-request conda-python-build: needs: conda-cpp-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-matrix-build.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-build.yaml@cuda-118 with: build_type: pull-request conda-python-tests: needs: conda-python-build secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 with: build_type: pull-request diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bd201e987f..c586ac769c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -16,7 +16,7 @@ on: jobs: conda-cpp-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-cpp-tests.yaml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} @@ -24,7 +24,7 @@ jobs: sha: ${{ inputs.sha }} conda-python-tests: secrets: inherit - uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@main + uses: rapidsai/shared-action-workflows/.github/workflows/conda-python-tests.yaml@cuda-118 with: build_type: nightly branch: ${{ inputs.branch }} From 6ced879866c0045023d126b02439bda07ee0a43e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 4 Jan 2023 15:00:39 -0600 Subject: [PATCH 02/10] Bump libfaiss version so it's new enough to build for GPU architectures 86 and 90. --- conda/recipes/libraft/conda_build_config.yaml | 2 +- dependencies.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 399dd198eb..901f6b18b3 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -20,7 +20,7 @@ gtest_version: - "=1.10.0" libfaiss_version: - - "1.7.0 *_cuda" + - "1.7.3 *_cuda" # The CTK libraries below are missing from the conda-forge::cudatoolkit # package. The "*_host_*" version specifiers correspond to `11.5` packages and the diff --git a/dependencies.yaml b/dependencies.yaml index 52054d9c7d..170764eb01 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -155,7 +155,7 @@ dependencies: - ucx>=1.13.0 - ucx-py=0.30.* - ucx-proc=*=gpu - - libfaiss>=1.7.0=cuda* + - libfaiss>=1.7.3=cuda* - faiss-proc=*=cuda - dask-cuda=23.02.* test_python: From 71895ae4a0ef7597ccaddf48498ecabd18d5166e Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 4 Jan 2023 15:03:02 -0600 Subject: [PATCH 03/10] Use 1.7.2 because 1.7.3 isn't available yet. Require only 1.7.1 as minimum version. --- conda/recipes/libraft/conda_build_config.yaml | 2 +- dependencies.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index 901f6b18b3..e8d19d896d 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -20,7 +20,7 @@ gtest_version: - "=1.10.0" libfaiss_version: - - "1.7.3 *_cuda" + - "1.7.2 *_cuda" # The CTK libraries below are missing from the conda-forge::cudatoolkit # package. The "*_host_*" version specifiers correspond to `11.5` packages and the diff --git a/dependencies.yaml b/dependencies.yaml index 170764eb01..508b1540f3 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -155,7 +155,7 @@ dependencies: - ucx>=1.13.0 - ucx-py=0.30.* - ucx-proc=*=gpu - - libfaiss>=1.7.3=cuda* + - libfaiss>=1.7.1=cuda* - faiss-proc=*=cuda - dask-cuda=23.02.* test_python: From 8ad6f19cc282efd842050294ab03bbd71baf2166 Mon Sep 17 00:00:00 2001 From: Bradley Dice Date: Wed, 4 Jan 2023 15:03:27 -0600 Subject: [PATCH 04/10] Bump conda environment. --- conda/environments/all_cuda-115_arch-x86_64.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-115_arch-x86_64.yaml index 18e0a8187f..a955719fc2 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-115_arch-x86_64.yaml @@ -32,7 +32,7 @@ dependencies: - libcusolver>=11.2.1.48,<=11.3.2.107 - libcusparse-dev>=11.7.0.31,<=11.7.0.107 - libcusparse>=11.7.0.31,<=11.7.0.107 -- libfaiss>=1.7.0=cuda* +- libfaiss>=1.7.1=cuda* - ninja - pytest - pytest-cov From bdd4ba0f2478256c90dfafb25c249fde3332db87 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 12 Jan 2023 15:45:45 -0500 Subject: [PATCH 05/10] rm redundant `.*` characters --- dependencies.yaml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/dependencies.yaml b/dependencies.yaml index 508b1540f3..0d07dd8fe8 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -53,12 +53,12 @@ dependencies: - matrix: arch: x86_64 packages: - - gcc_linux-64=9.* + - gcc_linux-64=9 - sysroot_linux-64==2.17 - matrix: arch: aarch64 packages: - - gcc_linux-aarch64=9.* + - gcc_linux-aarch64=9 - sysroot_linux-aarch64==2.17 checks: common: @@ -149,15 +149,15 @@ dependencies: common: - output_types: [conda] packages: - - rmm=23.02.* + - rmm=23.02 - dask>=2022.12.0 - distributed>=2022.12.0 - ucx>=1.13.0 - - ucx-py=0.30.* + - ucx-py=0.30 - ucx-proc=*=gpu - libfaiss>=1.7.1=cuda* - faiss-proc=*=cuda - - dask-cuda=23.02.* + - dask-cuda=23.02 test_python: common: - output_types: [conda, requirements] From 659c296e22ea4f653a7d97dd3f90c2275fe36977 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 12 Jan 2023 15:46:40 -0500 Subject: [PATCH 06/10] update `dependencies.yaml` & generated file --- ..._64.yaml => all_cuda-118_arch-x86_64.yaml} | 22 +++++++++---------- dependencies.yaml | 21 ++++++++++++++++-- 2 files changed, 30 insertions(+), 13 deletions(-) rename conda/environments/{all_cuda-115_arch-x86_64.yaml => all_cuda-118_arch-x86_64.yaml} (65%) diff --git a/conda/environments/all_cuda-115_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml similarity index 65% rename from conda/environments/all_cuda-115_arch-x86_64.yaml rename to conda/environments/all_cuda-118_arch-x86_64.yaml index a955719fc2..e0f83828a6 100644 --- a/conda/environments/all_cuda-115_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -12,9 +12,9 @@ dependencies: - clang-tools=11.1.0 - clang=11.1.0 - cmake>=3.23.1,!=3.25.0 -- cuda-profiler-api>=11.4.240,<=11.8.86 +- cuda-profiler-api=11.8.86 - cuda-python >=11.7.1,<12.0 -- cudatoolkit=11.5 +- cudatoolkit=11.8 - cupy - cxx-compiler - cython>=0.29,<0.30 @@ -24,14 +24,14 @@ dependencies: - doxygen>=1.8.20 - faiss-proc=*=cuda - gcc_linux-64=9.* -- libcublas-dev>=11.7.3.1,<=11.7.4.6 -- libcublas>=11.7.3.1,<=11.7.4.6 -- libcurand-dev>=10.2.6.48,<=10.2.7.107 -- libcurand>=10.2.6.48,<=10.2.7.107 -- libcusolver-dev>=11.2.1.48,<=11.3.2.107 -- libcusolver>=11.2.1.48,<=11.3.2.107 -- libcusparse-dev>=11.7.0.31,<=11.7.0.107 -- libcusparse>=11.7.0.31,<=11.7.0.107 +- libcublas-dev=11.11.3.6 +- libcublas=11.11.3.6 +- libcurand-dev=10.3.0.86 +- libcurand=10.3.0.86 +- libcusolver-dev=11.4.1.48 +- libcusolver=11.4.1.48 +- libcusparse-dev=11.7.5.86 +- libcusparse=11.7.5.86 - libfaiss>=1.7.1=cuda* - ninja - pytest @@ -45,4 +45,4 @@ dependencies: - ucx-proc=*=gpu - ucx-py=0.30.* - ucx>=1.13.0 -name: all_cuda-115_arch-x86_64 +name: all_cuda-118_arch-x86_64 diff --git a/dependencies.yaml b/dependencies.yaml index 0d07dd8fe8..ae900542c0 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -3,7 +3,7 @@ files: all: output: conda matrix: - cuda: ["11.5"] + cuda: ["11.8"] arch: [x86_64] includes: - build @@ -77,6 +77,19 @@ dependencies: specific: - output_types: conda matrices: + - matrix: + cuda: "11.8" + packages: + - cudatoolkit=11.8 + - cuda-profiler-api=11.8.86 + - libcublas-dev=11.11.3.6 + - libcublas=11.11.3.6 + - libcurand-dev=10.3.0.86 + - libcurand=10.3.0.86 + - libcusolver-dev=11.4.1.48 + - libcusolver=11.4.1.48 + - libcusparse-dev=11.7.5.86 + - libcusparse=11.7.5.86 - matrix: cuda: "11.5" packages: @@ -142,9 +155,13 @@ dependencies: py: "3.9" packages: - python=3.9 + - matrix: + py: "3.10" + packages: + - python=3.10 - matrix: packages: - - python>=3.8,<3.10 + - python>=3.8,<3.11 run: common: - output_types: [conda] From c91201454e9fa3a481f6c275a74504f4161317c9 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 12 Jan 2023 15:47:09 -0500 Subject: [PATCH 07/10] update `conda_build_config.yaml` versions --- conda/recipes/libraft/conda_build_config.yaml | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/conda/recipes/libraft/conda_build_config.yaml b/conda/recipes/libraft/conda_build_config.yaml index e8d19d896d..1012bddb40 100644 --- a/conda/recipes/libraft/conda_build_config.yaml +++ b/conda/recipes/libraft/conda_build_config.yaml @@ -23,39 +23,39 @@ libfaiss_version: - "1.7.2 *_cuda" # The CTK libraries below are missing from the conda-forge::cudatoolkit -# package. The "*_host_*" version specifiers correspond to `11.5` packages and the +# package. The "*_host_*" version specifiers correspond to `11.8` packages and the # "*_run_*" version specifiers correspond to `11.x` packages. libcublas_host_version: - - ">=11.7.3.1,<=11.7.4.6" + - "=11.11.3.6" libcublas_run_version: - - ">=11.5.2.43,<=11.11.3.6" + - ">=11.5.2.43,<12.0.0" libcurand_host_version: - - ">=10.2.6.48,<=10.2.7.107" + - "=10.3.0.86" libcurand_run_version: - - ">=10.2.5.43,<=10.3.0.86" + - ">=10.2.5.43,<10.3.1" libcusolver_host_version: - - ">=11.2.1.48,<=11.3.2.107" + - "=11.4.1.48" libcusolver_run_version: - - ">=11.2.0.43,<=11.4.1.48" + - ">=11.2.0.43,<11.4.2" libcusparse_host_version: - - ">=11.7.0.31,<=11.7.0.107" + - "=11.7.5.86" libcusparse_run_version: - - ">=11.6.0.43,<=11.7.5.86" + - ">=11.6.0.43,<12.0.0" # `cuda-profiler-api` only has `11.8.0` and `12.0.0` packages for all # architectures. The "*_host_*" version specifiers correspond to `11.8` packages and the # "*_run_*" version specifiers correspond to `11.x` packages. cuda_profiler_api_host_version: - - ">=11.8.86,<12" + - "=11.8.86" cuda_profiler_api_run_version: - ">=11.4.240,<12" From 18a55e3a108d7fb6239d38378654e91d3e8efba5 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 12 Jan 2023 15:47:25 -0500 Subject: [PATCH 08/10] update miscellaneous CUDA/Python versions --- README.md | 2 +- docs/source/build.md | 2 +- python/pylibraft/setup.py | 1 + python/raft-dask/setup.py | 1 + 4 files changed, 4 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 8e0da6cd6d..34d66cbbc3 100755 --- a/README.md +++ b/README.md @@ -277,7 +277,7 @@ Several CMake targets can be made available by adding components in the table be The easiest way to build RAFT from source is to use the `build.sh` script at the root of the repository: 1. Create an environment with the needed dependencies: ``` -mamba env create --name raft_dev_env -f conda/environments/all_cuda-115_arch-x86_64.yaml +mamba env create --name raft_dev_env -f conda/environments/all_cuda-118_arch-x86_64.yaml mamba activate raft_dev_env ``` ``` diff --git a/docs/source/build.md b/docs/source/build.md index c88cf6c162..8d63682211 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -183,7 +183,7 @@ Currently, shared libraries are provided for the `libraft-nn` and `libraft-dista Conda environment scripts are provided for installing the necessary dependencies for building and using the Python APIs. It is preferred to use `mamba`, as it provides significant speedup over `conda`. In addition you will have to manually install `nvcc` as it will not be installed as part of the conda environment. The following example will install create and install dependencies for a CUDA 11.5 conda environment: ```bash -mamba env create --name raft_env_name -f conda/environments/all_cuda-115_arch-x86_64.yaml +mamba env create --name raft_env_name -f conda/environments/all_cuda-118_arch-x86_64.yaml mamba activate raft_env_name ``` diff --git a/python/pylibraft/setup.py b/python/pylibraft/setup.py index 15889fcd71..079825a30c 100644 --- a/python/pylibraft/setup.py +++ b/python/pylibraft/setup.py @@ -69,6 +69,7 @@ def get_versions(): "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], author="NVIDIA Corporation", include_package_data=True, diff --git a/python/raft-dask/setup.py b/python/raft-dask/setup.py index 7009a9ab44..02bbced9a3 100644 --- a/python/raft-dask/setup.py +++ b/python/raft-dask/setup.py @@ -73,6 +73,7 @@ def get_versions(): "Programming Language :: Python", "Programming Language :: Python :: 3.8", "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", ], author="NVIDIA Corporation", include_package_data=True, From b6d0f86ac758833480e34786d51e719b44d49116 Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Thu, 12 Jan 2023 15:52:22 -0500 Subject: [PATCH 09/10] update generated files --- conda/environments/all_cuda-118_arch-x86_64.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml index e0f83828a6..87b7075935 100644 --- a/conda/environments/all_cuda-118_arch-x86_64.yaml +++ b/conda/environments/all_cuda-118_arch-x86_64.yaml @@ -18,12 +18,12 @@ dependencies: - cupy - cxx-compiler - cython>=0.29,<0.30 -- dask-cuda=23.02.* +- dask-cuda=23.02 - dask>=2022.12.0 - distributed>=2022.12.0 - doxygen>=1.8.20 - faiss-proc=*=cuda -- gcc_linux-64=9.* +- gcc_linux-64=9 - libcublas-dev=11.11.3.6 - libcublas=11.11.3.6 - libcurand-dev=10.3.0.86 @@ -36,13 +36,13 @@ dependencies: - ninja - pytest - pytest-cov -- rmm=23.02.* +- rmm=23.02 - scikit-build>=0.13.1 - scikit-learn - scipy - sphinx-markdown-tables - sysroot_linux-64==2.17 - ucx-proc=*=gpu -- ucx-py=0.30.* +- ucx-py=0.30 - ucx>=1.13.0 name: all_cuda-118_arch-x86_64 From 3100871984cdc80cf5a126beb0fe23cae4a2f7de Mon Sep 17 00:00:00 2001 From: AJ Schmidt Date: Sun, 15 Jan 2023 11:15:12 -0500 Subject: [PATCH 10/10] update doc reference to `11.8` --- docs/source/build.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/build.md b/docs/source/build.md index 8d63682211..4052e49cf8 100644 --- a/docs/source/build.md +++ b/docs/source/build.md @@ -180,7 +180,7 @@ Currently, shared libraries are provided for the `libraft-nn` and `libraft-dista ### Python -Conda environment scripts are provided for installing the necessary dependencies for building and using the Python APIs. It is preferred to use `mamba`, as it provides significant speedup over `conda`. In addition you will have to manually install `nvcc` as it will not be installed as part of the conda environment. The following example will install create and install dependencies for a CUDA 11.5 conda environment: +Conda environment scripts are provided for installing the necessary dependencies for building and using the Python APIs. It is preferred to use `mamba`, as it provides significant speedup over `conda`. In addition you will have to manually install `nvcc` as it will not be installed as part of the conda environment. The following example will install create and install dependencies for a CUDA 11.8 conda environment: ```bash mamba env create --name raft_env_name -f conda/environments/all_cuda-118_arch-x86_64.yaml