Use rapids-build-backend (#4393)

Contributes to rapidsai/build-planning#31 Contributes to rapidsai/dependency-file-generator#89 Proposes introducing `rapids-build-backend` as this project's build backend, to reduce the complexity of various CI/build scripts. Authors: - Kyle Edwards (https://github.com/KyleFromNVIDIA) - James Lamb (https://github.com/jameslamb) Approvers: - Mike Sarahan (https://github.com/msarahan) - Vyas Ramasubramani (https://github.com/vyasr) - Brad Rees (https://github.com/BradReesWork) URL: #4393
rapidsai · Jul 1, 2024 · 538a2ce · 538a2ce
1 parent 25113cc
commit 538a2ce
Show file tree

Hide file tree

Showing 48 changed files with 491 additions and 238 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -41,13 +41,8 @@ repos:
       - id: clang-format
         types_or: [c, c++, cuda]
         args: ["-fallback-style=none", "-style=file", "-i"]
-  - repo: https://github.com/rapidsai/dependency-file-generator
-    rev: v1.8.0
-    hooks:
-        - id: rapids-dependency-file-generator
-          args: ["--clean"]
   - repo: https://github.com/rapidsai/pre-commit-hooks
-    rev: v0.0.3
+    rev: v0.2.0
     hooks:
       - id: verify-copyright
         files: |
@@ -58,6 +53,12 @@ repos:
               [.]flake8[.]cython$|
               meta[.]yaml$|
               setup[.]cfg$
+      - id: verify-alpha-spec
+  - repo: https://github.com/rapidsai/dependency-file-generator
+    rev: v1.13.11
+    hooks:
+        - id: rapids-dependency-file-generator
+          args: ["--clean"]
   - repo: local
     hooks:
       - id: nx-cugraph-meta-data-update

diff --git a/build.sh b/build.sh
@@ -109,7 +109,7 @@ BUILD_CPP_MTMG_TESTS=OFF
 BUILD_ALL_GPU_ARCH=0
 BUILD_WITH_CUGRAPHOPS=ON
 CMAKE_GENERATOR_OPTION="-G Ninja"
-PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps"
+PYTHON_ARGS_FOR_INSTALL="-m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true"
 
 # Set defaults for vars that may not have been defined externally
 #  FIXME: if PREFIX is not set, check CONDA_PREFIX, but there is no fallback

diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh
@@ -13,10 +13,8 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
-version=$(rapids-generate-version)
-
 rapids-logger "Begin cpp build"
 
-RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild conda/recipes/libcugraph
+RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libcugraph
 
 rapids-upload-conda-to-s3 cpp
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -8,7 +8,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key docs \
+  --file-key docs \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n docs

diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -15,20 +15,11 @@ rapids-print-env
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
-export RAPIDS_PACKAGE_VERSION=${version}
-echo "${version}" > VERSION
+rapids-generate-version > ./VERSION
+export RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION)
 
 rapids-logger "Begin py build"
 
-package_dir="python"
-for package_name in pylibcugraph cugraph cugraph-pyg cugraph-dgl; do
-  underscore_package_name=$(echo "${package_name}" | tr "-" "_")
-  sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/${package_name}/${underscore_package_name}/_version.py"
-done
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" "${package_dir}/nx-cugraph/_nx_cugraph/_version.py"
-
 # TODO: Remove `--no-test` flags once importing on a CPU
 # node works correctly
 rapids-conda-retry mambabuild \
@@ -56,10 +47,6 @@ rapids-conda-retry mambabuild \
 # built on each CUDA platform to ensure they are included in each set of
 # artifacts, since test scripts only install from one set of artifacts based on
 # the CUDA version used for the test run.
-version_file_cugraph_service_client="python/cugraph-service/client/cugraph_service_client/_version.py"
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_cugraph_service_client}
-version_file_cugraph_service_server="python/cugraph-service/server/cugraph_service_server/_version.py"
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_cugraph_service_server}
 rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \

diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -5,54 +5,13 @@ set -euo pipefail
 
 package_name=$1
 package_dir=$2
-underscore_package_name=$(echo "${package_name}" | tr "-" "_")
 
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
-
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# This is the version of the suffix with a preceding hyphen. It's used
-# everywhere except in the final wheel name.
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
-
-# Patch project metadata files to include the CUDA version suffix and version override.
-version_package_name="$underscore_package_name"
-if [[ "${version_package_name}" = "nx_cugraph" ]]; then
-    version_package_name="_nx_cugraph"
-fi
-pyproject_file="${package_dir}/pyproject.toml"
-version_file="${package_dir}/${version_package_name}/_version.py"
-
-sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-echo "${version}" > VERSION
-sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file}
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
-for dep in rmm cudf cugraph raft-dask pylibcugraph pylibcugraphops pylibwholegraph pylibraft ucx-py; do
-    sed -r -i "s/${dep}==(.*)\"/${dep}${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
-done
-
-# dask-cuda & rapids-dask-dependency doesn't get a suffix, but it does get an alpha spec.
-for dep in dask-cuda rapids-dask-dependency; do
-    sed -r -i "s/${dep}==(.*)\"/${dep}==\1${alpha_spec}\"/g" ${pyproject_file}
-done
-
-
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
-fi
+rapids-generate-version > ./VERSION
 
 cd "${package_dir}"
 

diff --git a/ci/check_style.sh b/ci/check_style.sh
@@ -8,7 +8,7 @@ rapids-logger "Create checks conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key checks \
+  --file-key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n checks

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -74,12 +74,12 @@ DEPENDENCIES=(
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml python/cugraph-{pyg,dgl}/conda/*.yaml; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}"
-    sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
+    sed_runner "/-.* ucx-py==/ s/==.*/==${NEXT_UCX_PY_VERSION}.*,>=0.0.0a0/g" "${FILE}"
   done
   for FILE in python/**/pyproject.toml python/**/**/pyproject.toml; do
-    sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" "${FILE}"
-    sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_VERSION}.*\"/g" "${FILE}"
+    sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" "${FILE}"
+    sed_runner "/\"ucx-py==/ s/==.*\"/==${NEXT_UCX_PY_VERSION}.*,>=0.0.0a0\"/g" "${FILE}"
   done
 done
 

diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -11,7 +11,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_cpp \
+  --file-key test_cpp \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
@@ -8,7 +8,7 @@ set -Eeuo pipefail
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_notebooks \
+  --file-key test_notebooks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -11,7 +11,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
+  --file-key test_python \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -16,22 +16,22 @@ dependencies:
 - cuda-nvtx
 - cuda-version=11.8
 - cudatoolkit
-- cudf==24.8.*
+- cudf==24.8.*,>=0.0.0a0
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-cuda==24.8.*
-- dask-cudf==24.8.*
+- dask-cuda==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
 - doxygen
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
 - graphviz
 - ipython
-- libcudf==24.8.*
-- libcugraphops==24.8.*
-- libraft-headers==24.8.*
-- libraft==24.8.*
-- librmm==24.8.*
+- libcudf==24.8.*,>=0.0.0a0
+- libcugraphops==24.8.*,>=0.0.0a0
+- libraft-headers==24.8.*,>=0.0.0a0
+- libraft==24.8.*,>=0.0.0a0
+- librmm==24.8.*,>=0.0.0a0
 - nbsphinx
 - nccl>=2.9.9
 - networkx>=2.5.1
@@ -47,21 +47,22 @@ dependencies:
 - pandas
 - pre-commit
 - pydata-sphinx-theme
-- pylibcugraphops==24.8.*
-- pylibraft==24.8.*
-- pylibwholegraph==24.8.*
+- pylibcugraphops==24.8.*,>=0.0.0a0
+- pylibraft==24.8.*,>=0.0.0a0
+- pylibwholegraph==24.8.*,>=0.0.0a0
 - pytest
 - pytest-benchmark
 - pytest-cov
 - pytest-mpl
 - pytest-xdist
 - python-louvain
 - pytorch>=2.0,<2.2.0a0
-- raft-dask==24.8.*
-- rapids-dask-dependency==24.8.*
+- raft-dask==24.8.*,>=0.0.0a0
+- rapids-build-backend>=0.3.1,<0.4.0.dev0
+- rapids-dask-dependency==24.8.*,>=0.0.0a0
 - recommonmark
 - requests
-- rmm==24.8.*
+- rmm==24.8.*,>=0.0.0a0
 - scikit-build-core>=0.7.0
 - scikit-learn>=0.23.1
 - scipy
@@ -72,7 +73,7 @@ dependencies:
 - sphinxcontrib-websupport
 - thriftpy2<=0.5.0
 - ucx-proc=*=gpu
-- ucx-py==0.39.*
+- ucx-py==0.39.*,>=0.0.0a0
 - wget
 - wheel
 name: all_cuda-118_arch-x86_64
diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml
@@ -18,26 +18,26 @@ dependencies:
 - cuda-nvtx-dev
 - cuda-profiler-api
 - cuda-version=12.2
-- cudf==24.8.*
+- cudf==24.8.*,>=0.0.0a0
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-cuda==24.8.*
-- dask-cudf==24.8.*
+- dask-cuda==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
 - doxygen
 - fsspec>=0.6.0
 - gcc_linux-64=11.*
 - graphviz
 - ipython
 - libcublas-dev
-- libcudf==24.8.*
-- libcugraphops==24.8.*
+- libcudf==24.8.*,>=0.0.0a0
+- libcugraphops==24.8.*,>=0.0.0a0
 - libcurand-dev
 - libcusolver-dev
 - libcusparse-dev
-- libraft-headers==24.8.*
-- libraft==24.8.*
-- librmm==24.8.*
+- libraft-headers==24.8.*,>=0.0.0a0
+- libraft==24.8.*,>=0.0.0a0
+- librmm==24.8.*,>=0.0.0a0
 - nbsphinx
 - nccl>=2.9.9
 - networkx>=2.5.1
@@ -52,21 +52,22 @@ dependencies:
 - pandas
 - pre-commit
 - pydata-sphinx-theme
-- pylibcugraphops==24.8.*
-- pylibraft==24.8.*
-- pylibwholegraph==24.8.*
+- pylibcugraphops==24.8.*,>=0.0.0a0
+- pylibraft==24.8.*,>=0.0.0a0
+- pylibwholegraph==24.8.*,>=0.0.0a0
 - pytest
 - pytest-benchmark
 - pytest-cov
 - pytest-mpl
 - pytest-xdist
 - python-louvain
 - pytorch>=2.0,<2.2.0a0
-- raft-dask==24.8.*
-- rapids-dask-dependency==24.8.*
+- raft-dask==24.8.*,>=0.0.0a0
+- rapids-build-backend>=0.3.1,<0.4.0.dev0
+- rapids-dask-dependency==24.8.*,>=0.0.0a0
 - recommonmark
 - requests
-- rmm==24.8.*
+- rmm==24.8.*,>=0.0.0a0
 - scikit-build-core>=0.7.0
 - scikit-learn>=0.23.1
 - scipy
@@ -77,7 +78,7 @@ dependencies:
 - sphinxcontrib-websupport
 - thriftpy2<=0.5.0
 - ucx-proc=*=gpu
-- ucx-py==0.39.*
+- ucx-py==0.39.*,>=0.0.0a0
 - wget
 - wheel
 name: all_cuda-122_arch-x86_64
diff --git a/conda/recipes/cugraph-dgl/meta.yaml b/conda/recipes/cugraph-dgl/meta.yaml
@@ -21,6 +21,7 @@ build:
 requirements:
   host:
     - python
+    - rapids-build-backend>=0.3.1,<0.4.0.dev0
   run:
     - cugraph ={{ version }}
     - dgl >=1.1.0.cu*

diff --git a/conda/recipes/cugraph-equivariant/meta.yaml b/conda/recipes/cugraph-equivariant/meta.yaml
@@ -21,6 +21,7 @@ build:
 requirements:
   host:
     - python
+    - rapids-build-backend>=0.3.1,<0.4.0.dev0
   run:
     - pylibcugraphops ={{ minor_version }}
     - python

diff --git a/conda/recipes/cugraph-pyg/meta.yaml b/conda/recipes/cugraph-pyg/meta.yaml
@@ -25,6 +25,7 @@ requirements:
     - cython >=3.0.0
     - python
     - scikit-build-core >=0.7.0
+    - rapids-build-backend>=0.3.1,<0.4.0.dev0
   run:
     - rapids-dask-dependency ={{ minor_version }}
     - numba >=0.57

diff --git a/conda/recipes/cugraph-service/install_cugraph-service-server.sh b/conda/recipes/cugraph-service/install_cugraph-service-server.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 
 # This assumes the script is executed from the root of the repo directory
 
@@ -13,6 +13,14 @@
 
 cd "${SRC_DIR}/python/cugraph-service/server"
 mkdir -p pip_cache
-$PYTHON -m pip install --no-build-isolation --no-index --no-deps --ignore-installed --cache-dir ./pip_cache . -vv
+$PYTHON -m pip install \
+    --no-build-isolation \
+    --no-index \
+    --no-deps \
+    --ignore-installed \
+    --cache-dir ./pip_cache \
+    --config-settings rapidsai.disable-cuda=true \
+    -vv \
+    .
 
 #./build.sh cugraph-service