use rapids-build-backend (#234)

Contributes to rapidsai/build-planning#31 Contributes to rapidsai/build-planning#69 Contributes to rapidsai/dependency-file-generator#89 Proposes: * introducing `rapids-build-backend` as this project's build backend, to reduce the complexity of various CI/build scripts * using `pip install ./dist/*.whl` instead of `pip install --find-links ./dist` in CI, to reduce the risk of the types of bugs described in rapidsai/build-planning#69 Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Ray Douglass (https://github.com/raydouglass) - Peter Andreas Entschev (https://github.com/pentschev) URL: #234
rapidsai · Jun 10, 2024 · 1e6d80c · 1e6d80c
1 parent deb7d07
commit 1e6d80c
Show file tree

Hide file tree

Showing 23 changed files with 152 additions and 118 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -42,7 +42,7 @@ repos:
                 files: \.(h|cpp)$
                 # exclude: path/to/myfile.h
       - repo: https://github.com/rapidsai/dependency-file-generator
-        rev: v1.8.0
+        rev: v1.13.11
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]

diff --git a/build.sh b/build.sh
@@ -226,12 +226,12 @@ if buildAll || hasArg ucxx; then
 
     cd ${REPODIR}/python/
     SKBUILD_CMAKE_ARGS="-DCMAKE_PREFIX_PATH=${INSTALL_PREFIX};-DCMAKE_BUILD_TYPE=${BUILD_TYPE};${SKBUILD_EXTRA_CMAKE_ARGS}" \
-        python -m pip install --no-build-isolation --no-deps .
+        python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true .
 fi
 
 # Build and install the distributed_ucxx Python package
 if buildAll || hasArg distributed_ucxx; then
 
     cd ${REPODIR}/python/distributed-ucxx/
-    python -m pip install --no-build-isolation --no-deps .
+    python -m pip install --no-build-isolation --no-deps --config-settings rapidsai.disable-cuda=true .
 fi
diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -10,7 +10,7 @@ ENV_YAML_DIR="$(mktemp -d)"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key docs \
+  --file-key docs \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee "${ENV_YAML_DIR}/env.yaml"
 
 rapids-mamba-retry env create --yes -f "${ENV_YAML_DIR}/env.yaml" -n docs

diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -22,45 +22,11 @@ rm -rf /usr/include/uct
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-commit=$(git rev-parse HEAD)
-
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# This is the version of the suffix with a preceding hyphen. It's used
-# everywhere except in the final wheel name.
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
-
-# Patch project metadata files to include the CUDA version suffix and version override.
-pyproject_file="${package_dir}/pyproject.toml"
-
-sed -i -E "s/^name = \"${package_name}(.*)?\"$/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-echo "${version}" > VERSION
-sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name//-/_}/_version.py"
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
-# Add -cuXX to package name
-sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" ${pyproject_file}
-sed -r -i "s/rmm(.*)\"/rmm${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file}
-sed -r -i "s/cudf(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file}
-sed -r -i "s/libucx(.*)\"/libucx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file}
-
-# Update cupy package name (different suffix from RAPIDS)
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
-fi
+rapids-generate-version > ./VERSION
 
 if [[ ${package_name} == "distributed-ucxx" ]]; then
-    sed -r -i "s/\"ucxx(.*)\"/\"ucxx${PACKAGE_CUDA_SUFFIX}\1${alpha_spec}\"/g" ${pyproject_file}
-
     python -m pip wheel "${package_dir}/" -w "${package_dir}/dist" -vvv --no-deps --disable-pip-version-check
 
     RAPIDS_PY_WHEEL_NAME="distributed_ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 ${package_dir}/dist

diff --git a/ci/check_style.sh b/ci/check_style.sh
@@ -10,7 +10,7 @@ rapids-logger "Create checks conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key checks \
+  --file-key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n checks

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -35,9 +35,7 @@ function sed_runner() {
 echo "${NEXT_FULL_TAG}" > VERSION
 
 # bump RAPIDS libs
-sed_runner "/- librmm =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml
-sed_runner "/- rmm =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml
-sed_runner "/- rapids-dask-dependency =/ s/=.*/=${NEXT_RAPIDS_VERSION}/g" conda/recipes/ucxx/meta.yaml
+sed_runner "/^rapids_version:$/ {n;s/.*/  - \"${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*\"/}" conda/recipes/ucxx/conda_build_config.yaml
 
 DEPENDENCIES=(
   cudf
@@ -49,10 +47,10 @@ DEPENDENCIES=(
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
   done
-  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml
-  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*\"/g" python/distributed-ucxx/pyproject.toml
+  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/pyproject.toml
+  sed_runner "/\"${DEP}==/ s/==.*\"/==${NEXT_RAPIDS_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/distributed-ucxx/pyproject.toml
 done
 
 UCXX_DEPENDENCIES=(
@@ -61,10 +59,10 @@ UCXX_DEPENDENCIES=(
 )
 for DEP in "${UCXX_DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/,>=0.0.0a0/g" "${FILE}"
   done
-  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/pyproject.toml
-  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*\"/g" python/distributed-ucxx/pyproject.toml
+  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/pyproject.toml
+  sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}\.*,>=0.0.0a0\"/g" python/distributed-ucxx/pyproject.toml
 done
 
 # rapids-cmake version

diff --git a/ci/test_cpp.sh b/ci/test_cpp.sh
@@ -12,7 +12,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_cpp \
+  --file-key test_cpp \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -12,7 +12,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
+  --file-key test_python \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

diff --git a/ci/test_python_distributed.sh b/ci/test_python_distributed.sh
@@ -12,7 +12,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python_distributed \
+  --file-key test_python_distributed \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

diff --git a/ci/test_wheel_distributed_ucxx.sh b/ci/test_wheel_distributed_ucxx.sh
@@ -11,7 +11,11 @@ RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
 RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./dist
 ucxx_wheelhouse=$(RAPIDS_PY_WHEEL_NAME="ucxx_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 ./local-ucxx-dep)
-python -m pip install -v "$(echo ./dist/${package_name}_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
+
+python -m pip install \
+    -v \
+    "${ucxx_wheelhouse}"/ucxx_${RAPIDS_PY_CUDA_SUFFIX}*.whl \
+    "$(echo ./dist/${package_name}_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]"
 
 rapids-logger "Distributed Tests"
 

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -14,15 +14,15 @@ dependencies:
 - cmake>=3.26.4
 - cuda-version=11.8
 - cudatoolkit
-- cudf==24.8.*
+- cudf==24.8.*,>=0.0.0a0
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-cuda==24.8.*
-- dask-cudf==24.8.*
+- dask-cuda==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
 - doxygen=1.9.1
 - fmt>=10.1.1,<11
-- librmm==24.8.*
+- librmm==24.8.*,>=0.0.0a0
 - libtool
 - ninja
 - numba>=0.57.1
@@ -35,8 +35,9 @@ dependencies:
 - pytest-rerunfailures
 - pytest==7.*
 - python>=3.9,<3.12
-- rapids-dask-dependency==24.8.*
-- rmm==24.8.*
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.8.*,>=0.0.0a0
+- rmm==24.8.*,>=0.0.0a0
 - scikit-build-core>=0.7.0
 - setuptools>=64.0.0
 - spdlog>=1.12.0,<1.13

diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml
@@ -14,15 +14,15 @@ dependencies:
 - cmake>=3.26.4
 - cuda-cudart-dev
 - cuda-version=12.2
-- cudf==24.8.*
+- cudf==24.8.*,>=0.0.0a0
 - cupy>=12.0.0
 - cxx-compiler
 - cython>=3.0.0
-- dask-cuda==24.8.*
-- dask-cudf==24.8.*
+- dask-cuda==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
 - doxygen=1.9.1
 - fmt>=10.1.1,<11
-- librmm==24.8.*
+- librmm==24.8.*,>=0.0.0a0
 - libtool
 - ninja
 - numba>=0.57.1
@@ -35,8 +35,9 @@ dependencies:
 - pytest-rerunfailures
 - pytest==7.*
 - python>=3.9,<3.12
-- rapids-dask-dependency==24.8.*
-- rmm==24.8.*
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
+- rapids-dask-dependency==24.8.*,>=0.0.0a0
+- rmm==24.8.*,>=0.0.0a0
 - scikit-build-core>=0.7.0
 - setuptools>=64.0.0
 - spdlog>=1.12.0,<1.13

diff --git a/conda/recipes/ucxx/conda_build_config.yaml b/conda/recipes/ucxx/conda_build_config.yaml
@@ -32,3 +32,6 @@ gmock:
 
 gtest:
   - ">=1.13.0"
+
+rapids_version:
+  - "24.8.*"
diff --git a/conda/recipes/ucxx/meta.yaml b/conda/recipes/ucxx/meta.yaml
@@ -49,9 +49,10 @@ requirements:
     - cuda-cudart-dev
     {% endif %}
     - cuda-version ={{ cuda_version }}
+    - rapids-build-backend >=0.3.0,<0.4.0.dev0
     - ucx
     - python
-    - librmm =24.08
+    - librmm {{ rapids_version }}
 
 outputs:
   - name: libucxx
@@ -225,7 +226,8 @@ outputs:
         - numpy 1.23
         - {{ pin_subpackage('libucxx', exact=True) }}
         - ucx
-        - rmm =24.08
+        - rapids-build-backend >=0.3.0,<0.4.0.dev0
+        - rmm {{ rapids_version }}
         - cuda-version {{ cuda_version }}
       run:
         - python * *_cpython
@@ -273,11 +275,12 @@ outputs:
       host:
         - python
         - pip
+        - rapids-build-backend >=0.3.0,<0.4.0.dev0
         - setuptools
         - wheel
       run:
         - python * *_cpython
-        - rapids-dask-dependency =24.08
+        - rapids-dask-dependency {{ rapids_version }}
         - {{ pin_subpackage('ucxx', max_pin='x.x') }}
     test:
       imports: