use rapids-build-backend (#1343)

Contributes to rapidsai/build-planning#31
Contributes to rapidsai/dependency-file-generator#89

Proposes introducing `rapids-build-backend` as this project's build backend, to reduce the complexity of various CI/build scripts.

Authors:
  - James Lamb (https://github.com/jameslamb)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Lawrence Mitchell (https://github.com/wence-)
  - Ray Douglass (https://github.com/raydouglass)

URL: #1343

.pre-commit-config.yaml

@@ -33,7 +33,7 @@ repos:
                 args: ["--module=dask_cuda", "--ignore-missing-imports"]
                 pass_filenames: false
       - repo: https://github.com/rapidsai/dependency-file-generator
-        rev: v1.8.0
+        rev: v1.13.11
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]

ci/build_docs.sh

@@ -7,7 +7,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
     --output conda \
-    --file_key docs \
+    --file-key docs \
     --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n docs

ci/build_python.sh

@@ -13,18 +13,12 @@ export CMAKE_GENERATOR=Ninja
 
 rapids-print-env
 
-package_name="dask_cuda"
-
-version=$(rapids-generate-version)
-commit=$(git rev-parse HEAD)
-
-echo "${version}" | tr -d '"' > VERSION
-sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_name}/_version.py"
+rapids-generate-version > ./VERSION
 
 rapids-logger "Begin py build"
 conda config --set path_conflict prevent
 
-RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   conda/recipes/dask-cuda
 
 rapids-upload-conda-to-s3 python

ci/build_wheel.sh

@@ -6,22 +6,7 @@ set -euo pipefail
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-commit=$(git rev-parse HEAD)
-
-echo "${version}" | tr -d '"' > VERSION
-sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "dask_cuda/_version.py"
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-        alpha_spec=',>=0.0.0a0'
-fi
-
-sed -r -i "s/rapids-dask-dependency==(.*)\"/rapids-dask-dependency==\1${alpha_spec}\"/g" pyproject.toml
+rapids-generate-version > ./VERSION
 
 python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
 

ci/check_style.sh

@@ -8,7 +8,7 @@ rapids-logger "Create checks conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key checks \
+  --file-key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n checks

ci/release/update-version.sh

@@ -36,8 +36,8 @@ function sed_runner() {
 echo "${NEXT_FULL_TAG}" | tr -d '"' > VERSION
 
 # Bump testing dependencies
-sed_runner "s/ucx-py==.*/ucx-py==${NEXT_UCXPY_VERSION}.*/g" dependencies.yaml
-sed_runner "s/ucxx==.*/ucxx==${NEXT_UCXPY_VERSION}.*/g" dependencies.yaml
+sed_runner "s/ucx-py==.*/ucx-py==${NEXT_UCXPY_VERSION}.*,>=0.0.0a0/g" dependencies.yaml
+sed_runner "s/ucxx==.*/ucxx==${NEXT_UCXPY_VERSION}.*,>=0.0.0a0/g" dependencies.yaml
 
 DEPENDENCIES=(
   cudf
@@ -47,7 +47,7 @@ DEPENDENCIES=(
 )
 for FILE in dependencies.yaml conda/environments/*.yaml; do
   for DEP in "${DEPENDENCIES[@]}"; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
   done
 done
 

ci/test_python.sh

@@ -8,7 +8,7 @@ set -euo pipefail
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
+  --file-key test_python \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

conda/environments/all_cuda-114_arch-x86_64.yaml

@@ -10,10 +10,10 @@ dependencies:
 - click >=8.1
 - cuda-version=11.4
 - cudatoolkit
-- cudf==24.8.*
-- dask-cudf==24.8.*
-- distributed-ucxx==0.39.*
-- kvikio==24.8.*
+- cudf==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
+- distributed-ucxx==0.39.*,>=0.0.0a0
+- kvikio==24.8.*,>=0.0.0a0
 - numactl-devel-cos7-x86_64
 - numba>=0.57
 - numpy>=1.23,<2.0a0
@@ -24,13 +24,14 @@ dependencies:
 - pytest
 - pytest-cov
 - python>=3.9,<3.12
-- rapids-dask-dependency==24.8.*
+- rapids-build-backend>=0.3.0,<0.4.0dev0
+- rapids-dask-dependency==24.8.*,>=0.0.0a0
 - setuptools>=64.0.0
 - sphinx
 - sphinx-click>=2.7.1
 - sphinx-rtd-theme>=0.5.1
 - ucx-proc=*=gpu
-- ucx-py==0.39.*
-- ucxx==0.39.*
+- ucx-py==0.39.*,>=0.0.0a0
+- ucxx==0.39.*,>=0.0.0a0
 - zict>=2.0.0
 name: all_cuda-114_arch-x86_64

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -10,10 +10,10 @@ dependencies:
 - click >=8.1
 - cuda-version=11.8
 - cudatoolkit
-- cudf==24.8.*
-- dask-cudf==24.8.*
-- distributed-ucxx==0.39.*
-- kvikio==24.8.*
+- cudf==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
+- distributed-ucxx==0.39.*,>=0.0.0a0
+- kvikio==24.8.*,>=0.0.0a0
 - numactl-devel-cos7-x86_64
 - numba>=0.57
 - numpy>=1.23,<2.0a0
@@ -24,13 +24,14 @@ dependencies:
 - pytest
 - pytest-cov
 - python>=3.9,<3.12
-- rapids-dask-dependency==24.8.*
+- rapids-build-backend>=0.3.0,<0.4.0dev0
+- rapids-dask-dependency==24.8.*,>=0.0.0a0
 - setuptools>=64.0.0
 - sphinx
 - sphinx-click>=2.7.1
 - sphinx-rtd-theme>=0.5.1
 - ucx-proc=*=gpu
-- ucx-py==0.39.*
-- ucxx==0.39.*
+- ucx-py==0.39.*,>=0.0.0a0
+- ucxx==0.39.*,>=0.0.0a0
 - zict>=2.0.0
 name: all_cuda-118_arch-x86_64

conda/environments/all_cuda-122_arch-x86_64.yaml

@@ -11,10 +11,10 @@ dependencies:
 - cuda-nvcc-impl
 - cuda-nvrtc
 - cuda-version=12.2
-- cudf==24.8.*
-- dask-cudf==24.8.*
-- distributed-ucxx==0.39.*
-- kvikio==24.8.*
+- cudf==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
+- distributed-ucxx==0.39.*,>=0.0.0a0
+- kvikio==24.8.*,>=0.0.0a0
 - numactl-devel-cos7-x86_64
 - numba>=0.57
 - numpy>=1.23,<2.0a0
@@ -25,13 +25,14 @@ dependencies:
 - pytest
 - pytest-cov
 - python>=3.9,<3.12
-- rapids-dask-dependency==24.8.*
+- rapids-build-backend>=0.3.0,<0.4.0dev0
+- rapids-dask-dependency==24.8.*,>=0.0.0a0
 - setuptools>=64.0.0
 - sphinx
 - sphinx-click>=2.7.1
 - sphinx-rtd-theme>=0.5.1
 - ucx-proc=*=gpu
-- ucx-py==0.39.*
-- ucxx==0.39.*
+- ucx-py==0.39.*,>=0.0.0a0
+- ucxx==0.39.*,>=0.0.0a0
 - zict>=2.0.0
 name: all_cuda-122_arch-x86_64

conda/recipes/dask-cuda/meta.yaml

@@ -29,6 +29,7 @@ requirements:
   host:
     - python
     - pip
+    - rapids-build-backend>=0.3.0,<0.4.0.dev0
   run:
     - python
     {% for r in data.get("project", {}).get("dependencies", []) %}

dask_cuda/_version.py

@@ -15,6 +15,16 @@
 import importlib.resources
 
 __version__ = (
-    importlib.resources.files("dask_cuda").joinpath("VERSION").read_text().strip()
+    importlib.resources.files(__package__).joinpath("VERSION").read_text().strip()
 )
-__git_commit__ = ""
+try:
+    __git_commit__ = (
+        importlib.resources.files(__package__)
+        .joinpath("GIT_COMMIT")
+        .read_text()
+        .strip()
+    )
+except FileNotFoundError:
+    __git_commit__ = ""
+
+__all__ = ["__git_commit__", "__version__"]

dask_cuda/tests/test_version.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+import dask_cuda
+
+
+def test_version_constants_are_populated():
+    # __git_commit__ will only be non-empty in a built distribution
+    assert isinstance(dask_cuda.__git_commit__, str)
+
+    # __version__ should always be non-empty
+    assert isinstance(dask_cuda.__version__, str)
+    assert len(dask_cuda.__version__) > 0

dependencies.yaml

@@ -74,6 +74,7 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
+          - rapids-build-backend>=0.3.0,<0.4.0dev0
           - setuptools>=64.0.0
   cuda_version:
     specific:
@@ -153,23 +154,23 @@ dependencies:
           - numpy>=1.23,<2.0a0
           - pandas>=1.3
           - pynvml>=11.0.0,<11.5
-          - rapids-dask-dependency==24.8.*
+          - rapids-dask-dependency==24.8.*,>=0.0.0a0
           - zict>=2.0.0
   test_python:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - cudf==24.8.*
-          - dask-cudf==24.8.*
-          - kvikio==24.8.*
           - pytest
           - pytest-cov
-          - ucx-py==0.39.*
       - output_types: [conda]
         packages:
-          - distributed-ucxx==0.39.*
+          - &cudf_conda cudf==24.8.*,>=0.0.0a0
+          - &dask_cudf_conda dask-cudf==24.8.*,>=0.0.0a0
+          - distributed-ucxx==0.39.*,>=0.0.0a0
+          - &kvikio_conda kvikio==24.8.*,>=0.0.0a0
+          - &ucx_py_conda ucx-py==0.39.*,>=0.0.0a0
           - ucx-proc=*=gpu
-          - ucxx==0.39.*
+          - ucxx==0.39.*,>=0.0.0a0
     specific:
       - output_types: conda
         matrices:
@@ -181,3 +182,23 @@ dependencies:
               arch: aarch64
             packages:
               - numactl-devel-cos7-aarch64
+      - output_types: [requirements, pyproject]
+        matrices:
+          # kvikio should be added to the CUDA-version-specific matrices once there are wheels available
+          # ref: https://github.com/rapidsai/kvikio/pull/369
+          - matrix: {cuda: "12.*"}
+            packages:
+              - cudf-cu12==24.8.*,>=0.0.0a0
+              - dask-cudf-cu12==24.8.*,>=0.0.0a0
+              - ucx-py-cu12==0.39.*,>=0.0.0a0
+          - matrix: {cuda: "11.*"}
+            packages:
+              - cudf-cu11==24.8.*,>=0.0.0a0
+              - dask-cudf-cu11==24.8.*,>=0.0.0a0
+              - ucx-py-cu11==0.39.*,>=0.0.0a0
+          - matrix:
+            packages:
+              - *cudf_conda
+              - *dask_cudf_conda
+              - *kvikio_conda
+              - *ucx_py_conda

pyproject.toml

@@ -1,6 +1,7 @@
 [build-system]
-build-backend = "setuptools.build_meta"
+build-backend = "rapids_build_backend.build"
 requires = [
+    "rapids-build-backend>=0.3.0,<0.4.0dev0",
     "setuptools>=64.0.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 
@@ -20,7 +21,7 @@ dependencies = [
     "numpy>=1.23,<2.0a0",
     "pandas>=1.3",
     "pynvml>=11.0.0,<11.5",
-    "rapids-dask-dependency==24.8.*",
+    "rapids-dask-dependency==24.8.*,>=0.0.0a0",
     "zict>=2.0.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 classifiers = [
@@ -49,12 +50,12 @@ docs = [
     "sphinx-rtd-theme>=0.5.1",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 test = [
-    "cudf==24.8.*",
-    "dask-cudf==24.8.*",
-    "kvikio==24.8.*",
+    "cudf==24.8.*,>=0.0.0a0",
+    "dask-cudf==24.8.*,>=0.0.0a0",
+    "kvikio==24.8.*,>=0.0.0a0",
     "pytest",
     "pytest-cov",
-    "ucx-py==0.39.*",
+    "ucx-py==0.39.*,>=0.0.0a0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project.urls]
@@ -129,6 +130,11 @@ filterwarnings = [
     "ignore:Dask DataFrame implementation is deprecated:DeprecationWarning",
 ]
 
+[tool.rapids-build-backend]
+build-backend = "setuptools.build_meta"
+dependencies-file = "dependencies.yaml"
+disable-cuda = true
+
 [tool.setuptools]
 license-files = ["LICENSE"]