use rapids-build-backend (#1044)

Contributes to rapidsai/build-planning#31
Contributes to rapidsai/dependency-file-generator#89
Contributes to rapidsai/build-planning#72

Proposes:

* introducing `rapids-build-backend` as this project's build backend, to reduce the complexity of various CI/build scripts
* making get-data-from-`pyproject.toml` code in conda recipe a bit stricter

## Notes for Reviewers

This reverts some of the workarounds introduced for docs builds in #1041. See this thread for context: #1041 (comment)

~To do that, I set 2 environment variables manually in in the readthedocs console ([link](https://readthedocs.org/dashboard/ucx-py/edit/)):~

* ~`RAPIDS_DISABLE_CUDA=true` = used to prevent needing `nvcc` in the docs-building environment~
* ~`RAPIDS_MATRIX_ENTRY='cuda=12.2'` = used to ensure that `pip install .` tries to install a real package (e.g. `libucx-cu12`)~

~I think this is the first place we're using the `rapids-build-backend` environment variable support.~

see #1044 (comment)

Authors:
  - James Lamb (https://github.com/jameslamb)

Approvers:
  - Bradley Dice (https://github.com/bdice)
  - Peter Andreas Entschev (https://github.com/pentschev)

URL: #1044

.pre-commit-config.yaml

@@ -24,7 +24,7 @@ repos:
                 args: ["--config=.flake8.cython"]
                 types: [cython]
       - repo: https://github.com/rapidsai/dependency-file-generator
-        rev: v1.8.0
+        rev: v1.13.11
         hooks:
               - id: rapids-dependency-file-generator
                 args: ["--clean"]

.readthedocs.yml

@@ -5,8 +5,13 @@ build:
   tools:
     python: "mambaforge-22.9"
   jobs:
-    post_checkout:
-      - bash ci/build_docs_pre_install.sh
+    post_create_environment:
+      - |
+        pip install \
+          --extra-index-url "https://pypi.anaconda.org/rapidsai-wheels-nightly/simple/" \
+          -C rapidsai.disable-cuda=true \
+          -C rapidsai.matrix-entry="cuda=12.2" \
+          .
 
 conda:
   environment: conda/environments/builddocs.yml

ci/build_python.sh

@@ -11,16 +11,12 @@ source rapids-date-string
 
 rapids-print-env
 
-version=$(rapids-generate-version)
-commit=$(git rev-parse HEAD)
-
-echo "${version}" > VERSION
-sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" ucp/_version.py
+rapids-generate-version > ./VERSION
 
 rapids-logger "Begin py build"
 conda config --set path_conflict prevent
 
-RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   conda/recipes/ucx-py
 
 rapids-upload-conda-to-s3 python

ci/build_wheel.sh

@@ -22,39 +22,11 @@ rm -rf /usr/include/uct
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-commit=$(git rev-parse HEAD)
+rapids-generate-version > ./VERSION
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# This is the version of the suffix with a preceding hyphen. It's used
-# everywhere except in the final wheel name.
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
-
-# Patch project metadata files to include the CUDA version suffix and version override.
-pyproject_file="pyproject.toml"
-
-sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-echo "${version}" > VERSION
-sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" ucp/_version.py
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
-sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
-sed -r -i "/\"libucx([=><]+)/ s/\"libucx/\"libucx${PACKAGE_CUDA_SUFFIX}/g" ${pyproject_file}
-
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
-fi
-
-python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check
+python -m pip wheel . -w dist --no-deps --disable-pip-version-check
 
 mkdir -p final_dist
 python -m auditwheel repair \

ci/check_style.sh

@@ -8,7 +8,7 @@ rapids-logger "Create checks conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key checks \
+  --file-key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n checks

ci/release/update-version.sh

@@ -42,7 +42,7 @@ DEPENDENCIES=(
 )
 for FILE in dependencies.yaml; do
   for DEP in "${DEPENDENCIES[@]}"; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_RAPIDS_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
   done
 done
 

ci/test_python.sh

@@ -11,7 +11,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
+  --file-key test_python \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

conda/environments/builddocs.yml

@@ -14,7 +14,4 @@ dependencies:
 - recommonmark
 - pandoc=<2.0.0
 - pip
-- pip:
-  - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple
-  - ../../
 - cython

conda/recipes/ucx-py/meta.yaml

@@ -18,7 +18,7 @@ build:
   number: {{ GIT_DESCRIBE_NUMBER }}
   string: py{{ py_version }}_{{ date_string }}_{{ GIT_DESCRIBE_HASH }}_{{ GIT_DESCRIBE_NUMBER }}
   script:
-    - {{ PYTHON }} -m pip install . -vv
+    - {{ PYTHON }} -m pip install --config-settings rapidsai.disable-cuda=true . -vv
   ignore_run_exports_from:
     - ucx
 
@@ -30,29 +30,32 @@ requirements:
     - python
     - pip
     - ucx
+    {% for build_req in data["build-system"]["requires"] %}
+    - {{ build_req }}
+    {% endfor %}
     # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead
-    {% for r in data.get("build-system", {}).get("requires", []) if not r.startswith("libucx") %}
-    - {{ r }}
+    {% for rbb_req in data["tool"]["rapids-build-backend"]["requires"] if not rbb_req.startswith("libucx") %}
+    - {{ rbb_req }}
     {% endfor %}
   run:
     - python
     - ucx >=1.15.0,<1.16.0
     # 'libucx' wheel dependency is unnecessary... the 'ucx' conda-forge package is used here instead
-    {% for r in data.get("project", {}).get("dependencies", []) if not r.startswith("libucx") %}
-    - {{ r }}
+    {% for req in data["project"]["dependencies"] if not req.startswith("libucx") %}
+    - {{ req }}
     {% endfor %}
 
 test:
   imports:
     - ucp
 
 about:
-  home: {{ data.get("project", {}).get("urls", {}).get("Homepage", "") }}
-  license: {{ data.get("project", {}).get("license", {}).get("text", "") }}
+  home: {{ data["project"]["urls"]["Homepage"] }}
+  license: {{ data["project"]["license"]["text"] }}
   license_file:
-    {% for e in data.get("tool", {}).get("setuptools", {}).get("license-files", []) %}
+    {% for e in data["tool"]["setuptools"]["license-files"] %}
     - ../../../{{ e }}
     {% endfor %}
-  summary: {{ data.get("project", {}).get("description", "") }}
-  dev_url: {{ data.get("project", {}).get("urls", {}).get("Source", "") }}
-  doc_url: {{ data.get("project", {}).get("urls", {}).get("Documentation", "") }}
+  summary: {{ data["project"]["description"] }}
+  dev_url: {{ data["project"]["urls"]["Source"] }}
+  doc_url: {{ data["project"]["urls"]["Documentation"] }}

dependencies.yaml

@@ -7,6 +7,7 @@ files:
       - cuda
       - cuda_version
       - py_version
+      - rapids_build_setuptools
       - run
       - test_python
   test_python:
@@ -25,6 +26,14 @@ files:
     pyproject_dir: .
     extras:
       table: build-system
+    includes:
+      - rapids_build_setuptools
+  py_rapids_build:
+    output: pyproject
+    pyproject_dir: .
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
     includes:
       - build_python
       - depends_on_ucx_build
@@ -109,8 +118,13 @@ dependencies:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
-          - setuptools>=64.0.0
           - cython>=3.0.0
+  rapids_build_setuptools:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - rapids-build-backend>=0.3.1,<0.4.0dev0
+          - setuptools>=64.0.0
   run:
     common:
       - output_types: [conda, requirements, pyproject]
@@ -136,8 +150,6 @@ dependencies:
           - matrix: {cuda: "11.*"}
             packages:
               - libucx-cu11==1.15.0
-          # NOTE: this fallback needs to be a real, suffixed version
-          #       so 'pip install .' (e.g. as used in docs builds) will work
           - matrix: null
             packages:
               - libucx==1.15.0
@@ -160,8 +172,6 @@ dependencies:
           - matrix: {cuda: "11.*"}
             packages:
               - libucx-cu11>=1.15.0,<1.16
-          # NOTE: this fallback needs to be a real, suffixed version
-          #       so "pip install ." (e.g. as used in docs builds) will work
           - matrix: null
             packages:
               - libucx>=1.15.0,<1.16
@@ -170,7 +180,6 @@ dependencies:
       - output_types: [conda, requirements, pyproject]
         packages:
           - cloudpickle
-          - cudf==24.8.*
           - dask
           - distributed
           - numba>=0.57
@@ -179,7 +188,20 @@ dependencies:
           - pytest-rerunfailures
       - output_types: [conda]
         packages:
+          - &cudf_conda cudf==24.8.*,>=0.0.0a0
           - cupy>=12.0.0
+    specific:
       - output_types: [requirements, pyproject]
-        packages:
-          - cupy-cuda11x>=12.0.0
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - cudf-cu12==24.8.*,>=0.0.0a0
+              - cupy-cuda12x>=12.0.0
+          - matrix: {cuda: "11.*"}
+            packages:
+              - cudf-cu11==24.8.*,>=0.0.0a0
+              - &cupy_cu11 cupy-cuda11x>=12.0.0
+          - matrix:
+            packages:
+              - *cudf_conda
+              - *cupy_cu11

pyproject.toml

@@ -13,10 +13,9 @@
 # limitations under the License.
 
 [build-system]
-build-backend = "setuptools.build_meta"
+build-backend = "rapids_build_backend.build"
 requires = [
-    "cython>=3.0.0",
-    "libucx==1.15.0",
+    "rapids-build-backend>=0.3.1,<0.4.0dev0",
     "setuptools>=64.0.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
 
@@ -50,7 +49,7 @@ classifiers = [
 [project.optional-dependencies]
 test = [
     "cloudpickle",
-    "cudf==24.8.*",
+    "cudf==24.8.*,>=0.0.0a0",
     "cupy-cuda11x>=12.0.0",
     "dask",
     "distributed",
@@ -112,6 +111,17 @@ skip = [
 [tool.pytest.ini_options]
 xfail_strict = true
 
+[tool.rapids-build-backend]
+build-backend = "setuptools.build_meta"
+commit-files = [
+    "ucp/COMMIT_FILE"
+]
+dependencies-file = "dependencies.yaml"
+requires = [
+    "cython>=3.0.0",
+    "libucx==1.15.0",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit dependencies.yaml and run `rapids-dependency-file-generator`.
+
 [tool.setuptools]
 license-files = ["LICENSE"]
 zip-safe = false

tests/test_version.py

@@ -9,8 +9,13 @@ def test_get_ucx_version():
     assert ucp.core._ctx is None
 
 
-def test_version_constant():
+def test_version_constants_are_populated():
+    # __git_commit__ will only be non-empty in a built distribution
+    assert isinstance(ucp.__git_commit__, str)
+
+    # __version__ should always be non-empty
     assert isinstance(ucp.__version__, str)
+    assert len(ucp.__version__) > 0
 
 
 def test_ucx_version_constant():

ucp/_version.py

@@ -14,5 +14,18 @@
 
 import importlib.resources
 
-__version__ = importlib.resources.files("ucp").joinpath("VERSION").read_text().strip()
-__git_commit__ = ""
+__version__ = (
+    importlib.resources.files(__package__).joinpath("VERSION").read_text().strip()
+)
+
+try:
+    __git_commit__ = (
+        importlib.resources.files(__package__)
+        .joinpath("GIT_COMMIT")
+        .read_text()
+        .strip()
+    )
+except FileNotFoundError:
+    __git_commit__ = ""
+
+__all__ = ["__git_commit__", "__version__"]