use rapids-build-backend (#600)

Contributes to rapidsai/build-planning#31 Contributes to rapidsai/dependency-file-generator#89 Proposes introducing `rapids-build-backend` as this project's build backend, to reduce the complexity of various CI/build scripts. Authors: - James Lamb (https://github.com/jameslamb) Approvers: - Bradley Dice (https://github.com/bdice) - Ajay Thorve (https://github.com/AjayThorve) URL: #600
rapidsai · Jun 10, 2024 · 2a2866c · 2a2866c
1 parent e01ed47
commit 2a2866c
Show file tree

Hide file tree

Showing 16 changed files with 104 additions and 73 deletions.
diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -15,7 +15,7 @@ repos:
                 args: ["--config=.flake8"]
                 files: python/.*$
       - repo: https://github.com/rapidsai/dependency-file-generator
-        rev: v1.8.0
+        rev: v1.13.11
         hooks:
             - id: rapids-dependency-file-generator
               args: ["--clean"]

diff --git a/build.sh b/build.sh
@@ -111,7 +111,8 @@ if (( ${NUMARGS} == 0 )) || hasArg cuxfilter; then
     echo "8"
     if [[ ${INSTALL_TARGET} != "" ]]; then
         python setup.py build_ext --inplace
-        python setup.py install --single-version-externally-managed --record=record.txt
+        RAPIDS_DISABLE_CUDA=true \
+            python setup.py install --single-version-externally-managed --record=record.txt
     else
         python setup.py build_ext --inplace --library-dir=${LIBCUXFILTER_BUILD_DIR}
     fi

diff --git a/ci/build_docs.sh b/ci/build_docs.sh
@@ -7,7 +7,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
     --output conda \
-    --file_key docs \
+    --file-key docs \
     --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n docs

diff --git a/ci/build_python.sh b/ci/build_python.sh
@@ -11,21 +11,14 @@ source rapids-date-string
 
 rapids-print-env
 
-package_name="cuxfilter"
-package_dir="python"
-
-version=$(rapids-generate-version)
-commit=$(git rev-parse HEAD)
-
-echo "${version}" > VERSION
-sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" "${package_dir}/${package_name}/_version.py"
+rapids-generate-version > ./VERSION
 
 rapids-logger "Begin py build"
 
 conda config --set path_conflict prevent
 # TODO: Remove `--no-test` flag once importing on a CPU
 # node works correctly
-RAPIDS_PACKAGE_VERSION=${version} rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   --no-test \
   conda/recipes/cuxfilter
 

diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh
@@ -9,40 +9,10 @@ package_dir="python"
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-commit=$(git rev-parse HEAD)
+rapids-generate-version > ./VERSION
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# This is the version of the suffix with a preceding hyphen. It's used
-# everywhere except in the final wheel name.
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
-
-# Patch project metadata files to include the CUDA version suffix and version override.
-pyproject_file="${package_dir}/pyproject.toml"
-version_file="${package_dir}/${package_name}/_version.py"
-
-sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-echo "${version}" > VERSION
-sed -i "/^__git_commit__/ s/= .*/= \"${commit}\"/g" ${version_file}
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
-sed -r -i "s/cudf==(.*)\"/cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
-sed -r -i "s/dask_cudf==(.*)\"/dask_cudf${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
-sed -r -i "s/cuspatial==(.*)\"/cuspatial${PACKAGE_CUDA_SUFFIX}==\1${alpha_spec}\"/g" ${pyproject_file}
-
-if [[ $PACKAGE_CUDA_SUFFIX == "-cu12" ]]; then
-    sed -i "s/cupy-cuda11x/cupy-cuda12x/g" ${pyproject_file}
-fi
-
 cd "${package_dir}"
 
 python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check

diff --git a/ci/check_style.sh b/ci/check_style.sh
@@ -8,7 +8,7 @@ rapids-logger "Create checks conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key checks \
+  --file-key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n checks

diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh
@@ -50,10 +50,10 @@ DEPENDENCIES=(
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml ci/utils/external_dependencies.yaml; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" "${FILE}"
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" "${FILE}"
   done
   for FILE in python/pyproject.toml; do
-    sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" "${FILE}"
+    sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" "${FILE}"
   done
 done
 

diff --git a/ci/test_notebooks.sh b/ci/test_notebooks.sh
@@ -8,7 +8,7 @@ set -euo pipefail
 rapids-logger "Generate notebook testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_notebooks \
+  --file-key test_notebooks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

diff --git a/ci/test_python.sh b/ci/test_python.sh
@@ -8,7 +8,7 @@ set -euo pipefail
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
+  --file-key test_python \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

diff --git a/conda/environments/all_cuda-118_arch-x86_64.yaml b/conda/environments/all_cuda-118_arch-x86_64.yaml
@@ -9,12 +9,12 @@ dependencies:
 - bokeh>=3.1
 - cuda-version=11.8
 - cudatoolkit
-- cudf==24.8.*
-- cugraph==24.8.*
+- cudf==24.8.*,>=0.0.0a0
+- cugraph==24.8.*,>=0.0.0a0
 - cupy>=12.0.0
-- cuspatial==24.8.*
-- dask-cuda==24.8.*
-- dask-cudf==24.8.*
+- cuspatial==24.8.*,>=0.0.0a0
+- dask-cuda==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
 - datashader>=0.15
 - geopandas>=0.11.0
 - holoviews>=1.16.0
@@ -39,7 +39,9 @@ dependencies:
 - pytest-cov
 - pytest-xdist
 - python>=3.9,<3.12
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
+- setuptools
 - sphinx-markdown-tables
 - sphinx>=7.2.5
 - sphinx_rtd_theme

diff --git a/conda/environments/all_cuda-122_arch-x86_64.yaml b/conda/environments/all_cuda-122_arch-x86_64.yaml
@@ -8,12 +8,12 @@ channels:
 dependencies:
 - bokeh>=3.1
 - cuda-version=12.2
-- cudf==24.8.*
-- cugraph==24.8.*
+- cudf==24.8.*,>=0.0.0a0
+- cugraph==24.8.*,>=0.0.0a0
 - cupy>=12.0.0
-- cuspatial==24.8.*
-- dask-cuda==24.8.*
-- dask-cudf==24.8.*
+- cuspatial==24.8.*,>=0.0.0a0
+- dask-cuda==24.8.*,>=0.0.0a0
+- dask-cudf==24.8.*,>=0.0.0a0
 - datashader>=0.15
 - geopandas>=0.11.0
 - holoviews>=1.16.0
@@ -38,7 +38,9 @@ dependencies:
 - pytest-cov
 - pytest-xdist
 - python>=3.9,<3.12
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
+- setuptools
 - sphinx-markdown-tables
 - sphinx>=7.2.5
 - sphinx_rtd_theme

diff --git a/conda/recipes/cuxfilter/meta.yaml b/conda/recipes/cuxfilter/meta.yaml
@@ -21,6 +21,7 @@ build:
 requirements:
   host:
     - python
+    - rapids-build-backend >=0.3.0,<0.4.0.dev0
     - setuptools
     - cuda-version ={{ cuda_version }}
   run:

diff --git a/dependencies.yaml b/dependencies.yaml
@@ -13,6 +13,7 @@ files:
       - docs
       - notebook
       - py_version
+      - rapids_build_setuptools
       - run
       - test_python
   test_python:
@@ -44,6 +45,14 @@ files:
     pyproject_dir: python
     extras:
       table: build-system
+    includes:
+      - rapids_build_setuptools
+  py_rapids_build:
+    output: pyproject
+    pyproject_dir: python
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
     includes:
       - build_wheels
   py_run:
@@ -72,7 +81,6 @@ dependencies:
       - output_types: pyproject
         packages:
           - wheel
-          - setuptools
   cuda_version:
     specific:
       - output_types: conda
@@ -141,8 +149,8 @@ dependencies:
           - notebook>=0.5.0
       - output_types: [conda]
         packages:
-          - cugraph==24.8.*
-          - dask-cuda==24.8.*
+          - cugraph==24.8.*,>=0.0.0a0
+          - dask-cuda==24.8.*,>=0.0.0a0
   py_version:
     specific:
       - output_types: conda
@@ -162,14 +170,17 @@ dependencies:
           - matrix:
             packages:
               - python>=3.9,<3.12
+  rapids_build_setuptools:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - rapids-build-backend>=0.3.0,<0.4.0.dev0
+          - setuptools
   run:
     common:
       - output_types: [conda, requirements, pyproject]
         packages:
           - bokeh>=3.1
-          - cudf==24.8.*
-          - cuspatial==24.8.*
-          - dask-cudf==24.8.*
           - datashader>=0.15
           - geopandas>=0.11.0
           - holoviews>=1.16.0
@@ -180,12 +191,33 @@ dependencies:
           - panel>=1.0
       - output_types: conda
         packages:
+          - &cudf_conda cudf==24.8.*,>=0.0.0a0
           - cupy>=12.0.0
+          - &cuspatial_conda cuspatial==24.8.*,>=0.0.0a0
+          - &dask_cudf_conda dask-cudf==24.8.*,>=0.0.0a0
           - nodejs>=18
           - libwebp-base
+    specific:
       - output_types: [requirements, pyproject]
-        packages:
-          - cupy-cuda11x>=12.0.0
+        matrices:
+          - matrix: {cuda: "12.*"}
+            packages:
+              - cudf-cu12==24.8.*,>=0.0.0a0
+              - cupy-cuda12x>=12.0.0
+              - cuspatial-cu12==24.8.*,>=0.0.0a0
+              - dask-cudf-cu12==24.8.*,>=0.0.0a0
+          - matrix: {cuda: "11.*"}
+            packages:
+              - cudf-cu11==24.8.*,>=0.0.0a0
+              - &cupy_cu11 cupy-cuda11x>=12.0.0
+              - cuspatial-cu11==24.8.*,>=0.0.0a0
+              - dask-cudf-cu11==24.8.*,>=0.0.0a0
+          - matrix:
+            packages:
+              - *cudf_conda
+              - *cupy_cu11
+              - *cuspatial_conda
+              - *dask_cudf_conda
   test_python:
     common:
       - output_types: [conda, requirements, pyproject]

diff --git a/python/cuxfilter/_version.py b/python/cuxfilter/_version.py
@@ -15,9 +15,20 @@
 import importlib.resources
 
 __version__ = (
-    importlib.resources.files("cuxfilter")
+    importlib.resources.files(__package__)
     .joinpath("VERSION")
     .read_text()
     .strip()
 )
-__git_commit__ = ""
+
+try:
+    __git_commit__ = (
+        importlib.resources.files(__package__)
+        .joinpath("GIT_COMMIT")
+        .read_text()
+        .strip()
+    )
+except FileNotFoundError:
+    __git_commit__ = ""
+
+__all__ = ["__git_commit__", "__version__"]
diff --git a/python/cuxfilter/tests/test_version.py b/python/cuxfilter/tests/test_version.py
@@ -0,0 +1,12 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+import cuxfilter
+
+
+def test_version_constants_are_populated():
+    # __git_commit__ will only be non-empty in a built distribution
+    assert isinstance(cuxfilter.__git_commit__, str)
+
+    # __version__ should always be non-empty
+    assert isinstance(cuxfilter.__version__, str)
+    assert len(cuxfilter.__version__) > 0
diff --git a/python/pyproject.toml b/python/pyproject.toml
@@ -1,10 +1,10 @@
 # Copyright (c) 2023, NVIDIA CORPORATION.
 
 [build-system]
-build-backend = "setuptools.build_meta"
+build-backend = "rapids_build_backend.build"
 requires = [
+    "rapids-build-backend>=0.3.0,<0.4.0.dev0",
     "setuptools",
-    "wheel",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
 
 [project]
@@ -19,10 +19,10 @@ license = { text = "Apache 2.0" }
 requires-python = ">=3.9"
 dependencies = [
     "bokeh>=3.1",
-    "cudf==24.8.*",
+    "cudf==24.8.*,>=0.0.0a0",
     "cupy-cuda11x>=12.0.0",
-    "cuspatial==24.8.*",
-    "dask-cudf==24.8.*",
+    "cuspatial==24.8.*,>=0.0.0a0",
+    "dask-cudf==24.8.*,>=0.0.0a0",
     "datashader>=0.15",
     "geopandas>=0.11.0",
     "holoviews>=1.16.0",
@@ -55,6 +55,13 @@ test = [
 Homepage = "https://github.com/rapidsai/cuxfilter"
 Documentation = "https://docs.rapids.ai/api/cuxfilter/stable/"
 
+[tool.rapids-build-backend]
+build-backend = "setuptools.build_meta"
+dependencies-file = "../dependencies.yaml"
+requires = [
+    "wheel",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../dependencies.yaml and run `rapids-dependency-file-generator`.
+
 [tool.setuptools]
 license-files = ["LICENSE"]