use rapids-build-backend (#181)

Contributes to rapidsai/build-planning#31
Contributes to rapidsai/dependency-file-generator#89

Proposes introducing `rapids-build-backend` as this project's build backend, to reduce the complexity of various CI/build scripts.

Authors:
  - James Lamb (https://github.com/jameslamb)

Approvers:
  - Bradley Dice (https://github.com/bdice)

URL: #181

.pre-commit-config.yaml

@@ -32,7 +32,7 @@ repos:
         types_or: [c, c++, cuda]
         args: ["-fallback-style=none", "-style=file", "-i"]
   - repo: https://github.com/rapidsai/dependency-file-generator
-    rev: v1.11.0
+    rev: v1.13.11
     hooks:
       - id: rapids-dependency-file-generator
         args: ["--clean"]

build.sh

@@ -271,7 +271,11 @@ if buildAll || hasArg pylibwholegraph; then
     if ! hasArg --compile-cmd; then
         cd ${REPODIR}/python/pylibwholegraph
         env LIBWHOLEGRAPH_DIR=${LIBWHOLEGRAPH_DIR} \
-        SKBUILD_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE};${EXTRA_CMAKE_ARGS/ /;}" ${PYTHON} -m pip install --no-build-isolation --no-deps .
+        SKBUILD_CMAKE_ARGS="-DCMAKE_BUILD_TYPE=${BUILD_TYPE};${EXTRA_CMAKE_ARGS/ /;}" ${PYTHON} -m pip install \
+            --no-build-isolation \
+            --no-deps \
+            --config-settings rapidsai.disable-cuda=true \
+            .
 
     else
         # just invoke cmake without going through scikit-build-core

ci/build_docs.sh

@@ -8,7 +8,7 @@ rapids-logger "Create test conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key docs \
+  --file-key docs \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n docs

ci/build_python.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2022-2023, NVIDIA CORPORATION.
+# Copyright (c) 2022-2024, NVIDIA CORPORATION.
 
 set -euo pipefail
 
@@ -17,19 +17,14 @@ PACKAGES="libwholegraph"
 
 CPP_CHANNEL=$(rapids-download-conda-from-s3 cpp)
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
-export RAPIDS_PACKAGE_VERSION=${version}
-echo "${version}" > VERSION
+rapids-generate-version > ./VERSION
 
 rapids-logger "Begin py build"
 
 # TODO: Remove `--no-test` flags once importing on a CPU
 # node works correctly
 rapids-logger "Begin pylibwholegraph build"
-version_file_pylibwholegraph="python/pylibwholegraph/pylibwholegraph/_version.py"
-sed -i "/^__git_commit__/ s/= .*/= \"${git_commit}\"/g" ${version_file_pylibwholegraph}
-rapids-conda-retry mambabuild \
+RAPIDS_PACKAGE_VERSION=$(head -1 ./VERSION) rapids-conda-retry mambabuild \
   --no-test \
   --channel "${CPP_CHANNEL}" \
   conda/recipes/pylibwholegraph

ci/build_wheel.sh

@@ -9,32 +9,10 @@ package_dir="python/pylibwholegraph"
 source rapids-configure-sccache
 source rapids-date-string
 
-version=$(rapids-generate-version)
-git_commit=$(git rev-parse HEAD)
+rapids-generate-version > ./VERSION
 
 RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})"
 
-# This is the version of the suffix with a preceding hyphen. It's used
-# everywhere except in the final wheel name.
-PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}"
-
-# Patch project metadata files to include the CUDA version suffix and version override.
-pyproject_file="${package_dir}/pyproject.toml"
-version_file="${package_dir}/${package_name}/_version.py"
-
-sed -i "s/name = \"${package_name}\"/name = \"${package_name}${PACKAGE_CUDA_SUFFIX}\"/g" ${pyproject_file}
-echo "${version}" > VERSION
-sed -i "/^__git_commit__ / s/= .*/= \"${git_commit}\"/g" ${version_file}
-
-# For nightlies we want to ensure that we're pulling in alphas as well. The
-# easiest way to do so is to augment the spec with a constraint containing a
-# min alpha version that doesn't affect the version bounds but does allow usage
-# of alpha versions for that dependency without --pre
-alpha_spec=''
-if ! rapids-is-release-build; then
-    alpha_spec=',>=0.0.0a0'
-fi
-
 cd "${package_dir}"
 
 # Hardcode the output dir

ci/check_style.sh

@@ -8,7 +8,7 @@ rapids-logger "Create checks conda environment"
 
 rapids-dependency-file-generator \
   --output conda \
-  --file_key checks \
+  --file-key checks \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n checks

ci/release/update-version.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Copyright (c) 2018-2023, NVIDIA CORPORATION.
+# Copyright (c) 2018-2024, NVIDIA CORPORATION.
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
 # You may obtain a copy of the License at
@@ -68,10 +68,10 @@ DEPENDENCIES=(
 )
 for DEP in "${DEPENDENCIES[@]}"; do
   for FILE in dependencies.yaml conda/environments/*.yaml; do
-    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*/g" ${FILE}
+    sed_runner "/-.* ${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0/g" ${FILE}
   done
   for FILE in python/**/pyproject.toml; do
-    sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*\"/g" ${FILE}
+    sed_runner "/\"${DEP}\(-cu[[:digit:]]\{2\}\)\{0,1\}==/ s/==.*\"/==${NEXT_SHORT_TAG_PEP440}.*,>=0.0.0a0\"/g" ${FILE}
   done
 done
 

ci/test_clang_tidy.sh

@@ -9,7 +9,7 @@ rapids-logger "Create clang-tidy conda environment"
 rapids-logger "Generate clang-tidy testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key clang_tidy \
+  --file-key clang_tidy \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n clang_tidy

ci/test_cpp.sh

@@ -11,7 +11,7 @@ cd "$(dirname "$(realpath "${BASH_SOURCE[0]}")")"/../
 rapids-logger "Generate C++ testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_cpp \
+  --file-key test_cpp \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch)" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

ci/test_python.sh

@@ -24,7 +24,7 @@ fi
 rapids-logger "Generate Python testing dependencies"
 rapids-dependency-file-generator \
   --output conda \
-  --file_key test_python \
+  --file-key test_python \
   --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=${ARCH};py=${RAPIDS_PY_VERSION}" | tee env.yaml
 
 rapids-mamba-retry env create --yes -f env.yaml -n test

conda/environments/all_cuda-118_arch-x86_64.yaml

@@ -24,8 +24,8 @@ dependencies:
 - graphviz
 - ipykernel
 - ipython
-- libraft-headers==24.8.*
-- librmm==24.8.*
+- libraft-headers==24.8.*,>=0.0.0a0
+- librmm==24.8.*,>=0.0.0a0
 - nanobind>=0.2.0
 - nbsphinx
 - nccl
@@ -41,6 +41,7 @@ dependencies:
 - python>=3.9,<3.12
 - pytorch-cuda=11.8
 - pytorch=2.0.0
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
 - scikit-build-core>=0.7.0
 - sphinx-copybutton

conda/environments/all_cuda-122_arch-x86_64.yaml

@@ -25,8 +25,8 @@ dependencies:
 - graphviz
 - ipykernel
 - ipython
-- libraft-headers==24.8.*
-- librmm==24.8.*
+- libraft-headers==24.8.*,>=0.0.0a0
+- librmm==24.8.*,>=0.0.0a0
 - nanobind>=0.2.0
 - nbsphinx
 - nccl
@@ -39,6 +39,7 @@ dependencies:
 - pytest-forked
 - pytest-xdist
 - python>=3.9,<3.12
+- rapids-build-backend>=0.3.0,<0.4.0.dev0
 - recommonmark
 - scikit-build-core>=0.7.0
 - sphinx-copybutton

conda/recipes/pylibwholegraph/meta.yaml

@@ -63,6 +63,7 @@ requirements:
     - cython
     - libwholegraph ={{ version }}
     - python
+    - rapids-build-backend >=0.3.0,<0.4.0.dev0
     - scikit-build-core {{ scikit_build_core_version }}
   run:
     - {{ pin_compatible('cuda-version', max_pin='x', min_pin='x') }}

dependencies.yaml

@@ -11,6 +11,7 @@ files:
       - cuda
       - cuda_version
       - py_version
+      - rapids_build_skbuild
       - run
       - test_python
       - docs
@@ -44,6 +45,7 @@ files:
       - cuda
       - cuda_version
       - py_version
+      - rapids_build_skbuild
       - run
       - pytorch_cpu
       - clang_tools
@@ -52,6 +54,14 @@ files:
     pyproject_dir: python/pylibwholegraph
     extras:
       table: build-system
+    includes:
+      - rapids_build_skbuild
+  py_rapids_build_pylibwholegraph:
+    output: pyproject
+    pyproject_dir: python/pylibwholegraph
+    extras:
+      table: tool.rapids-build-backend
+      key: requires
     includes:
       - python_build_wheel
 channels:
@@ -74,11 +84,10 @@ dependencies:
           - cxx-compiler
           - cython>=3.0.0
           - &doxygen doxygen==1.9.1
-          - libraft-headers==24.8.*
-          - librmm==24.8.*
+          - libraft-headers==24.8.*,>=0.0.0a0
+          - librmm==24.8.*,>=0.0.0a0
           - nanobind>=0.2.0
           - nccl
-          - scikit-build-core>=0.7.0
     specific:
       - output_types: conda
         matrices:
@@ -315,11 +324,21 @@ dependencies:
         packages:
           - clangxx==16.0.6
           - clang-tools==16.0.6
+  rapids_build_skbuild:
+    common:
+      - output_types: [conda, requirements, pyproject]
+        packages:
+          - rapids-build-backend>=0.3.0,<0.4.0.dev0
+      - output_types: conda
+        packages:
+          - scikit-build-core>=0.7.0
+      - output_types: [requirements, pyproject]
+        packages:
+          - scikit-build-core[pyproject]>=0.7.0
   python_build_wheel:
     common:
       - output_types: [pyproject]
         packages:
           - cmake>=3.26.4
           - cython>=3.0.0
           - ninja
-          - scikit-build-core[pyproject]>=0.7.0

python/pylibwholegraph/pylibwholegraph/_version.py

@@ -1,4 +1,4 @@
-# Copyright (c) 2023, NVIDIA CORPORATION.
+# Copyright (c) 2023-2024, NVIDIA CORPORATION.
 #
 # Licensed under the Apache License, Version 2.0 (the "License");
 # you may not use this file except in compliance with the License.
@@ -16,6 +16,16 @@
 import importlib.resources
 
 __version__ = (
-    importlib.resources.files("pylibwholegraph").joinpath("VERSION").read_text().strip()
+    importlib.resources.files(__package__).joinpath("VERSION").read_text().strip()
 )
-__git_commit__ = ""
+try:
+    __git_commit__ = (
+        importlib.resources.files(__package__)
+        .joinpath("GIT_COMMIT")
+        .read_text()
+        .strip()
+    )
+except FileNotFoundError:
+    __git_commit__ = ""
+
+__all__ = ["__git_commit__", "__version__"]

python/pylibwholegraph/pylibwholegraph/tests/test_version.py

@@ -0,0 +1,12 @@
+# Copyright (c) 2024, NVIDIA CORPORATION.
+
+import pylibwholegraph
+
+
+def test_version_constants_are_populated():
+    # __git_commit__ will only be non-empty in a built distribution
+    assert isinstance(pylibwholegraph.__git_commit__, str)
+
+    # __version__ should always be non-empty
+    assert isinstance(pylibwholegraph.__version__, str)
+    assert len(pylibwholegraph.__version__) > 0

python/pylibwholegraph/pyproject.toml

@@ -13,11 +13,9 @@
 # limitations under the License.
 
 [build-system]
-build-backend = "scikit_build_core.build"
+build-backend = "rapids_build_backend.build"
 requires = [
-    "cmake>=3.26.4",
-    "cython>=3.0.0",
-    "ninja",
+    "rapids-build-backend>=0.3.0,<0.4.0.dev0",
     "scikit-build-core[pyproject]>=0.7.0",
 ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
 
@@ -42,6 +40,15 @@ classifiers = [
     "Programming Language :: Python :: 3.11",
 ]
 
+[tool.rapids-build-backend]
+build-backend = "scikit_build_core.build"
+dependencies-file = "../../dependencies.yaml"
+requires = [
+    "cmake>=3.26.4",
+    "cython>=3.0.0",
+    "ninja",
+] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.
+
 [tool.scikit-build]
 build-dir = "build/{wheel_tag}"
 cmake.build-type = "Release"