diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index ed5f00c638..d634249556 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -66,10 +66,11 @@ jobs: branch: ${{ inputs.branch }} date: ${{ inputs.date }} sha: ${{ inputs.sha }} - wheel-build-cuml: + wheel-build-cpp: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: ${{ inputs.build_type || 'branch' }} branch: ${{ inputs.branch }} sha: ${{ inputs.sha }} @@ -81,8 +82,35 @@ jobs: extra-repo: rapidsai/cumlprims_mg extra-repo-sha: branch-24.08 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY - wheel-publish-cuml: - needs: wheel-build-cuml + wheel-publish-cpp: + needs: wheel-build-cpp + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + package-name: libcuml + package-type: cpp + wheel-build-python: + secrets: inherit + needs: wheel-build-cpp + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + with: + build_type: ${{ inputs.build_type || 'branch' }} + branch: ${{ inputs.branch }} + sha: ${{ inputs.sha }} + date: ${{ inputs.date }} + script: ci/build_wheel_python.sh + # Note that this approach to cloning repos obviates any modification to + # the CMake variables in get_cumlprims_mg.cmake since CMake will just use + # the clone as is. + extra-repo: rapidsai/cumlprims_mg + extra-repo-sha: branch-24.08 + extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + wheel-publish-python: + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-publish.yaml@branch-24.08 with: @@ -91,3 +119,4 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: cuml + package-type: python \ No newline at end of file diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 5468680f3d..bcd615fb00 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,7 +22,8 @@ jobs: - conda-python-tests-dask - conda-notebook-tests - docs-build - - wheel-build-cuml + - wheel-build-cpp + - wheel-build-python - wheel-tests-cuml - devcontainer secrets: inherit @@ -112,18 +113,29 @@ jobs: arch: "amd64" container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-cuml: + wheel-build-cpp: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 with: + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) build_type: pull-request - script: ci/build_wheel.sh + script: ci/build_wheel_cpp.sh + extra-repo: rapidsai/cumlprims_mg + extra-repo-sha: branch-24.08 + extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY + wheel-build-python: + needs: [checks, wheel-build-cpp] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.08 + with: + build_type: pull-request + script: ci/build_wheel_python.sh extra-repo: rapidsai/cumlprims_mg extra-repo-sha: branch-24.08 extra-repo-deploy-key: CUMLPRIMS_SSH_PRIVATE_DEPLOY_KEY wheel-tests-cuml: - needs: wheel-build-cuml + needs: wheel-build-python secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.08 with: diff --git a/ci/build_wheel_cpp.sh b/ci/build_wheel_cpp.sh new file mode 100644 index 0000000000..832b9759a6 --- /dev/null +++ b/ci/build_wheel_cpp.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Copyright (c) 2023-2024, NVIDIA CORPORATION. + +set -euo pipefail + +package_name="libcuml" +package_dir="python/libcuml" + +source rapids-configure-sccache +source rapids-date-string + +rapids-generate-version > ./VERSION + +cd "${package_dir}" + +python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +mkdir -p final_dist +python -m auditwheel repair -w final_dist dist/* + +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 cpp final_dist \ No newline at end of file diff --git a/ci/build_wheel.sh b/ci/build_wheel_python.sh similarity index 59% rename from ci/build_wheel.sh rename to ci/build_wheel_python.sh index db8393edeb..49e48667f6 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel_python.sh @@ -16,6 +16,17 @@ PACKAGE_CUDA_SUFFIX="-${RAPIDS_PY_CUDA_SUFFIX}" rapids-generate-version > ./VERSION + +# Downloads libcuml wheel from this current build, +# then ensures 'cuml' wheel builds always use the 'libcuml' just built in the same CI run. +# +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +RAPIDS_PY_WHEEL_NAME="libcuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libcuml_dist + +echo "libcuml-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libcuml_dist/libcuml_*.whl)" > /tmp/constraints.txt +export PIP_CONSTRAINT="/tmp/constraints.txt" + cd ${package_dir} SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DDISABLE_DEPRECATION_WARNINGS=ON;-DCPM_cumlprims_mg_SOURCE=${GITHUB_WORKSPACE}/cumlprims_mg/" \ @@ -28,4 +39,4 @@ SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DDISABLE_DEPRECATION_WARNINGS=ON;-DC mkdir -p final_dist python -m auditwheel repair -w final_dist dist/* -RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="cuml_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist diff --git a/dependencies.yaml b/dependencies.yaml index 4c4232cd5a..3e240e864f 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -66,14 +66,30 @@ files: - py_version - test_cuml - test_notebooks - py_build: + py_cpp_build: + output: pyproject + pyproject_dir: python/libcuml + extras: + table: build-system + includes: + - rapids_build_backend + py_wheel_build: output: pyproject pyproject_dir: python/cuml extras: table: build-system includes: - rapids_build_backend - py_rapids_build: + py_rapids_cpp_build: + output: pyproject + pyproject_dir: python/libcuml + extras: + table: tool.rapids-build-backend + key: requires + includes: + - common_build + - py_build + py_rapids_wheel_build: output: pyproject pyproject_dir: python/cuml extras: diff --git a/python/cuml/CMakeLists.txt b/python/cuml/CMakeLists.txt index f2541f7f04..fbee571b79 100644 --- a/python/cuml/CMakeLists.txt +++ b/python/cuml/CMakeLists.txt @@ -36,7 +36,6 @@ project( ################################################################################ # - User Options -------------------------------------------------------------- option(CUML_UNIVERSAL "Build all cuML Python components." ON) -option(FIND_CUML_CPP "Search for existing CUML C++ installations before defaulting to local files" OFF) option(SINGLEGPU "Disable all mnmg components and comms libraries" OFF) set(CUML_RAFT_CLONE_ON_PIN OFF) @@ -44,7 +43,6 @@ set(CUML_RAFT_CLONE_ON_PIN OFF) # todo: use CMAKE_MESSAGE_CONTEXT for prefix for logging. # https://github.com/rapidsai/cuml/issues/4843 message(VERBOSE "CUML_PY: Build only cuML CPU Python components.: ${CUML_CPU}") -message(VERBOSE "CUML_PY: Searching for existing CUML C++ installations before defaulting to local files: ${FIND_CUML_CPP}") message(VERBOSE "CUML_PY: Disabling all mnmg components and comms libraries: ${SINGLEGPU}") set(CUML_ALGORITHMS "ALL" CACHE STRING "Choose which algorithms are built cuML. Can specify individual algorithms or groups in a semicolon-separated list.") @@ -55,54 +53,18 @@ set(CUML_CPP_SRC "../../cpp") ################################################################################ # - Process User Options ------------------------------------------------------ -# If the user requested it, we attempt to find cuml. -if(FIND_CUML_CPP) - # We need to call get_treelite explicitly because we need the correct - # ${TREELITE_LIBS} definition for RF - include(rapids-cpm) - include(rapids-export) - rapids_cpm_init() - find_package(cuml ${CUML_VERSION} REQUIRED) - include(${CUML_CPP_SRC}/cmake/thirdparty/get_treelite.cmake) -else() - set(cuml_FOUND OFF) -endif() +# We need to call get_treelite explicitly because we need the correct +# ${TREELITE_LIBS} definition for RF +include(rapids-cpm) +include(rapids-export) +rapids_cpm_init() +find_package(cuml ${CUML_VERSION} REQUIRED) +include(${CUML_CPP_SRC}/cmake/thirdparty/get_treelite.cmake) include(rapids-cython-core) set(CUML_PYTHON_TREELITE_TARGET treelite::treelite) -if(NOT ${CUML_CPU}) - if(NOT cuml_FOUND) - set(BUILD_CUML_TESTS OFF) - set(BUILD_PRIMS_TESTS OFF) - set(BUILD_CUML_C_LIBRARY OFF) - set(BUILD_CUML_EXAMPLES OFF) - set(BUILD_CUML_BENCH OFF) - set(BUILD_CUML_PRIMS_BENCH OFF) - set(CUML_EXPORT_TREELITE_LINKAGE ON) - set(CUML_PYTHON_TREELITE_TARGET treelite::treelite_static) - - # Statically link dependencies if building wheels - set(CUDA_STATIC_RUNTIME ON) - set(CUDA_STATIC_MATH_LIBRARIES ON) - set(CUML_USE_RAFT_STATIC ON) - set(CUML_USE_FAISS_STATIC ON) - set(CUML_USE_TREELITE_STATIC ON) - set(CUML_USE_CUMLPRIMS_MG_STATIC ON) - # Don't install the static libs into wheels - set(CUML_EXCLUDE_RAFT_FROM_ALL ON) - set(RAFT_EXCLUDE_FAISS_FROM_ALL ON) - set(CUML_EXCLUDE_TREELITE_FROM_ALL ON) - set(CUML_EXCLUDE_CUMLPRIMS_MG_FROM_ALL ON) - - add_subdirectory(${CUML_CPP_SRC} cuml-cpp EXCLUDE_FROM_ALL) - - set(cython_lib_dir cuml) - install(TARGETS ${CUML_CPP_TARGET} DESTINATION ${cython_lib_dir}) - endif() -endif() - if(CUML_CPU) set(CUML_UNIVERSAL OFF) set(SINGLEGPU ON) diff --git a/python/cuml/cuml/__init__.py b/python/cuml/cuml/__init__.py index 62ab93c1b4..e856127b74 100644 --- a/python/cuml/cuml/__init__.py +++ b/python/cuml/cuml/__init__.py @@ -1,5 +1,5 @@ # -# Copyright (c) 2022-2023, NVIDIA CORPORATION. +# Copyright (c) 2022-2024, NVIDIA CORPORATION. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -14,6 +14,17 @@ # limitations under the License. # +# If libcuml was installed as a wheel, we must request it to load the library symbols. +# Otherwise, we assume that the library was installed in a system path that ld can find. +try: + import libcuml +except ModuleNotFoundError: + pass +else: + libcuml.load_library() + del libcuml + + from cuml.internals.base import Base, UniversalBase from cuml.internals.available_devices import is_cuda_available diff --git a/python/libcuml/CMakeLists.txt b/python/libcuml/CMakeLists.txt new file mode 100644 index 0000000000..00012aff4d --- /dev/null +++ b/python/libcuml/CMakeLists.txt @@ -0,0 +1,47 @@ +@@ -0,0 +1,49 @@ +# ============================================================================= +# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.26.4 FATAL_ERROR) + +include(../../rapids_config.cmake) + +project( + libcuml-python + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX +) + +# Check if cuml is already available. If so, it is the user's responsibility to ensure that the +# CMake package is also available at build time of the Python cuml package. +find_package(cuml "${RAPIDS_VERSION}") + +if(cuml_FOUND) + return() +endif() + +unset(cuml_FOUND) + +# Find Python early so that later commands can use it +find_package(Python 3.9 REQUIRED COMPONENTS Interpreter) + +set(BUILD_TESTS OFF) +set(BUILD_BENCHMARKS OFF) +set(cuml_BUILD_TESTUTIL OFF) +set(cuml_BUILD_STREAMS_TEST_UTIL OFF) +set(CUDA_STATIC_RUNTIME ON) + +set(CMAKE_LIBRARY_OUTPUT_DIRECTORY ${PROJECT_BINARY_DIR}/lib) + +add_subdirectory(../../cpp cuml-cpp) diff --git a/python/libcuml/LICENSE b/python/libcuml/LICENSE new file mode 120000 index 0000000000..30cff7403d --- /dev/null +++ b/python/libcuml/LICENSE @@ -0,0 +1 @@ +../../LICENSE \ No newline at end of file diff --git a/python/libcuml/README.md b/python/libcuml/README.md new file mode 120000 index 0000000000..fe84005413 --- /dev/null +++ b/python/libcuml/README.md @@ -0,0 +1 @@ +../../README.md \ No newline at end of file diff --git a/python/libcuml/VERSION b/python/libcuml/VERSION new file mode 120000 index 0000000000..558194c5a5 --- /dev/null +++ b/python/libcuml/VERSION @@ -0,0 +1 @@ +../../VERSION \ No newline at end of file diff --git a/python/libcuml/libcuml/__init__.py b/python/libcuml/libcuml/__init__.py new file mode 100644 index 0000000000..d86d2f1066 --- /dev/null +++ b/python/libcuml/libcuml/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libcuml._version import __git_commit__, __version__ +from libcuml.load import load_library diff --git a/python/libcuml/libcuml/_version.py b/python/libcuml/libcuml/_version.py new file mode 100644 index 0000000000..7dd732b490 --- /dev/null +++ b/python/libcuml/libcuml/_version.py @@ -0,0 +1,33 @@ +# Copyright (c) 2023-2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__) + .joinpath("VERSION") + .read_text() + .strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libcuml/libcuml/load.py b/python/libcuml/libcuml/load.py new file mode 100644 index 0000000000..a33c684af4 --- /dev/null +++ b/python/libcuml/libcuml/load.py @@ -0,0 +1,50 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + + +def load_library(): + # This is loading the libarrow shared library in situations where it comes from the + # pyarrow package (i.e. when installed as a wheel). + import pyarrow # noqa: F401 + + # Dynamically load libcuml.so. Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + libcuml_lib = None + try: + libcuml_lib = ctypes.CDLL("libcuml.so", ctypes.RTLD_GLOBAL) + except OSError: + # If neither of these directories contain the library, we assume we are in an + # environment where the C++ library is already installed somewhere else and the + # CMake build of the libcuml Python package was a no-op. Note that this approach + # won't work for real editable installs of the libcuml package, but that's not a + # use case I think we need to support. scikit-build-core has limited support for + # importlib.resources so there isn't a clean way to support that case yet. + for lib_dir in ("lib", "lib64"): + if os.path.isfile( + lib := os.path.join( + os.path.dirname(__file__), lib_dir, "libcuml.so" + ) + ): + libcuml_lib = ctypes.CDLL(lib, ctypes.RTLD_GLOBAL) + break + + # The caller almost never needs to do anything with this library, but no + # harm in offering the option since this object at least provides a handle + # to inspect where libcuml was loaded from. + return libcuml_lib diff --git a/python/libcuml/pyproject.toml b/python/libcuml/pyproject.toml new file mode 100644 index 0000000000..31a4c29433 --- /dev/null +++ b/python/libcuml/pyproject.toml @@ -0,0 +1,77 @@ +# Copyright (c) 2024, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.7.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libcudf" +dynamic = ["version"] +description = "cuDF - GPU Dataframe (C++)" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache 2.0" } +requires-python = ">=3.9" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] +dependencies = [ + "pyarrow>=16.1.0,<16.2.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project.urls] +Homepage = "https://github.com/rapidsai/cudf" + +[project.entry-points."cmake.prefix"] +libcudf = "libcudf" + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.minimum-version = "3.26.4" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libcudf"] +wheel.install-dir = "libcudf" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libcudf/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" +requires = [ + "cmake>=3.26.4,!=3.30.0", + "cuda-python", + "cython>=3.0.0", + "ninja", + "pylibraft==24.8.*,>=0.0.0a0", + "rmm==24.8.*,>=0.0.0a0", + "treelite==4.3.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. \ No newline at end of file