diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index a2945185..cd8f183d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -114,7 +114,18 @@ jobs: date: ${{ inputs.date }} package-name: cugraph-pyg package-type: python + wheel-build-libwholegraph: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.06 + with: + build_type: pull-request + script: ci/build_wheel_libwholegraph.sh + package-type: cpp + wheel-name: libwholegraph + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) wheel-build-pylibwholegraph: + needs: wheel-build-libwholegraph secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.06 with: diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index c6adc51a..e07af7aa 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -22,6 +22,7 @@ jobs: - conda-python-tests - docs-build - conda-notebook-tests + - wheel-build-libwholegraph - wheel-build-pylibwholegraph - wheel-tests-pylibwholegraph - wheel-build-cugraph-dgl @@ -136,10 +137,21 @@ jobs: build_type: pull-request container_image: "rapidsai/ci-conda:latest" run_script: "ci/build_docs.sh" - wheel-build-pylibwholegraph: + wheel-build-libwholegraph: needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.06 + with: + build_type: pull-request + script: ci/build_wheel_libwholegraph.sh + package-type: cpp + wheel-name: libwholegraph + # build for every combination of arch and CUDA version, but only for the latest Python + matrix_filter: group_by([.ARCH, (.CUDA_VER|split(".")|map(tonumber)|.[0])]) | map(max_by(.PY_VER|split(".")|map(tonumber))) + wheel-build-pylibwholegraph: + needs: [checks, wheel-build-libwholegraph] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-25.06 with: build_type: pull-request script: ci/build_wheel_pylibwholegraph.sh diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 46f0bd20..58ccbbff 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -5,30 +5,31 @@ set -euo pipefail package_name=$1 package_dir=$2 -underscore_package_name=$(echo "${package_name}" | tr "-" "_") +package_type=$3 # The set of shared libraries that should be packaged differs by project. # # Capturing that here in argument-parsing to allow this build_wheel.sh # script to be re-used by all wheel builds in the project. -case "${package_dir}" in - python/pylibwholegraph) - EXCLUDE_ARGS=( - --exclude libcuda.so.1 - --exclude libnvidia-ml.so.1 +# +EXCLUDE_ARGS=( + --exclude libcuda.so.1 + --exclude libnvidia-ml.so.1 + --exclude librapids_logger.so +) + +if [[ "${package_name}" != "libwholegraph" ]]; then + EXCLUDE_ARGS+=( + --exclude libwholegraph.so ) - ;; - *) - EXCLUDE_ARGS=() - ;; -esac +fi source rapids-configure-sccache source rapids-date-string rapids-generate-version > ./VERSION -RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" cd "${package_dir}" @@ -56,5 +57,5 @@ else -w "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" \ dist/* - RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" + RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 "${package_type}" "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" fi diff --git a/ci/build_wheel_cugraph-dgl.sh b/ci/build_wheel_cugraph-dgl.sh index e492eddb..dc29b871 100755 --- a/ci/build_wheel_cugraph-dgl.sh +++ b/ci/build_wheel_cugraph-dgl.sh @@ -5,5 +5,5 @@ set -euo pipefail package_dir="python/cugraph-dgl" -./ci/build_wheel.sh cugraph-dgl ${package_dir} +./ci/build_wheel.sh cugraph-dgl ${package_dir} python ./ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" diff --git a/ci/build_wheel_cugraph-pyg.sh b/ci/build_wheel_cugraph-pyg.sh index f1476714..686a1289 100755 --- a/ci/build_wheel_cugraph-pyg.sh +++ b/ci/build_wheel_cugraph-pyg.sh @@ -5,5 +5,5 @@ set -euo pipefail package_dir="python/cugraph-pyg" -./ci/build_wheel.sh cugraph-pyg ${package_dir} +./ci/build_wheel.sh cugraph-pyg ${package_dir} python ./ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" diff --git a/ci/build_wheel_libwholegraph.sh b/ci/build_wheel_libwholegraph.sh new file mode 100755 index 00000000..8c700914 --- /dev/null +++ b/ci/build_wheel_libwholegraph.sh @@ -0,0 +1,11 @@ +#!/bin/bash +# Copyright (c) 2024-2025, NVIDIA CORPORATION. + +set -euo pipefail + +package_dir="python/libwholegraph" + +export SKBUILD_CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON;-DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE;-DCUDA_STATIC_RUNTIME=ON" + +./ci/build_wheel.sh libwholegraph ${package_dir} cpp +./ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" diff --git a/ci/build_wheel_pylibwholegraph.sh b/ci/build_wheel_pylibwholegraph.sh index fc8da533..0b079246 100755 --- a/ci/build_wheel_pylibwholegraph.sh +++ b/ci/build_wheel_pylibwholegraph.sh @@ -5,7 +5,18 @@ set -euo pipefail package_dir="python/pylibwholegraph" -export SKBUILD_CMAKE_ARGS="-DBUILD_SHARED_LIBS=OFF;-DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE;-DCUDA_STATIC_RUNTIME=ON;-DWHOLEGRAPH_BUILD_WHEELS=ON" +RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen "${RAPIDS_CUDA_VERSION}")" -./ci/build_wheel.sh pylibwholegraph ${package_dir} +# Download the libcugraph wheel built in the previous step and make it +# available for pip to find. +RAPIDS_PY_WHEEL_NAME="libwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp /tmp/libwholegraph_dist +echo "libwholegraph-${RAPIDS_PY_CUDA_SUFFIX} @ file://$(echo /tmp/libwholegraph_dist/libwholegraph_*.whl)" > /tmp/constraints.txt + +# Using env variable PIP_CONSTRAINT is necessary to ensure the constraints +# are used when creating the isolated build environment. +export PIP_CONSTRAINT="/tmp/constraints.txt" + +export SKBUILD_CMAKE_ARGS="-DBUILD_SHARED_LIBS=ON;-DCMAKE_MESSAGE_LOG_LEVEL=VERBOSE;-DCUDA_STATIC_RUNTIME=ON;-DWHOLEGRAPH_BUILD_WHEELS=ON" + +./ci/build_wheel.sh pylibwholegraph ${package_dir} python ./ci/validate_wheel.sh ${package_dir} "${RAPIDS_WHEEL_BLD_OUTPUT_DIR}" diff --git a/ci/test_wheel_cugraph-dgl.sh b/ci/test_wheel_cugraph-dgl.sh index 5900dced..756b0308 100755 --- a/ci/test_wheel_cugraph-dgl.sh +++ b/ci/test_wheel_cugraph-dgl.sh @@ -8,7 +8,8 @@ package_name="cugraph-dgl" mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the pylibwholegraph and cugraph-dgl built in the previous step +# Download the libwholegraph, pylibwholegraph, and cugraph-dgl built in the previous step +RAPIDS_PY_WHEEL_NAME="libwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libwholegraph-dep RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-deps RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist @@ -28,6 +29,7 @@ rapids-pip-retry install \ -v \ --extra-index-url "${PYTORCH_URL}" \ --find-links "${DGL_URL}" \ + ./local-libwholegraph-dep/*.whl \ "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/cugraph_dgl_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" \ 'dgl==2.4.0' \ diff --git a/ci/test_wheel_cugraph-pyg.sh b/ci/test_wheel_cugraph-pyg.sh index 4f521083..04d4c20f 100755 --- a/ci/test_wheel_cugraph-pyg.sh +++ b/ci/test_wheel_cugraph-pyg.sh @@ -8,13 +8,15 @@ package_name="cugraph-pyg" mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -# Download the pylibwholegraph and cugraph-pyg built in the previous step +# Download the libwholegraph, pylibwholegraph, and cugraph-pyg built in the previous step +RAPIDS_PY_WHEEL_NAME="libwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libwholegraph-dep RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./local-deps RAPIDS_PY_WHEEL_NAME="${package_name}_${RAPIDS_PY_CUDA_SUFFIX}" RAPIDS_PY_WHEEL_PURE="1" rapids-download-wheels-from-s3 python ./dist # echo to expand wildcard before adding `[extra]` requires for pip rapids-pip-retry install \ -v \ + ./local-libwholegraph-dep/*.whl \ "$(echo ./local-deps/pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}*.whl)" \ "$(echo ./dist/cugraph_pyg_${RAPIDS_PY_CUDA_SUFFIX}*.whl)[test]" diff --git a/ci/test_wheel_pylibwholegraph.sh b/ci/test_wheel_pylibwholegraph.sh index f3b646db..ae7bbbad 100755 --- a/ci/test_wheel_pylibwholegraph.sh +++ b/ci/test_wheel_pylibwholegraph.sh @@ -7,6 +7,7 @@ set -E # ERR traps are inherited by subcommands mkdir -p ./dist RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" +RAPIDS_PY_WHEEL_NAME="libwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 cpp ./local-libwholegraph-dep RAPIDS_PY_WHEEL_NAME="pylibwholegraph_${RAPIDS_PY_CUDA_SUFFIX}" rapids-download-wheels-from-s3 python ./dist # determine pytorch source @@ -26,6 +27,7 @@ rapids-logger "Installing Packages" rapids-pip-retry install \ --extra-index-url ${INDEX_URL} \ "$(echo ./dist/pylibwholegraph*.whl)[test]" \ + ./local-libwholegraph-dep/*.whl \ 'torch>=2.3' rapids-logger "pytest pylibwholegraph" diff --git a/dependencies.yaml b/dependencies.yaml index f3fc3080..0ebd9e8b 100644 --- a/dependencies.yaml +++ b/dependencies.yaml @@ -108,6 +108,34 @@ files: - depends_on_pylibwholegraph - test_python_pylibwholegraph + py_build_libwholegraph: + output: pyproject + pyproject_dir: python/libwholegraph + extras: + table: build-system + includes: + - rapids_build_skbuild + + py_rapids_build_libwholegraph: + output: pyproject + pyproject_dir: python/libwholegraph + extras: + table: tool.rapids-build-backend + key: requires + includes: + - common_build + - depends_on_libraft + - depends_on_librmm + + py_run_libwholegraph: + output: pyproject + pyproject_dir: python/libwholegraph + extras: + table: project + includes: + - depends_on_rapids_logger + - depends_on_libraft + py_build_pylibwholegraph: output: pyproject pyproject_dir: python/pylibwholegraph @@ -115,6 +143,7 @@ files: table: build-system includes: - rapids_build_skbuild + py_rapids_build_pylibwholegraph: output: pyproject pyproject_dir: python/pylibwholegraph @@ -124,6 +153,10 @@ files: includes: - common_build - python_build_cythonize + - depends_on_libwholegraph + - depends_on_libraft + - depends_on_librmm + py_run_pylibwholegraph: output: pyproject pyproject_dir: python/pylibwholegraph @@ -131,6 +164,7 @@ files: table: project includes: - python_run_pylibwholegraph + - depends_on_libwholegraph py_test_pylibwholegraph: output: pyproject pyproject_dir: python/pylibwholegraph @@ -367,6 +401,11 @@ dependencies: - output_types: [conda, pyproject, requirements] packages: - cython>=3.0.0 + python_run_libwholegraph: + common: + - output_types: [conda, pyproject, requirements] + packages: + - depends_on_rapids_logger python_run_pylibwholegraph: common: - output_types: [conda, pyproject, requirements] @@ -524,11 +563,91 @@ dependencies: - pylibwholegraph-cu11==25.6.*,>=0.0.0a0 - {matrix: null, packages: [*pylibwholegraph_unsuffixed]} + depends_on_libraft: + common: + - output_types: conda + packages: + - &libraft_unsuffixed libraft==25.6.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libraft-cu12==25.6.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libraft-cu11==25.6.*,>=0.0.0a0 + - {matrix: null, packages: [*libraft_unsuffixed]} + + depends_on_librmm: + common: + - output_types: conda + packages: + - librmm==25.6.*,>=0.0.0a0 + - &librmm_unsuffixed librmm==25.6.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - librmm-cu12==25.6.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - librmm-cu11==25.6.*,>=0.0.0a0 + - {matrix: null, packages: [*librmm_unsuffixed]} + + depends_on_rapids_logger: + common: + - output_types: [conda, requirements, pyproject] + packages: + - rapids-logger==0.1.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + depends_on_libwholegraph: common: - output_types: conda packages: - - libwholegraph==25.6.*,>=0.0.0a0 + - &libwholegraph_unsuffixed libwholegraph==25.6.*,>=0.0.0a0 + - output_types: requirements + packages: + # pip recognizes the index as a global option for the requirements.txt file + - --extra-index-url=https://pypi.nvidia.com + - --extra-index-url=https://pypi.anaconda.org/rapidsai-wheels-nightly/simple + specific: + - output_types: [requirements, pyproject] + matrices: + - matrix: + cuda: "12.*" + cuda_suffixed: "true" + packages: + - libwholegraph-cu12==25.6.*,>=0.0.0a0 + - matrix: + cuda: "11.*" + cuda_suffixed: "true" + packages: + - libwholegraph-cu11==25.6.*,>=0.0.0a0 + - {matrix: null, packages: [*libwholegraph_unsuffixed]} depends_on_libwholegraph_tests: common: diff --git a/python/libwholegraph/CMakeLists.txt b/python/libwholegraph/CMakeLists.txt new file mode 100644 index 00000000..20a7cbab --- /dev/null +++ b/python/libwholegraph/CMakeLists.txt @@ -0,0 +1,34 @@ +# ============================================================================= +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except +# in compliance with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software distributed under the License +# is distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express +# or implied. See the License for the specific language governing permissions and limitations under +# the License. +# ============================================================================= + +cmake_minimum_required(VERSION 3.30.4 FATAL_ERROR) + +include(../../cmake/rapids_config.cmake) +include(rapids-cmake) +include(rapids-cuda) + +rapids_cuda_init_architectures(LIBWHOLEGRAPH) + +project( + LIBWHOLEGRAPH + VERSION "${RAPIDS_VERSION}" + LANGUAGES CXX CUDA +) + +# * Dependencies + +SET(BUILD_TESTS OFF) +SET(BUILD_BENCHMARKS OFF) + +add_subdirectory(../../cpp/ libwholegraph) diff --git a/python/libwholegraph/README.md b/python/libwholegraph/README.md new file mode 100644 index 00000000..ae42a26d --- /dev/null +++ b/python/libwholegraph/README.md @@ -0,0 +1 @@ +../../README.md diff --git a/python/libwholegraph/libwholegraph/VERSION b/python/libwholegraph/libwholegraph/VERSION new file mode 120000 index 00000000..d62dc733 --- /dev/null +++ b/python/libwholegraph/libwholegraph/VERSION @@ -0,0 +1 @@ +../../../VERSION \ No newline at end of file diff --git a/python/libwholegraph/libwholegraph/__init__.py b/python/libwholegraph/libwholegraph/__init__.py new file mode 100644 index 00000000..086b300a --- /dev/null +++ b/python/libwholegraph/libwholegraph/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from libwholegraph._version import __git_commit__, __version__ +from libwholegraph.load import load_library diff --git a/python/libwholegraph/libwholegraph/_version.py b/python/libwholegraph/libwholegraph/_version.py new file mode 100644 index 00000000..da66c0d5 --- /dev/null +++ b/python/libwholegraph/libwholegraph/_version.py @@ -0,0 +1,30 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import importlib.resources + +__version__ = ( + importlib.resources.files(__package__).joinpath("VERSION").read_text().strip() +) +try: + __git_commit__ = ( + importlib.resources.files(__package__) + .joinpath("GIT_COMMIT") + .read_text() + .strip() + ) +except FileNotFoundError: + __git_commit__ = "" + +__all__ = ["__git_commit__", "__version__"] diff --git a/python/libwholegraph/libwholegraph/load.py b/python/libwholegraph/libwholegraph/load.py new file mode 100644 index 00000000..af26cd50 --- /dev/null +++ b/python/libwholegraph/libwholegraph/load.py @@ -0,0 +1,100 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +import ctypes +import os + +# Loading with RTLD_LOCAL adds the library itself to the loader's +# loaded library cache without loading any symbols into the global +# namespace. This allows libraries that express a dependency on +# this library to be loaded later and successfully satisfy this dependency +# without polluting the global symbol table with symbols from +# libwholegraph that could conflict with symbols from other DSOs. +PREFERRED_LOAD_FLAG = ctypes.RTLD_LOCAL + + +def _load_system_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + Raises ``OSError`` if library cannot be loaded. + """ + return ctypes.CDLL(soname, PREFERRED_LOAD_FLAG) + + +def _load_wheel_installation(soname: str): + """Try to dlopen() the library indicated by ``soname`` + + Returns ``None`` if the library cannot be loaded. + """ + if os.path.isfile(lib := os.path.join(os.path.dirname(__file__), "lib64", soname)): + return ctypes.CDLL(lib, PREFERRED_LOAD_FLAG) + return None + + +def load_library(): + """Dynamically load libwholegraph.so and its dependencies""" + try: + # these libraries must be loaded before libwholegraph because + # libwholegraph references their symbols + import libraft + import rapids_logger + + libraft.load_library() + rapids_logger.load_library() + except ModuleNotFoundError: + # 'libwholegraph' has a runtime dependency on 'libraft' et al.. However, + # that dependency might be satisfied by a conda package + # (which does not have any Python modules), instead of a wheel. + # + # In that situation, assume that a shared object is in a place where + # the loader can find it. + pass + + prefer_system_installation = ( + os.getenv("RAPIDS_LIBWHOLEGRAPH_PREFER_SYSTEM_LIBRARY", "false").lower() + != "false" + ) + + libs_to_return = [] + for soname in ["libwholegraph.so"]: + libwholegraph_lib = None + if prefer_system_installation: + # Prefer a system library if one is present to + # avoid clobbering symbols that other packages might expect, but if no + # other library is present use the one in the wheel. + try: + libwholegraph_lib = _load_system_installation(soname) + except OSError: + libwholegraph_lib = _load_wheel_installation(soname) + else: + # Prefer the libraries bundled in this package. If they aren't found + # (which might be the case in builds where the library was prebuilt before + # packaging the wheel), look for a system installation. + try: + libwholegraph_lib = _load_wheel_installation(soname) + if libwholegraph_lib is None: + libwholegraph_lib = _load_system_installation(soname) + except OSError: + # If none of the searches above succeed, just silently return None + # and rely on other mechanisms (like RPATHs on other DSOs) to + # help the loader find the library. + pass + if libwholegraph_lib: + libs_to_return.append(libwholegraph_lib) + + # The caller almost never needs to do anything with these libraries, but no + # harm in offering the option since these objects at least provide handles + # to inspect where libcugraph was loaded from. + + return libs_to_return diff --git a/python/libwholegraph/pyproject.toml b/python/libwholegraph/pyproject.toml new file mode 100644 index 00000000..cf3394d2 --- /dev/null +++ b/python/libwholegraph/pyproject.toml @@ -0,0 +1,90 @@ +# Copyright (c) 2025, NVIDIA CORPORATION. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +[build-system] +build-backend = "rapids_build_backend.build" +requires = [ + "rapids-build-backend>=0.3.0,<0.4.0.dev0", + "scikit-build-core[pyproject]>=0.10.0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + +[project] +name = "libwholegraph" +dynamic = ["version"] +description = "wholegraph" +readme = { file = "README.md", content-type = "text/markdown" } +authors = [ + { name = "NVIDIA Corporation" }, +] +license = { text = "Apache-2.0" } +requires-python = ">=3.10" +classifiers = [ + "Intended Audience :: Developers", + "Topic :: Database", + "Topic :: Scientific/Engineering", + "License :: OSI Approved :: Apache Software License", + "Programming Language :: C++", + "Environment :: GPU :: NVIDIA CUDA", +] +dependencies = [ + "libraft==25.6.*,>=0.0.0a0", + "rapids-logger==0.1.*,>=0.0.0a0", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. + + +[tool.setuptools.dynamic] +version = {file = "libwholegraph/VERSION"} + +[project.urls] +Homepage = "https://github.com/rapidsai/cugraph-gnn" + +[project.entry-points."cmake.prefix"] +# this tells scikit-build-core to add this to cmake.prefix so +# find_package(libwholegraph) can find the files from this wheel +libwholegraph = "libwholegraph" + +[tool.pydistcheck] +select = [ + "distro-too-large-compressed", +] + +# detect when package size grows significantly +max_allowed_size_compressed = '0.4G' + +[tool.scikit-build] +build-dir = "build/{wheel_tag}" +cmake.build-type = "Release" +cmake.version = "CMakeLists.txt" +minimum-version = "build-system.requires" +ninja.make-fallback = true +sdist.reproducible = true +wheel.packages = ["libwholegraph"] +wheel.install-dir = "libwholegraph" +wheel.py-api = "py3" + +[tool.scikit-build.metadata.version] +provider = "scikit_build_core.metadata.regex" +input = "libwholegraph/VERSION" +regex = "(?P.*)" + +[tool.rapids-build-backend] +build-backend = "scikit_build_core.build" +dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" +requires = [ + "cmake>=3.30.4", + "libraft==25.6.*,>=0.0.0a0", + "librmm==25.6.*,>=0.0.0a0", + "ninja", +] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. diff --git a/python/pylibwholegraph/CMakeLists.txt b/python/pylibwholegraph/CMakeLists.txt index 4fd8fc00..3263f958 100644 --- a/python/pylibwholegraph/CMakeLists.txt +++ b/python/pylibwholegraph/CMakeLists.txt @@ -27,54 +27,11 @@ project( LANGUAGES CXX CUDA ) -# ################################################################################################## -# * User Options ------------------------------------------------------------ - -option(WHOLEGRAPH_BUILD_WHEELS "Whether we're building a wheel for pypi" OFF) - -# ################################################################################################## -# * Base Rapids Options ----------------------------------------------------- - -set(CMAKE_EXPORT_COMPILE_COMMANDS ON) - -# default build type -rapids_cmake_build_type(Release) - -rapids_cpm_init() - -include(../../cpp/cmake/thirdparty/get_raft.cmake) - -# ################################################################################################## -# * Dependencies ------------------------------------------------------------ - -# use _ROOT here to take precedence over any other package -if(DEFINED ENV{LIBWHOLEGRAPH_DIR}) - set(wholegraph_ROOT "$ENV{LIBWHOLEGRAPH_DIR}") -endif() -find_package(wholegraph "${RAPIDS_VERSION}") -message("WholeGraph") -if(WHOLEGRAPH_FOUND) - message(STATUS "PYLIBWHOLEGRAPH: using pre-built wholegraph C++ package") -elseif(WHOLEGRAPH_BUILD_WHEELS) - # statically link dependencies if building wheels - message(STATUS "PYLIBWHOLEGRAPH: build wheels") - add_subdirectory(../../cpp/ libwholegraph EXCLUDE_FROM_ALL) -else() - message(FATAL_ERROR "PYLIBWHOLEGRAPH: could not find wholegraph package in " - "cmake prefix ${CMAKE_PREFIX_PATH} or user dir $ENV{LIBWHOLEGRAPH_DIR}" - ) -endif() +find_package(wholegraph "${RAPIDS_VERSION}" REQUIRED) include(rapids-cython-core) rapids_cython_init() -# ################################################################################################## -# * Compiler options --------------------------------------------------------- - -message(STATUS "PYLIBWHOLEGRAPH: DEFAULT_CXX_FLAGS='${DEFAULT_CXX_FLAGS}'") -message(STATUS "PYLIBWHOLEGRAPH: CXX_FLAGS='${CXX_FLAGS}'") -message(STATUS "PYLIBWHOLEGRAPH: CXX_DEFINITIONS='${CXX_DEFINITIONS}'") - # ################################################################################################## # * Cython modules ----------------------------------------------------------- diff --git a/python/pylibwholegraph/pylibwholegraph/__init__.py b/python/pylibwholegraph/pylibwholegraph/__init__.py index e600c30e..efb078f6 100644 --- a/python/pylibwholegraph/pylibwholegraph/__init__.py +++ b/python/pylibwholegraph/pylibwholegraph/__init__.py @@ -1,4 +1,4 @@ -# Copyright (c) 2022-2024, NVIDIA CORPORATION. +# Copyright (c) 2022-2025, NVIDIA CORPORATION. # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at @@ -12,3 +12,14 @@ # limitations under the License. from pylibwholegraph._version import __git_commit__, __version__ + +# If libwholegraph was installed as a wheel, we must request it to load the +# library symbols. Otherwise, we assume that the library was installed in a +# system path that ld can find. +try: + import libwholegraph +except ModuleNotFoundError: + pass +else: + libwholegraph.load_library() + del libwholegraph diff --git a/python/pylibwholegraph/pyproject.toml b/python/pylibwholegraph/pyproject.toml index 0a1ce3b0..c6cc49ce 100644 --- a/python/pylibwholegraph/pyproject.toml +++ b/python/pylibwholegraph/pyproject.toml @@ -38,6 +38,7 @@ classifiers = [ "Programming Language :: Python :: 3.12", ] dependencies = [ + "libwholegraph==25.6.*,>=0.0.0a0", "numpy>=1.23,<3.0a0", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`. @@ -55,9 +56,13 @@ test = [ [tool.rapids-build-backend] build-backend = "scikit_build_core.build" dependencies-file = "../../dependencies.yaml" +matrix-entry = "cuda_suffixed=true" requires = [ "cmake>=3.30.4", "cython>=3.0.0", + "libraft==25.6.*,>=0.0.0a0", + "librmm==25.6.*,>=0.0.0a0", + "libwholegraph==25.6.*,>=0.0.0a0", "ninja", ] # This list was generated by `rapids-dependency-file-generator`. To make changes, edit ../../dependencies.yaml and run `rapids-dependency-file-generator`.