diff --git a/contrib/Dockerfile.rockylinux8 b/contrib/Dockerfile.rockylinux8 new file mode 100644 index 0000000000..52443dbd3b --- /dev/null +++ b/contrib/Dockerfile.rockylinux8 @@ -0,0 +1,355 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +ARG BASE_IMAGE +ARG BASE_IMAGE_TAG +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} + +ARG DEFAULT_PYTHON_VERSION="3.14" +ARG ARCH="x86_64" +ARG UCX_REF="v1.20.x" +ARG LIBFABRIC_VERSION="v1.21.0" +ARG HWLOC_VERSION="2.12.2" +ARG BUILD_TYPE="release" +ARG URING_TAG="liburing-2.13" + +RUN yum groupinstall -y 'Development Tools' && \ + dnf config-manager --set-enabled powertools && \ + dnf install -y epel-release && \ + dnf install -y \ + boost \ + boost-devel \ + clang-devel \ + gcc-toolset-11 \ + cmake \ + dkms \ + flex \ + gflags \ + glibc-headers \ + gcc-c++ \ + libaio \ + libaio-devel \ + libtool-ltdl \ + ninja-build \ + openssl \ + openssl-devel \ + protobuf-compiler \ + protobuf-c-devel \ + protobuf-devel \ + libibverbs \ + libibverbs-devel \ + rdma-core \ + rdma-core-devel \ + libibumad \ + libibumad-devel \ + numactl-devel \ + librdmacm-devel \ + pciutils-devel \ + wget \ + zlib + +ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH +ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-11/root/usr/lib64:$LD_LIBRARY_PATH +ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-11/root/usr/lib64/pkgconfig:${PKG_CONFIG_PATH}" + +# Build latest hwloc from source to avoid older RHEL8 API limitations. +RUN cd /tmp && \ + HWLOC_SERIES="$(echo ${HWLOC_VERSION} | cut -d. -f1,2)" && \ + wget -q "https://download.open-mpi.org/release/hwloc/v${HWLOC_SERIES}/hwloc-${HWLOC_VERSION}.tar.gz" && \ + tar -xzf "hwloc-${HWLOC_VERSION}.tar.gz" && \ + cd "hwloc-${HWLOC_VERSION}" && \ + ./configure --prefix=/usr/local --disable-nvml && \ + make -j"$(nproc)" && \ + make install && \ + ldconfig && \ + rm -rf "/tmp/hwloc-${HWLOC_VERSION}" "/tmp/hwloc-${HWLOC_VERSION}.tar.gz" + +# Build OpenSSL 3.x +RUN yum install -y perl-IPC-Cmd perl-Test-Simple perl-Data-Dumper +RUN cd /tmp && \ + wget -q https://www.openssl.org/source/openssl-3.0.16.tar.gz && \ + tar -xzf openssl-3.0.16.tar.gz && \ + cd openssl-3.0.16 && \ + ./Configure --prefix=/usr/local/openssl3 --openssldir=/usr/local/openssl3 \ + shared zlib linux-$ARCH && \ + make -j$(nproc) && \ + make install_sw && \ + echo "/usr/local/openssl3/lib64" > /etc/ld.so.conf.d/openssl3.conf && \ + echo "/usr/local/openssl3/lib" >> /etc/ld.so.conf.d/openssl3.conf && \ + ldconfig && \ + rm -rf /tmp/openssl-3.0.16* + +# Set environment variables to use the new OpenSSL +ENV PKG_CONFIG_PATH="/usr/local/openssl3/lib64/pkgconfig:/usr/local/openssl3/lib/pkgconfig:$PKG_CONFIG_PATH" +ENV LD_LIBRARY_PATH="/usr/local/openssl3/lib64:/usr/local/openssl3/lib:$LD_LIBRARY_PATH" +ENV OPENSSL_ROOT_DIR="/usr/local/openssl3" +ENV OPENSSL_LIBRARIES="/usr/local/openssl3/lib64:/usr/local/openssl3/lib" +ENV OPENSSL_INCLUDE_DIR="/usr/local/openssl3/include" + +WORKDIR /workspace + +RUN git clone --recurse-submodules -b v1.73.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ + cd grpc && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DgRPC_SSL_PROVIDER=package ../.. && \ + make -j$(nproc) && \ + make install + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ + cd etcd-cpp-apiv3 && \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ + mkdir build && cd build && \ + cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release -DETCD_CMAKE_CXX_STANDARD=17 && \ + make -j$(nproc) && make install + +# The base image libcurl is linked against openssl 1.x, so we need to build from source +# in order to use openssl 3.x. This is needed to build aws-sdk-cpp. +RUN wget https://curl.se/download/curl-8.5.0.tar.gz && \ + tar xzf curl-8.5.0.tar.gz && cd curl-8.5.0 && \ + ./configure --prefix=/usr/local --with-ssl=/usr/local/openssl3 --enable-shared && \ + make -j$(nproc) && make install + +RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 +RUN mkdir aws_sdk_build && cd aws_sdk_build && \ + export LDFLAGS="-L/usr/local/openssl3/lib64 -L/usr/local/openssl3/lib" && \ + export CFLAGS="-I/usr/local/openssl3/include" && \ + export CXXFLAGS="-I/usr/local/openssl3/include" && \ + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_PREFIX_PATH="/usr/local/openssl3;/usr/local" \ + -DCURL_LIBRARY=/usr/local/lib/libcurl.so \ + -DCURL_INCLUDE_DIR=/usr/local/include \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + make -j${NPROC:-$(nproc)} && make install + +RUN git clone https://github.com/axboe/liburing.git && \ + cd liburing && \ + git checkout $URING_TAG && \ + ./configure --cc=gcc --cxx=g++ && \ + make -j library && make liburing.pc && make install && \ + ldconfig && \ + cd .. + +ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig:$PKG_CONFIG_PATH + +RUN git clone https://github.com/nvidia/gusli.git && \ + cd gusli && \ + sed -i '/^LFLAGS_ALL =/{/-lrt/ b; s/$/ -lrt/}' Makefile && \ + make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ + cd .. + +# Required for Azure SDK +RUN git clone --depth 1 https://gitlab.gnome.org/GNOME/libxml2.git --branch 2.15 && \ + cd libxml2 && \ + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DLIBXML2_WITH_PYTHON=OFF && \ + cmake --build build -j"$(nproc)" && \ + cmake --install build + +RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ + cd azure-sdk-for-cpp/ && \ + mkdir build && cd build && \ + export LDFLAGS="-L/usr/local/openssl3/lib64 -L/usr/local/openssl3/lib" && \ + export CFLAGS="-isystem /usr/local/openssl3/include" && \ + export CXXFLAGS="-isystem /usr/local/openssl3/include" && \ + AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DCMAKE_PREFIX_PATH="/usr/local/openssl3;/usr/local" \ + -DCURL_LIBRARY=/usr/local/lib/libcurl.so \ + -DCURL_INCLUDE_DIR=/usr/local/include \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + cmake --build . --target azure-storage-blobs azure-identity && \ + cmake --install sdk/core && \ + cmake --install sdk/storage/azure-storage-common && \ + cmake --install sdk/storage/azure-storage-blobs && \ + cmake --install sdk/identity + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=1.86.0 \ + RUSTARCH=${ARCH}-unknown-linux-gnu + +RUN wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init" && \ + chmod +x rustup-init && \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${RUSTARCH} && \ + case "$ARCH" in \ + aarch64) RUSTUP_SHA256="c64b33db2c6b9385817ec0e49a84bcfe018ed6e328fe755c3c809580cc70ce7a" ;; \ + x86_64) RUSTUP_SHA256="a3339fb004c3d0bb9862ba0bce001861fe5cbde9c10d16591eb3f39ee6cd3e7f" ;; \ + *) echo "Unsupported architecture for Rust: $ARCH" && exit 1 ;; \ + esac && \ + echo "$RUSTUP_SHA256 *rustup-init" | sha256sum -c - && \ + rm rustup-init && \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +RUN wget --tries=3 --waitretry=5 https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host-3.2.0-125000_25.10_rhel8.${ARCH}.rpm -O doca-host.rpm && \ + rpm -i doca-host.rpm && \ + dnf install -y libnl3-devel && \ + cd /usr/share/doca-host-3.2.0/repo/Packages/ && \ + rpm -ivh --nodeps doca-sdk-common-*rpm && \ + rpm -ivh --nodeps doca-sdk-rdma-*rpm && \ + rpm -ivh --nodeps doca-sdk-verbs-*rpm && \ + rpm -ivh --nodeps doca-sdk-gpunetio-*rpm && \ + # Check that gpunetio development package is installed correctly + pkg-config --cflags --libs doca-gpunetio + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +ENV CUDA_PATH=/usr/local/cuda + +WORKDIR /workspace + +RUN rm -rf /usr/lib/ucx +RUN rm -rf /opt/hpcx/ucx + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/NVIDIA/gdrcopy.git && \ + cd gdrcopy && \ + git fetch --tags --depth=1 && \ + latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") && \ + git checkout "$latest_tag" && \ + cd packages && \ + CUDA=/usr/local/cuda ./build-rpm-packages.sh && \ + rpm -Uvh gdrcopy-kmod-*.el8.noarch.rpm && \ + rpm -Uvh gdrcopy-*.el8.$ARCH.rpm && \ + rpm -Uvh gdrcopy-devel-*.el8.noarch.rpm + +RUN cd /usr/local/src && \ + git clone https://github.com/openucx/ucx.git && \ + cd ucx && \ + git checkout $UCX_REF && \ + ./autogen.sh && \ + ./contrib/configure-release-mt \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa && \ + make -j && \ + make -j install-strip && \ + ldconfig + +# Build libfabric from source +RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ + "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 && \ + tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 && \ + cd libfabric-* && \ + ./autogen.sh && \ + ./configure --prefix=/usr/local \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa \ + --with-cuda=/usr/local/cuda \ + --enable-cuda-dlopen \ + --with-gdrcopy \ + --enable-gdrcopy-dlopen && \ + make -j$(nproc) && \ + make install && \ + ldconfig + +# Ensure pkg-config and linker can discover source-installed libs under /usr/local. +ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig +RUN printf "/usr/local/lib64\n/usr/local/lib\n" > /etc/ld.so.conf.d/usr-local.conf && ldconfig + +# By default, uv downloads python packages to $HOME/.cache/uv and hard links them +# from the virtual environment. This means that the files reside in /root/.cache/uv, +# which is not what we want since some systems mount user home dir into /root, +# in which case the venv is broken when the container is started. +# Set a custom cache directory inside /workspace to avoid this. +ENV UV_CACHE_DIR=/workspace/.cache/uv +RUN mkdir -p $UV_CACHE_DIR +# Create a new virtual environment +ENV VIRTUAL_ENV=/workspace/.venv +RUN rm -rf $VIRTUAL_ENV && uv venv $VIRTUAL_ENV --python $DEFAULT_PYTHON_VERSION +# Activate the virtual environment +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Install python dependencies +RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click setuptools tabulate auditwheel tomlkit +# Install PyTorch +RUN export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + uv pip install torch torchvision torchaudio +# Upgrade setuptools to latest version for compatibility with PEP 639 (license format) +RUN uv pip install --upgrade 'setuptools>=80.9.0' +# Meson fails to download dependencies without truststore on Rocky Linux 8 +RUN uv pip install truststore + +COPY . /workspace/nixl +WORKDIR /workspace/nixl + +RUN rm -rf build && \ + mkdir build && \ + meson setup build/ --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE \ + -Dcudapath_lib="/usr/local/cuda/lib64" \ + -Dcudapath_inc="/usr/local/cuda/include" && \ + cd build && \ + ninja && \ + ninja install + +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/plugins:$LD_LIBRARY_PATH +ENV NIXL_PLUGIN_DIR=/usr/local/nixl/lib64/plugins + +RUN echo "/usr/local/nixl/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf && \ + echo "/usr/local/nixl/lib/$ARCH-linux-gnu/plugins" >> /etc/ld.so.conf.d/nixl.conf && \ + ldconfig + +# Create the wheel +# No need to specifically add path to libcuda.so here, meson finds the stubs and links them +ARG WHL_PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14" +ARG WHL_PLATFORM="manylinux_2_28_$ARCH" +RUN dnf install -y python3.11-devel python3.12-devel +RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \ + export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + export UV_INDEX_STRATEGY=unsafe-best-match && \ + rm -rf dist && mkdir -p dist && \ + for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do \ + export PATH=$VIRTUAL_ENV/bin:$PATH && \ + ./contrib/build-wheel.sh \ + --python-version $PYTHON_VERSION \ + --platform $WHL_PLATFORM \ + --ucx-plugins-dir /usr/lib64/ucx \ + --nixl-plugins-dir $NIXL_PLUGIN_DIR \ + --output-dir dist || exit 1; \ + done + +# Copy the meta package wheel to the dist directory, which will be used to push to PyPI. +# Only do this for the CUDA 12 builds, since by default nixl depends on nixl-cu12. +RUN if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ + cp build/src/bindings/python/nixl-meta/nixl*.whl dist/; \ + fi + +RUN uv pip install dist/nixl*cp${DEFAULT_PYTHON_VERSION//./}*.whl build/src/bindings/python/nixl-meta/nixl*.whl diff --git a/contrib/Dockerfile.rockylinux9 b/contrib/Dockerfile.rockylinux9 new file mode 100644 index 0000000000..c303c90242 --- /dev/null +++ b/contrib/Dockerfile.rockylinux9 @@ -0,0 +1,326 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +ARG BASE_IMAGE +ARG BASE_IMAGE_TAG +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} + +ARG DEFAULT_PYTHON_VERSION="3.14" +ARG ARCH="x86_64" +ARG UCX_REF="v1.20.x" +ARG LIBFABRIC_VERSION="v1.21.0" +ARG HWLOC_VERSION="2.12.2" +ARG BUILD_TYPE="release" +ARG URING_TAG="liburing-2.13" + +RUN dnf install -y dnf-plugins-core && \ + dnf groupinstall -y "Development Tools" && \ + dnf config-manager --set-enabled crb && \ + dnf install -y epel-release && \ + dnf install -y \ + autoconf \ + automake \ + boost \ + boost-devel \ + bzip2 \ + clang-devel \ + cmake \ + diffutils \ + distribution-gpg-keys-copr \ + dkms \ + flex \ + gflags \ + gcc \ + glibc-headers \ + gcc-c++ \ + git \ + libaio \ + libaio-devel \ + libcurl-devel \ + libtool \ + libtool-ltdl \ + make \ + ninja-build \ + numactl-devel \ + openssl \ + openssl-devel \ + pciutils-devel \ + perl \ + perl-Data-Dumper \ + perl-IPC-Cmd \ + perl-Test-Simple \ + pkgconfig \ + protobuf-compiler \ + protobuf-c-devel \ + protobuf-devel \ + rdma-core \ + rdma-core-devel \ + libibverbs \ + libibverbs-devel \ + libibumad \ + libibumad-devel \ + librdmacm-devel \ + tar \ + which \ + wget \ + xz \ + zlib && \ + dnf clean all + +# Build latest hwloc from source to avoid API limitations. +RUN cd /tmp && \ + HWLOC_SERIES="$(echo ${HWLOC_VERSION} | cut -d. -f1,2)" && \ + wget -q "https://download.open-mpi.org/release/hwloc/v${HWLOC_SERIES}/hwloc-${HWLOC_VERSION}.tar.gz" && \ + tar -xzf "hwloc-${HWLOC_VERSION}.tar.gz" && \ + cd "hwloc-${HWLOC_VERSION}" && \ + ./configure --prefix=/usr/local --disable-nvml && \ + make -j"$(nproc)" && \ + make install && \ + ldconfig && \ + rm -rf "/tmp/hwloc-${HWLOC_VERSION}" "/tmp/hwloc-${HWLOC_VERSION}.tar.gz" + +WORKDIR /workspace + +RUN git clone --recurse-submodules -b v1.73.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ + cd grpc && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DgRPC_SSL_PROVIDER=package ../.. && \ + make -j$(nproc) && \ + make install + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ + cd etcd-cpp-apiv3 && \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ + mkdir build && cd build && \ + cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release -DETCD_CMAKE_CXX_STANDARD=17 && \ + make -j$(nproc) && make install + +RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 +RUN mkdir aws_sdk_build && cd aws_sdk_build && \ + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + make -j${NPROC:-$(nproc)} && make install + +RUN git clone https://github.com/axboe/liburing.git && \ + cd liburing && \ + git checkout $URING_TAG && \ + ./configure --cc=gcc --cxx=g++ && \ + make -j library && make liburing.pc && make install && \ + ldconfig && \ + cd .. + +ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig:$PKG_CONFIG_PATH + +RUN git clone https://github.com/nvidia/gusli.git && \ + cd gusli && \ + sed -i '/^LFLAGS_ALL =/{/-lrt/ b; s/$/ -lrt/}' Makefile && \ + make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ + cd .. + +# Required for Azure SDK +RUN git clone --depth 1 https://gitlab.gnome.org/GNOME/libxml2.git --branch 2.15 && \ + cd libxml2 && \ + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DLIBXML2_WITH_PYTHON=OFF && \ + cmake --build build -j"$(nproc)" && \ + cmake --install build + +RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ + cd azure-sdk-for-cpp/ && \ + mkdir build && cd build && \ + AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + cmake --build . --target azure-storage-blobs azure-identity && \ + cmake --install sdk/core && \ + cmake --install sdk/storage/azure-storage-common && \ + cmake --install sdk/storage/azure-storage-blobs && \ + cmake --install sdk/identity + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=1.86.0 \ + RUSTARCH=${ARCH}-unknown-linux-gnu + +RUN wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init" && \ + chmod +x rustup-init && \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${RUSTARCH} && \ + case "$ARCH" in \ + aarch64) RUSTUP_SHA256="c64b33db2c6b9385817ec0e49a84bcfe018ed6e328fe755c3c809580cc70ce7a" ;; \ + x86_64) RUSTUP_SHA256="a3339fb004c3d0bb9862ba0bce001861fe5cbde9c10d16591eb3f39ee6cd3e7f" ;; \ + *) echo "Unsupported architecture for Rust: $ARCH" && exit 1 ;; \ + esac && \ + echo "$RUSTUP_SHA256 *rustup-init" | sha256sum -c - && \ + rm rustup-init && \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +RUN wget --tries=3 --waitretry=5 "https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host-3.2.0-125000_25.10_rhel9.${ARCH}.rpm" -O doca-host.rpm && \ + rpm -i doca-host.rpm && \ + dnf install -y libnl3-devel && \ + cd /usr/share/doca-host-3.2.0/repo/Packages/ && \ + rpm -ivh --nodeps doca-sdk-common-*rpm && \ + rpm -ivh --nodeps doca-sdk-rdma-*rpm && \ + rpm -ivh --nodeps doca-sdk-verbs-*rpm && \ + rpm -ivh --nodeps doca-sdk-gpunetio-*rpm && \ + # Check that gpunetio development package is installed correctly + pkg-config --cflags --libs doca-gpunetio + +ENV LD_LIBRARY_PATH=/usr/lib:/usr/local/lib:$LD_LIBRARY_PATH + +ENV CUDA_PATH=/usr/local/cuda + +WORKDIR /workspace + +RUN rm -rf /usr/lib/ucx +RUN rm -rf /opt/hpcx/ucx + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/NVIDIA/gdrcopy.git && \ + cd gdrcopy && \ + git fetch --tags --depth=1 && \ + latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") && \ + git checkout "$latest_tag" && \ + cd packages && \ + CUDA=/usr/local/cuda ./build-rpm-packages.sh && \ + rpm -Uvh gdrcopy-kmod-*.el9.noarch.rpm && \ + rpm -Uvh gdrcopy-*.el9.$ARCH.rpm && \ + rpm -Uvh gdrcopy-devel-*.el9.noarch.rpm + +RUN cd /usr/local/src && \ + git clone https://github.com/openucx/ucx.git && \ + cd ucx && \ + git checkout $UCX_REF && \ + ./autogen.sh && \ + ./contrib/configure-release-mt \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa && \ + make -j && \ + make -j install-strip && \ + ldconfig + +# Build libfabric from source +RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ + "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 && \ + tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 && \ + cd libfabric-* && \ + ./autogen.sh && \ + ./configure --prefix=/usr/local \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa \ + --with-cuda=/usr/local/cuda \ + --enable-cuda-dlopen \ + --with-gdrcopy \ + --enable-gdrcopy-dlopen && \ + make -j$(nproc) && \ + make install && \ + ldconfig + +# Ensure pkg-config and linker can discover source-installed libs under /usr/local. +ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig +RUN printf "/usr/local/lib64\n/usr/local/lib\n" > /etc/ld.so.conf.d/usr-local.conf && ldconfig + +# By default, uv downloads python packages to $HOME/.cache/uv and hard links them +# from the virtual environment. This means that the files reside in /root/.cache/uv, +# which is not what we want since some systems mount user home dir into /root, +# in which case the venv is broken when the container is started. +# Set a custom cache directory inside /workspace to avoid this. +ENV UV_CACHE_DIR=/workspace/.cache/uv +RUN mkdir -p $UV_CACHE_DIR +# Create a new virtual environment +ENV VIRTUAL_ENV=/workspace/.venv +RUN rm -rf $VIRTUAL_ENV && uv venv $VIRTUAL_ENV --python $DEFAULT_PYTHON_VERSION +# Activate the virtual environment +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Install python dependencies +RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click setuptools tabulate auditwheel tomlkit +# Install PyTorch +RUN export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + uv pip install torch torchvision torchaudio +# Upgrade setuptools to latest version for compatibility with PEP 639 (license format) +RUN uv pip install --upgrade 'setuptools>=80.9.0' + +COPY . /workspace/nixl +WORKDIR /workspace/nixl + +RUN rm -rf build && \ + mkdir build && \ + meson setup build/ --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE \ + -Dcudapath_lib="/usr/local/cuda/lib64" \ + -Dcudapath_inc="/usr/local/cuda/include" && \ + cd build && \ + ninja && \ + ninja install + +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/plugins:$LD_LIBRARY_PATH +ENV NIXL_PLUGIN_DIR=/usr/local/nixl/lib64/plugins + +RUN echo "/usr/local/nixl/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf && \ + echo "/usr/local/nixl/lib/$ARCH-linux-gnu/plugins" >> /etc/ld.so.conf.d/nixl.conf && \ + ldconfig + +# Create the wheel +# No need to specifically add path to libcuda.so here, meson finds the stubs and links them +ARG WHL_PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14" +ARG WHL_PLATFORM="manylinux_2_28_$ARCH" +RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \ + export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + export UV_INDEX_STRATEGY=unsafe-best-match && \ + rm -rf dist && mkdir -p dist && \ + for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do \ + export PATH=$VIRTUAL_ENV/bin:$PATH && \ + ./contrib/build-wheel.sh \ + --python-version $PYTHON_VERSION \ + --platform $WHL_PLATFORM \ + --ucx-plugins-dir /usr/lib64/ucx \ + --nixl-plugins-dir $NIXL_PLUGIN_DIR \ + --output-dir dist || exit 1 ; \ + done + +# Copy the meta package wheel to the dist directory, which will be used to push to PyPI. +# Only do this for the CUDA 12 builds, since by default nixl depends on nixl-cu12. +RUN if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ + cp build/src/bindings/python/nixl-meta/nixl*.whl dist/; \ + fi + +RUN uv pip install dist/nixl*cp${DEFAULT_PYTHON_VERSION//./}*.whl build/src/bindings/python/nixl-meta/nixl*.whl diff --git a/contrib/build-wheel.sh b/contrib/build-wheel.sh index f4f25e56f5..2f5b4ea158 100755 --- a/contrib/build-wheel.sh +++ b/contrib/build-wheel.sh @@ -84,7 +84,16 @@ if [ "$CUDA_MAJOR" -ne 12 ] && [ "$CUDA_MAJOR" -ne 13 ]; then fi PKG_NAME="nixl-cu${CUDA_MAJOR}" ./contrib/tomlutil.py --wheel-name $PKG_NAME pyproject.toml -uv build --wheel --out-dir $TMP_DIR --python $PYTHON_VERSION + +# Required for publishing manylinux_x86_64 wheels: avoid x86_64_v2+ codegen. +if [ "$ARCH" = "x86_64" ]; then + export CFLAGS="-march=x86-64 -mtune=generic ${CFLAGS:-}" + export CXXFLAGS="-march=x86-64 -mtune=generic ${CXXFLAGS:-}" +fi + +uv build --wheel --out-dir $TMP_DIR --python $PYTHON_VERSION \ + -Csetup-args=-Dbuild_examples=false \ + -Csetup-args=-Dbuild_tests=false # Bundle libraries mkdir $TMP_DIR/dist diff --git a/contrib/create-wheels.sh b/contrib/create-wheels.sh new file mode 100755 index 0000000000..14c52f0ef2 --- /dev/null +++ b/contrib/create-wheels.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Exit on error and print commands +set -e +set -x + +CUDA_VERSIONS="13.0.2 12.8.1" +PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14" + +arch=$(uname -m) +[ "$arch" = "arm64" ] && arch="aarch64" + +# Remove any existing dist and wheels directories +rm -rf dist/* +rm -rf wheels/* +mkdir -p wheels + +# Remove any existing container +docker rm temp-nixl || true + +for cuda_version in ${CUDA_VERSIONS} +do + tag="nixl-wheels-${cuda_version}" + ./contrib/build-container.sh \ + --base-image 'nvcr.io/nvidia/cuda' \ + --base-image-tag "${cuda_version}-devel-rockylinux9" \ + --wheel-base manylinux_2_34 \ + --python-versions "${PYTHON_VERSIONS}" \ + --tag $tag \ + --arch $arch \ + --dockerfile contrib/Dockerfile.rockylinux9 + docker create --name temp-nixl $tag + docker cp temp-nixl:/workspace/nixl/dist/ wheels/ + # Move all .whl files from wheels/dist subdirectories at any depth to wheels/ + find wheels/dist -type f -name '*.whl' -exec mv {} wheels/ \; + rm -rf wheels/dist + docker rm temp-nixl +done diff --git a/pyproject.toml b/pyproject.toml index 104c1d3bfb..45791ca9cf 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ # limitations under the License. [build-system] -requires = ["meson-python", "pybind11", "patchelf", "pyyaml", "types-PyYAML", "pytest", "build", "setuptools>=80.9.0"] +requires = ["meson-python", "pybind11", "patchelf", "pyyaml", "types-PyYAML", "pytest", "build", "truststore", "setuptools>=80.9.0"] build-backend = "mesonpy" [project]