From 0ad05c345e13b611b238fe3b6d217210952f400e Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 17 Dec 2025 01:52:49 +0100 Subject: [PATCH 01/15] Scripts to package wheels using Rocky Linux 8 Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux8 | 301 +++++++++++++++++++++++++++++++++ contrib/create-wheels.sh | 40 +++++ pyproject.toml | 2 +- 3 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 contrib/Dockerfile.rockylinux8 create mode 100755 contrib/create-wheels.sh diff --git a/contrib/Dockerfile.rockylinux8 b/contrib/Dockerfile.rockylinux8 new file mode 100644 index 0000000000..7e49a65054 --- /dev/null +++ b/contrib/Dockerfile.rockylinux8 @@ -0,0 +1,301 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +ARG BASE_IMAGE +ARG BASE_IMAGE_TAG +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} + +ARG DEFAULT_PYTHON_VERSION="3.14" +ARG ARCH="x86_64" +ARG UCX_REF="v1.20.x" +ARG LIBFABRIC_VERSION="v1.21.0" +ARG BUILD_TYPE="release" + +RUN yum groupinstall -y 'Development Tools' && \ + dnf config-manager --set-enabled powertools && \ + dnf install -y epel-release && \ + dnf install -y \ + boost \ + boost-devel \ + clang-devel \ + gcc-toolset-11 \ + cmake \ + dkms \ + flex \ + gflags \ + glibc-headers \ + gcc-c++ \ + libaio \ + libaio-devel \ + libtool-ltdl \ + ninja-build \ + openssl \ + openssl-devel \ + protobuf-compiler \ + protobuf-c-devel \ + protobuf-devel \ + libibverbs \ + libibverbs-devel \ + rdma-core \ + rdma-core-devel \ + libibumad \ + libibumad-devel \ + numactl-devel \ + librdmacm-devel \ + hwloc \ + hwloc-devel \ + wget \ + zlib + +ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH +ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-11/root/usr/lib64:$LD_LIBRARY_PATH +ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-11/root/usr/lib64/pkgconfig:${PKG_CONFIG_PATH}" + +# Build OpenSSL 3.x +RUN yum install -y perl-IPC-Cmd perl-Test-Simple perl-Data-Dumper +RUN cd /tmp && \ + wget -q https://www.openssl.org/source/openssl-3.0.16.tar.gz && \ + tar -xzf openssl-3.0.16.tar.gz && \ + cd openssl-3.0.16 && \ + ./Configure --prefix=/usr/local/openssl3 --openssldir=/usr/local/openssl3 \ + shared zlib linux-$ARCH && \ + make -j$(nproc) && \ + make install_sw && \ + echo "/usr/local/openssl3/lib64" > /etc/ld.so.conf.d/openssl3.conf && \ + echo "/usr/local/openssl3/lib" >> /etc/ld.so.conf.d/openssl3.conf && \ + ldconfig && \ + rm -rf /tmp/openssl-3.0.16* + +# Set environment variables to use the new OpenSSL +ENV PKG_CONFIG_PATH="/usr/local/openssl3/lib64/pkgconfig:/usr/local/openssl3/lib/pkgconfig:$PKG_CONFIG_PATH" +ENV LD_LIBRARY_PATH="/usr/local/openssl3/lib64:/usr/local/openssl3/lib:$LD_LIBRARY_PATH" +ENV OPENSSL_ROOT_DIR="/usr/local/openssl3" +ENV OPENSSL_LIBRARIES="/usr/local/openssl3/lib64:/usr/local/openssl3/lib" +ENV OPENSSL_INCLUDE_DIR="/usr/local/openssl3/include" + +WORKDIR /workspace + +RUN git clone --recurse-submodules -b v1.73.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ + cd grpc && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DgRPC_SSL_PROVIDER=package ../.. && \ + make -j$(nproc) && \ + make install + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ + cd etcd-cpp-apiv3 && \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ + mkdir build && cd build && \ + cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release -DETCD_CMAKE_CXX_STANDARD=17 && \ + make -j$(nproc) && make install + +# The base image libcurl is linked against openssl 1.x, so we need to build from source +# in order to use openssl 3.x. This is needed to build aws-sdk-cpp. +RUN wget https://curl.se/download/curl-8.5.0.tar.gz && \ + tar xzf curl-8.5.0.tar.gz && cd curl-8.5.0 && \ + ./configure --prefix=/usr/local --with-ssl=/usr/local/openssl3 --enable-shared && \ + make -j$(nproc) && make install + +RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.581 +RUN mkdir aws_sdk_build && cd aws_sdk_build && \ + export LDFLAGS="-L/usr/local/openssl3/lib64 -L/usr/local/openssl3/lib" && \ + export CFLAGS="-I/usr/local/openssl3/include" && \ + export CXXFLAGS="-I/usr/local/openssl3/include" && \ + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_PREFIX_PATH="/usr/local/openssl3;/usr/local" \ + -DCURL_LIBRARY=/usr/local/lib/libcurl.so \ + -DCURL_INCLUDE_DIR=/usr/local/include \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + make -j${NPROC:-$(nproc)} && make install + +RUN git clone https://github.com/nvidia/gusli.git && \ + cd gusli && \ + sed -i '/^LFLAGS_ALL =/{/-lrt/ b; s/$/ -lrt/}' Makefile && \ + make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ + cd .. + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=1.86.0 \ + RUSTARCH=${ARCH}-unknown-linux-gnu + +RUN wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init" && \ + chmod +x rustup-init && \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${RUSTARCH} && \ + case "$ARCH" in \ + aarch64) RUSTUP_SHA256="c64b33db2c6b9385817ec0e49a84bcfe018ed6e328fe755c3c809580cc70ce7a" ;; \ + x86_64) RUSTUP_SHA256="a3339fb004c3d0bb9862ba0bce001861fe5cbde9c10d16591eb3f39ee6cd3e7f" ;; \ + *) echo "Unsupported architecture for Rust: $ARCH" && exit 1 ;; \ + esac && \ + echo "$RUSTUP_SHA256 *rustup-init" | sha256sum -c - && \ + rm rustup-init && \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +RUN wget --tries=3 --waitretry=5 https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host-3.2.0-125000_25.10_rhel8.${ARCH}.rpm -O doca-host.rpm && \ + rpm -i doca-host.rpm && \ + dnf install -y libnl3-devel && \ + cd /usr/share/doca-host-3.2.0/repo/Packages/ && \ + rpm -ivh --nodeps doca-sdk-common-*rpm && \ + rpm -ivh --nodeps doca-sdk-rdma-*rpm && \ + rpm -ivh --nodeps doca-sdk-verbs-*rpm && \ + rpm -ivh --nodeps doca-sdk-gpunetio-*rpm && \ + # Check that gpunetio development package is installed correctly + pkg-config --cflags --libs doca-gpunetio + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +ENV CUDA_PATH=/usr/local/cuda + +WORKDIR /workspace + +RUN rm -rf /usr/lib/ucx +RUN rm -rf /opt/hpcx/ucx + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/NVIDIA/gdrcopy.git && \ + cd gdrcopy && \ + git fetch --tags --depth=1 && \ + latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") && \ + git checkout "$latest_tag" && \ + cd packages && \ + CUDA=/usr/local/cuda ./build-rpm-packages.sh && \ + rpm -Uvh gdrcopy-kmod-*.el8.noarch.rpm && \ + rpm -Uvh gdrcopy-*.el8.$ARCH.rpm && \ + rpm -Uvh gdrcopy-devel-*.el8.noarch.rpm + +RUN cd /usr/local/src && \ + git clone https://github.com/openucx/ucx.git && \ + cd ucx && \ + git checkout $UCX_REF && \ + ./autogen.sh && \ + ./contrib/configure-release-mt \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa && \ + make -j && \ + make -j install-strip && \ + ldconfig + +# Build libfabric from source +RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ + "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 && \ + tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 && \ + cd libfabric-* && \ + ./autogen.sh && \ + ./configure --prefix=/usr/local \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa \ + --with-cuda=/usr/local/cuda \ + --enable-cuda-dlopen \ + --with-gdrcopy \ + --enable-gdrcopy-dlopen && \ + make -j$(nproc) && \ + make install && \ + ldconfig + +# By default, uv downloads python packages to $HOME/.cache/uv and hard links them +# from the virtual environment. This means that the files reside in /root/.cache/uv, +# which is not what we want since some systems mount user home dir into /root, +# in which case the venv is broken when the container is started. +# Set a custom cache directory inside /workspace to avoid this. +ENV UV_CACHE_DIR=/workspace/.cache/uv +RUN mkdir -p $UV_CACHE_DIR +# Create a new virtual environment +ENV VIRTUAL_ENV=/workspace/.venv +RUN rm -rf $VIRTUAL_ENV && uv venv $VIRTUAL_ENV --python $DEFAULT_PYTHON_VERSION +# Activate the virtual environment +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Install python dependencies +RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click setuptools tabulate auditwheel tomlkit +# Install PyTorch +RUN export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + uv pip install torch torchvision torchaudio +# Upgrade setuptools to latest version for compatibility with PEP 639 (license format) +RUN uv pip install --upgrade 'setuptools>=80.9.0' +# Meson fails to download dependencies without truststore on Rocky Linux 8 +RUN uv pip install truststore + +COPY . /workspace/nixl +WORKDIR /workspace/nixl + +RUN rm -rf build && \ + mkdir build && \ + meson setup build/ --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE \ + -Dcudapath_lib="/usr/local/cuda/lib64" \ + -Dcudapath_inc="/usr/local/cuda/include" && \ + cd build && \ + ninja && \ + ninja install + +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/plugins:$LD_LIBRARY_PATH +ENV NIXL_PLUGIN_DIR=/usr/local/nixl/lib64/plugins + +RUN echo "/usr/local/nixl/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf && \ + echo "/usr/local/nixl/lib/$ARCH-linux-gnu/plugins" >> /etc/ld.so.conf.d/nixl.conf && \ + ldconfig + +# Create the wheel +# No need to specifically add path to libcuda.so here, meson finds the stubs and links them +ARG WHL_PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14" +ARG WHL_PLATFORM="manylinux_2_28_$ARCH" +RUN dnf install -y python3.11-devel python3.12-devel +RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \ + export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + export UV_INDEX_STRATEGY=unsafe-best-match && \ + mkdir -p dist && \ + for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do \ + export PATH=$VIRTUAL_ENV/bin:$PATH && \ + ./contrib/build-wheel.sh \ + --python-version $PYTHON_VERSION \ + --platform $WHL_PLATFORM \ + --ucx-plugins-dir /usr/lib64/ucx \ + --nixl-plugins-dir $NIXL_PLUGIN_DIR \ + --output-dir dist || exit 1; \ + done + +# Copy the meta package wheel to the dist directory, which will be used to push to PyPI. +# Only do this for the CUDA 12 builds, since by default nixl depends on nixl-cu12. +RUN if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ + cp build/src/bindings/python/nixl-meta/nixl*.whl dist/; \ + fi + +RUN uv pip install dist/nixl*cp${DEFAULT_PYTHON_VERSION//./}*.whl build/src/bindings/python/nixl-meta/nixl*.whl diff --git a/contrib/create-wheels.sh b/contrib/create-wheels.sh new file mode 100755 index 0000000000..d7984aec7b --- /dev/null +++ b/contrib/create-wheels.sh @@ -0,0 +1,40 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 + +# Exit on error and print commands +set -e +set -x + +CUDA_VERSIONS="13.0.2 12.8.1" +PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14" + +arch=$(uname -m) +[ "$arch" = "arm64" ] && arch="aarch64" + +# Remove any existing dist and wheels directories +rm -rf dist/* +rm -rf wheels/* +mkdir -p wheels + +# Remove any existing container +docker rm temp-nixl || true + +for cuda_version in ${CUDA_VERSIONS} +do + tag="nixl-wheels-${cuda_version}" + ./contrib/build-container.sh \ + --base-image 'nvcr.io/nvidia/cuda' \ + --base-image-tag "${cuda_version}-devel-rockylinux8" \ + --wheel-base manylinux_2_28 \ + --python-versions "${PYTHON_VERSIONS}" \ + --tag $tag \ + --arch $arch \ + --dockerfile contrib/Dockerfile.rockylinux8 + docker create --name temp-nixl $tag + docker cp temp-nixl:/workspace/nixl/dist/ wheels/ + # Move all .whl files from wheels/dist subdirectories at any depth to wheels/ + find wheels/dist -type f -name '*.whl' -exec mv {} wheels/ \; + rm -rf wheels/dist + docker rm temp-nixl +done diff --git a/pyproject.toml b/pyproject.toml index 08b3cc516c..fb0de95e67 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ # limitations under the License. [build-system] -requires = ["meson-python", "pybind11", "patchelf", "pyyaml", "types-PyYAML", "pytest", "build", "setuptools>=80.9.0"] +requires = ["meson-python", "pybind11", "patchelf", "pyyaml", "types-PyYAML", "pytest", "build", "truststore", "setuptools>=80.9.0"] build-backend = "mesonpy" [project] From cb5f75c2fa54495ea9fbfcbdc7edda7eefcb2efc Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Tue, 10 Mar 2026 22:54:15 +0100 Subject: [PATCH 02/15] Initial rocky 9 dockerfile Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux9 | 317 +++++++++++++++++++++++++++++++++ contrib/create-wheels.sh | 6 +- 2 files changed, 320 insertions(+), 3 deletions(-) create mode 100644 contrib/Dockerfile.rockylinux9 diff --git a/contrib/Dockerfile.rockylinux9 b/contrib/Dockerfile.rockylinux9 new file mode 100644 index 0000000000..469f5f0739 --- /dev/null +++ b/contrib/Dockerfile.rockylinux9 @@ -0,0 +1,317 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +ARG BASE_IMAGE +ARG BASE_IMAGE_TAG +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} + +ARG DEFAULT_PYTHON_VERSION="3.14" +ARG ARCH="x86_64" +ARG UCX_REF="v1.20.x" +ARG LIBFABRIC_VERSION="v1.21.0" +ARG HWLOC_VERSION="2.12.2" +ARG BUILD_TYPE="release" +ARG URING_TAG="liburing-2.13" + +RUN dnf install -y dnf-plugins-core && \ + dnf groupinstall -y "Development Tools" && \ + dnf config-manager --set-enabled crb && \ + dnf install -y epel-release && \ + dnf install -y \ + autoconf \ + automake \ + boost \ + boost-devel \ + bzip2 \ + clang-devel \ + cmake \ + diffutils \ + distribution-gpg-keys-copr \ + dkms \ + flex \ + gflags \ + gcc \ + glibc-headers \ + gcc-c++ \ + git \ + libaio \ + libaio-devel \ + libtool \ + libtool-ltdl \ + make \ + ninja-build \ + numactl-devel \ + openssl \ + openssl-devel \ + pciutils-devel \ + perl \ + perl-Data-Dumper \ + perl-IPC-Cmd \ + perl-Test-Simple \ + pkgconfig \ + protobuf-compiler \ + protobuf-c-devel \ + protobuf-devel \ + rdma-core \ + rdma-core-devel \ + libibverbs \ + libibverbs-devel \ + libibumad \ + libibumad-devel \ + librdmacm-devel \ + tar \ + which \ + wget \ + xz \ + zlib && \ + dnf clean all + +# Build latest hwloc from source to avoid API limitations. +RUN cd /tmp && \ + HWLOC_SERIES="$(echo ${HWLOC_VERSION} | cut -d. -f1,2)" && \ + wget -q "https://download.open-mpi.org/release/hwloc/v${HWLOC_SERIES}/hwloc-${HWLOC_VERSION}.tar.gz" && \ + tar -xzf "hwloc-${HWLOC_VERSION}.tar.gz" && \ + cd "hwloc-${HWLOC_VERSION}" && \ + ./configure --prefix=/usr/local --disable-nvml && \ + make -j"$(nproc)" && \ + make install && \ + ldconfig && \ + rm -rf "/tmp/hwloc-${HWLOC_VERSION}" "/tmp/hwloc-${HWLOC_VERSION}.tar.gz" + +WORKDIR /workspace + +RUN git clone --recurse-submodules -b v1.73.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ + cd grpc && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DgRPC_SSL_PROVIDER=package ../.. && \ + make -j$(nproc) && \ + make install + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ + cd etcd-cpp-apiv3 && \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ + mkdir build && cd build && \ + cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release -DETCD_CMAKE_CXX_STANDARD=17 && \ + make -j$(nproc) && make install + +RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 +RUN mkdir aws_sdk_build && cd aws_sdk_build && \ + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + make -j${NPROC:-$(nproc)} && make install + +RUN git clone https://github.com/axboe/liburing.git && \ + cd liburing && \ + git checkout $URING_TAG && \ + ./configure --cc=gcc --cxx=g++ && \ + make -j library && make liburing.pc && make install && \ + ldconfig && \ + cd .. + +ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig:$PKG_CONFIG_PATH + +RUN git clone https://github.com/nvidia/gusli.git && \ + cd gusli && \ + sed -i '/^LFLAGS_ALL =/{/-lrt/ b; s/$/ -lrt/}' Makefile && \ + make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ + cd .. + +# Required for Azure SDK +RUN git clone --depth 1 https://gitlab.gnome.org/GNOME/libxml2.git --branch 2.15 && \ + cd libxml2 && \ + ACLOCAL_PATH=/usr/share/aclocal ./autogen.sh && make -j && \ + make install + +RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ + cd azure-sdk-for-cpp/ && \ + mkdir build && cd build && \ + AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + cmake --build . --target azure-storage-blobs azure-identity && \ + cmake --install sdk/core && \ + cmake --install sdk/storage/azure-storage-common && \ + cmake --install sdk/storage/azure-storage-blobs && \ + cmake --install sdk/identity + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=1.86.0 \ + RUSTARCH=${ARCH}-unknown-linux-gnu + +RUN wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init" && \ + chmod +x rustup-init && \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${RUSTARCH} && \ + case "$ARCH" in \ + aarch64) RUSTUP_SHA256="c64b33db2c6b9385817ec0e49a84bcfe018ed6e328fe755c3c809580cc70ce7a" ;; \ + x86_64) RUSTUP_SHA256="a3339fb004c3d0bb9862ba0bce001861fe5cbde9c10d16591eb3f39ee6cd3e7f" ;; \ + *) echo "Unsupported architecture for Rust: $ARCH" && exit 1 ;; \ + esac && \ + echo "$RUSTUP_SHA256 *rustup-init" | sha256sum -c - && \ + rm rustup-init && \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +RUN wget --tries=3 --waitretry=5 "https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host-3.2.0-125000_25.10_rhel9.${ARCH}.rpm" -O doca-host.rpm && \ + rpm -i doca-host.rpm && \ + dnf install -y libnl3-devel && \ + cd /usr/share/doca-host-3.2.0/repo/Packages/ && \ + rpm -ivh --nodeps doca-sdk-common-*rpm && \ + rpm -ivh --nodeps doca-sdk-rdma-*rpm && \ + rpm -ivh --nodeps doca-sdk-verbs-*rpm && \ + rpm -ivh --nodeps doca-sdk-gpunetio-*rpm && \ + # Check that gpunetio development package is installed correctly + pkg-config --cflags --libs doca-gpunetio + +ENV LD_LIBRARY_PATH=/usr/lib:/usr/local/lib:$LD_LIBRARY_PATH + +ENV CUDA_PATH=/usr/local/cuda + +WORKDIR /workspace + +RUN rm -rf /usr/lib/ucx +RUN rm -rf /opt/hpcx/ucx + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/NVIDIA/gdrcopy.git && \ + cd gdrcopy && \ + git fetch --tags --depth=1 && \ + latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") && \ + git checkout "$latest_tag" && \ + cd packages && \ + CUDA=/usr/local/cuda ./build-rpm-packages.sh && \ + rpm -Uvh gdrcopy-kmod-*.el9.noarch.rpm && \ + rpm -Uvh gdrcopy-*.el9.$ARCH.rpm && \ + rpm -Uvh gdrcopy-devel-*.el9.noarch.rpm + +RUN cd /usr/local/src && \ + git clone https://github.com/openucx/ucx.git && \ + cd ucx && \ + git checkout $UCX_REF && \ + ./autogen.sh && \ + ./contrib/configure-release-mt \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa && \ + make -j && \ + make -j install-strip && \ + ldconfig + +# Build libfabric from source +RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ + "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 && \ + tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 && \ + cd libfabric-* && \ + ./autogen.sh && \ + ./configure --prefix=/usr/local \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa \ + --with-cuda=/usr/local/cuda \ + --enable-cuda-dlopen \ + --with-gdrcopy \ + --enable-gdrcopy-dlopen && \ + make -j$(nproc) && \ + make install && \ + ldconfig + +# By default, uv downloads python packages to $HOME/.cache/uv and hard links them +# from the virtual environment. This means that the files reside in /root/.cache/uv, +# which is not what we want since some systems mount user home dir into /root, +# in which case the venv is broken when the container is started. +# Set a custom cache directory inside /workspace to avoid this. +ENV UV_CACHE_DIR=/workspace/.cache/uv +RUN mkdir -p $UV_CACHE_DIR +# Create a new virtual environment +ENV VIRTUAL_ENV=/workspace/.venv +RUN rm -rf $VIRTUAL_ENV && uv venv $VIRTUAL_ENV --python $DEFAULT_PYTHON_VERSION +# Activate the virtual environment +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Install python dependencies +RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click setuptools tabulate auditwheel tomlkit +# Install PyTorch +RUN export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + uv pip install torch torchvision torchaudio +# Upgrade setuptools to latest version for compatibility with PEP 639 (license format) +RUN uv pip install --upgrade 'setuptools>=80.9.0' + +COPY . /workspace/nixl +WORKDIR /workspace/nixl + +RUN rm -rf build && \ + mkdir build && \ + meson setup build/ --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE \ + -Dcudapath_lib="/usr/local/cuda/lib64" \ + -Dcudapath_inc="/usr/local/cuda/include" && \ + cd build && \ + ninja && \ + ninja install + +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/plugins:$LD_LIBRARY_PATH +ENV NIXL_PLUGIN_DIR=/usr/local/nixl/lib64/plugins + +RUN echo "/usr/local/nixl/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf && \ + echo "/usr/local/nixl/lib/$ARCH-linux-gnu/plugins" >> /etc/ld.so.conf.d/nixl.conf && \ + ldconfig + +# Create the wheel +# No need to specifically add path to libcuda.so here, meson finds the stubs and links them +ARG WHL_PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14" +ARG WHL_PLATFORM="manylinux_2_28_$ARCH" +RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \ + export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + export UV_INDEX_STRATEGY=unsafe-best-match && \ + rm -rf dist && mkdir -p dist && \ + for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do \ + export PATH=$VIRTUAL_ENV/bin:$PATH && \ + ./contrib/build-wheel.sh \ + --python-version $PYTHON_VERSION \ + --platform $WHL_PLATFORM \ + --ucx-plugins-dir /usr/lib64/ucx \ + --nixl-plugins-dir $NIXL_PLUGIN_DIR \ + --output-dir dist ; \ + done + +# Copy the meta package wheel to the dist directory, which will be used to push to PyPI. +# Only do this for the CUDA 12 builds, since by default nixl depends on nixl-cu12. +RUN if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ + cp build/src/bindings/python/nixl-meta/nixl*.whl dist/; \ + fi + +RUN uv pip install dist/nixl*cp${DEFAULT_PYTHON_VERSION//./}*.whl build/src/bindings/python/nixl-meta/nixl*.whl diff --git a/contrib/create-wheels.sh b/contrib/create-wheels.sh index d7984aec7b..81df5493ae 100755 --- a/contrib/create-wheels.sh +++ b/contrib/create-wheels.sh @@ -25,12 +25,12 @@ do tag="nixl-wheels-${cuda_version}" ./contrib/build-container.sh \ --base-image 'nvcr.io/nvidia/cuda' \ - --base-image-tag "${cuda_version}-devel-rockylinux8" \ - --wheel-base manylinux_2_28 \ + --base-image-tag "${cuda_version}-devel-rockylinux9" \ + --wheel-base manylinux_2_34 \ --python-versions "${PYTHON_VERSIONS}" \ --tag $tag \ --arch $arch \ - --dockerfile contrib/Dockerfile.rockylinux8 + --dockerfile contrib/Dockerfile.rockylinux9 docker create --name temp-nixl $tag docker cp temp-nixl:/workspace/nixl/dist/ wheels/ # Move all .whl files from wheels/dist subdirectories at any depth to wheels/ From 466405fbb4769a283737d9000f4cfea837096dae Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Tue, 10 Mar 2026 23:07:35 +0100 Subject: [PATCH 03/15] Add curl Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux9 | 2 ++ 1 file changed, 2 insertions(+) diff --git a/contrib/Dockerfile.rockylinux9 b/contrib/Dockerfile.rockylinux9 index 469f5f0739..5e0a38d4c0 100644 --- a/contrib/Dockerfile.rockylinux9 +++ b/contrib/Dockerfile.rockylinux9 @@ -38,6 +38,7 @@ RUN dnf install -y dnf-plugins-core && \ bzip2 \ clang-devel \ cmake \ + curl \ diffutils \ distribution-gpg-keys-copr \ dkms \ @@ -49,6 +50,7 @@ RUN dnf install -y dnf-plugins-core && \ git \ libaio \ libaio-devel \ + libcurl-devel \ libtool \ libtool-ltdl \ make \ From ac7ef3099d30d95e7b33759deee49e69904c490f Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Tue, 10 Mar 2026 23:11:50 +0100 Subject: [PATCH 04/15] Install curl devel Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux9 | 1 - 1 file changed, 1 deletion(-) diff --git a/contrib/Dockerfile.rockylinux9 b/contrib/Dockerfile.rockylinux9 index 5e0a38d4c0..2a4843e1fc 100644 --- a/contrib/Dockerfile.rockylinux9 +++ b/contrib/Dockerfile.rockylinux9 @@ -38,7 +38,6 @@ RUN dnf install -y dnf-plugins-core && \ bzip2 \ clang-devel \ cmake \ - curl \ diffutils \ distribution-gpg-keys-copr \ dkms \ From 4ea91872cec324c7f0522a06f8df8fde49462589 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Tue, 10 Mar 2026 23:24:19 +0100 Subject: [PATCH 05/15] Fix libxml2 Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux9 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/Dockerfile.rockylinux9 b/contrib/Dockerfile.rockylinux9 index 2a4843e1fc..b6298a371c 100644 --- a/contrib/Dockerfile.rockylinux9 +++ b/contrib/Dockerfile.rockylinux9 @@ -143,8 +143,12 @@ RUN git clone https://github.com/nvidia/gusli.git && \ # Required for Azure SDK RUN git clone --depth 1 https://gitlab.gnome.org/GNOME/libxml2.git --branch 2.15 && \ cd libxml2 && \ - ACLOCAL_PATH=/usr/share/aclocal ./autogen.sh && make -j && \ - make install + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DLIBXML2_WITH_PYTHON=OFF && \ + cmake --build build -j"$(nproc)" && \ + cmake --install build RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ cd azure-sdk-for-cpp/ && \ From 3bc560fca9b0349e188fa5a3f4b71c8a45b9dffd Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Tue, 10 Mar 2026 23:50:27 +0100 Subject: [PATCH 06/15] Build wheel for x86_64 generic not v2 Signed-off-by: Ovidiu Mara --- contrib/build-wheel.sh | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/contrib/build-wheel.sh b/contrib/build-wheel.sh index f4f25e56f5..effb2bae9b 100755 --- a/contrib/build-wheel.sh +++ b/contrib/build-wheel.sh @@ -84,6 +84,13 @@ if [ "$CUDA_MAJOR" -ne 12 ] && [ "$CUDA_MAJOR" -ne 13 ]; then fi PKG_NAME="nixl-cu${CUDA_MAJOR}" ./contrib/tomlutil.py --wheel-name $PKG_NAME pyproject.toml + +# Required for publishing manylinux_x86_64 wheels: avoid x86_64_v2+ codegen. +if [ "$ARCH" = "x86_64" ]; then + export CFLAGS="-march=x86-64 -mtune=generic ${CFLAGS:-}" + export CXXFLAGS="-march=x86-64 -mtune=generic ${CXXFLAGS:-}" +fi + uv build --wheel --out-dir $TMP_DIR --python $PYTHON_VERSION # Bundle libraries From 4bac6bca05342d07e687756f0b6286e2f00c3d09 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Tue, 10 Mar 2026 23:58:28 +0100 Subject: [PATCH 07/15] Do not build examples and tests in wheel env Signed-off-by: Ovidiu Mara --- contrib/build-wheel.sh | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/contrib/build-wheel.sh b/contrib/build-wheel.sh index effb2bae9b..2f5b4ea158 100755 --- a/contrib/build-wheel.sh +++ b/contrib/build-wheel.sh @@ -91,7 +91,9 @@ if [ "$ARCH" = "x86_64" ]; then export CXXFLAGS="-march=x86-64 -mtune=generic ${CXXFLAGS:-}" fi -uv build --wheel --out-dir $TMP_DIR --python $PYTHON_VERSION +uv build --wheel --out-dir $TMP_DIR --python $PYTHON_VERSION \ + -Csetup-args=-Dbuild_examples=false \ + -Csetup-args=-Dbuild_tests=false # Bundle libraries mkdir $TMP_DIR/dist From 3808309078f1c879103a80f62e2cb24c195dcfda Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 11 Mar 2026 00:09:33 +0100 Subject: [PATCH 08/15] Set pkg config path explicitly Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux9 | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/contrib/Dockerfile.rockylinux9 b/contrib/Dockerfile.rockylinux9 index b6298a371c..b9055030d3 100644 --- a/contrib/Dockerfile.rockylinux9 +++ b/contrib/Dockerfile.rockylinux9 @@ -255,6 +255,10 @@ RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ make install && \ ldconfig +# Ensure pkg-config and linker can discover source-installed libs under /usr/local. +ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig +RUN printf "/usr/local/lib64\n/usr/local/lib\n" > /etc/ld.so.conf.d/usr-local.conf && ldconfig + # By default, uv downloads python packages to $HOME/.cache/uv and hard links them # from the virtual environment. This means that the files reside in /root/.cache/uv, # which is not what we want since some systems mount user home dir into /root, From 66e8b5bddf57a2f3c9013e1cfb6ce8fe3dd6aace Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 11 Mar 2026 02:01:01 +0100 Subject: [PATCH 09/15] Remove rockylinux8 file Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux8 | 301 --------------------------------- 1 file changed, 301 deletions(-) delete mode 100644 contrib/Dockerfile.rockylinux8 diff --git a/contrib/Dockerfile.rockylinux8 b/contrib/Dockerfile.rockylinux8 deleted file mode 100644 index 7e49a65054..0000000000 --- a/contrib/Dockerfile.rockylinux8 +++ /dev/null @@ -1,301 +0,0 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. -# SPDX-License-Identifier: Apache-2.0 -# -# Licensed under the Apache License, Version 2.0 (the "License"); -# you may not use this file except in compliance with the License. -# You may obtain a copy of the License at -# -# http://www.apache.org/licenses/LICENSE-2.0 -# -# Unless required by applicable law or agreed to in writing, software -# distributed under the License is distributed on an "AS IS" BASIS, -# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -# See the License for the specific language governing permissions and -# limitations under the License. - - -ARG BASE_IMAGE -ARG BASE_IMAGE_TAG -FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} - -ARG DEFAULT_PYTHON_VERSION="3.14" -ARG ARCH="x86_64" -ARG UCX_REF="v1.20.x" -ARG LIBFABRIC_VERSION="v1.21.0" -ARG BUILD_TYPE="release" - -RUN yum groupinstall -y 'Development Tools' && \ - dnf config-manager --set-enabled powertools && \ - dnf install -y epel-release && \ - dnf install -y \ - boost \ - boost-devel \ - clang-devel \ - gcc-toolset-11 \ - cmake \ - dkms \ - flex \ - gflags \ - glibc-headers \ - gcc-c++ \ - libaio \ - libaio-devel \ - libtool-ltdl \ - ninja-build \ - openssl \ - openssl-devel \ - protobuf-compiler \ - protobuf-c-devel \ - protobuf-devel \ - libibverbs \ - libibverbs-devel \ - rdma-core \ - rdma-core-devel \ - libibumad \ - libibumad-devel \ - numactl-devel \ - librdmacm-devel \ - hwloc \ - hwloc-devel \ - wget \ - zlib - -ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH -ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-11/root/usr/lib64:$LD_LIBRARY_PATH -ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-11/root/usr/lib64/pkgconfig:${PKG_CONFIG_PATH}" - -# Build OpenSSL 3.x -RUN yum install -y perl-IPC-Cmd perl-Test-Simple perl-Data-Dumper -RUN cd /tmp && \ - wget -q https://www.openssl.org/source/openssl-3.0.16.tar.gz && \ - tar -xzf openssl-3.0.16.tar.gz && \ - cd openssl-3.0.16 && \ - ./Configure --prefix=/usr/local/openssl3 --openssldir=/usr/local/openssl3 \ - shared zlib linux-$ARCH && \ - make -j$(nproc) && \ - make install_sw && \ - echo "/usr/local/openssl3/lib64" > /etc/ld.so.conf.d/openssl3.conf && \ - echo "/usr/local/openssl3/lib" >> /etc/ld.so.conf.d/openssl3.conf && \ - ldconfig && \ - rm -rf /tmp/openssl-3.0.16* - -# Set environment variables to use the new OpenSSL -ENV PKG_CONFIG_PATH="/usr/local/openssl3/lib64/pkgconfig:/usr/local/openssl3/lib/pkgconfig:$PKG_CONFIG_PATH" -ENV LD_LIBRARY_PATH="/usr/local/openssl3/lib64:/usr/local/openssl3/lib:$LD_LIBRARY_PATH" -ENV OPENSSL_ROOT_DIR="/usr/local/openssl3" -ENV OPENSSL_LIBRARIES="/usr/local/openssl3/lib64:/usr/local/openssl3/lib" -ENV OPENSSL_INCLUDE_DIR="/usr/local/openssl3/include" - -WORKDIR /workspace - -RUN git clone --recurse-submodules -b v1.73.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ - cd grpc && \ - mkdir -p cmake/build && \ - cd cmake/build && \ - cmake -DgRPC_INSTALL=ON \ - -DgRPC_BUILD_TESTS=OFF \ - -DBUILD_SHARED_LIBS=ON \ - -DCMAKE_CXX_STANDARD=17 \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DgRPC_SSL_PROVIDER=package ../.. && \ - make -j$(nproc) && \ - make install - -ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH - -RUN cd /workspace && \ - git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ - cd etcd-cpp-apiv3 && \ - sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ - mkdir build && cd build && \ - cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release -DETCD_CMAKE_CXX_STANDARD=17 && \ - make -j$(nproc) && make install - -# The base image libcurl is linked against openssl 1.x, so we need to build from source -# in order to use openssl 3.x. This is needed to build aws-sdk-cpp. -RUN wget https://curl.se/download/curl-8.5.0.tar.gz && \ - tar xzf curl-8.5.0.tar.gz && cd curl-8.5.0 && \ - ./configure --prefix=/usr/local --with-ssl=/usr/local/openssl3 --enable-shared && \ - make -j$(nproc) && make install - -RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.581 -RUN mkdir aws_sdk_build && cd aws_sdk_build && \ - export LDFLAGS="-L/usr/local/openssl3/lib64 -L/usr/local/openssl3/lib" && \ - export CFLAGS="-I/usr/local/openssl3/include" && \ - export CXXFLAGS="-I/usr/local/openssl3/include" && \ - cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \ - -DCMAKE_PREFIX_PATH="/usr/local/openssl3;/usr/local" \ - -DCURL_LIBRARY=/usr/local/lib/libcurl.so \ - -DCURL_INCLUDE_DIR=/usr/local/include \ - -DOPENSSL_USE_STATIC_LIBS=OFF && \ - make -j${NPROC:-$(nproc)} && make install - -RUN git clone https://github.com/nvidia/gusli.git && \ - cd gusli && \ - sed -i '/^LFLAGS_ALL =/{/-lrt/ b; s/$/ -lrt/}' Makefile && \ - make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ - cd .. - -COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ - -ENV RUSTUP_HOME=/usr/local/rustup \ - CARGO_HOME=/usr/local/cargo \ - PATH=/usr/local/cargo/bin:$PATH \ - RUST_VERSION=1.86.0 \ - RUSTARCH=${ARCH}-unknown-linux-gnu - -RUN wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init" && \ - chmod +x rustup-init && \ - ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${RUSTARCH} && \ - case "$ARCH" in \ - aarch64) RUSTUP_SHA256="c64b33db2c6b9385817ec0e49a84bcfe018ed6e328fe755c3c809580cc70ce7a" ;; \ - x86_64) RUSTUP_SHA256="a3339fb004c3d0bb9862ba0bce001861fe5cbde9c10d16591eb3f39ee6cd3e7f" ;; \ - *) echo "Unsupported architecture for Rust: $ARCH" && exit 1 ;; \ - esac && \ - echo "$RUSTUP_SHA256 *rustup-init" | sha256sum -c - && \ - rm rustup-init && \ - chmod -R a+w $RUSTUP_HOME $CARGO_HOME - -RUN wget --tries=3 --waitretry=5 https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host-3.2.0-125000_25.10_rhel8.${ARCH}.rpm -O doca-host.rpm && \ - rpm -i doca-host.rpm && \ - dnf install -y libnl3-devel && \ - cd /usr/share/doca-host-3.2.0/repo/Packages/ && \ - rpm -ivh --nodeps doca-sdk-common-*rpm && \ - rpm -ivh --nodeps doca-sdk-rdma-*rpm && \ - rpm -ivh --nodeps doca-sdk-verbs-*rpm && \ - rpm -ivh --nodeps doca-sdk-gpunetio-*rpm && \ - # Check that gpunetio development package is installed correctly - pkg-config --cflags --libs doca-gpunetio - -ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH - -ENV CUDA_PATH=/usr/local/cuda - -WORKDIR /workspace - -RUN rm -rf /usr/lib/ucx -RUN rm -rf /opt/hpcx/ucx - -RUN cd /workspace && \ - git clone --depth 1 https://github.com/NVIDIA/gdrcopy.git && \ - cd gdrcopy && \ - git fetch --tags --depth=1 && \ - latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") && \ - git checkout "$latest_tag" && \ - cd packages && \ - CUDA=/usr/local/cuda ./build-rpm-packages.sh && \ - rpm -Uvh gdrcopy-kmod-*.el8.noarch.rpm && \ - rpm -Uvh gdrcopy-*.el8.$ARCH.rpm && \ - rpm -Uvh gdrcopy-devel-*.el8.noarch.rpm - -RUN cd /usr/local/src && \ - git clone https://github.com/openucx/ucx.git && \ - cd ucx && \ - git checkout $UCX_REF && \ - ./autogen.sh && \ - ./contrib/configure-release-mt \ - --enable-shared \ - --disable-static \ - --disable-doxygen-doc \ - --enable-optimizations \ - --enable-cma \ - --enable-devel-headers \ - --with-cuda=/usr/local/cuda \ - --with-verbs \ - --with-dm \ - --with-gdrcopy=/usr/local \ - --with-efa && \ - make -j && \ - make -j install-strip && \ - ldconfig - -# Build libfabric from source -RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ - "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 && \ - tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 && \ - cd libfabric-* && \ - ./autogen.sh && \ - ./configure --prefix=/usr/local \ - --disable-verbs \ - --disable-psm3 \ - --disable-opx \ - --disable-usnic \ - --disable-rstream \ - --enable-efa \ - --with-cuda=/usr/local/cuda \ - --enable-cuda-dlopen \ - --with-gdrcopy \ - --enable-gdrcopy-dlopen && \ - make -j$(nproc) && \ - make install && \ - ldconfig - -# By default, uv downloads python packages to $HOME/.cache/uv and hard links them -# from the virtual environment. This means that the files reside in /root/.cache/uv, -# which is not what we want since some systems mount user home dir into /root, -# in which case the venv is broken when the container is started. -# Set a custom cache directory inside /workspace to avoid this. -ENV UV_CACHE_DIR=/workspace/.cache/uv -RUN mkdir -p $UV_CACHE_DIR -# Create a new virtual environment -ENV VIRTUAL_ENV=/workspace/.venv -RUN rm -rf $VIRTUAL_ENV && uv venv $VIRTUAL_ENV --python $DEFAULT_PYTHON_VERSION -# Activate the virtual environment -ENV PATH="$VIRTUAL_ENV/bin:$PATH" -# Install python dependencies -RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click setuptools tabulate auditwheel tomlkit -# Install PyTorch -RUN export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ - uv pip install torch torchvision torchaudio -# Upgrade setuptools to latest version for compatibility with PEP 639 (license format) -RUN uv pip install --upgrade 'setuptools>=80.9.0' -# Meson fails to download dependencies without truststore on Rocky Linux 8 -RUN uv pip install truststore - -COPY . /workspace/nixl -WORKDIR /workspace/nixl - -RUN rm -rf build && \ - mkdir build && \ - meson setup build/ --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE \ - -Dcudapath_lib="/usr/local/cuda/lib64" \ - -Dcudapath_inc="/usr/local/cuda/include" && \ - cd build && \ - ninja && \ - ninja install - -ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/:$LD_LIBRARY_PATH -ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/plugins:$LD_LIBRARY_PATH -ENV NIXL_PLUGIN_DIR=/usr/local/nixl/lib64/plugins - -RUN echo "/usr/local/nixl/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf && \ - echo "/usr/local/nixl/lib/$ARCH-linux-gnu/plugins" >> /etc/ld.so.conf.d/nixl.conf && \ - ldconfig - -# Create the wheel -# No need to specifically add path to libcuda.so here, meson finds the stubs and links them -ARG WHL_PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14" -ARG WHL_PLATFORM="manylinux_2_28_$ARCH" -RUN dnf install -y python3.11-devel python3.12-devel -RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \ - export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ - export UV_INDEX_STRATEGY=unsafe-best-match && \ - mkdir -p dist && \ - for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do \ - export PATH=$VIRTUAL_ENV/bin:$PATH && \ - ./contrib/build-wheel.sh \ - --python-version $PYTHON_VERSION \ - --platform $WHL_PLATFORM \ - --ucx-plugins-dir /usr/lib64/ucx \ - --nixl-plugins-dir $NIXL_PLUGIN_DIR \ - --output-dir dist || exit 1; \ - done - -# Copy the meta package wheel to the dist directory, which will be used to push to PyPI. -# Only do this for the CUDA 12 builds, since by default nixl depends on nixl-cu12. -RUN if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ - cp build/src/bindings/python/nixl-meta/nixl*.whl dist/; \ - fi - -RUN uv pip install dist/nixl*cp${DEFAULT_PYTHON_VERSION//./}*.whl build/src/bindings/python/nixl-meta/nixl*.whl From 51c0fd349b538cb3a3cfa80d9e05488f82c990b5 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 11 Mar 2026 02:02:09 +0100 Subject: [PATCH 10/15] Revert "Remove rockylinux8 file" This reverts commit 66e8b5bddf57a2f3c9013e1cfb6ce8fe3dd6aace. --- contrib/Dockerfile.rockylinux8 | 301 +++++++++++++++++++++++++++++++++ 1 file changed, 301 insertions(+) create mode 100644 contrib/Dockerfile.rockylinux8 diff --git a/contrib/Dockerfile.rockylinux8 b/contrib/Dockerfile.rockylinux8 new file mode 100644 index 0000000000..7e49a65054 --- /dev/null +++ b/contrib/Dockerfile.rockylinux8 @@ -0,0 +1,301 @@ +# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + + +ARG BASE_IMAGE +ARG BASE_IMAGE_TAG +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} + +ARG DEFAULT_PYTHON_VERSION="3.14" +ARG ARCH="x86_64" +ARG UCX_REF="v1.20.x" +ARG LIBFABRIC_VERSION="v1.21.0" +ARG BUILD_TYPE="release" + +RUN yum groupinstall -y 'Development Tools' && \ + dnf config-manager --set-enabled powertools && \ + dnf install -y epel-release && \ + dnf install -y \ + boost \ + boost-devel \ + clang-devel \ + gcc-toolset-11 \ + cmake \ + dkms \ + flex \ + gflags \ + glibc-headers \ + gcc-c++ \ + libaio \ + libaio-devel \ + libtool-ltdl \ + ninja-build \ + openssl \ + openssl-devel \ + protobuf-compiler \ + protobuf-c-devel \ + protobuf-devel \ + libibverbs \ + libibverbs-devel \ + rdma-core \ + rdma-core-devel \ + libibumad \ + libibumad-devel \ + numactl-devel \ + librdmacm-devel \ + hwloc \ + hwloc-devel \ + wget \ + zlib + +ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH +ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-11/root/usr/lib64:$LD_LIBRARY_PATH +ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-11/root/usr/lib64/pkgconfig:${PKG_CONFIG_PATH}" + +# Build OpenSSL 3.x +RUN yum install -y perl-IPC-Cmd perl-Test-Simple perl-Data-Dumper +RUN cd /tmp && \ + wget -q https://www.openssl.org/source/openssl-3.0.16.tar.gz && \ + tar -xzf openssl-3.0.16.tar.gz && \ + cd openssl-3.0.16 && \ + ./Configure --prefix=/usr/local/openssl3 --openssldir=/usr/local/openssl3 \ + shared zlib linux-$ARCH && \ + make -j$(nproc) && \ + make install_sw && \ + echo "/usr/local/openssl3/lib64" > /etc/ld.so.conf.d/openssl3.conf && \ + echo "/usr/local/openssl3/lib" >> /etc/ld.so.conf.d/openssl3.conf && \ + ldconfig && \ + rm -rf /tmp/openssl-3.0.16* + +# Set environment variables to use the new OpenSSL +ENV PKG_CONFIG_PATH="/usr/local/openssl3/lib64/pkgconfig:/usr/local/openssl3/lib/pkgconfig:$PKG_CONFIG_PATH" +ENV LD_LIBRARY_PATH="/usr/local/openssl3/lib64:/usr/local/openssl3/lib:$LD_LIBRARY_PATH" +ENV OPENSSL_ROOT_DIR="/usr/local/openssl3" +ENV OPENSSL_LIBRARIES="/usr/local/openssl3/lib64:/usr/local/openssl3/lib" +ENV OPENSSL_INCLUDE_DIR="/usr/local/openssl3/include" + +WORKDIR /workspace + +RUN git clone --recurse-submodules -b v1.73.0 --depth 1 --shallow-submodules https://github.com/grpc/grpc && \ + cd grpc && \ + mkdir -p cmake/build && \ + cd cmake/build && \ + cmake -DgRPC_INSTALL=ON \ + -DgRPC_BUILD_TESTS=OFF \ + -DBUILD_SHARED_LIBS=ON \ + -DCMAKE_CXX_STANDARD=17 \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DgRPC_SSL_PROVIDER=package ../.. && \ + make -j$(nproc) && \ + make install + +ENV LD_LIBRARY_PATH=/usr/local/lib:/usr/local/lib64:$LD_LIBRARY_PATH + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git && \ + cd etcd-cpp-apiv3 && \ + sed -i '/^find_dependency(cpprestsdk)$/d' etcd-cpp-api-config.in.cmake && \ + mkdir build && cd build && \ + cmake .. -DBUILD_ETCD_CORE_ONLY=ON -DCMAKE_BUILD_TYPE=Release -DETCD_CMAKE_CXX_STANDARD=17 && \ + make -j$(nproc) && make install + +# The base image libcurl is linked against openssl 1.x, so we need to build from source +# in order to use openssl 3.x. This is needed to build aws-sdk-cpp. +RUN wget https://curl.se/download/curl-8.5.0.tar.gz && \ + tar xzf curl-8.5.0.tar.gz && cd curl-8.5.0 && \ + ./configure --prefix=/usr/local --with-ssl=/usr/local/openssl3 --enable-shared && \ + make -j$(nproc) && make install + +RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.581 +RUN mkdir aws_sdk_build && cd aws_sdk_build && \ + export LDFLAGS="-L/usr/local/openssl3/lib64 -L/usr/local/openssl3/lib" && \ + export CFLAGS="-I/usr/local/openssl3/include" && \ + export CXXFLAGS="-I/usr/local/openssl3/include" && \ + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DCMAKE_PREFIX_PATH="/usr/local/openssl3;/usr/local" \ + -DCURL_LIBRARY=/usr/local/lib/libcurl.so \ + -DCURL_INCLUDE_DIR=/usr/local/include \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + make -j${NPROC:-$(nproc)} && make install + +RUN git clone https://github.com/nvidia/gusli.git && \ + cd gusli && \ + sed -i '/^LFLAGS_ALL =/{/-lrt/ b; s/$/ -lrt/}' Makefile && \ + make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ + cd .. + +COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ + +ENV RUSTUP_HOME=/usr/local/rustup \ + CARGO_HOME=/usr/local/cargo \ + PATH=/usr/local/cargo/bin:$PATH \ + RUST_VERSION=1.86.0 \ + RUSTARCH=${ARCH}-unknown-linux-gnu + +RUN wget --tries=3 --waitretry=5 "https://static.rust-lang.org/rustup/archive/1.28.1/${RUSTARCH}/rustup-init" && \ + chmod +x rustup-init && \ + ./rustup-init -y --no-modify-path --profile minimal --default-toolchain $RUST_VERSION --default-host ${RUSTARCH} && \ + case "$ARCH" in \ + aarch64) RUSTUP_SHA256="c64b33db2c6b9385817ec0e49a84bcfe018ed6e328fe755c3c809580cc70ce7a" ;; \ + x86_64) RUSTUP_SHA256="a3339fb004c3d0bb9862ba0bce001861fe5cbde9c10d16591eb3f39ee6cd3e7f" ;; \ + *) echo "Unsupported architecture for Rust: $ARCH" && exit 1 ;; \ + esac && \ + echo "$RUSTUP_SHA256 *rustup-init" | sha256sum -c - && \ + rm rustup-init && \ + chmod -R a+w $RUSTUP_HOME $CARGO_HOME + +RUN wget --tries=3 --waitretry=5 https://www.mellanox.com/downloads/DOCA/DOCA_v3.2.0/host/doca-host-3.2.0-125000_25.10_rhel8.${ARCH}.rpm -O doca-host.rpm && \ + rpm -i doca-host.rpm && \ + dnf install -y libnl3-devel && \ + cd /usr/share/doca-host-3.2.0/repo/Packages/ && \ + rpm -ivh --nodeps doca-sdk-common-*rpm && \ + rpm -ivh --nodeps doca-sdk-rdma-*rpm && \ + rpm -ivh --nodeps doca-sdk-verbs-*rpm && \ + rpm -ivh --nodeps doca-sdk-gpunetio-*rpm && \ + # Check that gpunetio development package is installed correctly + pkg-config --cflags --libs doca-gpunetio + +ENV LD_LIBRARY_PATH=/usr/local/lib:$LD_LIBRARY_PATH + +ENV CUDA_PATH=/usr/local/cuda + +WORKDIR /workspace + +RUN rm -rf /usr/lib/ucx +RUN rm -rf /opt/hpcx/ucx + +RUN cd /workspace && \ + git clone --depth 1 https://github.com/NVIDIA/gdrcopy.git && \ + cd gdrcopy && \ + git fetch --tags --depth=1 && \ + latest_tag=$(git describe --tags "$(git rev-list --tags --max-count=1)") && \ + git checkout "$latest_tag" && \ + cd packages && \ + CUDA=/usr/local/cuda ./build-rpm-packages.sh && \ + rpm -Uvh gdrcopy-kmod-*.el8.noarch.rpm && \ + rpm -Uvh gdrcopy-*.el8.$ARCH.rpm && \ + rpm -Uvh gdrcopy-devel-*.el8.noarch.rpm + +RUN cd /usr/local/src && \ + git clone https://github.com/openucx/ucx.git && \ + cd ucx && \ + git checkout $UCX_REF && \ + ./autogen.sh && \ + ./contrib/configure-release-mt \ + --enable-shared \ + --disable-static \ + --disable-doxygen-doc \ + --enable-optimizations \ + --enable-cma \ + --enable-devel-headers \ + --with-cuda=/usr/local/cuda \ + --with-verbs \ + --with-dm \ + --with-gdrcopy=/usr/local \ + --with-efa && \ + make -j && \ + make -j install-strip && \ + ldconfig + +# Build libfabric from source +RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ + "https://github.com/ofiwg/libfabric/releases/download/${LIBFABRIC_VERSION}/libfabric-${LIBFABRIC_VERSION#v}.tar.bz2" -O libfabric.tar.bz2 && \ + tar xjf libfabric.tar.bz2 && rm libfabric.tar.bz2 && \ + cd libfabric-* && \ + ./autogen.sh && \ + ./configure --prefix=/usr/local \ + --disable-verbs \ + --disable-psm3 \ + --disable-opx \ + --disable-usnic \ + --disable-rstream \ + --enable-efa \ + --with-cuda=/usr/local/cuda \ + --enable-cuda-dlopen \ + --with-gdrcopy \ + --enable-gdrcopy-dlopen && \ + make -j$(nproc) && \ + make install && \ + ldconfig + +# By default, uv downloads python packages to $HOME/.cache/uv and hard links them +# from the virtual environment. This means that the files reside in /root/.cache/uv, +# which is not what we want since some systems mount user home dir into /root, +# in which case the venv is broken when the container is started. +# Set a custom cache directory inside /workspace to avoid this. +ENV UV_CACHE_DIR=/workspace/.cache/uv +RUN mkdir -p $UV_CACHE_DIR +# Create a new virtual environment +ENV VIRTUAL_ENV=/workspace/.venv +RUN rm -rf $VIRTUAL_ENV && uv venv $VIRTUAL_ENV --python $DEFAULT_PYTHON_VERSION +# Activate the virtual environment +ENV PATH="$VIRTUAL_ENV/bin:$PATH" +# Install python dependencies +RUN uv pip install --upgrade meson meson-python pybind11 patchelf pyYAML click setuptools tabulate auditwheel tomlkit +# Install PyTorch +RUN export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + uv pip install torch torchvision torchaudio +# Upgrade setuptools to latest version for compatibility with PEP 639 (license format) +RUN uv pip install --upgrade 'setuptools>=80.9.0' +# Meson fails to download dependencies without truststore on Rocky Linux 8 +RUN uv pip install truststore + +COPY . /workspace/nixl +WORKDIR /workspace/nixl + +RUN rm -rf build && \ + mkdir build && \ + meson setup build/ --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE \ + -Dcudapath_lib="/usr/local/cuda/lib64" \ + -Dcudapath_inc="/usr/local/cuda/include" && \ + cd build && \ + ninja && \ + ninja install + +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/:$LD_LIBRARY_PATH +ENV LD_LIBRARY_PATH=/usr/local/nixl/lib64/plugins:$LD_LIBRARY_PATH +ENV NIXL_PLUGIN_DIR=/usr/local/nixl/lib64/plugins + +RUN echo "/usr/local/nixl/lib/$ARCH-linux-gnu" > /etc/ld.so.conf.d/nixl.conf && \ + echo "/usr/local/nixl/lib/$ARCH-linux-gnu/plugins" >> /etc/ld.so.conf.d/nixl.conf && \ + ldconfig + +# Create the wheel +# No need to specifically add path to libcuda.so here, meson finds the stubs and links them +ARG WHL_PYTHON_VERSIONS="3.10,3.11,3.12,3.13,3.14" +ARG WHL_PLATFORM="manylinux_2_28_$ARCH" +RUN dnf install -y python3.11-devel python3.12-devel +RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \ + export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ + export UV_INDEX_STRATEGY=unsafe-best-match && \ + mkdir -p dist && \ + for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do \ + export PATH=$VIRTUAL_ENV/bin:$PATH && \ + ./contrib/build-wheel.sh \ + --python-version $PYTHON_VERSION \ + --platform $WHL_PLATFORM \ + --ucx-plugins-dir /usr/lib64/ucx \ + --nixl-plugins-dir $NIXL_PLUGIN_DIR \ + --output-dir dist || exit 1; \ + done + +# Copy the meta package wheel to the dist directory, which will be used to push to PyPI. +# Only do this for the CUDA 12 builds, since by default nixl depends on nixl-cu12. +RUN if [ "$(echo $CUDA_VERSION | cut -d. -f1)" = "12" ]; then \ + cp build/src/bindings/python/nixl-meta/nixl*.whl dist/; \ + fi + +RUN uv pip install dist/nixl*cp${DEFAULT_PYTHON_VERSION//./}*.whl build/src/bindings/python/nixl-meta/nixl*.whl From 120c6b6f4f36e28aef555730f1ad2b5b94886067 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 11 Mar 2026 02:10:15 +0100 Subject: [PATCH 11/15] Update rockylinux8 Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux8 | 57 +++++++++++++++++++++++++++++++--- contrib/create-wheels.sh | 4 +-- 2 files changed, 55 insertions(+), 6 deletions(-) diff --git a/contrib/Dockerfile.rockylinux8 b/contrib/Dockerfile.rockylinux8 index 7e49a65054..f757a066bf 100644 --- a/contrib/Dockerfile.rockylinux8 +++ b/contrib/Dockerfile.rockylinux8 @@ -23,6 +23,7 @@ ARG ARCH="x86_64" ARG UCX_REF="v1.20.x" ARG LIBFABRIC_VERSION="v1.21.0" ARG BUILD_TYPE="release" +ARG URING_TAG="liburing-2.13" RUN yum groupinstall -y 'Development Tools' && \ dnf config-manager --set-enabled powertools && \ @@ -55,8 +56,7 @@ RUN yum groupinstall -y 'Development Tools' && \ libibumad-devel \ numactl-devel \ librdmacm-devel \ - hwloc \ - hwloc-devel \ + pciutils-devel \ wget \ zlib @@ -64,6 +64,18 @@ ENV PATH=/opt/rh/gcc-toolset-11/root/usr/bin:$PATH ENV LD_LIBRARY_PATH=/opt/rh/gcc-toolset-11/root/usr/lib64:$LD_LIBRARY_PATH ENV PKG_CONFIG_PATH="/opt/rh/gcc-toolset-11/root/usr/lib64/pkgconfig:${PKG_CONFIG_PATH}" +# Build latest hwloc from source to avoid older RHEL8 API limitations. +RUN cd /tmp && \ + HWLOC_SERIES="$(echo ${HWLOC_VERSION} | cut -d. -f1,2)" && \ + wget -q "https://download.open-mpi.org/release/hwloc/v${HWLOC_SERIES}/hwloc-${HWLOC_VERSION}.tar.gz" && \ + tar -xzf "hwloc-${HWLOC_VERSION}.tar.gz" && \ + cd "hwloc-${HWLOC_VERSION}" && \ + ./configure --prefix=/usr/local --disable-nvml && \ + make -j"$(nproc)" && \ + make install && \ + ldconfig && \ + rm -rf "/tmp/hwloc-${HWLOC_VERSION}" "/tmp/hwloc-${HWLOC_VERSION}.tar.gz" + # Build OpenSSL 3.x RUN yum install -y perl-IPC-Cmd perl-Test-Simple perl-Data-Dumper RUN cd /tmp && \ @@ -119,24 +131,57 @@ RUN wget https://curl.se/download/curl-8.5.0.tar.gz && \ ./configure --prefix=/usr/local --with-ssl=/usr/local/openssl3 --enable-shared && \ make -j$(nproc) && make install -RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.581 +RUN git clone --recurse-submodules --depth 1 --shallow-submodules https://github.com/aws/aws-sdk-cpp.git --branch 1.11.760 RUN mkdir aws_sdk_build && cd aws_sdk_build && \ export LDFLAGS="-L/usr/local/openssl3/lib64 -L/usr/local/openssl3/lib" && \ export CFLAGS="-I/usr/local/openssl3/include" && \ export CXXFLAGS="-I/usr/local/openssl3/include" && \ - cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \ + cmake ../aws-sdk-cpp/ -DCMAKE_BUILD_TYPE=Release -DBUILD_ONLY="s3;s3-crt" -DENABLE_TESTING=OFF -DCMAKE_INSTALL_PREFIX=/usr/local \ -DCMAKE_PREFIX_PATH="/usr/local/openssl3;/usr/local" \ -DCURL_LIBRARY=/usr/local/lib/libcurl.so \ -DCURL_INCLUDE_DIR=/usr/local/include \ -DOPENSSL_USE_STATIC_LIBS=OFF && \ make -j${NPROC:-$(nproc)} && make install +RUN git clone https://github.com/axboe/liburing.git && \ + cd liburing && \ + git checkout $URING_TAG && \ + ./configure --cc=gcc --cxx=g++ && \ + make -j library && make liburing.pc && make install && \ + ldconfig && \ + cd .. + +ENV PKG_CONFIG_PATH=/usr/lib/pkgconfig:$PKG_CONFIG_PATH + RUN git clone https://github.com/nvidia/gusli.git && \ cd gusli && \ sed -i '/^LFLAGS_ALL =/{/-lrt/ b; s/$/ -lrt/}' Makefile && \ make all BUILD_RELEASE=1 BUILD_FOR_UNITEST=0 VERBOSE=1 ALLOW_USE_URING=0 && \ cd .. +# Required for Azure SDK +RUN git clone --depth 1 https://gitlab.gnome.org/GNOME/libxml2.git --branch 2.15 && \ + cd libxml2 && \ + ACLOCAL_PATH=/usr/share/aclocal ./autogen.sh && make -j && \ + make install + +RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ + cd azure-sdk-for-cpp/ && \ + mkdir build && cd build && \ + export LDFLAGS="-L/usr/local/openssl3/lib64 -L/usr/local/openssl3/lib" && \ + export CFLAGS="-isystem /usr/local/openssl3/include" && \ + export CXXFLAGS="-isystem /usr/local/openssl3/include" && \ + AZURE_SDK_DISABLE_AUTO_VCPKG=1 cmake .. -DCMAKE_BUILD_TYPE=Release -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DDISABLE_AMQP=ON -DDISABLE_AZURE_CORE_OPENTELEMETRY=ON \ + -DCMAKE_PREFIX_PATH="/usr/local/openssl3;/usr/local" \ + -DCURL_LIBRARY=/usr/local/lib/libcurl.so \ + -DCURL_INCLUDE_DIR=/usr/local/include \ + -DOPENSSL_USE_STATIC_LIBS=OFF && \ + cmake --build . --target azure-storage-blobs azure-identity && \ + cmake --install sdk/core && \ + cmake --install sdk/storage/azure-storage-common && \ + cmake --install sdk/storage/azure-storage-blobs && \ + cmake --install sdk/identity + COPY --from=ghcr.io/astral-sh/uv:latest /uv /uvx /bin/ ENV RUSTUP_HOME=/usr/local/rustup \ @@ -231,6 +276,10 @@ RUN wget --tries=3 --waitretry=5 --timeout=30 --read-timeout=60 \ make install && \ ldconfig +# Ensure pkg-config and linker can discover source-installed libs under /usr/local. +ENV PKG_CONFIG_PATH=/usr/local/lib64/pkgconfig:/usr/local/lib/pkgconfig:/usr/lib64/pkgconfig:/usr/lib/pkgconfig +RUN printf "/usr/local/lib64\n/usr/local/lib\n" > /etc/ld.so.conf.d/usr-local.conf && ldconfig + # By default, uv downloads python packages to $HOME/.cache/uv and hard links them # from the virtual environment. This means that the files reside in /root/.cache/uv, # which is not what we want since some systems mount user home dir into /root, diff --git a/contrib/create-wheels.sh b/contrib/create-wheels.sh index 81df5493ae..456c864b56 100755 --- a/contrib/create-wheels.sh +++ b/contrib/create-wheels.sh @@ -25,12 +25,12 @@ do tag="nixl-wheels-${cuda_version}" ./contrib/build-container.sh \ --base-image 'nvcr.io/nvidia/cuda' \ - --base-image-tag "${cuda_version}-devel-rockylinux9" \ + --base-image-tag "${cuda_version}-devel-rockylinux8" \ --wheel-base manylinux_2_34 \ --python-versions "${PYTHON_VERSIONS}" \ --tag $tag \ --arch $arch \ - --dockerfile contrib/Dockerfile.rockylinux9 + --dockerfile contrib/Dockerfile.rockylinux8 docker create --name temp-nixl $tag docker cp temp-nixl:/workspace/nixl/dist/ wheels/ # Move all .whl files from wheels/dist subdirectories at any depth to wheels/ From ab113ec049b7a41f3cba43783a8ef68ad2fdc868 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 11 Mar 2026 02:16:45 +0100 Subject: [PATCH 12/15] Merge fixes Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux8 | 3 ++- contrib/Dockerfile.rockylinux9 | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/contrib/Dockerfile.rockylinux8 b/contrib/Dockerfile.rockylinux8 index f757a066bf..2be3a58137 100644 --- a/contrib/Dockerfile.rockylinux8 +++ b/contrib/Dockerfile.rockylinux8 @@ -22,6 +22,7 @@ ARG DEFAULT_PYTHON_VERSION="3.14" ARG ARCH="x86_64" ARG UCX_REF="v1.20.x" ARG LIBFABRIC_VERSION="v1.21.0" +ARG HWLOC_VERSION="2.12.2" ARG BUILD_TYPE="release" ARG URING_TAG="liburing-2.13" @@ -330,7 +331,7 @@ RUN dnf install -y python3.11-devel python3.12-devel RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \ export UV_INDEX="https://download.pytorch.org/whl/cu$(echo $CUDA_VERSION | cut -d. -f1,2 | tr -d .)" && \ export UV_INDEX_STRATEGY=unsafe-best-match && \ - mkdir -p dist && \ + rm -rf dist && mkdir -p dist && \ for PYTHON_VERSION in "${PYTHON_VERSIONS[@]}"; do \ export PATH=$VIRTUAL_ENV/bin:$PATH && \ ./contrib/build-wheel.sh \ diff --git a/contrib/Dockerfile.rockylinux9 b/contrib/Dockerfile.rockylinux9 index b9055030d3..c303c90242 100644 --- a/contrib/Dockerfile.rockylinux9 +++ b/contrib/Dockerfile.rockylinux9 @@ -314,7 +314,7 @@ RUN IFS=',' read -ra PYTHON_VERSIONS <<< "$WHL_PYTHON_VERSIONS" && \ --platform $WHL_PLATFORM \ --ucx-plugins-dir /usr/lib64/ucx \ --nixl-plugins-dir $NIXL_PLUGIN_DIR \ - --output-dir dist ; \ + --output-dir dist || exit 1 ; \ done # Copy the meta package wheel to the dist directory, which will be used to push to PyPI. From cbb1d03aeef306c702f288392b0a9a720ec45784 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 11 Mar 2026 02:33:55 +0100 Subject: [PATCH 13/15] Fix libxml build Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux8 | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/contrib/Dockerfile.rockylinux8 b/contrib/Dockerfile.rockylinux8 index 2be3a58137..efd64fb4db 100644 --- a/contrib/Dockerfile.rockylinux8 +++ b/contrib/Dockerfile.rockylinux8 @@ -163,8 +163,12 @@ RUN git clone https://github.com/nvidia/gusli.git && \ # Required for Azure SDK RUN git clone --depth 1 https://gitlab.gnome.org/GNOME/libxml2.git --branch 2.15 && \ cd libxml2 && \ - ACLOCAL_PATH=/usr/share/aclocal ./autogen.sh && make -j && \ - make install + cmake -S . -B build \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=/usr/local \ + -DLIBXML2_WITH_PYTHON=OFF && \ + cmake --build build -j"$(nproc)" && \ + cmake --install build RUN git clone --depth 1 https://github.com/Azure/azure-sdk-for-cpp.git --branch azure-storage-blobs_12.15.0 && \ cd azure-sdk-for-cpp/ && \ From 1f899777c2675c38a0f56b61b3a47b3303415133 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 11 Mar 2026 02:52:30 +0100 Subject: [PATCH 14/15] Switch back to rocky 9 Signed-off-by: Ovidiu Mara --- contrib/create-wheels.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/create-wheels.sh b/contrib/create-wheels.sh index 456c864b56..81df5493ae 100755 --- a/contrib/create-wheels.sh +++ b/contrib/create-wheels.sh @@ -25,12 +25,12 @@ do tag="nixl-wheels-${cuda_version}" ./contrib/build-container.sh \ --base-image 'nvcr.io/nvidia/cuda' \ - --base-image-tag "${cuda_version}-devel-rockylinux8" \ + --base-image-tag "${cuda_version}-devel-rockylinux9" \ --wheel-base manylinux_2_34 \ --python-versions "${PYTHON_VERSIONS}" \ --tag $tag \ --arch $arch \ - --dockerfile contrib/Dockerfile.rockylinux8 + --dockerfile contrib/Dockerfile.rockylinux9 docker create --name temp-nixl $tag docker cp temp-nixl:/workspace/nixl/dist/ wheels/ # Move all .whl files from wheels/dist subdirectories at any depth to wheels/ From e1d52a9d8b172690b1d0a2497c90c99d2d6176dc Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 11 Mar 2026 02:55:14 +0100 Subject: [PATCH 15/15] Fix copyright Signed-off-by: Ovidiu Mara --- contrib/Dockerfile.rockylinux8 | 2 +- contrib/create-wheels.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/Dockerfile.rockylinux8 b/contrib/Dockerfile.rockylinux8 index efd64fb4db..52443dbd3b 100644 --- a/contrib/Dockerfile.rockylinux8 +++ b/contrib/Dockerfile.rockylinux8 @@ -1,4 +1,4 @@ -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # # Licensed under the Apache License, Version 2.0 (the "License"); diff --git a/contrib/create-wheels.sh b/contrib/create-wheels.sh index 81df5493ae..14c52f0ef2 100755 --- a/contrib/create-wheels.sh +++ b/contrib/create-wheels.sh @@ -1,5 +1,5 @@ #!/bin/bash -# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-FileCopyrightText: Copyright (c) 2025-2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. # SPDX-License-Identifier: Apache-2.0 # Exit on error and print commands