From 6f1bb07e8852b141d5b4426d2f0e983e02cd6e8a Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Wed, 23 Jul 2025 17:58:43 +0200 Subject: [PATCH 1/6] Add option to set build type for nixlbench container, default to release Signed-off-by: Ovidiu Mara --- benchmark/nixlbench/contrib/Dockerfile | 5 +-- benchmark/nixlbench/contrib/build.sh | 46 ++++++++++++++++---------- 2 files changed, 31 insertions(+), 20 deletions(-) diff --git a/benchmark/nixlbench/contrib/Dockerfile b/benchmark/nixlbench/contrib/Dockerfile index c01c434892..5a6aededf5 100644 --- a/benchmark/nixlbench/contrib/Dockerfile +++ b/benchmark/nixlbench/contrib/Dockerfile @@ -102,6 +102,7 @@ ARG ARCH="x86_64" ARG DEFAULT_PYTHON_VERSION ARG WHL_PYTHON_VERSIONS="3.12" ARG WHL_PLATFORM="manylinux_2_39_$ARCH" +ARG BUILD_TYPE="release" WORKDIR /workspace RUN git clone https://github.com/etcd-cpp-apiv3/etcd-cpp-apiv3.git &&\ @@ -123,7 +124,7 @@ RUN uv venv $VIRTUAL_ENV --python $DEFAULT_PYTHON_VERSION && \ RUN rm -rf build && \ mkdir build && \ - uv run meson setup build/ --prefix=/usr/local/nixl && \ + uv run meson setup build/ --prefix=/usr/local/nixl --buildtype=$BUILD_TYPE && \ cd build && \ ninja && \ ninja install @@ -156,7 +157,7 @@ RUN ls -ll /workspace/nixlbench RUN rm -rf build && \ mkdir build && \ - uv run meson setup build -Dnixl_path=/usr/local/nixl/ -Dprefix=/usr/local/nixlbench && \ + uv run meson setup build -Dnixl_path=/usr/local/nixl/ -Dprefix=/usr/local/nixlbench --buildtype=$BUILD_TYPE && \ cd build && ninja && ninja install WORKDIR /workspace/nixl diff --git a/benchmark/nixlbench/contrib/build.sh b/benchmark/nixlbench/contrib/build.sh index 1bce3d86b9..a111e8f819 100755 --- a/benchmark/nixlbench/contrib/build.sh +++ b/benchmark/nixlbench/contrib/build.sh @@ -24,6 +24,7 @@ NIXL_BENCH_BUILD_CONTEXT_ARGS="--build-context nixlbench=$BUILD_CONTEXT/" DOCKER_FILE="${SOURCE_DIR}/Dockerfile" UCX_SRC="" UCX_BUILD_CONTEXT_ARGS="" +BUILD_TYPE="release" commit_id=$(git rev-parse --short HEAD) # Get latest TAG and add COMMIT_ID for dev @@ -35,12 +36,22 @@ fi BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base BASE_IMAGE_TAG=25.03-cuda12.8-devel-ubuntu24.04 -ARCH=$(uname -m) -[ "$ARCH" = "arm64" ] && ARCH="aarch64" -WHL_BASE=manylinux_2_39 -WHL_PLATFORM=${WHL_BASE}_${ARCH} WHL_PYTHON_VERSIONS="3.12" OS="ubuntu24" +ARCH=$(uname -m) +case "$ARCH" in + x86_64) + WHL_PLATFORM="manylinux_2_39_x86_64" + ;; + aarch64) + WHL_PLATFORM="manylinux_2_39_aarch64" + ;; + *) + echo "Unsupported architecture: $ARCH" + exit 1 + ;; +esac + get_options() { while :; do @@ -65,6 +76,14 @@ get_options() { missing_requirement $1 fi ;; + --build-type) + if [ "$2" ]; then + BUILD_TYPE=$2 + shift + else + missing_requirement $1 + fi + ;; --nixl) if [ "$2" ]; then NIXL_BUILD_CONTEXT_ARGS="--build-context nixl=$2" @@ -111,15 +130,6 @@ get_options() { missing_requirement $1 fi ;; - --arch) - if [ "$2" ]; then - ARCH=$2 - WHL_PLATFORM=${WHL_BASE}_${ARCH} - shift - else - missing_requirement $1 - fi - ;; --) shift break @@ -155,11 +165,11 @@ show_build_options() { echo "Building NIXLBench Image" echo "NIXL Source: ${NIXL_SRC}" echo "UCX Source: ${UCX_SRC} (optional)" + echo "Build Type: ${BUILD_TYPE}" echo "Image Tag: ${TAG}" echo "Build Context: ${BUILD_CONTEXT}" echo "Build Context Args: ${BUILD_CONTEXT_ARGS}" echo "Base Image: ${BASE_IMAGE}:${BASE_IMAGE_TAG}" - echo "Container arch: ${ARCH}" echo "Python Versions for wheel build: ${WHL_PYTHON_VERSIONS}" echo "Wheel Platform: ${WHL_PLATFORM}" } @@ -174,7 +184,6 @@ show_help() { echo " [--os [ubuntu24|ubuntu22] to select Ubuntu version]" echo " [--python-versions python versions to build for, comma separated]" echo " [--tag tag for image]" - echo " [--arch [x86_64|aarch64] to select target architecture]" exit 0 } @@ -192,8 +201,9 @@ get_options "$@" BUILD_ARGS+=" --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg BASE_IMAGE_TAG=$BASE_IMAGE_TAG" BUILD_ARGS+=" --build-arg WHL_PYTHON_VERSIONS=$WHL_PYTHON_VERSIONS" BUILD_ARGS+=" --build-arg WHL_PLATFORM=$WHL_PLATFORM" -BUILD_ARGS+=" --build-arg ARCH=$ARCH" +BUILD_ARGS+=" --build-arg TARGETARCH=$ARCH" +BUILD_ARGS+=" --build-arg BUILD_TYPE=$BUILD_TYPE" show_build_options - -docker build --platform linux/$ARCH -f $DOCKER_FILE $BUILD_ARGS $TAG $NO_CACHE $BUILD_CONTEXT_ARGS $BUILD_CONTEXT --progress plain +set -x +docker build -f $DOCKER_FILE $BUILD_ARGS $TAG $NO_CACHE $BUILD_ARGS $BUILD_CONTEXT_ARGS $BUILD_CONTEXT --progress plain From 703511191ab449a7305382cf70816e65eec666be Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Mon, 28 Jul 2025 14:33:26 +0200 Subject: [PATCH 2/6] Add --build-type to help Signed-off-by: Ovidiu Mara --- benchmark/nixlbench/contrib/build.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/benchmark/nixlbench/contrib/build.sh b/benchmark/nixlbench/contrib/build.sh index a111e8f819..72ada5ae3e 100755 --- a/benchmark/nixlbench/contrib/build.sh +++ b/benchmark/nixlbench/contrib/build.sh @@ -180,6 +180,7 @@ show_help() { echo " [--base-image-tag base image tag]" echo " [--nixlbench path/to/nixlbench/source/dir]" echo " [--ucx path/to/ucx/source/dir]" + echo " [--build-type [debug|release]]" echo " [--no-cache disable docker build cache]" echo " [--os [ubuntu24|ubuntu22] to select Ubuntu version]" echo " [--python-versions python versions to build for, comma separated]" From cb0126849f20cb070d7af7742cb79e682a0724f7 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Sat, 2 Aug 2025 14:23:08 +0200 Subject: [PATCH 3/6] Cleanup build script Signed-off-by: Ovidiu Mara --- benchmark/nixlbench/contrib/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/nixlbench/contrib/build.sh b/benchmark/nixlbench/contrib/build.sh index 72ada5ae3e..7f97d29620 100755 --- a/benchmark/nixlbench/contrib/build.sh +++ b/benchmark/nixlbench/contrib/build.sh @@ -207,4 +207,4 @@ BUILD_ARGS+=" --build-arg BUILD_TYPE=$BUILD_TYPE" show_build_options set -x -docker build -f $DOCKER_FILE $BUILD_ARGS $TAG $NO_CACHE $BUILD_ARGS $BUILD_CONTEXT_ARGS $BUILD_CONTEXT --progress plain +docker build -f $DOCKER_FILE $BUILD_ARGS $TAG $NO_CACHE $BUILD_CONTEXT_ARGS $BUILD_CONTEXT From ba8e85eaaf80333374fee8f8fceb0a01db2950f2 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Sun, 3 Aug 2025 10:09:49 +0200 Subject: [PATCH 4/6] Revert unrelated changes Signed-off-by: Ovidiu Mara --- benchmark/nixlbench/contrib/build.sh | 37 ++++++++++++++-------------- 1 file changed, 19 insertions(+), 18 deletions(-) diff --git a/benchmark/nixlbench/contrib/build.sh b/benchmark/nixlbench/contrib/build.sh index 7f97d29620..62d269def9 100755 --- a/benchmark/nixlbench/contrib/build.sh +++ b/benchmark/nixlbench/contrib/build.sh @@ -36,22 +36,12 @@ fi BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base BASE_IMAGE_TAG=25.03-cuda12.8-devel-ubuntu24.04 +ARCH=$(uname -m) +[ "$ARCH" = "arm64" ] && ARCH="aarch64" +WHL_BASE=manylinux_2_39 +WHL_PLATFORM=${WHL_BASE}_${ARCH} WHL_PYTHON_VERSIONS="3.12" OS="ubuntu24" -ARCH=$(uname -m) -case "$ARCH" in - x86_64) - WHL_PLATFORM="manylinux_2_39_x86_64" - ;; - aarch64) - WHL_PLATFORM="manylinux_2_39_aarch64" - ;; - *) - echo "Unsupported architecture: $ARCH" - exit 1 - ;; -esac - get_options() { while :; do @@ -130,6 +120,15 @@ get_options() { missing_requirement $1 fi ;; + --arch) + if [ "$2" ]; then + ARCH=$2 + WHL_PLATFORM=${WHL_BASE}_${ARCH} + shift + else + missing_requirement $1 + fi + ;; --) shift break @@ -170,6 +169,7 @@ show_build_options() { echo "Build Context: ${BUILD_CONTEXT}" echo "Build Context Args: ${BUILD_CONTEXT_ARGS}" echo "Base Image: ${BASE_IMAGE}:${BASE_IMAGE_TAG}" + echo "Container arch: ${ARCH}" echo "Python Versions for wheel build: ${WHL_PYTHON_VERSIONS}" echo "Wheel Platform: ${WHL_PLATFORM}" } @@ -180,11 +180,12 @@ show_help() { echo " [--base-image-tag base image tag]" echo " [--nixlbench path/to/nixlbench/source/dir]" echo " [--ucx path/to/ucx/source/dir]" - echo " [--build-type [debug|release]]" + echo " [--build-type [debug|release] to select build type]" echo " [--no-cache disable docker build cache]" echo " [--os [ubuntu24|ubuntu22] to select Ubuntu version]" echo " [--python-versions python versions to build for, comma separated]" echo " [--tag tag for image]" + echo " [--arch [x86_64|aarch64] to select target architecture]" exit 0 } @@ -202,9 +203,9 @@ get_options "$@" BUILD_ARGS+=" --build-arg BASE_IMAGE=$BASE_IMAGE --build-arg BASE_IMAGE_TAG=$BASE_IMAGE_TAG" BUILD_ARGS+=" --build-arg WHL_PYTHON_VERSIONS=$WHL_PYTHON_VERSIONS" BUILD_ARGS+=" --build-arg WHL_PLATFORM=$WHL_PLATFORM" -BUILD_ARGS+=" --build-arg TARGETARCH=$ARCH" +BUILD_ARGS+=" --build-arg ARCH=$ARCH" BUILD_ARGS+=" --build-arg BUILD_TYPE=$BUILD_TYPE" show_build_options -set -x -docker build -f $DOCKER_FILE $BUILD_ARGS $TAG $NO_CACHE $BUILD_CONTEXT_ARGS $BUILD_CONTEXT + +docker build --platform linux/$ARCH -f $DOCKER_FILE $BUILD_ARGS $TAG $NO_CACHE $BUILD_CONTEXT_ARGS $BUILD_CONTEXT --progress plain From a686342375cba732ab044a42bfdbc90c0e82beff Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Tue, 5 Aug 2025 14:49:14 +0200 Subject: [PATCH 5/6] Nixlbench: also build ucx in debug mode if nixl and nixlbench are built in debug mode Signed-off-by: Ovidiu Mara --- benchmark/nixlbench/contrib/Dockerfile | 3 +++ 1 file changed, 3 insertions(+) diff --git a/benchmark/nixlbench/contrib/Dockerfile b/benchmark/nixlbench/contrib/Dockerfile index beb6f49d98..d4e6b98a9f 100644 --- a/benchmark/nixlbench/contrib/Dockerfile +++ b/benchmark/nixlbench/contrib/Dockerfile @@ -47,6 +47,7 @@ RUN apt-get update -y && \ libtool \ libz-dev \ flex \ + valgrind \ libgtest-dev \ build-essential @@ -74,6 +75,7 @@ RUN echo "INFO: Using UCX from base image (UCX=${UCX})." # --- Stage 2b: Represents building UCX from source --- FROM os_setup_stage AS ucx_custom_image +ARG BUILD_TYPE="release" ARG NPROC RUN mkdir -p /workspace/ucx COPY --from=ucx . /workspace/ucx @@ -90,6 +92,7 @@ RUN echo "INFO: Starting custom UCX build..." && \ ./autogen.sh && \ echo "INFO: Building UCX..." && \ ./contrib/configure-release --with-cuda=/usr/local/cuda \ + $(if [ "$BUILD_TYPE" = "debug" ]; then echo "--enable-debug"; fi) \ --enable-mt \ --without-go && \ make -j${NPROC:-$(nproc)} && \ From 6102b66b0af3a5e546458f65fb9ace5598d67b10 Mon Sep 17 00:00:00 2001 From: Ovidiu Mara Date: Mon, 11 Aug 2025 18:50:26 +0200 Subject: [PATCH 6/6] Remove valgrind Signed-off-by: Ovidiu Mara --- benchmark/nixlbench/contrib/Dockerfile | 1 - 1 file changed, 1 deletion(-) diff --git a/benchmark/nixlbench/contrib/Dockerfile b/benchmark/nixlbench/contrib/Dockerfile index d4e6b98a9f..af866185a3 100644 --- a/benchmark/nixlbench/contrib/Dockerfile +++ b/benchmark/nixlbench/contrib/Dockerfile @@ -47,7 +47,6 @@ RUN apt-get update -y && \ libtool \ libz-dev \ flex \ - valgrind \ libgtest-dev \ build-essential