From f20263fd5f78f221f848c4c36391d96d35785c86 Mon Sep 17 00:00:00 2001 From: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Date: Fri, 22 May 2026 11:09:24 -0700 Subject: [PATCH 1/3] build(container): bump nixl_gdrcopy_ref to v2.5.2 (kernel >=6.15 fix) (#9705) Signed-off-by: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Signed-off-by: Jie Hao --- container/context.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/container/context.yaml b/container/context.yaml index ceb995a585c4..a23bfaff3577 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -29,7 +29,7 @@ dynamo: nixl_ref: 0.10.1 nixl_ucx_ref: v1.20.x - nixl_gdrcopy_ref: v2.5.1 + nixl_gdrcopy_ref: v2.5.2 nixl_ucx_efa_ref: 9d2b88a1f67faf9876f267658bd077b379b8bb76 nixl_libfabric_ref: v2.3.0 enable_kvbm: "true" From 0bc5f5ff7dbb5a6679b8cbbac9103d0d841ab16d Mon Sep 17 00:00:00 2001 From: Yifan Jiang <19356972+yifjiang@users.noreply.github.com> Date: Tue, 26 May 2026 13:43:22 -0700 Subject: [PATCH 2/3] feat(container): configurable libfabric repo + v2.5.1 overlay for EFA (#9727) Signed-off-by: Yifan Jiang Co-authored-by: Yifan Jiang Signed-off-by: Jie Hao --- container/context.yaml | 3 +- container/templates/args.Dockerfile | 1 + container/templates/aws.Dockerfile | 62 ++++++++++++++++++-- container/templates/wheel_builder.Dockerfile | 14 +++-- 4 files changed, 71 insertions(+), 9 deletions(-) diff --git a/container/context.yaml b/container/context.yaml index a23bfaff3577..f89a0c6f616f 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -31,7 +31,8 @@ dynamo: nixl_ucx_ref: v1.20.x nixl_gdrcopy_ref: v2.5.2 nixl_ucx_efa_ref: 9d2b88a1f67faf9876f267658bd077b379b8bb76 - nixl_libfabric_ref: v2.3.0 + nixl_libfabric_repo: https://github.com/ofiwg/libfabric.git + nixl_libfabric_ref: v2.5.1 enable_kvbm: "true" enable_media_ffmpeg: "false" enable_gpu_memory_service: "true" diff --git a/container/templates/args.Dockerfile b/container/templates/args.Dockerfile index d3c217d3baf7..2ddc48f1de23 100644 --- a/container/templates/args.Dockerfile +++ b/container/templates/args.Dockerfile @@ -74,6 +74,7 @@ ARG NIXL_REF={{ context[framework].nixl_ref }} {% endif -%} {% if device == "cuda" %} ARG NIXL_GDRCOPY_REF={{ context.dynamo.nixl_gdrcopy_ref }} +ARG NIXL_LIBFABRIC_REPO={{ context.dynamo.nixl_libfabric_repo }} ARG NIXL_LIBFABRIC_REF={{ context.dynamo.nixl_libfabric_ref }} {% endif %} diff --git a/container/templates/aws.Dockerfile b/container/templates/aws.Dockerfile index 07b34d83c22c..1933c3cecdef 100644 --- a/container/templates/aws.Dockerfile +++ b/container/templates/aws.Dockerfile @@ -36,14 +36,68 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update && \ ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify && \ rm -rf /tmp/efa && \ - rm -rf /opt/amazon/aws-ofi-nccl && \ + # Disable the EFA installer's aws-ofi-nccl plugin: it crashes TRT-LLM at engine init. + # The plugin is installed at /opt/amazon/ofi-nccl (no `aws-` prefix), but ld.so picks + # it up via /etc/ld.so.conf.d/aws-ofi-nccl.conf (which DOES carry the `aws-` prefix). + # Remove both, and also the cuda-dl-base location /opt/amazon/aws-ofi-nccl if present, + # before re-running ldconfig. + rm -rf /opt/amazon/aws-ofi-nccl /opt/amazon/ofi-nccl \ + /etc/ld.so.conf.d/aws-ofi-nccl.conf && \ ldconfig ENV EFA_VERSION="${EFA_VERSION}" +ARG NIXL_LIBFABRIC_REF + +# Copy the wheel_builder-built libfabric and register it with the dynamic linker +# ONLY if the EFA-bundled libfabric is older than NIXL_LIBFABRIC_REF. +# When a future EFA installer ships libfabric >= the version we build, the +# version comparison evaluates to false and this becomes a no-op automatically. +RUN --mount=from=wheel_builder,source=/usr/local/libfabric,target=/tmp/libfabric_build \ + EFA_PC=$(find /opt/amazon/efa -path '*/pkgconfig/libfabric.pc' 2>/dev/null | head -n1) && \ + EFA_LIBFABRIC_RAW=$(cat "$EFA_PC" 2>/dev/null | grep '^Version:' | awk '{print $2}') && \ + EFA_LIBFABRIC_VER=$(echo "$EFA_LIBFABRIC_RAW" | grep -oE '^[0-9]+\.[0-9]+(\.[0-9]+)?') && \ + REF_VER=$(echo "${NIXL_LIBFABRIC_REF}" | sed 's/^v//') && \ + if [ -n "$EFA_LIBFABRIC_VER" ] && [ -n "$REF_VER" ] && \ + [ "$(printf '%s\n' "$EFA_LIBFABRIC_VER" "$REF_VER" | sort -V | head -n1)" = "$EFA_LIBFABRIC_VER" ] && \ + [ "$EFA_LIBFABRIC_VER" != "$REF_VER" ]; then \ + cp -Pf /tmp/libfabric_build/lib/libfabric.so* /opt/amazon/efa/lib/ && \ + if [ -d /opt/amazon/efa/lib64 ]; then \ + cp -Pf /tmp/libfabric_build/lib/libfabric.so* /opt/amazon/efa/lib64/; \ + fi && \ + cp -f /tmp/libfabric_build/bin/fi_info /opt/amazon/efa/bin/fi_info && \ + ldconfig && \ + echo "[aws] libfabric overlay: ${REF_VER} (overwrites EFA stock ${EFA_LIBFABRIC_RAW})"; \ + else \ + echo "[aws] libfabric overlay: skipped (EFA stock ${EFA_LIBFABRIC_RAW:-unknown} >= ${REF_VER})"; \ + fi + +{% if framework == "trtllm" %} +# After the upstream mesonpy refactor, libplugin_LIBFABRIC.so lands under the +# Dynamo venv while the rest of the NIXL plugin set (GDS/UCX/POSIX) remains at +# the canonical arch-specific location. Copy LIBFABRIC alongside the others so +# NIXL_PLUGIN_DIR resolves every backend from a single directory, and expose a +# stable arch-agnostic alias at /opt/nvidia/nvda_nixl/plugins. +# +# Also clear LD_PRELOAD (the upstream trtllm_runtime stage's ai-dynamo/nixl#1668 +# workaround force-loads TRT-LLM's bundled NIXL 0.9.0; that conflicts with the +# Dynamo-built NIXL 0.10.1 plugins). LIBFABRIC goes through libfabric directly +# (not UCX), so it is unaffected by the UCX 1.20.0 hang that LD_PRELOAD works +# around — and LIBFABRIC is the recommended backend for EFA. +RUN set -e && \ + arch_libdir=$(find /opt/nvidia/nvda_nixl/lib -maxdepth 1 -type d -name '*-linux-gnu' | head -1) && \ + [ -n "$arch_libdir" ] || { echo "ERROR: no arch-specific NIXL plugin dir under /opt/nvidia/nvda_nixl/lib" >&2; exit 1; } && \ + venv_lf=$(find /opt/dynamo/venv -path '*.nixl_cu13.mesonpy.libs/plugins/libplugin_LIBFABRIC.so' | head -1) && \ + [ -n "$venv_lf" ] || { echo "ERROR: no libplugin_LIBFABRIC.so under /opt/dynamo/venv" >&2; exit 1; } && \ + cp -Pf "$venv_lf" "$arch_libdir/plugins/" && \ + ln -sfT "$arch_libdir/plugins" /opt/nvidia/nvda_nixl/plugins && \ + [ -f /opt/nvidia/nvda_nixl/plugins/libplugin_LIBFABRIC.so ] || { echo "ERROR: LIBFABRIC plugin not visible via /opt/nvidia/nvda_nixl/plugins" >&2; ls -la /opt/nvidia/nvda_nixl/plugins/ >&2; exit 1; } && \ + echo "[aws] NIXL plugins consolidated under /opt/nvidia/nvda_nixl/plugins -> $arch_libdir/plugins" + +ENV LD_PRELOAD="" +ENV NIXL_PLUGIN_DIR=/opt/nvidia/nvda_nixl/plugins +{% endif %} + {% if target == "runtime" %} USER dynamo {% endif %} - -ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] -CMD [] diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 03eca70f26ad..2a8d25373fe9 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -116,10 +116,13 @@ RUN apt-get update -y \ {% if device == "cuda" %} # Install system dependencies # Cache dnf downloads; sharing=locked avoids dnf/rpm races with concurrent builds. +# --setopt=tsflags=nocontexts: skip SELinux file-context labeling. The manylinux +# image lacks the SELinux policy store that some compute nodes expect; without +# this flag, dnf fails with "ValueError: SELinux policy is not managed". RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked \ - dnf install -y almalinux-release-synergy && \ + dnf install -y --setopt=tsflags=nocontexts almalinux-release-synergy && \ dnf config-manager --set-enabled powertools && \ - dnf install -y \ + dnf install -y --setopt=tsflags=nocontexts \ # Autotools (required for UCX, libfabric ./autogen.sh and ./configure) autoconf \ automake \ @@ -273,7 +276,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token apt-get update -y && apt-get install -y build-essential pkg-config xz-utils git yasm; \ apt-get clean && rm -rf /var/lib/apt/lists/*; \ elif [ "$DEVICE" = "cuda" ]; then \ - dnf install -y pkg-config xz git yasm; \ + dnf install -y --setopt=tsflags=nocontexts pkg-config xz git yasm; \ fi && \ # nv-codec-headers: provides the NVENC/NVDEC API headers ffmpeg compiles against. # Header-only, no runtime dep here; libcuda/libnvidia-encode are loaded at runtime @@ -388,6 +391,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token ldconfig {% if device == "cuda" %} +ARG NIXL_LIBFABRIC_REPO ARG NIXL_LIBFABRIC_REF RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token \ --mount=type=secret,id=aws-role-arn,env=AWS_ROLE_ARN \ @@ -397,7 +401,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token eval $(/tmp/use-sccache.sh setup-env); \ fi && \ cd /usr/local/src && \ - git clone https://github.com/ofiwg/libfabric.git && \ + git clone "${NIXL_LIBFABRIC_REPO}" && \ cd libfabric && \ git checkout $NIXL_LIBFABRIC_REF && \ ./autogen.sh && \ @@ -417,6 +421,8 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token /tmp/use-sccache.sh show-stats "LIBFABRIC" && \ echo "/usr/local/libfabric/lib" > /etc/ld.so.conf.d/libfabric.conf && \ ldconfig + +ENV PKG_CONFIG_PATH="/usr/local/libfabric/lib/pkgconfig:${PKG_CONFIG_PATH}" {% endif %} {% if framework == "vllm" and device == "cuda" %} From 6a3b62a671e8124b3a2a37e9e076efa47ceedea1 Mon Sep 17 00:00:00 2001 From: Erez Zarum Date: Thu, 28 May 2026 07:37:25 +0300 Subject: [PATCH 3/3] feat(container): libfabric2.5.1 for EFA and TRTLLM (#10047) Signed-off-by: Erez Zarum Signed-off-by: Jie Hao --- container/templates/aws.Dockerfile | 37 +++++++++++------------------- 1 file changed, 14 insertions(+), 23 deletions(-) diff --git a/container/templates/aws.Dockerfile b/container/templates/aws.Dockerfile index 1933c3cecdef..6132c787dd97 100644 --- a/container/templates/aws.Dockerfile +++ b/container/templates/aws.Dockerfile @@ -36,13 +36,7 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update && \ ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify && \ rm -rf /tmp/efa && \ - # Disable the EFA installer's aws-ofi-nccl plugin: it crashes TRT-LLM at engine init. - # The plugin is installed at /opt/amazon/ofi-nccl (no `aws-` prefix), but ld.so picks - # it up via /etc/ld.so.conf.d/aws-ofi-nccl.conf (which DOES carry the `aws-` prefix). - # Remove both, and also the cuda-dl-base location /opt/amazon/aws-ofi-nccl if present, - # before re-running ldconfig. - rm -rf /opt/amazon/aws-ofi-nccl /opt/amazon/ofi-nccl \ - /etc/ld.so.conf.d/aws-ofi-nccl.conf && \ + rm -rf /opt/amazon/aws-ofi-nccl /etc/ld.so.conf.d/aws-ofi-nccl.conf && \ ldconfig ENV EFA_VERSION="${EFA_VERSION}" @@ -61,11 +55,11 @@ RUN --mount=from=wheel_builder,source=/usr/local/libfabric,target=/tmp/libfabric if [ -n "$EFA_LIBFABRIC_VER" ] && [ -n "$REF_VER" ] && \ [ "$(printf '%s\n' "$EFA_LIBFABRIC_VER" "$REF_VER" | sort -V | head -n1)" = "$EFA_LIBFABRIC_VER" ] && \ [ "$EFA_LIBFABRIC_VER" != "$REF_VER" ]; then \ - cp -Pf /tmp/libfabric_build/lib/libfabric.so* /opt/amazon/efa/lib/ && \ - if [ -d /opt/amazon/efa/lib64 ]; then \ - cp -Pf /tmp/libfabric_build/lib/libfabric.so* /opt/amazon/efa/lib64/; \ - fi && \ - cp -f /tmp/libfabric_build/bin/fi_info /opt/amazon/efa/bin/fi_info && \ + rm -rf /opt/amazon/efa && \ + cp -Pfr /tmp/libfabric_build /opt/amazon/efa && \ + sed -i 's|^prefix=.*|prefix=/opt/amazon/efa|' /opt/amazon/efa/lib/pkgconfig/libfabric.pc && \ + echo "/opt/amazon/efa/lib" > /etc/ld.so.conf.d/000_efa.conf && \ + rm -f /etc/ld.so.conf.d/efa.conf && \ ldconfig && \ echo "[aws] libfabric overlay: ${REF_VER} (overwrites EFA stock ${EFA_LIBFABRIC_RAW})"; \ else \ @@ -84,18 +78,15 @@ RUN --mount=from=wheel_builder,source=/usr/local/libfabric,target=/tmp/libfabric # Dynamo-built NIXL 0.10.1 plugins). LIBFABRIC goes through libfabric directly # (not UCX), so it is unaffected by the UCX 1.20.0 hang that LD_PRELOAD works # around — and LIBFABRIC is the recommended backend for EFA. -RUN set -e && \ - arch_libdir=$(find /opt/nvidia/nvda_nixl/lib -maxdepth 1 -type d -name '*-linux-gnu' | head -1) && \ - [ -n "$arch_libdir" ] || { echo "ERROR: no arch-specific NIXL plugin dir under /opt/nvidia/nvda_nixl/lib" >&2; exit 1; } && \ - venv_lf=$(find /opt/dynamo/venv -path '*.nixl_cu13.mesonpy.libs/plugins/libplugin_LIBFABRIC.so' | head -1) && \ - [ -n "$venv_lf" ] || { echo "ERROR: no libplugin_LIBFABRIC.so under /opt/dynamo/venv" >&2; exit 1; } && \ - cp -Pf "$venv_lf" "$arch_libdir/plugins/" && \ - ln -sfT "$arch_libdir/plugins" /opt/nvidia/nvda_nixl/plugins && \ - [ -f /opt/nvidia/nvda_nixl/plugins/libplugin_LIBFABRIC.so ] || { echo "ERROR: LIBFABRIC plugin not visible via /opt/nvidia/nvda_nixl/plugins" >&2; ls -la /opt/nvidia/nvda_nixl/plugins/ >&2; exit 1; } && \ - echo "[aws] NIXL plugins consolidated under /opt/nvidia/nvda_nixl/plugins -> $arch_libdir/plugins" +RUN --mount=from=wheel_builder,source=/opt/nvidia/nvda_nixl,target=/tmp/nvda_nixl \ + rm -rf /opt/nvidia/nvda_nixl && \ + cp -Pfr /tmp/nvda_nixl /opt/nvidia/nvda_nixl && \ + export LD_PRELOAD=/opt/nvidia/nvda_nixl/lib64/libnixl.so && \ + export NIXL_PLUGIN_DIR=/opt/nvidia/nvda_nixl/lib64/plugins && \ + ldconfig -ENV LD_PRELOAD="" -ENV NIXL_PLUGIN_DIR=/opt/nvidia/nvda_nixl/plugins +ENV LD_PRELOAD=/opt/nvidia/nvda_nixl/lib64/libnixl.so +ENV NIXL_PLUGIN_DIR=/opt/nvidia/nvda_nixl/lib64/plugins {% endif %} {% if target == "runtime" %}