diff --git a/container/context.yaml b/container/context.yaml index a5dc8305843f..15c529186b7f 100644 --- a/container/context.yaml +++ b/container/context.yaml @@ -31,7 +31,8 @@ dynamo: nixl_ucx_ref: v1.20.x nixl_gdrcopy_ref: v2.5.2 nixl_ucx_efa_ref: 9d2b88a1f67faf9876f267658bd077b379b8bb76 - nixl_libfabric_ref: v2.3.0 + nixl_libfabric_repo: https://github.com/ofiwg/libfabric.git + nixl_libfabric_ref: v2.5.1 # Keep in sync with upstream NIXL's contrib/Dockerfile.manylinux. hwloc_version: 2.12.2 enable_kvbm: "true" diff --git a/container/templates/args.Dockerfile b/container/templates/args.Dockerfile index c48587c94ee1..4e21b5ba1ea2 100644 --- a/container/templates/args.Dockerfile +++ b/container/templates/args.Dockerfile @@ -71,6 +71,7 @@ ARG NIXL_REF={{ context[framework].nixl_ref }} {% endif -%} {% if device == "cuda" %} ARG NIXL_GDRCOPY_REF={{ context.dynamo.nixl_gdrcopy_ref }} +ARG NIXL_LIBFABRIC_REPO={{ context.dynamo.nixl_libfabric_repo }} ARG NIXL_LIBFABRIC_REF={{ context.dynamo.nixl_libfabric_ref }} ARG HWLOC_VERSION={{ context.dynamo.hwloc_version }} {% endif %} diff --git a/container/templates/aws.Dockerfile b/container/templates/aws.Dockerfile index 07b34d83c22c..1933c3cecdef 100644 --- a/container/templates/aws.Dockerfile +++ b/container/templates/aws.Dockerfile @@ -36,14 +36,68 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \ apt-get update && \ ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify && \ rm -rf /tmp/efa && \ - rm -rf /opt/amazon/aws-ofi-nccl && \ + # Disable the EFA installer's aws-ofi-nccl plugin: it crashes TRT-LLM at engine init. + # The plugin is installed at /opt/amazon/ofi-nccl (no `aws-` prefix), but ld.so picks + # it up via /etc/ld.so.conf.d/aws-ofi-nccl.conf (which DOES carry the `aws-` prefix). + # Remove both, and also the cuda-dl-base location /opt/amazon/aws-ofi-nccl if present, + # before re-running ldconfig. + rm -rf /opt/amazon/aws-ofi-nccl /opt/amazon/ofi-nccl \ + /etc/ld.so.conf.d/aws-ofi-nccl.conf && \ ldconfig ENV EFA_VERSION="${EFA_VERSION}" +ARG NIXL_LIBFABRIC_REF + +# Copy the wheel_builder-built libfabric and register it with the dynamic linker +# ONLY if the EFA-bundled libfabric is older than NIXL_LIBFABRIC_REF. +# When a future EFA installer ships libfabric >= the version we build, the +# version comparison evaluates to false and this becomes a no-op automatically. +RUN --mount=from=wheel_builder,source=/usr/local/libfabric,target=/tmp/libfabric_build \ + EFA_PC=$(find /opt/amazon/efa -path '*/pkgconfig/libfabric.pc' 2>/dev/null | head -n1) && \ + EFA_LIBFABRIC_RAW=$(cat "$EFA_PC" 2>/dev/null | grep '^Version:' | awk '{print $2}') && \ + EFA_LIBFABRIC_VER=$(echo "$EFA_LIBFABRIC_RAW" | grep -oE '^[0-9]+\.[0-9]+(\.[0-9]+)?') && \ + REF_VER=$(echo "${NIXL_LIBFABRIC_REF}" | sed 's/^v//') && \ + if [ -n "$EFA_LIBFABRIC_VER" ] && [ -n "$REF_VER" ] && \ + [ "$(printf '%s\n' "$EFA_LIBFABRIC_VER" "$REF_VER" | sort -V | head -n1)" = "$EFA_LIBFABRIC_VER" ] && \ + [ "$EFA_LIBFABRIC_VER" != "$REF_VER" ]; then \ + cp -Pf /tmp/libfabric_build/lib/libfabric.so* /opt/amazon/efa/lib/ && \ + if [ -d /opt/amazon/efa/lib64 ]; then \ + cp -Pf /tmp/libfabric_build/lib/libfabric.so* /opt/amazon/efa/lib64/; \ + fi && \ + cp -f /tmp/libfabric_build/bin/fi_info /opt/amazon/efa/bin/fi_info && \ + ldconfig && \ + echo "[aws] libfabric overlay: ${REF_VER} (overwrites EFA stock ${EFA_LIBFABRIC_RAW})"; \ + else \ + echo "[aws] libfabric overlay: skipped (EFA stock ${EFA_LIBFABRIC_RAW:-unknown} >= ${REF_VER})"; \ + fi + +{% if framework == "trtllm" %} +# After the upstream mesonpy refactor, libplugin_LIBFABRIC.so lands under the +# Dynamo venv while the rest of the NIXL plugin set (GDS/UCX/POSIX) remains at +# the canonical arch-specific location. Copy LIBFABRIC alongside the others so +# NIXL_PLUGIN_DIR resolves every backend from a single directory, and expose a +# stable arch-agnostic alias at /opt/nvidia/nvda_nixl/plugins. +# +# Also clear LD_PRELOAD (the upstream trtllm_runtime stage's ai-dynamo/nixl#1668 +# workaround force-loads TRT-LLM's bundled NIXL 0.9.0; that conflicts with the +# Dynamo-built NIXL 0.10.1 plugins). LIBFABRIC goes through libfabric directly +# (not UCX), so it is unaffected by the UCX 1.20.0 hang that LD_PRELOAD works +# around — and LIBFABRIC is the recommended backend for EFA. +RUN set -e && \ + arch_libdir=$(find /opt/nvidia/nvda_nixl/lib -maxdepth 1 -type d -name '*-linux-gnu' | head -1) && \ + [ -n "$arch_libdir" ] || { echo "ERROR: no arch-specific NIXL plugin dir under /opt/nvidia/nvda_nixl/lib" >&2; exit 1; } && \ + venv_lf=$(find /opt/dynamo/venv -path '*.nixl_cu13.mesonpy.libs/plugins/libplugin_LIBFABRIC.so' | head -1) && \ + [ -n "$venv_lf" ] || { echo "ERROR: no libplugin_LIBFABRIC.so under /opt/dynamo/venv" >&2; exit 1; } && \ + cp -Pf "$venv_lf" "$arch_libdir/plugins/" && \ + ln -sfT "$arch_libdir/plugins" /opt/nvidia/nvda_nixl/plugins && \ + [ -f /opt/nvidia/nvda_nixl/plugins/libplugin_LIBFABRIC.so ] || { echo "ERROR: LIBFABRIC plugin not visible via /opt/nvidia/nvda_nixl/plugins" >&2; ls -la /opt/nvidia/nvda_nixl/plugins/ >&2; exit 1; } && \ + echo "[aws] NIXL plugins consolidated under /opt/nvidia/nvda_nixl/plugins -> $arch_libdir/plugins" + +ENV LD_PRELOAD="" +ENV NIXL_PLUGIN_DIR=/opt/nvidia/nvda_nixl/plugins +{% endif %} + {% if target == "runtime" %} USER dynamo {% endif %} - -ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"] -CMD [] diff --git a/container/templates/wheel_builder.Dockerfile b/container/templates/wheel_builder.Dockerfile index 086ed05abdaf..19e07cbf64f9 100644 --- a/container/templates/wheel_builder.Dockerfile +++ b/container/templates/wheel_builder.Dockerfile @@ -116,10 +116,13 @@ RUN apt-get update -y \ {% if device == "cuda" %} # Install system dependencies # Cache dnf downloads; sharing=locked avoids dnf/rpm races with concurrent builds. +# --setopt=tsflags=nocontexts: skip SELinux file-context labeling. The manylinux +# image lacks the SELinux policy store that some compute nodes expect; without +# this flag, dnf fails with "ValueError: SELinux policy is not managed". RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked \ - dnf install -y almalinux-release-synergy && \ + dnf install -y --setopt=tsflags=nocontexts almalinux-release-synergy && \ dnf config-manager --set-enabled powertools && \ - dnf install -y \ + dnf install -y --setopt=tsflags=nocontexts \ # Autotools (required for UCX, libfabric ./autogen.sh and ./configure) autoconf \ automake \ @@ -266,7 +269,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token apt-get update -y && apt-get install -y build-essential pkg-config xz-utils; \ apt-get clean && rm -rf /var/lib/apt/lists/*; \ elif [ "$DEVICE" = "cuda" ]; then \ - dnf install -y pkg-config xz; \ + dnf install -y --setopt=tsflags=nocontexts pkg-config xz; \ fi && \ cd /tmp && \ curl --retry 5 --retry-delay 3 -LO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz && \ @@ -362,6 +365,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token ldconfig {% if device == "cuda" %} +ARG NIXL_LIBFABRIC_REPO ARG NIXL_LIBFABRIC_REF RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token \ --mount=type=secret,id=aws-role-arn,env=AWS_ROLE_ARN \ @@ -371,7 +375,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token eval $(/tmp/use-sccache.sh setup-env); \ fi && \ cd /usr/local/src && \ - git clone https://github.com/ofiwg/libfabric.git && \ + git clone "${NIXL_LIBFABRIC_REPO}" && \ cd libfabric && \ git checkout $NIXL_LIBFABRIC_REF && \ ./autogen.sh && \ @@ -391,6 +395,8 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token /tmp/use-sccache.sh show-stats "LIBFABRIC" && \ echo "/usr/local/libfabric/lib" > /etc/ld.so.conf.d/libfabric.conf && \ ldconfig + +ENV PKG_CONFIG_PATH="/usr/local/libfabric/lib/pkgconfig:${PKG_CONFIG_PATH}" {% endif %} {% if framework == "vllm" and device == "cuda" %}