feat(container): configurable libfabric repo + v2.5.1 overlay for EFA

container/context.yaml

@@ -31,7 +31,8 @@ dynamo:
   nixl_ucx_ref: v1.20.x
   nixl_gdrcopy_ref: v2.5.2
   nixl_ucx_efa_ref: 9d2b88a1f67faf9876f267658bd077b379b8bb76
-  nixl_libfabric_ref: v2.3.0
+  nixl_libfabric_repo: https://github.com/ofiwg/libfabric.git
+  nixl_libfabric_ref: v2.5.1
   # Keep in sync with upstream NIXL's contrib/Dockerfile.manylinux.
   hwloc_version: 2.12.2
   enable_kvbm: "true"

container/templates/args.Dockerfile

@@ -71,6 +71,7 @@ ARG NIXL_REF={{ context[framework].nixl_ref }}
 {% endif -%}
 {% if device == "cuda" %}
 ARG NIXL_GDRCOPY_REF={{ context.dynamo.nixl_gdrcopy_ref }}
+ARG NIXL_LIBFABRIC_REPO={{ context.dynamo.nixl_libfabric_repo }}
 ARG NIXL_LIBFABRIC_REF={{ context.dynamo.nixl_libfabric_ref }}
 ARG HWLOC_VERSION={{ context.dynamo.hwloc_version }}
 {% endif %}

container/templates/aws.Dockerfile

@@ -36,14 +36,68 @@ RUN --mount=type=cache,target=/var/cache/apt,sharing=locked \
     apt-get update && \
     ./efa_installer.sh -y --skip-kmod --skip-limit-conf --no-verify && \
     rm -rf /tmp/efa && \
-    rm -rf /opt/amazon/aws-ofi-nccl && \
+    # Disable the EFA installer's aws-ofi-nccl plugin: it crashes TRT-LLM at engine init.
+    # The plugin is installed at /opt/amazon/ofi-nccl (no `aws-` prefix), but ld.so picks
+    # it up via /etc/ld.so.conf.d/aws-ofi-nccl.conf (which DOES carry the `aws-` prefix).
+    # Remove both, and also the cuda-dl-base location /opt/amazon/aws-ofi-nccl if present,
+    # before re-running ldconfig.
+    rm -rf /opt/amazon/aws-ofi-nccl /opt/amazon/ofi-nccl \
+           /etc/ld.so.conf.d/aws-ofi-nccl.conf && \
     ldconfig
 
 ENV EFA_VERSION="${EFA_VERSION}"
 
+ARG NIXL_LIBFABRIC_REF
+
+# Copy the wheel_builder-built libfabric and register it with the dynamic linker
+# ONLY if the EFA-bundled libfabric is older than NIXL_LIBFABRIC_REF.
+# When a future EFA installer ships libfabric >= the version we build, the
+# version comparison evaluates to false and this becomes a no-op automatically.
+RUN --mount=from=wheel_builder,source=/usr/local/libfabric,target=/tmp/libfabric_build \
+    EFA_PC=$(find /opt/amazon/efa -path '*/pkgconfig/libfabric.pc' 2>/dev/null | head -n1) && \
+    EFA_LIBFABRIC_RAW=$(cat "$EFA_PC" 2>/dev/null | grep '^Version:' | awk '{print $2}') && \
+    EFA_LIBFABRIC_VER=$(echo "$EFA_LIBFABRIC_RAW" | grep -oE '^[0-9]+\.[0-9]+(\.[0-9]+)?') && \
+    REF_VER=$(echo "${NIXL_LIBFABRIC_REF}" | sed 's/^v//') && \
+    if [ -n "$EFA_LIBFABRIC_VER" ] && [ -n "$REF_VER" ] && \
+       [ "$(printf '%s\n' "$EFA_LIBFABRIC_VER" "$REF_VER" | sort -V | head -n1)" = "$EFA_LIBFABRIC_VER" ] && \
+       [ "$EFA_LIBFABRIC_VER" != "$REF_VER" ]; then \
+        cp -Pf /tmp/libfabric_build/lib/libfabric.so* /opt/amazon/efa/lib/ && \
+        if [ -d /opt/amazon/efa/lib64 ]; then \
+            cp -Pf /tmp/libfabric_build/lib/libfabric.so* /opt/amazon/efa/lib64/; \
+        fi && \
+        cp -f /tmp/libfabric_build/bin/fi_info /opt/amazon/efa/bin/fi_info && \
+        ldconfig && \
+        echo "[aws] libfabric overlay: ${REF_VER} (overwrites EFA stock ${EFA_LIBFABRIC_RAW})"; \
+    else \
+        echo "[aws] libfabric overlay: skipped (EFA stock ${EFA_LIBFABRIC_RAW:-unknown} >= ${REF_VER})"; \
+    fi
+
+{% if framework == "trtllm" %}
+# After the upstream mesonpy refactor, libplugin_LIBFABRIC.so lands under the
+# Dynamo venv while the rest of the NIXL plugin set (GDS/UCX/POSIX) remains at
+# the canonical arch-specific location. Copy LIBFABRIC alongside the others so
+# NIXL_PLUGIN_DIR resolves every backend from a single directory, and expose a
+# stable arch-agnostic alias at /opt/nvidia/nvda_nixl/plugins.
+#
+# Also clear LD_PRELOAD (the upstream trtllm_runtime stage's ai-dynamo/nixl#1668
+# workaround force-loads TRT-LLM's bundled NIXL 0.9.0; that conflicts with the
+# Dynamo-built NIXL 0.10.1 plugins). LIBFABRIC goes through libfabric directly
+# (not UCX), so it is unaffected by the UCX 1.20.0 hang that LD_PRELOAD works
+# around — and LIBFABRIC is the recommended backend for EFA.
+RUN set -e && \
+    arch_libdir=$(find /opt/nvidia/nvda_nixl/lib -maxdepth 1 -type d -name '*-linux-gnu' | head -1) && \
+    [ -n "$arch_libdir" ] || { echo "ERROR: no arch-specific NIXL plugin dir under /opt/nvidia/nvda_nixl/lib" >&2; exit 1; } && \
+    venv_lf=$(find /opt/dynamo/venv -path '*.nixl_cu13.mesonpy.libs/plugins/libplugin_LIBFABRIC.so' | head -1) && \
+    [ -n "$venv_lf" ] || { echo "ERROR: no libplugin_LIBFABRIC.so under /opt/dynamo/venv" >&2; exit 1; } && \
+    cp -Pf "$venv_lf" "$arch_libdir/plugins/" && \
+    ln -sfT "$arch_libdir/plugins" /opt/nvidia/nvda_nixl/plugins && \
+    [ -f /opt/nvidia/nvda_nixl/plugins/libplugin_LIBFABRIC.so ] || { echo "ERROR: LIBFABRIC plugin not visible via /opt/nvidia/nvda_nixl/plugins" >&2; ls -la /opt/nvidia/nvda_nixl/plugins/ >&2; exit 1; } && \
+    echo "[aws] NIXL plugins consolidated under /opt/nvidia/nvda_nixl/plugins -> $arch_libdir/plugins"
+
+ENV LD_PRELOAD=""
+ENV NIXL_PLUGIN_DIR=/opt/nvidia/nvda_nixl/plugins
+{% endif %}
+
 {% if target == "runtime" %}
 USER dynamo
 {% endif %}
-
-ENTRYPOINT ["/opt/nvidia/nvidia_entrypoint.sh"]
-CMD []

container/templates/wheel_builder.Dockerfile

@@ -116,10 +116,13 @@ RUN apt-get update -y \
 {% if device == "cuda" %}
 # Install system dependencies
 # Cache dnf downloads; sharing=locked avoids dnf/rpm races with concurrent builds.
+# --setopt=tsflags=nocontexts: skip SELinux file-context labeling. The manylinux
+# image lacks the SELinux policy store that some compute nodes expect; without
+# this flag, dnf fails with "ValueError: SELinux policy is not managed".
 RUN --mount=type=cache,target=/var/cache/dnf,sharing=locked \
-    dnf install -y almalinux-release-synergy && \
+    dnf install -y --setopt=tsflags=nocontexts almalinux-release-synergy && \
     dnf config-manager --set-enabled powertools && \
-    dnf install -y \
+    dnf install -y --setopt=tsflags=nocontexts \
         # Autotools (required for UCX, libfabric ./autogen.sh and ./configure)
         autoconf \
         automake \
@@ -266,7 +269,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token
     apt-get update -y && apt-get install -y build-essential pkg-config xz-utils; \
     apt-get clean && rm -rf /var/lib/apt/lists/*; \
     elif [ "$DEVICE" = "cuda" ]; then \
-    dnf install -y pkg-config xz; \
+    dnf install -y --setopt=tsflags=nocontexts pkg-config xz; \
     fi && \
     cd /tmp && \
     curl --retry 5 --retry-delay 3 -LO https://ffmpeg.org/releases/ffmpeg-${FFMPEG_VERSION}.tar.xz && \
@@ -362,6 +365,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token
      ldconfig
 
 {% if device == "cuda" %}
+ARG NIXL_LIBFABRIC_REPO
 ARG NIXL_LIBFABRIC_REF
 RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token \
     --mount=type=secret,id=aws-role-arn,env=AWS_ROLE_ARN \
@@ -371,7 +375,7 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token
         eval $(/tmp/use-sccache.sh setup-env); \
     fi && \
     cd /usr/local/src && \
-    git clone https://github.com/ofiwg/libfabric.git && \
+    git clone "${NIXL_LIBFABRIC_REPO}" && \
     cd libfabric && \
     git checkout $NIXL_LIBFABRIC_REF && \
     ./autogen.sh && \
@@ -391,6 +395,8 @@ RUN --mount=type=secret,id=aws-web-identity-token,target=/run/secrets/aws-token
     /tmp/use-sccache.sh show-stats "LIBFABRIC" && \
     echo "/usr/local/libfabric/lib" > /etc/ld.so.conf.d/libfabric.conf && \
     ldconfig
+
+ENV PKG_CONFIG_PATH="/usr/local/libfabric/lib/pkgconfig:${PKG_CONFIG_PATH}"
 {% endif %}
 
 {% if framework == "vllm" and device == "cuda" %}