ai-dynamo · lishapira · Aug 31, 2026 · Aug 24, 2026 · Aug 24, 2026 · Aug 24, 2026
@@ -8,6 +8,7 @@ CI_FILES=(
     ".ci/dockerfiles/Dockerfile.base"
     ".ci/dockerfiles/Dockerfile.gpu-test"
     ".ci/dockerfiles/Dockerfile.build_helper"
+    ".ci/patches/nixl_ep_vllm_release_test.patch"
     ".gitlab/build.sh"
     ".ci/scripts/common.sh"
     "contrib/Dockerfile.manylinux"

@@ -27,6 +27,10 @@ ARG PRE_INSTALLED_ENV
 ARG PRE_INSTALLED_NIXL_ENV
 ARG PRE_INSTALLED_UCX_ENV
 ARG UCX_VERSION=v1.23.x
+# Optional vLLM Elastic EP environment.
+ARG BUILD_VLLM_ELASTIC_TEST=false
+# Empty selects the latest stable release when the base image is built.
+ARG VLLM_REF
 # When set (e.g. "address"), build the Abseil/gRPC/etcd dependency stack with the
 # matching -fsanitize flags so AddressSanitizer NIXL builds have an ABI-compatible
 # (instrumented) Abseil. Empty for normal images.
@@ -66,6 +70,7 @@ RUN if ! getent group "${_GID}" > /dev/null 2>&1; then \
 RUN mkdir -p ${NIXL_INSTALL_DIR} && chown -R "${_UID}":"${_GID}" ${NIXL_INSTALL_DIR}
 COPY --chown="${_UID}":"${_GID}" .ci/scripts/ /.ci/scripts/
 COPY --chown="${_UID}":"${_GID}" .gitlab/build.sh /.gitlab/build.sh
+COPY --chown="${_UID}":"${_GID}" .ci/patches/nixl_ep_vllm_release_test.patch /tmp/
 
 # Configure sudo access
 RUN mkdir -p /etc/sudoers.d && \
@@ -79,10 +84,38 @@ USER ${_LOGIN}
 # Set environment variables
 ENV HOME=${_HOME}
 ENV USER=${_LOGIN}
+ENV VLLM_ELASTIC_TEST_DIR=${_HOME}/vllm
 
 RUN /.gitlab/build.sh ${NIXL_INSTALL_DIR}
 RUN sudo rm -rf /.gitlab/build.sh /.ci/scripts/
 
+# Install editable vLLM source with matching CUDA 13 precompiled binaries.
+# Resolve the latest stable release unless explicitly overridden.
+RUN if [ "${BUILD_VLLM_ELASTIC_TEST}" = "true" ]; then \
+        UV="${HOME}/.local/bin/uv"; \
+        VENV_PY="${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python"; \
+        if [ -z "${VLLM_REF}" ]; then \
+            VLLM_REF="$(python3 -c 'import json, urllib.request; r = urllib.request.Request("https://api.github.com/repos/vllm-project/vllm/releases/latest", headers={"Accept": "application/vnd.github+json", "User-Agent": "nixl-ci"}); print(json.load(urllib.request.urlopen(r, timeout=30))["tag_name"])')"; \
+        fi && \
+        echo "Using vLLM release ${VLLM_REF}" && \
+        git clone --depth 1 --branch "${VLLM_REF}" \
+            https://github.com/vllm-project/vllm.git "${VLLM_ELASTIC_TEST_DIR}" && \
+        git -C "${VLLM_ELASTIC_TEST_DIR}" apply \
+            /tmp/nixl_ep_vllm_release_test.patch && \
+        "${UV}" venv "${VLLM_ELASTIC_TEST_DIR}/.venv" && \
+        VLLM_USE_PRECOMPILED=1 \
+        VLLM_PRECOMPILED_WHEEL_VARIANT=cu130 \
+        VLLM_PRECOMPILED_WHEEL_COMMIT="$(git -C "${VLLM_ELASTIC_TEST_DIR}" rev-parse HEAD)" \
+        "${UV}" pip install --python "${VENV_PY}" \
+            --editable "${VLLM_ELASTIC_TEST_DIR}" --torch-backend=cu130 && \
+        "${UV}" pip install --python "${VENV_PY}" \
+            pytest tblib "ray[default]>=2.48.0" && \
+        "${VENV_PY}" -c 'import glob, os, sys, torch, vllm; \
+so = glob.glob(os.path.join(os.path.dirname(vllm.__file__), "_C*.so")); \
+print("vLLM", vllm.__version__, "torch", torch.__version__, torch.version.cuda, so); \
+sys.exit(0 if so else 1)'; \
+    fi
+
 # Set working directory
 WORKDIR ${_HOME}
 

@@ -21,6 +21,7 @@ ARG PRE_INSTALLED_UCX_ENV
 ARG UCX_VERSION=v1.23.x
 ARG HAS_GPU=true
 ARG BUILD_NIXL_EP=false
+ARG NIXL_PYTHON
 ARG WORKSPACE=/workspace/nixl
 
 

@@ -163,7 +163,7 @@ their own nightly/manual trigger. They split into two groups:
   - `nixl-ci-non-gpu` — `.ci/jenkins/lib/build-matrix.yaml`
   - `nixl-ci-gpu` — `.ci/jenkins/lib/test-matrix.yaml`
   - `nixl-ci-dl-gpu` — `.ci/jenkins/lib/test-dl-matrix.yaml` (dlcluster.nvidia.com)
-  - `nixl-ci-dl-gpu-ep` — `.ci/jenkins/lib/test-dl-ep-matrix.yaml` (nixl_ep elastic tests on dlcluster.nvidia.com)
+  - `nixl-ci-dl-gpu-ep` — `.ci/jenkins/lib/test-dl-ep-matrix.yaml` (NIXL EP tests on dlcluster.nvidia.com)
   - `nixl-ci-build-wheel` — `.ci/jenkins/lib/build-wheel-matrix.yaml`
   - `nixl-ci-test-sanitizers` — `.ci/jenkins/lib/test-sanitizer-matrix.yaml` (ASan/UBSan + TSan)
   - `nixl-ci-build-container-pr` — `.ci/jenkins/lib/build-container-pr-matrix.yaml`
@@ -253,7 +253,7 @@ NEW_TAG=$(git log -1 --format=%h -- "${CI_FILES[@]}")
 
 This returns the short git commit hash of the most recent commit that touched
 any of the CI source files (`Dockerfile.base`, `Dockerfile.gpu-test`,
-`Dockerfile.build_helper`, `build.sh`, `common.sh`, `Dockerfile.manylinux`). It
+`Dockerfile.build_helper`, `nixl_ep_vllm_release_test.patch`, `build.sh`, `common.sh`, `Dockerfile.manylinux`). It
 then patches all six YAML files in the Jenkins workspace with `sed` before the
 matrix library reads them. No commit or push is made — the patch exists only in
 the workspace.

@@ -2,7 +2,7 @@
 #
 # DLCluster GPU EP Test Matrix Configuration for dlcluster.nvidia.com
 #
-# Runs the nixl_ep elastic tests on the DL cluster. This job is split out from
+# Runs nixl_ep native elastic.py and vLLM+nixl_ep tests on the DL cluster. This job is split out from
 # nixl-ci-dl-gpu so the EP-specific image build (BUILD_NIXL_EP=true) and EP
 # Slurm reservation do not affect the existing DL test flow.
 #
@@ -12,7 +12,7 @@
 #                  build_helper_dl_ep)
 # - Matrix Axes: aarch64 (UCX version from the UCX_VER job parameter)
 # - Run Steps: build PR image with BUILD_NIXL_EP=true, allocate Slurm, run
-#              .gitlab/test_ep.sh (elastic.py NVLink + RDMA on 4 GPUs)
+#              .gitlab/test_ep.sh (nixl_ep native elastic.py and vLLM+nixl_ep tests on 4 GPUs)
 #
 # When Modified:
 # - Adding/removing Docker images: Affects available test environments
@@ -28,7 +28,7 @@ job: nixl-ci-dl-gpu-ep
 # Fail job if one of the steps fails or continue
 failFast: false
 
-timeout_minutes: 240
+timeout_minutes: 300
 
 registry_host: artifactory.nvidia.com
 registry_auth: svc-nixl-new-artifactory-token
@@ -48,17 +48,19 @@ credentials:
 env:
   ARTIFACTORY_PATH: /sw-nbu-swx-nixl-docker-local/ci
   NIXL_INSTALL_DIR: /opt/nixl
+  NIXL_EP_PYTHON: /home/svc-nixl/vllm/.venv/bin/python
   NIXL_BUILD_DIR: nixl_build
   SLURM_NODES: 1
   SLURM_PARTITION: gb200nvl72_cx8
   SLURM_HEAD_NODE: dlcluster.nvidia.com
   SLURM_HEAD_USER: svc-nixl
   SLURM_ACCOUNT: 'blackwell'
-  SLURM_JOB_TIMEOUT: '01:30:00'
+  SLURM_JOB_TIMEOUT: '02:00:00'
   SLURM_IMMEDIATE_TIMEOUT: 3600
   SSH_CREDENTIALS_ID: 'svc-nixl-ssh_key'
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
-  TEST_TIMEOUT: 50
+  TEST_TIMEOUT: 90
+  VLLM_HF_ENDPOINT: "https://urm.nvidia.com/artifactory/api/huggingfaceml/huggingface-remote"
   STORAGE_DRIVER: overlay
   # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files;
   # the "CI_MANAGED" placeholder must not be hand-edited.
@@ -79,7 +81,7 @@ runs_on_dockers:
      name: 'nixl-ci-dl-gpu-ep-base-pytorch26.06-cuda13.3-ubuntu24.04',
      tag: "${CI_IMAGE_TAG}",
      arch: "aarch64",
-     build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg BASE_IMAGE=nvcr.io/nvidia/pytorch:26.06-py3 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache'
+     build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg BASE_IMAGE=nvcr.io/nvidia/pytorch:26.06-py3 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg BUILD_VLLM_ELASTIC_TEST=true --build-arg ARCH=${arch} --pull --no-cache'
     }
 
   - {
@@ -119,6 +121,7 @@ steps:
       --build-arg NIXL_BUILD_DIR=${NIXL_BUILD_DIR} \
       --build-arg HAS_GPU=true \
       --build-arg BUILD_NIXL_EP=true \
+      --build-arg NIXL_PYTHON=${NIXL_EP_PYTHON} \
       --build-arg BASE_IMAGE=${registry_host}${registry_path}/${arch}/nixl-ci-dl-gpu-ep-base-pytorch26.06-cuda13.3-ubuntu24.04:${CI_IMAGE_TAG} \
       --tag ${PR_IMAGE} \
       -f .ci/dockerfiles/Dockerfile.gpu-test .
@@ -148,7 +151,7 @@ steps:
         "--account=${SLURM_ACCOUNT}"
       ]
 
-  - name: Run DL EP elastic tests
+  - name: Run DL NIXL EP tests
     containerSelector: "{name: 'build_helper_dl_ep'}"
     timeout: "${TEST_TIMEOUT}"
     parallel: false
@@ -157,7 +160,7 @@ steps:
     run: run
     args:
       jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_ep_${BUILD_NUMBER}.txt"
-      testScript: ".gitlab/test_ep.sh ${NIXL_INSTALL_DIR}"
+      testScript: "HF_ENDPOINT=${VLLM_HF_ENDPOINT} HF_HUB_DISABLE_XET=1 .gitlab/test_ep.sh ${NIXL_INSTALL_DIR}"
       headNode: "${SLURM_HEAD_NODE}"
       headUser: "${SLURM_HEAD_USER}"
       dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-dl-gpu-ep-test:${BUILD_NUMBER}"

@@ -320,7 +320,7 @@
                     parent-credentials: true
         script-path: "{jjb_jenkinsfile}"  # Path to Jenkinsfile that defines the build steps
 
-# Template for the DLCluster GPU EP test job that runs nixl_ep elastic tests on dlcluster.nvidia.com
+# Template for the DLCluster job that runs nixl_ep native elastic.py and vLLM+nixl_ep tests
 - job-template:
     name: "{jjb_proj}-dl-gpu-ep"      # Will be expanded to 'nixl-ci-dl-gpu-ep'
     project-type: pipeline
@@ -1119,7 +1119,7 @@
         - "{jjb_proj}-test-llm-container"   # Create LLM container test job (SLURM)
         - "{jjb_proj}-gpu"        # Create gpu job
         - "{jjb_proj}-dl-gpu"   # Create dl-gpu job for dlcluster.nvidia.com
-        - "{jjb_proj}-dl-gpu-ep"  # Create dl-gpu-ep job for nixl_ep elastic tests on dlcluster.nvidia.com
+        - "{jjb_proj}-dl-gpu-ep"  # Create dl-gpu-ep job for NIXL EP on dlcluster.nvidia.com
         - "{jjb_proj}-build-wheel"  # Create build wheel job
         - "{jjb_proj}-build-wheel-nightly"  # Create nightly wheel verification job
         - "{jjb_proj}-cleanup-artifacts"  # Create Artifactory cleanup job

@@ -0,0 +1,26 @@
+diff --git a/tests/distributed/test_elastic_ep.py b/tests/distributed/test_elastic_ep.py
+--- a/tests/distributed/test_elastic_ep.py
++++ b/tests/distributed/test_elastic_ep.py
+@@ -175,6 +175,9 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str
+         "--enable-expert-parallel",
+         "--all2all-backend",
+-        "allgather_reducescatter",
++        os.getenv(
++            "VLLM_TEST_ELASTIC_EP_ALL2ALL_BACKEND",
++            "allgather_reducescatter",
++        ),
+         "--enable-elastic-ep",
+         "--enable-eplb",
+         "--eplb-config.num_redundant_experts",
+@@ -204,4 +207,8 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str
+-    return args
+-
+-
++    attention_backend = os.getenv("VLLM_ATTENTION_BACKEND")
++    if attention_backend:
++        args.extend(["--attention-backend", attention_backend])
++
++    return args
++
++
+ @pytest.mark.parametrize(
@@ -438,9 +438,27 @@ else
     if [ "${BUILD_NIXL_EP}" = "true" ]; then
         EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS} -Dbuild_nixl_ep=true"
     fi
+    # When NIXL_PYTHON is set (currently only by test-dl-ep-matrix.yaml), build and
+    # install NIXL EP against vLLM's Python/Torch venv to prevent ABI mismatches.
+    # Only NIXL's Meson build uses this venv; dependency builds keep system Python.
+    # Other jobs leave NIXL_PYTHON_ARGS empty and keep the existing build behavior.
+    NIXL_PYTHON_ARGS=()
+    NIXL_PYTHON_NATIVE_FILE=""
+    if [ -n "${NIXL_PYTHON:-}" ]; then
+        if [ ! -x "${NIXL_PYTHON}" ]; then
+            echo "ERROR: NIXL_PYTHON is not executable: ${NIXL_PYTHON}" >&2
+            exit 1
+        fi
+        NIXL_PYTHON_NATIVE_FILE=$(mktemp)
+        printf "[binaries]\npython = '%s'\n" "${NIXL_PYTHON}" > "${NIXL_PYTHON_NATIVE_FILE}"
+        NIXL_PYTHON_ARGS=(--native-file "${NIXL_PYTHON_NATIVE_FILE}" -Dpython.install_env=venv)
+    fi
     # shellcheck disable=SC2086
-    meson setup ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug
+    meson setup "${NIXL_PYTHON_ARGS[@]}" ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug
     ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} && ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install
+    if [ -n "${NIXL_PYTHON_NATIVE_FILE}" ]; then
+        rm -f "${NIXL_PYTHON_NATIVE_FILE}"
+    fi
     mkdir -p dist && cp ${NIXL_BUILD_DIR}/src/bindings/python/nixl-meta/nixl-*.whl dist/
 
     # TODO(kapila): Copy the nixl.pc file to the install directory if needed.

@@ -14,13 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# nixl_ep elastic CI: run only EP tests (invoked from nixl-ci-dl-gpu-ep flow).
+# NIXL EP CI: run nixl_ep native elastic.py and vLLM+nixl_ep tests.
 
 # shellcheck disable=SC1091
 . "$(dirname "$0")/../.ci/scripts/common.sh"
 
 set -e
 set -x
+set -o pipefail
 
 INSTALL_DIR=$1
 
@@ -29,6 +30,14 @@ if [ -z "$INSTALL_DIR" ]; then
     exit 1
 fi
 
+: "${VLLM_ELASTIC_TEST_DIR:?vLLM Elastic EP test environment is not installed}"
+VLLM_PYTHON="${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python"
+
+if [ ! -x "${VLLM_PYTHON}" ]; then
+    echo "ERROR: vLLM Python environment is missing: ${VLLM_PYTHON}" >&2
+    exit 1
+fi
+
 ARCH=$(uname -m)
 [ "$ARCH" = "arm64" ] && ARCH="aarch64"
 
@@ -40,9 +49,8 @@ export NIXL_PLUGIN_DIR=${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins
 export NIXL_PREFIX=${INSTALL_DIR}
 export NIXL_DEBUG_LOGGING=yes
 
-# Make `import nixl_ep` resolve the source-tree dispatcher, which loads the
-# CUDA-versioned backend (nixl_ep_cu*) from the source install under ${INSTALL_DIR}.
-export PYTHONPATH="${PWD}/src/bindings/python/nixl-meta:${INSTALL_DIR}/lib/python3/dist-packages${PYTHONPATH:+:$PYTHONPATH}"
+# Load the source dispatcher and the backend installed in the shared venv.
+export PYTHONPATH="${PWD}/src/bindings/python/nixl-meta${PYTHONPATH:+:$PYTHONPATH}"
 
 echo "==== Show system info ===="
 env
@@ -67,7 +75,7 @@ run_elastic_test() {
             export UCX_TLS=^rc_gda
         fi
         PYTHONPATH="${NIXL_BUILD_DIR}/${EP_SRC_DIR}:${EP_SRC_DIR}/tests:${EP_SRC_DIR}/tests/elastic${PYTHONPATH:+:$PYTHONPATH}" \
-        timeout 300 python3 ${EP_SRC_DIR}/tests/elastic/elastic.py \
+        timeout 300 "${VLLM_PYTHON}" ${EP_SRC_DIR}/tests/elastic/elastic.py \
             --plan "$plan_file" \
             --num-processes 4 \
             --num-experts-per-rank 32 \
@@ -103,3 +111,38 @@ else
 fi
 
 echo "==== nixl_ep elastic tests done ===="
+
+echo "==== Running vLLM Elastic EP test ===="
+(
+    # Avoid SPCx loading HPC-X UCX 1.21; NIXL EP requires UCX >=1.22.
+    unset NCCL_NET_PLUGIN
+    unset UCX_NET_DEVICES
+    # TODO: remove this override when vLLM updates FlashInfer with
+    # https://github.com/flashinfer-ai/flashinfer/pull/4377.
+    # FlashInfer 0.6.16.post3's default MLA backend fails to JIT on CUDA 13.3.
+    export VLLM_ATTENTION_BACKEND=CUTLASS_MLA
+    export PATH="${VLLM_ELASTIC_TEST_DIR}/.venv/bin:${PATH}"
+    VLLM_LOG="${PWD}/elastic_ep_vllm_single_node.log"
+    VLLM_REF="$(git -C "${VLLM_ELASTIC_TEST_DIR}" describe --tags --exact-match HEAD)"
+    VLLM_COMMIT="$(git -C "${VLLM_ELASTIC_TEST_DIR}" rev-parse HEAD)"
+
+    echo "vLLM source: VLLM_REF=${VLLM_REF} VLLM_COMMIT=${VLLM_COMMIT}"
+
+    # Run vLLM's 2 -> 4 -> 2 Elastic EP scaling test with NIXL EP (Cover eager heavy traffic and CUDA graphs testing).
+    (
+        cd "${VLLM_ELASTIC_TEST_DIR}"
+        VLLM_NIXL_EP_MAX_NUM_RANKS=4 \
+        VLLM_TEST_ELASTIC_EP_ALL2ALL_BACKEND=nixl_ep \
+        timeout 4500 "${VLLM_PYTHON}" -m pytest \
+            "tests/distributed/test_elastic_ep.py::test_elastic_ep_scaling[enforce_eager_heavy]" \
+            "tests/distributed/test_elastic_ep.py::test_elastic_ep_scaling[cuda_graphs_heavy]" \
+            -v -s --tb=short 2>&1 | tee "${VLLM_LOG}"
+    )
+
+    if grep -Eiq '(^|[[:space:]])[0-9]+ skipped|SKIPPED' "${VLLM_LOG}"; then
+        echo "ERROR: vLLM Elastic EP test was skipped" >&2
+        exit 1
+    fi
+
+    echo "==== vLLM Elastic EP test done ===="
+)