CI: add VLLM+nixl_ep test to EP CI job (first version)

.ci/cidemo-init.sh

@@ -8,6 +8,7 @@ CI_FILES=(
     ".ci/dockerfiles/Dockerfile.base"
     ".ci/dockerfiles/Dockerfile.gpu-test"
     ".ci/dockerfiles/Dockerfile.build_helper"
+    ".ci/patches/nixl_ep_vllm_release_test.patch"
     ".gitlab/build.sh"
     ".ci/scripts/common.sh"
     "contrib/Dockerfile.manylinux"

.ci/dockerfiles/Dockerfile.base

@@ -27,6 +27,10 @@ ARG PRE_INSTALLED_ENV
 ARG PRE_INSTALLED_NIXL_ENV
 ARG PRE_INSTALLED_UCX_ENV
 ARG UCX_VERSION=v1.22.x
+# Build the vLLM Elastic EP test environment only in the dedicated EP base
+# image. Other Dockerfile.base users do not download or install vLLM.
+ARG BUILD_VLLM_ELASTIC_TEST=false
+ARG VLLM_REF=releases/v0.27.1
 # When set (e.g. "address"), build the Abseil/gRPC/etcd dependency stack with the
 # matching -fsanitize flags so AddressSanitizer NIXL builds have an ABI-compatible
 # (instrumented) Abseil. Empty for normal images.
@@ -66,6 +70,7 @@ RUN if ! getent group "${_GID}" > /dev/null 2>&1; then \
 RUN mkdir -p ${NIXL_INSTALL_DIR} && chown -R "${_UID}":"${_GID}" ${NIXL_INSTALL_DIR}
 COPY --chown="${_UID}":"${_GID}" .ci/scripts/ /.ci/scripts/
 COPY --chown="${_UID}":"${_GID}" .gitlab/build.sh /.gitlab/build.sh
+COPY --chown="${_UID}":"${_GID}" .ci/patches/nixl_ep_vllm_release_test.patch /tmp/
 
 # Configure sudo access
 RUN mkdir -p /etc/sudoers.d && \
@@ -79,10 +84,41 @@ USER ${_LOGIN}
 # Set environment variables
 ENV HOME=${_HOME}
 ENV USER=${_LOGIN}
+ENV VLLM_ELASTIC_TEST_DIR=${_HOME}/vllm
+ENV VLLM_REF=${VLLM_REF}
 
 RUN /.gitlab/build.sh ${NIXL_INSTALL_DIR}
 RUN sudo rm -rf /.gitlab/build.sh /.ci/scripts/
 
+# Prepare vLLM Elastic EP tests in the dedicated EP base image.
+# The PR image builds NIXL once for both native and vLLM tests.
+# DL EP CI runs on GB200, so build only vLLM SM100 kernels. MAX_JOBS is bounded
+# by the builder memory limit in .ci/jenkins/lib/test-dl-ep-matrix.yaml.
+RUN if [ "${BUILD_VLLM_ELASTIC_TEST}" = "true" ]; then \
+        export LD_LIBRARY_PATH="/opt/hpcx/ucx/lib:/opt/hpcx/ucc/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}" && \
+        echo "builder cgroup memory.max: $(cat /sys/fs/cgroup/memory.max 2>/dev/null || echo unreadable) nproc: $(nproc)" && \
+        git clone --depth 1 --branch "${VLLM_REF}" \
+            https://github.com/vllm-project/vllm.git "${VLLM_ELASTIC_TEST_DIR}" && \
+        git -C "${VLLM_ELASTIC_TEST_DIR}" apply \
+            /tmp/nixl_ep_vllm_release_test.patch && \
+        "${HOME}/.local/bin/uv" venv \
+            "${VLLM_ELASTIC_TEST_DIR}/.venv" && \
+        "${HOME}/.local/bin/uv" pip install \
+            --python "${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python" \
+            --torch-backend=cu130 \
+            -r "${VLLM_ELASTIC_TEST_DIR}/requirements/build/cuda.txt" && \
+        "${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python" -c \
+            'import torch; print(torch.__version__, torch.version.cuda)' && \
+        TORCH_CUDA_ARCH_LIST=10.0 MAX_JOBS=4 VERBOSE=1 \
+            "${HOME}/.local/bin/uv" --verbose pip install \
+            --python "${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python" \
+            --editable "${VLLM_ELASTIC_TEST_DIR}" --torch-backend=cu130 \
+            --no-build-isolation && \
+        "${HOME}/.local/bin/uv" pip install \
+            --python "${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python" \
+            pytest tblib "ray[default]>=2.48.0"; \
+    fi
+
 # Set working directory
 WORKDIR ${_HOME}
 

.ci/dockerfiles/Dockerfile.gpu-test

@@ -21,6 +21,7 @@ ARG PRE_INSTALLED_UCX_ENV
 ARG UCX_VERSION=v1.22.x
 ARG HAS_GPU=true
 ARG BUILD_NIXL_EP=false
+ARG NIXL_PYTHON
 ARG WORKSPACE=/workspace/nixl
 
 

.ci/docs/ci-overview.md

@@ -163,7 +163,7 @@ their own nightly/manual trigger. They split into two groups:
   - `nixl-ci-non-gpu` — `.ci/jenkins/lib/build-matrix.yaml`
   - `nixl-ci-gpu` — `.ci/jenkins/lib/test-matrix.yaml`
   - `nixl-ci-dl-gpu` — `.ci/jenkins/lib/test-dl-matrix.yaml` (dlcluster.nvidia.com)
-  - `nixl-ci-dl-gpu-ep` — `.ci/jenkins/lib/test-dl-ep-matrix.yaml` (nixl_ep elastic tests on dlcluster.nvidia.com)
+  - `nixl-ci-dl-gpu-ep` — `.ci/jenkins/lib/test-dl-ep-matrix.yaml` (NIXL EP tests: native `elastic.py` and vLLM Elastic EP on dlcluster.nvidia.com)
   - `nixl-ci-build-wheel` — `.ci/jenkins/lib/build-wheel-matrix.yaml`
   - `nixl-ci-test-sanitizers` — `.ci/jenkins/lib/test-sanitizer-matrix.yaml` (ASan/UBSan + TSan)
   - `nixl-ci-build-container-pr` — `.ci/jenkins/lib/build-container-pr-matrix.yaml`
@@ -253,10 +253,10 @@ NEW_TAG=$(git log -1 --format=%h -- "${CI_FILES[@]}")
 
 This returns the short git commit hash of the most recent commit that touched
 any of the CI source files (`Dockerfile.base`, `Dockerfile.gpu-test`,
-`Dockerfile.build_helper`, `build.sh`, `common.sh`, `Dockerfile.manylinux`). It
-then patches all six YAML files in the Jenkins workspace with `sed` before the
-matrix library reads them. No commit or push is made — the patch exists only in
-the workspace.
+`Dockerfile.build_helper`, `nixl_ep_vllm_release_test.patch`, `build.sh`,
+`common.sh`, `Dockerfile.manylinux`). It then patches all six YAML files in the
+Jenkins workspace with `sed` before the matrix library reads them. No commit or
+push is made — the patch exists only in the workspace.
 
 **Caching behaviour:** the derived tag is stable as long as the CI source files
 are unchanged. Two PRs that both leave the CI files untouched get the same tag

.ci/jenkins/lib/test-dl-ep-matrix.yaml

@@ -2,17 +2,18 @@
 #
 # DLCluster GPU EP Test Matrix Configuration for dlcluster.nvidia.com
 #
-# Runs the nixl_ep elastic tests on the DL cluster. This job is split out from
-# nixl-ci-dl-gpu so the EP-specific image build (BUILD_NIXL_EP=true) and EP
-# Slurm reservation do not affect the existing DL test flow.
+# Runs the native nixl_ep elastic tests and vLLM Elastic EP on the same DL
+# cluster allocation. This job is split out from nixl-ci-dl-gpu so the
+# EP-specific image build and Slurm reservation do not affect the existing DL
+# test flow.
 #
 # Key Components:
 # - Job Configuration: Defines timeout, failure behavior, and server resources
 # - Docker Images: Dedicated EP base + build_helper (nixl-ci-dl-gpu-ep-base-... and
 #                  build_helper_dl_ep)
 # - Matrix Axes: aarch64 (UCX version from the UCX_VER job parameter)
-# - Run Steps: build PR image with BUILD_NIXL_EP=true, allocate Slurm, run
-#              .gitlab/test_ep.sh (elastic.py NVLink + RDMA on 4 GPUs)
+# - Run Steps: build PR image with NIXL EP and vLLM, allocate Slurm, run
+#              .gitlab/test_ep.sh (elastic.py plus vLLM Elastic EP on 4 GPUs)
 #
 # When Modified:
 # - Adding/removing Docker images: Affects available test environments
@@ -28,7 +29,7 @@ job: nixl-ci-dl-gpu-ep
 # Fail job if one of the steps fails or continue
 failFast: false
 
-timeout_minutes: 240
+timeout_minutes: 600
 
 registry_host: artifactory.nvidia.com
 registry_auth: svc-nixl-new-artifactory-token
@@ -38,8 +39,9 @@ kubernetes:
   cloud: il-ipp-blossom-prod
   namespace: nbu-swx-nixl
   imagePullSecrets: "['artifactory-pull-secret']"
-  limits: "{memory: 16Gi, cpu: 16000m}"
-  requests: "{memory: 8Gi, cpu: 8000m}"
+  # The EP base image compiles vLLM from source, so this job needs the memory budget of the build jobs rather than the test jobs.
+  limits: "{memory: 32Gi, cpu: 16000m}"
+  requests: "{memory: 32Gi, cpu: 16000m}"
   privileged: true
 
 credentials:
@@ -48,17 +50,19 @@ credentials:
 env:
   ARTIFACTORY_PATH: /sw-nbu-swx-nixl-docker-local/ci
   NIXL_INSTALL_DIR: /opt/nixl
+  NIXL_EP_PYTHON: /home/svc-nixl/vllm/.venv/bin/python
   NIXL_BUILD_DIR: nixl_build
   SLURM_NODES: 1
   SLURM_PARTITION: gb200nvl72_cx8
   SLURM_HEAD_NODE: dlcluster.nvidia.com
   SLURM_HEAD_USER: svc-nixl
   SLURM_ACCOUNT: 'blackwell'
-  SLURM_JOB_TIMEOUT: '01:30:00'
+  SLURM_JOB_TIMEOUT: '03:30:00'
   SLURM_IMMEDIATE_TIMEOUT: 3600
   SSH_CREDENTIALS_ID: 'svc-nixl-ssh_key'
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
-  TEST_TIMEOUT: 50
+  TEST_TIMEOUT: 180
+  VLLM_HF_ENDPOINT: "https://urm.nvidia.com/artifactory/api/huggingfaceml/huggingface-remote"
   STORAGE_DRIVER: overlay
   # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files;
   # the "CI_MANAGED" placeholder must not be hand-edited.
@@ -70,16 +74,16 @@ empty_volumes:
 pvc_volumes:
   - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false}
 
-# Docker images for DL EP testing.
-# Dedicated EP base + build_helper images (separate artifactory paths from the
-# nixl-ci-dl-gpu job).
+# Docker images for DL EP testing. The dedicated EP base contains vLLM and its
+# Torch environment; the PR image builds NIXL EP against that same Torch.
+# Both test suites therefore use the same image on the same allocation.
 runs_on_dockers:
   - {
      file: '.ci/dockerfiles/Dockerfile.base',
      name: 'nixl-ci-dl-gpu-ep-base-pytorch26.06-cuda13.3-ubuntu24.04',
      tag: "${CI_IMAGE_TAG}",
      arch: "aarch64",
-     build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg BASE_IMAGE=nvcr.io/nvidia/pytorch:26.06-py3 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache'
+     build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg BASE_IMAGE=nvcr.io/nvidia/pytorch:26.06-py3 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg BUILD_VLLM_ELASTIC_TEST=true --build-arg ARCH=${arch} --pull --no-cache'
     }
 
   - {
@@ -118,6 +122,7 @@ steps:
       --build-arg NIXL_BUILD_DIR=${NIXL_BUILD_DIR} \
       --build-arg HAS_GPU=true \
       --build-arg BUILD_NIXL_EP=true \
+      --build-arg NIXL_PYTHON=${NIXL_EP_PYTHON} \
       --build-arg BASE_IMAGE=${registry_host}${registry_path}/${arch}/nixl-ci-dl-gpu-ep-base-pytorch26.06-cuda13.3-ubuntu24.04:${CI_IMAGE_TAG} \
       --tag ${PR_IMAGE} \
       -f .ci/dockerfiles/Dockerfile.gpu-test .
@@ -147,7 +152,7 @@ steps:
         "--account=${SLURM_ACCOUNT}"
       ]
 
-  - name: Run DL EP elastic tests
+  - name: Run DL NIXL EP tests (native elastic.py and vLLM Elastic EP)
     containerSelector: "{name: 'build_helper_dl_ep'}"
     timeout: "${TEST_TIMEOUT}"
     parallel: false
@@ -156,7 +161,7 @@ steps:
     run: run
     args:
       jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_ep_${BUILD_NUMBER}.txt"
-      testScript: ".gitlab/test_ep.sh ${NIXL_INSTALL_DIR}"
+      testScript: "HF_ENDPOINT=${VLLM_HF_ENDPOINT} .gitlab/test_ep.sh ${NIXL_INSTALL_DIR}"
       headNode: "${SLURM_HEAD_NODE}"
       headUser: "${SLURM_HEAD_USER}"
       dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-dl-gpu-ep-test:${BUILD_NUMBER}"

.ci/patches/nixl_ep_vllm_release_test.patch

@@ -0,0 +1,14 @@
+diff --git a/tests/distributed/test_elastic_ep.py b/tests/distributed/test_elastic_ep.py
+--- a/tests/distributed/test_elastic_ep.py
++++ b/tests/distributed/test_elastic_ep.py
+@@ -175,6 +175,9 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str
+         "--enable-expert-parallel",
+         "--all2all-backend",
+-        "allgather_reducescatter",
++        os.getenv(
++            "VLLM_TEST_ELASTIC_EP_ALL2ALL_BACKEND",
++            "allgather_reducescatter",
++        ),
+         "--enable-elastic-ep",
+         "--enable-eplb",
+         "--eplb-config.num_redundant_experts",

.gitlab/build.sh

@@ -430,9 +430,24 @@ else
     if [ "${BUILD_NIXL_EP}" = "true" ]; then
         EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS} -Dbuild_nixl_ep=true"
     fi
+    # Select the venv only for NIXL's Meson build; dependency builds keep system Python.
+    NIXL_PYTHON_ARGS=()
+    NIXL_PYTHON_NATIVE_FILE=""
+    if [ -n "${NIXL_PYTHON:-}" ]; then
+        if [ ! -x "${NIXL_PYTHON}" ]; then
+            echo "ERROR: NIXL_PYTHON is not executable: ${NIXL_PYTHON}" >&2
+            exit 1
+        fi
+        NIXL_PYTHON_NATIVE_FILE=$(mktemp)
+        printf "[binaries]\npython = '%s'\n" "${NIXL_PYTHON}" > "${NIXL_PYTHON_NATIVE_FILE}"
+        NIXL_PYTHON_ARGS=(--native-file "${NIXL_PYTHON_NATIVE_FILE}" -Dpython.install_env=venv)
+    fi
     # shellcheck disable=SC2086
-    meson setup ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug
+    meson setup "${NIXL_PYTHON_ARGS[@]}" ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug
     ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} && ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install
+    if [ -n "${NIXL_PYTHON_NATIVE_FILE}" ]; then
+        rm -f "${NIXL_PYTHON_NATIVE_FILE}"
+    fi
     mkdir -p dist && cp ${NIXL_BUILD_DIR}/src/bindings/python/nixl-meta/nixl-*.whl dist/
 
     # TODO(kapila): Copy the nixl.pc file to the install directory if needed.

.gitlab/test_ep.sh

@@ -14,13 +14,14 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# nixl_ep elastic CI: run only EP tests (invoked from nixl-ci-dl-gpu-ep flow).
+# NIXL EP CI: run native elastic tests and vLLM Elastic EP on one allocation.
 
 # shellcheck disable=SC1091
 . "$(dirname "$0")/../.ci/scripts/common.sh"
 
 set -e
 set -x
+set -o pipefail
 
 INSTALL_DIR=$1
 
@@ -29,6 +30,14 @@ if [ -z "$INSTALL_DIR" ]; then
     exit 1
 fi
 
+: "${VLLM_ELASTIC_TEST_DIR:?vLLM Elastic EP test environment is not installed}"
+VLLM_PYTHON="${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python"
+
+if [ ! -x "${VLLM_PYTHON}" ]; then
+    echo "ERROR: vLLM Python environment is missing: ${VLLM_PYTHON}" >&2
+    exit 1
+fi
+
 ARCH=$(uname -m)
 [ "$ARCH" = "arm64" ] && ARCH="aarch64"
 
@@ -41,8 +50,8 @@ export NIXL_PREFIX=${INSTALL_DIR}
 export NIXL_DEBUG_LOGGING=yes
 
 # Make `import nixl_ep` resolve the source-tree dispatcher, which loads the
-# CUDA-versioned backend (nixl_ep_cu*) from the source install under ${INSTALL_DIR}.
-export PYTHONPATH="${PWD}/src/bindings/python/nixl-meta:${INSTALL_DIR}/lib/python3/dist-packages${PYTHONPATH:+:$PYTHONPATH}"
+# CUDA-versioned backend (nixl_ep_cu*) installed in the shared venv.
+export PYTHONPATH="${PWD}/src/bindings/python/nixl-meta${PYTHONPATH:+:$PYTHONPATH}"
 
 echo "==== Show system info ===="
 env
@@ -67,7 +76,7 @@ run_elastic_test() {
             export UCX_TLS=^rc_gda
         fi
         PYTHONPATH="${NIXL_BUILD_DIR}/${EP_SRC_DIR}:${EP_SRC_DIR}/tests:${EP_SRC_DIR}/tests/elastic${PYTHONPATH:+:$PYTHONPATH}" \
-        timeout 300 python3 ${EP_SRC_DIR}/tests/elastic/elastic.py \
+        timeout 300 "${VLLM_PYTHON}" ${EP_SRC_DIR}/tests/elastic/elastic.py \
             --plan "$plan_file" \
             --num-processes 4 \
             --num-experts-per-rank 32 \
@@ -103,3 +112,58 @@ else
 fi
 
 echo "==== nixl_ep elastic tests done ===="
+
+echo "==== Running vLLM Elastic EP test ===="
+export LD_LIBRARY_PATH="/opt/hpcx/ucx/lib:/opt/hpcx/ucc/lib${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}}"
+VLLM_LOG="${PWD}/elastic_ep_vllm_single_node.log"
+VLLM_COMMIT="$(git -C "${VLLM_ELASTIC_TEST_DIR}" rev-parse HEAD)"
+
+echo "vLLM source: VLLM_REF=${VLLM_REF:-unknown} VLLM_COMMIT=${VLLM_COMMIT}"
+
+# Verify that the vLLM environment can use the NIXL and NIXL EP artifacts that
+# were built in this PR image. This makes an unavailable backend fail before
+# pytest can report the test as skipped.
+"${VLLM_PYTHON}" - <<'PY'
+from importlib.metadata import PackageNotFoundError, version
+
+import nixl
+import nixl_ep
+import torch
+import vllm
+from vllm.distributed.eplb.eplb_communicator import has_nixl
+
+try:
+    nixl_version = version("nixl")
+except PackageNotFoundError:
+    nixl_version = "source tree"
+
+assert torch.cuda.is_available(), "CUDA is unavailable"
+assert torch.cuda.device_count() >= 4, "vLLM Elastic EP requires four GPUs"
+assert has_nixl(), "vLLM cannot load NIXL"
+
+print("vLLM:", vllm.__version__)
+print("NIXL:", nixl_version, nixl.__file__)
+print("NIXL EP:", nixl_ep.__file__)
+print("Torch/CUDA:", torch.__version__, torch.version.cuda)
+print("GPU:", torch.cuda.get_device_name())
+print("Visible GPUs:", torch.cuda.device_count())
+PY
+
+# Run vLLM's 2 -> 4 -> 2 Elastic EP scaling test with NIXL EP.
+(
+    cd "${VLLM_ELASTIC_TEST_DIR}"
+    VLLM_NIXL_EP_MAX_NUM_RANKS=4 \
+    VLLM_TEST_ELASTIC_EP_ALL2ALL_BACKEND=nixl_ep \
+    VLLM_TEST_ELASTIC_EP_INITIAL_DP=2 \
+    VLLM_TEST_ELASTIC_EP_TARGET_DP=4 \
+    timeout 7200 "${VLLM_PYTHON}" -m pytest \
+        tests/distributed/test_elastic_ep.py::test_elastic_ep_scaling \
+        -v -s --tb=short 2>&1 | tee "${VLLM_LOG}"
+)
+
+if grep -Eiq '(^|[[:space:]])[0-9]+ skipped|SKIPPED' "${VLLM_LOG}"; then
+    echo "ERROR: vLLM Elastic EP test was skipped" >&2
+    exit 1
+fi
+
+echo "==== vLLM Elastic EP test done ===="