diff --git a/.ci/cidemo-init.sh b/.ci/cidemo-init.sh index 6f879dacf9..a9a22e118e 100755 --- a/.ci/cidemo-init.sh +++ b/.ci/cidemo-init.sh @@ -8,6 +8,7 @@ CI_FILES=( ".ci/dockerfiles/Dockerfile.base" ".ci/dockerfiles/Dockerfile.gpu-test" ".ci/dockerfiles/Dockerfile.build_helper" + ".ci/patches/nixl_ep_vllm_release_test.patch" ".gitlab/build.sh" ".ci/scripts/common.sh" "contrib/Dockerfile.manylinux" diff --git a/.ci/dockerfiles/Dockerfile.base b/.ci/dockerfiles/Dockerfile.base index 43c3016653..2a7980d03b 100644 --- a/.ci/dockerfiles/Dockerfile.base +++ b/.ci/dockerfiles/Dockerfile.base @@ -27,6 +27,10 @@ ARG PRE_INSTALLED_ENV ARG PRE_INSTALLED_NIXL_ENV ARG PRE_INSTALLED_UCX_ENV ARG UCX_VERSION=v1.23.x +# Optional vLLM Elastic EP environment. +ARG BUILD_VLLM_ELASTIC_TEST=false +# Empty selects the latest stable release when the base image is built. +ARG VLLM_REF # When set (e.g. "address"), build the Abseil/gRPC/etcd dependency stack with the # matching -fsanitize flags so AddressSanitizer NIXL builds have an ABI-compatible # (instrumented) Abseil. Empty for normal images. @@ -66,6 +70,7 @@ RUN if ! getent group "${_GID}" > /dev/null 2>&1; then \ RUN mkdir -p ${NIXL_INSTALL_DIR} && chown -R "${_UID}":"${_GID}" ${NIXL_INSTALL_DIR} COPY --chown="${_UID}":"${_GID}" .ci/scripts/ /.ci/scripts/ COPY --chown="${_UID}":"${_GID}" .gitlab/build.sh /.gitlab/build.sh +COPY --chown="${_UID}":"${_GID}" .ci/patches/nixl_ep_vllm_release_test.patch /tmp/ # Configure sudo access RUN mkdir -p /etc/sudoers.d && \ @@ -79,10 +84,38 @@ USER ${_LOGIN} # Set environment variables ENV HOME=${_HOME} ENV USER=${_LOGIN} +ENV VLLM_ELASTIC_TEST_DIR=${_HOME}/vllm RUN /.gitlab/build.sh ${NIXL_INSTALL_DIR} RUN sudo rm -rf /.gitlab/build.sh /.ci/scripts/ +# Install editable vLLM source with matching CUDA 13 precompiled binaries. +# Resolve the latest stable release unless explicitly overridden. +RUN if [ "${BUILD_VLLM_ELASTIC_TEST}" = "true" ]; then \ + UV="${HOME}/.local/bin/uv"; \ + VENV_PY="${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python"; \ + if [ -z "${VLLM_REF}" ]; then \ + VLLM_REF="$(python3 -c 'import json, urllib.request; r = urllib.request.Request("https://api.github.com/repos/vllm-project/vllm/releases/latest", headers={"Accept": "application/vnd.github+json", "User-Agent": "nixl-ci"}); print(json.load(urllib.request.urlopen(r, timeout=30))["tag_name"])')"; \ + fi && \ + echo "Using vLLM release ${VLLM_REF}" && \ + git clone --depth 1 --branch "${VLLM_REF}" \ + https://github.com/vllm-project/vllm.git "${VLLM_ELASTIC_TEST_DIR}" && \ + git -C "${VLLM_ELASTIC_TEST_DIR}" apply \ + /tmp/nixl_ep_vllm_release_test.patch && \ + "${UV}" venv "${VLLM_ELASTIC_TEST_DIR}/.venv" && \ + VLLM_USE_PRECOMPILED=1 \ + VLLM_PRECOMPILED_WHEEL_VARIANT=cu130 \ + VLLM_PRECOMPILED_WHEEL_COMMIT="$(git -C "${VLLM_ELASTIC_TEST_DIR}" rev-parse HEAD)" \ + "${UV}" pip install --python "${VENV_PY}" \ + --editable "${VLLM_ELASTIC_TEST_DIR}" --torch-backend=cu130 && \ + "${UV}" pip install --python "${VENV_PY}" \ + pytest tblib "ray[default]>=2.48.0" && \ + "${VENV_PY}" -c 'import glob, os, sys, torch, vllm; \ +so = glob.glob(os.path.join(os.path.dirname(vllm.__file__), "_C*.so")); \ +print("vLLM", vllm.__version__, "torch", torch.__version__, torch.version.cuda, so); \ +sys.exit(0 if so else 1)'; \ + fi + # Set working directory WORKDIR ${_HOME} diff --git a/.ci/dockerfiles/Dockerfile.gpu-test b/.ci/dockerfiles/Dockerfile.gpu-test index e0597a25fb..45440b87a5 100644 --- a/.ci/dockerfiles/Dockerfile.gpu-test +++ b/.ci/dockerfiles/Dockerfile.gpu-test @@ -21,6 +21,7 @@ ARG PRE_INSTALLED_UCX_ENV ARG UCX_VERSION=v1.23.x ARG HAS_GPU=true ARG BUILD_NIXL_EP=false +ARG NIXL_PYTHON ARG WORKSPACE=/workspace/nixl diff --git a/.ci/docs/ci-overview.md b/.ci/docs/ci-overview.md index 41de11b896..4bed24c29a 100644 --- a/.ci/docs/ci-overview.md +++ b/.ci/docs/ci-overview.md @@ -163,7 +163,7 @@ their own nightly/manual trigger. They split into two groups: - `nixl-ci-non-gpu` — `.ci/jenkins/lib/build-matrix.yaml` - `nixl-ci-gpu` — `.ci/jenkins/lib/test-matrix.yaml` - `nixl-ci-dl-gpu` — `.ci/jenkins/lib/test-dl-matrix.yaml` (dlcluster.nvidia.com) - - `nixl-ci-dl-gpu-ep` — `.ci/jenkins/lib/test-dl-ep-matrix.yaml` (nixl_ep elastic tests on dlcluster.nvidia.com) + - `nixl-ci-dl-gpu-ep` — `.ci/jenkins/lib/test-dl-ep-matrix.yaml` (NIXL EP tests on dlcluster.nvidia.com) - `nixl-ci-build-wheel` — `.ci/jenkins/lib/build-wheel-matrix.yaml` - `nixl-ci-test-sanitizers` — `.ci/jenkins/lib/test-sanitizer-matrix.yaml` (ASan/UBSan + TSan) - `nixl-ci-build-container-pr` — `.ci/jenkins/lib/build-container-pr-matrix.yaml` @@ -253,7 +253,7 @@ NEW_TAG=$(git log -1 --format=%h -- "${CI_FILES[@]}") This returns the short git commit hash of the most recent commit that touched any of the CI source files (`Dockerfile.base`, `Dockerfile.gpu-test`, -`Dockerfile.build_helper`, `build.sh`, `common.sh`, `Dockerfile.manylinux`). It +`Dockerfile.build_helper`, `nixl_ep_vllm_release_test.patch`, `build.sh`, `common.sh`, `Dockerfile.manylinux`). It then patches all six YAML files in the Jenkins workspace with `sed` before the matrix library reads them. No commit or push is made — the patch exists only in the workspace. diff --git a/.ci/jenkins/lib/test-dl-ep-matrix.yaml b/.ci/jenkins/lib/test-dl-ep-matrix.yaml index 8bb44168a8..58c182902a 100644 --- a/.ci/jenkins/lib/test-dl-ep-matrix.yaml +++ b/.ci/jenkins/lib/test-dl-ep-matrix.yaml @@ -2,7 +2,7 @@ # # DLCluster GPU EP Test Matrix Configuration for dlcluster.nvidia.com # -# Runs the nixl_ep elastic tests on the DL cluster. This job is split out from +# Runs nixl_ep native elastic.py and vLLM+nixl_ep tests on the DL cluster. This job is split out from # nixl-ci-dl-gpu so the EP-specific image build (BUILD_NIXL_EP=true) and EP # Slurm reservation do not affect the existing DL test flow. # @@ -12,7 +12,7 @@ # build_helper_dl_ep) # - Matrix Axes: aarch64 (UCX version from the UCX_VER job parameter) # - Run Steps: build PR image with BUILD_NIXL_EP=true, allocate Slurm, run -# .gitlab/test_ep.sh (elastic.py NVLink + RDMA on 4 GPUs) +# .gitlab/test_ep.sh (nixl_ep native elastic.py and vLLM+nixl_ep tests on 4 GPUs) # # When Modified: # - Adding/removing Docker images: Affects available test environments @@ -28,7 +28,7 @@ job: nixl-ci-dl-gpu-ep # Fail job if one of the steps fails or continue failFast: false -timeout_minutes: 240 +timeout_minutes: 300 registry_host: artifactory.nvidia.com registry_auth: svc-nixl-new-artifactory-token @@ -48,17 +48,19 @@ credentials: env: ARTIFACTORY_PATH: /sw-nbu-swx-nixl-docker-local/ci NIXL_INSTALL_DIR: /opt/nixl + NIXL_EP_PYTHON: /home/svc-nixl/vllm/.venv/bin/python NIXL_BUILD_DIR: nixl_build SLURM_NODES: 1 SLURM_PARTITION: gb200nvl72_cx8 SLURM_HEAD_NODE: dlcluster.nvidia.com SLURM_HEAD_USER: svc-nixl SLURM_ACCOUNT: 'blackwell' - SLURM_JOB_TIMEOUT: '01:30:00' + SLURM_JOB_TIMEOUT: '02:00:00' SLURM_IMMEDIATE_TIMEOUT: 3600 SSH_CREDENTIALS_ID: 'svc-nixl-ssh_key' JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}" - TEST_TIMEOUT: 50 + TEST_TIMEOUT: 90 + VLLM_HF_ENDPOINT: "https://urm.nvidia.com/artifactory/api/huggingfaceml/huggingface-remote" STORAGE_DRIVER: overlay # Auto-derived and patched by cidemo-init.sh at CI time from the CI source files; # the "CI_MANAGED" placeholder must not be hand-edited. @@ -79,7 +81,7 @@ runs_on_dockers: name: 'nixl-ci-dl-gpu-ep-base-pytorch26.06-cuda13.3-ubuntu24.04', tag: "${CI_IMAGE_TAG}", arch: "aarch64", - build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg BASE_IMAGE=nvcr.io/nvidia/pytorch:26.06-py3 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache' + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg BASE_IMAGE=nvcr.io/nvidia/pytorch:26.06-py3 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg BUILD_VLLM_ELASTIC_TEST=true --build-arg ARCH=${arch} --pull --no-cache' } - { @@ -119,6 +121,7 @@ steps: --build-arg NIXL_BUILD_DIR=${NIXL_BUILD_DIR} \ --build-arg HAS_GPU=true \ --build-arg BUILD_NIXL_EP=true \ + --build-arg NIXL_PYTHON=${NIXL_EP_PYTHON} \ --build-arg BASE_IMAGE=${registry_host}${registry_path}/${arch}/nixl-ci-dl-gpu-ep-base-pytorch26.06-cuda13.3-ubuntu24.04:${CI_IMAGE_TAG} \ --tag ${PR_IMAGE} \ -f .ci/dockerfiles/Dockerfile.gpu-test . @@ -148,7 +151,7 @@ steps: "--account=${SLURM_ACCOUNT}" ] - - name: Run DL EP elastic tests + - name: Run DL NIXL EP tests containerSelector: "{name: 'build_helper_dl_ep'}" timeout: "${TEST_TIMEOUT}" parallel: false @@ -157,7 +160,7 @@ steps: run: run args: jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_ep_${BUILD_NUMBER}.txt" - testScript: ".gitlab/test_ep.sh ${NIXL_INSTALL_DIR}" + testScript: "HF_ENDPOINT=${VLLM_HF_ENDPOINT} HF_HUB_DISABLE_XET=1 .gitlab/test_ep.sh ${NIXL_INSTALL_DIR}" headNode: "${SLURM_HEAD_NODE}" headUser: "${SLURM_HEAD_USER}" dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-dl-gpu-ep-test:${BUILD_NUMBER}" diff --git a/.ci/jenkins/pipeline/proj-jjb.yaml b/.ci/jenkins/pipeline/proj-jjb.yaml index 1b61c115c0..a11e3dcc8a 100644 --- a/.ci/jenkins/pipeline/proj-jjb.yaml +++ b/.ci/jenkins/pipeline/proj-jjb.yaml @@ -320,7 +320,7 @@ parent-credentials: true script-path: "{jjb_jenkinsfile}" # Path to Jenkinsfile that defines the build steps -# Template for the DLCluster GPU EP test job that runs nixl_ep elastic tests on dlcluster.nvidia.com +# Template for the DLCluster job that runs nixl_ep native elastic.py and vLLM+nixl_ep tests - job-template: name: "{jjb_proj}-dl-gpu-ep" # Will be expanded to 'nixl-ci-dl-gpu-ep' project-type: pipeline @@ -1119,7 +1119,7 @@ - "{jjb_proj}-test-llm-container" # Create LLM container test job (SLURM) - "{jjb_proj}-gpu" # Create gpu job - "{jjb_proj}-dl-gpu" # Create dl-gpu job for dlcluster.nvidia.com - - "{jjb_proj}-dl-gpu-ep" # Create dl-gpu-ep job for nixl_ep elastic tests on dlcluster.nvidia.com + - "{jjb_proj}-dl-gpu-ep" # Create dl-gpu-ep job for NIXL EP on dlcluster.nvidia.com - "{jjb_proj}-build-wheel" # Create build wheel job - "{jjb_proj}-build-wheel-nightly" # Create nightly wheel verification job - "{jjb_proj}-cleanup-artifacts" # Create Artifactory cleanup job diff --git a/.ci/patches/nixl_ep_vllm_release_test.patch b/.ci/patches/nixl_ep_vllm_release_test.patch new file mode 100644 index 0000000000..69c3954885 --- /dev/null +++ b/.ci/patches/nixl_ep_vllm_release_test.patch @@ -0,0 +1,26 @@ +diff --git a/tests/distributed/test_elastic_ep.py b/tests/distributed/test_elastic_ep.py +--- a/tests/distributed/test_elastic_ep.py ++++ b/tests/distributed/test_elastic_ep.py +@@ -175,6 +175,9 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str + "--enable-expert-parallel", + "--all2all-backend", +- "allgather_reducescatter", ++ os.getenv( ++ "VLLM_TEST_ELASTIC_EP_ALL2ALL_BACKEND", ++ "allgather_reducescatter", ++ ), + "--enable-elastic-ep", + "--enable-eplb", + "--eplb-config.num_redundant_experts", +@@ -204,4 +207,8 @@ def _base_serve_args(dp_size: int = 2, enforce_eager: bool = False) -> list[str +- return args +- +- ++ attention_backend = os.getenv("VLLM_ATTENTION_BACKEND") ++ if attention_backend: ++ args.extend(["--attention-backend", attention_backend]) ++ ++ return args ++ ++ + @pytest.mark.parametrize( diff --git a/.gitlab/build.sh b/.gitlab/build.sh index 3acdc860ef..b1ef2fa848 100755 --- a/.gitlab/build.sh +++ b/.gitlab/build.sh @@ -438,9 +438,27 @@ else if [ "${BUILD_NIXL_EP}" = "true" ]; then EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS} -Dbuild_nixl_ep=true" fi + # When NIXL_PYTHON is set (currently only by test-dl-ep-matrix.yaml), build and + # install NIXL EP against vLLM's Python/Torch venv to prevent ABI mismatches. + # Only NIXL's Meson build uses this venv; dependency builds keep system Python. + # Other jobs leave NIXL_PYTHON_ARGS empty and keep the existing build behavior. + NIXL_PYTHON_ARGS=() + NIXL_PYTHON_NATIVE_FILE="" + if [ -n "${NIXL_PYTHON:-}" ]; then + if [ ! -x "${NIXL_PYTHON}" ]; then + echo "ERROR: NIXL_PYTHON is not executable: ${NIXL_PYTHON}" >&2 + exit 1 + fi + NIXL_PYTHON_NATIVE_FILE=$(mktemp) + printf "[binaries]\npython = '%s'\n" "${NIXL_PYTHON}" > "${NIXL_PYTHON_NATIVE_FILE}" + NIXL_PYTHON_ARGS=(--native-file "${NIXL_PYTHON_NATIVE_FILE}" -Dpython.install_env=venv) + fi # shellcheck disable=SC2086 - meson setup ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug + meson setup "${NIXL_PYTHON_ARGS[@]}" ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} && ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install + if [ -n "${NIXL_PYTHON_NATIVE_FILE}" ]; then + rm -f "${NIXL_PYTHON_NATIVE_FILE}" + fi mkdir -p dist && cp ${NIXL_BUILD_DIR}/src/bindings/python/nixl-meta/nixl-*.whl dist/ # TODO(kapila): Copy the nixl.pc file to the install directory if needed. diff --git a/.gitlab/test_ep.sh b/.gitlab/test_ep.sh index 56378cce07..307fb96a17 100755 --- a/.gitlab/test_ep.sh +++ b/.gitlab/test_ep.sh @@ -14,13 +14,14 @@ # See the License for the specific language governing permissions and # limitations under the License. -# nixl_ep elastic CI: run only EP tests (invoked from nixl-ci-dl-gpu-ep flow). +# NIXL EP CI: run nixl_ep native elastic.py and vLLM+nixl_ep tests. # shellcheck disable=SC1091 . "$(dirname "$0")/../.ci/scripts/common.sh" set -e set -x +set -o pipefail INSTALL_DIR=$1 @@ -29,6 +30,14 @@ if [ -z "$INSTALL_DIR" ]; then exit 1 fi +: "${VLLM_ELASTIC_TEST_DIR:?vLLM Elastic EP test environment is not installed}" +VLLM_PYTHON="${VLLM_ELASTIC_TEST_DIR}/.venv/bin/python" + +if [ ! -x "${VLLM_PYTHON}" ]; then + echo "ERROR: vLLM Python environment is missing: ${VLLM_PYTHON}" >&2 + exit 1 +fi + ARCH=$(uname -m) [ "$ARCH" = "arm64" ] && ARCH="aarch64" @@ -40,9 +49,8 @@ export NIXL_PLUGIN_DIR=${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins export NIXL_PREFIX=${INSTALL_DIR} export NIXL_DEBUG_LOGGING=yes -# Make `import nixl_ep` resolve the source-tree dispatcher, which loads the -# CUDA-versioned backend (nixl_ep_cu*) from the source install under ${INSTALL_DIR}. -export PYTHONPATH="${PWD}/src/bindings/python/nixl-meta:${INSTALL_DIR}/lib/python3/dist-packages${PYTHONPATH:+:$PYTHONPATH}" +# Load the source dispatcher and the backend installed in the shared venv. +export PYTHONPATH="${PWD}/src/bindings/python/nixl-meta${PYTHONPATH:+:$PYTHONPATH}" echo "==== Show system info ====" env @@ -67,7 +75,7 @@ run_elastic_test() { export UCX_TLS=^rc_gda fi PYTHONPATH="${NIXL_BUILD_DIR}/${EP_SRC_DIR}:${EP_SRC_DIR}/tests:${EP_SRC_DIR}/tests/elastic${PYTHONPATH:+:$PYTHONPATH}" \ - timeout 300 python3 ${EP_SRC_DIR}/tests/elastic/elastic.py \ + timeout 300 "${VLLM_PYTHON}" ${EP_SRC_DIR}/tests/elastic/elastic.py \ --plan "$plan_file" \ --num-processes 4 \ --num-experts-per-rank 32 \ @@ -103,3 +111,38 @@ else fi echo "==== nixl_ep elastic tests done ====" + +echo "==== Running vLLM Elastic EP test ====" +( + # Avoid SPCx loading HPC-X UCX 1.21; NIXL EP requires UCX >=1.22. + unset NCCL_NET_PLUGIN + unset UCX_NET_DEVICES + # TODO: remove this override when vLLM updates FlashInfer with + # https://github.com/flashinfer-ai/flashinfer/pull/4377. + # FlashInfer 0.6.16.post3's default MLA backend fails to JIT on CUDA 13.3. + export VLLM_ATTENTION_BACKEND=CUTLASS_MLA + export PATH="${VLLM_ELASTIC_TEST_DIR}/.venv/bin:${PATH}" + VLLM_LOG="${PWD}/elastic_ep_vllm_single_node.log" + VLLM_REF="$(git -C "${VLLM_ELASTIC_TEST_DIR}" describe --tags --exact-match HEAD)" + VLLM_COMMIT="$(git -C "${VLLM_ELASTIC_TEST_DIR}" rev-parse HEAD)" + + echo "vLLM source: VLLM_REF=${VLLM_REF} VLLM_COMMIT=${VLLM_COMMIT}" + + # Run vLLM's 2 -> 4 -> 2 Elastic EP scaling test with NIXL EP (Cover eager heavy traffic and CUDA graphs testing). + ( + cd "${VLLM_ELASTIC_TEST_DIR}" + VLLM_NIXL_EP_MAX_NUM_RANKS=4 \ + VLLM_TEST_ELASTIC_EP_ALL2ALL_BACKEND=nixl_ep \ + timeout 4500 "${VLLM_PYTHON}" -m pytest \ + "tests/distributed/test_elastic_ep.py::test_elastic_ep_scaling[enforce_eager_heavy]" \ + "tests/distributed/test_elastic_ep.py::test_elastic_ep_scaling[cuda_graphs_heavy]" \ + -v -s --tb=short 2>&1 | tee "${VLLM_LOG}" + ) + + if grep -Eiq '(^|[[:space:]])[0-9]+ skipped|SKIPPED' "${VLLM_LOG}"; then + echo "ERROR: vLLM Elastic EP test was skipped" >&2 + exit 1 + fi + + echo "==== vLLM Elastic EP test done ====" +)