diff --git a/.ci/cidemo-init.sh b/.ci/cidemo-init.sh index 8034766317..a7815a9c27 100755 --- a/.ci/cidemo-init.sh +++ b/.ci/cidemo-init.sh @@ -13,6 +13,7 @@ CI_FILES=( BUILD_MATRIX_YAML=".ci/jenkins/lib/build-matrix.yaml" TEST_MATRIX_YAML=".ci/jenkins/lib/test-matrix.yaml" TEST_DL_MATRIX_YAML=".ci/jenkins/lib/test-dl-matrix.yaml" +TEST_DL_EP_MATRIX_YAML=".ci/jenkins/lib/test-dl-ep-matrix.yaml" SANITIZER_MATRIX_YAML=".ci/jenkins/lib/test-sanitizer-matrix.yaml" # Function to extract CI_IMAGE_TAG from a YAML file @@ -61,22 +62,26 @@ echo "CI files were modified. Checking if CI_IMAGE_TAG was increased..." current_build_image_tag=$(get_ci_image_tag "$BUILD_MATRIX_YAML" "") current_test_image_tag=$(get_ci_image_tag "$TEST_MATRIX_YAML" "") current_test_dl_image_tag=$(get_ci_image_tag "$TEST_DL_MATRIX_YAML" "") +current_test_dl_ep_image_tag=$(get_ci_image_tag "$TEST_DL_EP_MATRIX_YAML" "") current_sanitizer_image_tag=$(get_ci_image_tag "$SANITIZER_MATRIX_YAML" "") previous_build_image_tag=$(get_ci_image_tag "$BUILD_MATRIX_YAML" "HEAD~1") previous_test_image_tag=$(get_ci_image_tag "$TEST_MATRIX_YAML" "HEAD~1") previous_test_dl_image_tag=$(get_ci_image_tag "$TEST_DL_MATRIX_YAML" "HEAD~1") +previous_test_dl_ep_image_tag=$(get_ci_image_tag "$TEST_DL_EP_MATRIX_YAML" "HEAD~1") previous_sanitizer_image_tag=$(get_ci_image_tag "$SANITIZER_MATRIX_YAML" "HEAD~1") echo "Build Matrix CI_IMAGE_TAG: $previous_build_image_tag -> $current_build_image_tag" echo "Test Matrix CI_IMAGE_TAG: $previous_test_image_tag -> $current_test_image_tag" echo "Test DL Matrix CI_IMAGE_TAG: $previous_test_dl_image_tag -> $current_test_dl_image_tag" +echo "Test DL EP Matrix CI_IMAGE_TAG: $previous_test_dl_ep_image_tag -> $current_test_dl_ep_image_tag" echo "Sanitizer Matrix CI_IMAGE_TAG: $previous_sanitizer_image_tag -> $current_sanitizer_image_tag" # Check if CI_IMAGE_TAG was changed in all files build_tag_changed=false test_tag_changed=false test_dl_tag_changed=false +test_dl_ep_tag_changed=false sanitizer_tag_changed=false if [ "$current_build_image_tag" != "$previous_build_image_tag" ]; then @@ -94,12 +99,17 @@ if [ "$current_test_dl_image_tag" != "$previous_test_dl_image_tag" ]; then test_dl_tag_changed=true fi +if [ "$current_test_dl_ep_image_tag" != "$previous_test_dl_ep_image_tag" ]; then + echo "✓ CI_IMAGE_TAG in test-dl-ep-matrix.yaml was updated" + test_dl_ep_tag_changed=true +fi + if [ "$current_sanitizer_image_tag" != "$previous_sanitizer_image_tag" ]; then echo "✓ CI_IMAGE_TAG in test-sanitizer-matrix.yaml was updated" sanitizer_tag_changed=true fi -if [ "$build_tag_changed" = false ] || [ "$test_tag_changed" = false ] || [ "$test_dl_tag_changed" = false ] || [ "$sanitizer_tag_changed" = false ]; then +if [ "$build_tag_changed" = false ] || [ "$test_tag_changed" = false ] || [ "$test_dl_tag_changed" = false ] || [ "$test_dl_ep_tag_changed" = false ] || [ "$sanitizer_tag_changed" = false ]; then echo "" echo "❌ ERROR: You have changed CI files but forgot to increase CI_IMAGE_TAG!" echo "" @@ -116,6 +126,7 @@ if [ "$build_tag_changed" = false ] || [ "$test_tag_changed" = false ] || [ "$te echo " - $BUILD_MATRIX_YAML (line 46)" echo " - $TEST_MATRIX_YAML (line 53)" echo " - $TEST_DL_MATRIX_YAML (line 52)" + echo " - $TEST_DL_EP_MATRIX_YAML" echo " - $SANITIZER_MATRIX_YAML" echo "" exit 1 diff --git a/.ci/dockerfiles/Dockerfile.gpu-test b/.ci/dockerfiles/Dockerfile.gpu-test index 036c947110..aada7e00ea 100644 --- a/.ci/dockerfiles/Dockerfile.gpu-test +++ b/.ci/dockerfiles/Dockerfile.gpu-test @@ -20,6 +20,7 @@ ARG PRE_INSTALLED_NIXL_ENV ARG PRE_INSTALLED_UCX_ENV ARG UCX_VERSION=v1.21.x ARG HAS_GPU=true +ARG BUILD_NIXL_EP=false ARG WORKSPACE=/workspace/nixl diff --git a/.ci/jenkins/lib/build-matrix.yaml b/.ci/jenkins/lib/build-matrix.yaml index bb949b2a6f..c967cb1848 100644 --- a/.ci/jenkins/lib/build-matrix.yaml +++ b/.ci/jenkins/lib/build-matrix.yaml @@ -44,7 +44,7 @@ env: TEST_TIMEOUT: 30 UCX_TLS: "^shm" STORAGE_DRIVER: 'overlay' - CI_IMAGE_TAG: "20260607-1" + CI_IMAGE_TAG: "20260625-1" runs_on_dockers: - { diff --git a/.ci/jenkins/lib/test-dl-ep-matrix.yaml b/.ci/jenkins/lib/test-dl-ep-matrix.yaml new file mode 100644 index 0000000000..c917f9cef5 --- /dev/null +++ b/.ci/jenkins/lib/test-dl-ep-matrix.yaml @@ -0,0 +1,168 @@ +--- +# +# DLCluster GPU EP Test Matrix Configuration for dlcluster.nvidia.com +# +# Runs the nixl_ep elastic tests on the DL cluster. This job is split out from +# nixl-ci-dl-gpu so the EP-specific image build (BUILD_NIXL_EP=true) and EP +# Slurm reservation do not affect the existing DL test flow. +# +# Key Components: +# - Job Configuration: Defines timeout, failure behavior, and server resources +# - Docker Images: Dedicated EP base + build_helper (nixl-ci-dl-gpu-ep-base-... and +# build_helper_dl_ep) +# - Matrix Axes: aarch64, UCX master + v1.21.x +# - Run Steps: build PR image with BUILD_NIXL_EP=true, allocate Slurm, run +# .gitlab/test_ep.sh (elastic.py NVLink + RDMA on 4 GPUs) +# +# When Modified: +# - Adding/removing Docker images: Affects available test environments +# - Modifying matrix axes: Changes test variations +# - Adjusting resource limits: Impacts test performance and resource allocation +# - Adding/removing steps: Changes the test pipeline sequence +# +# Note: Changes to this file are tested as part of the PR CI flow no need to test them manually. + + +job: nixl-ci-dl-gpu-ep + +# Fail job if one of the steps fails or continue +failFast: false + +timeout_minutes: 240 + +registry_host: artifactory.nvidia.com +registry_auth: svc-nixl-new-artifactory-token +registry_path: /sw-nbu-swx-nixl-docker-local/ci + +kubernetes: + cloud: il-ipp-blossom-prod + namespace: nbu-swx-nixl + imagePullSecrets: "['artifactory-pull-secret']" + limits: "{memory: 16Gi, cpu: 16000m}" + requests: "{memory: 8Gi, cpu: 8000m}" + privileged: true + +credentials: + - {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'} + +env: + ARTIFACTORY_PATH: /sw-nbu-swx-nixl-docker-local/ci + NIXL_INSTALL_DIR: /opt/nixl + NIXL_BUILD_DIR: nixl_build + SLURM_NODES: 1 + SLURM_PARTITION: gb200nvl72_cx8 + SLURM_HEAD_NODE: dlcluster.nvidia.com + SLURM_HEAD_USER: svc-nixl + SLURM_ACCOUNT: 'blackwell' + SLURM_JOB_TIMEOUT: '01:30:00' + SLURM_IMMEDIATE_TIMEOUT: 3600 + SSH_CREDENTIALS_ID: 'svc-nixl-ssh_key' + JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}" + TEST_TIMEOUT: 50 + STORAGE_DRIVER: overlay + CI_IMAGE_TAG: "20260625-1" + +empty_volumes: + - {mountPath: /var/lib/containers/storage, memory: false} + +pvc_volumes: + - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false} + +# Docker images for DL EP testing. +# Dedicated EP base + build_helper images (separate artifactory paths from the +# nixl-ci-dl-gpu job). +runs_on_dockers: + - { + file: '.ci/dockerfiles/Dockerfile.base', + name: 'nixl-ci-dl-gpu-ep-base-25.10-cuda13.0-ubuntu24.04', + tag: "${CI_IMAGE_TAG}", + arch: "aarch64", + build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.10-cuda13.0-devel-ubuntu24.04 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache' + } + + - { + file: '.ci/dockerfiles/Dockerfile.build_helper', + name: 'build_helper_dl_ep', + arch: "aarch64", + tag: "${CI_IMAGE_TAG}", + build_args: '--build-arg BASE_IMAGE=dockerhub.nvidia.com/ubuntu:24.04 --build-arg ARCH=${arch}' + } + +matrix: + axes: + arch: + - aarch64 + ucx_version: + - master + - v1.21.x + +taskName: "${name}/${arch}/ucx-${ucx_version}/${axis_index}" + +steps: + - name: Compiling NIXL EP Docker Image for DL + containerSelector: "{name: 'build_helper_dl_ep'}" + credentialsId: "svc-nixl-new-artifactory-token" + parallel: false + run: | + set -x + export PR_IMAGE=${registry_host}${registry_path}/pr/${arch}/nixl-ci-dl-gpu-ep-test-${ucx_version}:${BUILD_NUMBER} + rm -rf /etc/containers/storage.conf && rm -f /usr/share/containers/storage.conf + podman build --network host --creds ${REPO_USER}:${REPO_PASS} \ + --build-arg UCX_VERSION=${ucx_version} \ + --build-arg PRE_INSTALLED_ENV="true" \ + --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} \ + --build-arg NIXL_BUILD_DIR=${NIXL_BUILD_DIR} \ + --build-arg HAS_GPU=true \ + --build-arg BUILD_NIXL_EP=true \ + --build-arg BASE_IMAGE=${registry_host}${registry_path}/${arch}/nixl-ci-dl-gpu-ep-base-25.10-cuda13.0-ubuntu24.04:${CI_IMAGE_TAG} \ + --tag ${PR_IMAGE} \ + -f .ci/dockerfiles/Dockerfile.gpu-test . + podman push --creds ${REPO_USER}:${REPO_PASS} ${PR_IMAGE} + + - name: Allocate DL EP Environment + containerSelector: "{name: 'build_helper_dl_ep'}" + parallel: false + shell: action + module: slurmCI + run: allocation + args: + partition: "${SLURM_PARTITION}" + headNode: "${SLURM_HEAD_NODE}" + headUser: "${SLURM_HEAD_USER}" + nodes: "${SLURM_NODES}" + jobTimeout: "${SLURM_JOB_TIMEOUT}" + immediateTimeout: "${SLURM_IMMEDIATE_TIMEOUT}" + jobName: "nixl-ci-ep-${ucx_version}-${BUILD_NUMBER}" + jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_ep_${ucx_version}_${BUILD_NUMBER}.txt" + credentialsId: "${SSH_CREDENTIALS_ID}" + extraArgs: [ + "--account=${SLURM_ACCOUNT}" + ] + + - name: Run DL EP elastic tests + containerSelector: "{name: 'build_helper_dl_ep'}" + timeout: "${TEST_TIMEOUT}" + parallel: false + shell: action + module: slurmCI + run: run + args: + jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_ep_${ucx_version}_${BUILD_NUMBER}.txt" + testScript: ".gitlab/test_ep.sh ${NIXL_INSTALL_DIR}" + headNode: "${SLURM_HEAD_NODE}" + headUser: "${SLURM_HEAD_USER}" + dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-dl-gpu-ep-test-${ucx_version}:${BUILD_NUMBER}" + credentialsId: "${SSH_CREDENTIALS_ID}" + containerName: "nixl-ci-ep-${ucx_version}-${BUILD_NUMBER}" + +pipeline_stop: + containerSelector: "{name: 'build_helper_dl_ep'}" + parallel: false + shell: action + module: slurmCI + run: stopAllForBuild + args: + credentialsId: "${SSH_CREDENTIALS_ID}" + headNode: "${SLURM_HEAD_NODE}" + headUser: "${SLURM_HEAD_USER}" + jobIdDir: "${JOB_ID_FILE_ROOT}" diff --git a/.ci/jenkins/lib/test-dl-matrix.yaml b/.ci/jenkins/lib/test-dl-matrix.yaml index 911d06f349..16e5d34280 100644 --- a/.ci/jenkins/lib/test-dl-matrix.yaml +++ b/.ci/jenkins/lib/test-dl-matrix.yaml @@ -54,7 +54,7 @@ env: JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}" TEST_TIMEOUT: 50 STORAGE_DRIVER: overlay - CI_IMAGE_TAG: "20260607-1" + CI_IMAGE_TAG: "20260625-1" empty_volumes: - {mountPath: /var/lib/containers/storage, memory: false} diff --git a/.ci/jenkins/lib/test-matrix.yaml b/.ci/jenkins/lib/test-matrix.yaml index c381aec267..2c6ca98633 100644 --- a/.ci/jenkins/lib/test-matrix.yaml +++ b/.ci/jenkins/lib/test-matrix.yaml @@ -52,7 +52,7 @@ env: SCCTL_CREDENTIALS_ID: 'svc-nixl-scctl' JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}" STORAGE_DRIVER: overlay - CI_IMAGE_TAG: "20260607-1" + CI_IMAGE_TAG: "20260625-1" empty_volumes: - {mountPath: /root, memory: false} diff --git a/.ci/jenkins/lib/test-sanitizer-matrix.yaml b/.ci/jenkins/lib/test-sanitizer-matrix.yaml index 62a502057b..e6318a0cb5 100644 --- a/.ci/jenkins/lib/test-sanitizer-matrix.yaml +++ b/.ci/jenkins/lib/test-sanitizer-matrix.yaml @@ -42,7 +42,7 @@ env: TEST_TIMEOUT: 90 UCX_TLS: "^shm" STORAGE_DRIVER: 'overlay' - CI_IMAGE_TAG: "20260607-1" + CI_IMAGE_TAG: "20260625-1" runs_on_dockers: # TSan-instrumented dependency image for the tsan build (DEPS_SANITIZE=thread diff --git a/.ci/jenkins/pipeline/proj-jjb.yaml b/.ci/jenkins/pipeline/proj-jjb.yaml index 868bc08586..661d8a1e6f 100644 --- a/.ci/jenkins/pipeline/proj-jjb.yaml +++ b/.ci/jenkins/pipeline/proj-jjb.yaml @@ -125,6 +125,12 @@ string(name: 'sha1', value: githubHelper.getMergedSHA()), string(name: 'githubData', value: VARIABLE_FROM_POST) ], propagate: false, wait: true + }}, 'dl-gpu-ep': {{ + def jobName = 'nixl-ci-dl-gpu-ep' + build job: jobName, parameters: [ + string(name: 'sha1', value: githubHelper.getMergedSHA()), + string(name: 'githubData', value: VARIABLE_FROM_POST) + ], propagate: false, wait: true }}, wheel: {{ def buildJob = 'nixl-ci-build-wheel' build job: buildJob, parameters: [ @@ -324,6 +330,67 @@ parent-credentials: true script-path: "{jjb_jenkinsfile}" # Path to Jenkinsfile that defines the build steps +# Template for the DLCluster GPU EP test job that runs nixl_ep elastic tests on dlcluster.nvidia.com +- job-template: + name: "{jjb_proj}-dl-gpu-ep" # Will be expanded to 'nixl-ci-dl-gpu-ep' + project-type: pipeline + disabled: false + properties: + # Similar properties as dispatcher job + - github: + url: "{jjb_gh_url}" + - build-discarder: + days-to-keep: 14 + num-to-keep: 1000 + - inject: + keep-system-variables: true + properties-content: | + jjb_proj={jjb_proj}-dl-gpu-ep + description: Do NOT edit this job through the Web GUI ! + concurrent: true + sandbox: true + # Test job parameters + parameters: + - string: + name: "sha1" + default: "{jjb_branch}" # Default to 'main' branch + description: "Commit to be checked, usually set by PR" + - string: + name: "githubData" + default: "" + description: "Variables from post" + - string: + name: "conf_file" + default: ".ci/jenkins/lib/test-dl-ep-matrix.yaml" # DLCluster EP test matrix configuration + description: "Job config file" + - bool: + name: "build_dockers" + default: false + description: "Force rebuild docker containers" + - string: + name: "DEBUG" + default: 0 + description: "Enable debug prints and traces, valid values are 0-9" + # SCM configuration for the build job + pipeline-scm: + scm: + - git: + url: "{jjb_git}" + branches: ['$sha1'] + shallow-clone: false + do-not-fetch-tags: false + # Configure refspec to handle branches, PRs, and tags + refspec: "{jjb_gh_refspec}" + browser: githubweb + browser-url: "{jjb_git}" + # Handle git submodules + submodule: + disable: false + recursive: true + tracking: true + parent-credentials: true + script-path: "{jjb_jenkinsfile}" # Path to Jenkinsfile that defines the build steps + # Template for the DLCluster GPU test job that runs on dlcluster.nvidia.com - job-template: name: "{jjb_proj}-dl-gpu" # Will be expanded to 'nixl-ci-dl-gpu' @@ -561,4 +628,5 @@ - "{jjb_proj}-build-container" # Create container builder job - "{jjb_proj}-gpu" # Create gpu job - "{jjb_proj}-dl-gpu" # Create dl-gpu job for dlcluster.nvidia.com + - "{jjb_proj}-dl-gpu-ep" # Create dl-gpu-ep job for nixl_ep elastic tests on dlcluster.nvidia.com - "{jjb_proj}-build-wheel" # Create build wheel job diff --git a/.gitlab/build.sh b/.gitlab/build.sh index 9de4804cd3..d2905b51f0 100755 --- a/.gitlab/build.sh +++ b/.gitlab/build.sh @@ -406,6 +406,9 @@ export UCX_TLS=^cuda_ipc if [ -n "$PRE_INSTALLED_NIXL_ENV" ]; then echo "PRE_INSTALLED_NIXL_ENV is set, skipping compilation" else + if [ "${BUILD_NIXL_EP}" = "true" ]; then + EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS} -Dbuild_nixl_ep=true" + fi # shellcheck disable=SC2086 meson setup ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} && ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install diff --git a/.gitlab/test_ep.sh b/.gitlab/test_ep.sh new file mode 100755 index 0000000000..7521a70cfc --- /dev/null +++ b/.gitlab/test_ep.sh @@ -0,0 +1,140 @@ +#!/bin/bash +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# nixl_ep elastic CI: run only EP tests (invoked from nixl-ci-dl-gpu-ep flow). + +# shellcheck disable=SC1091 +. "$(dirname "$0")/../.ci/scripts/common.sh" + +set -e +set -x + +INSTALL_DIR=$1 + +if [ -z "$INSTALL_DIR" ]; then + echo "Usage: $0 " + exit 1 +fi + +ARCH=$(uname -m) +[ "$ARCH" = "arm64" ] && ARCH="aarch64" + +export LD_LIBRARY_PATH=${INSTALL_DIR}/lib:${INSTALL_DIR}/lib/$ARCH-linux-gnu:${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins:/usr/local/lib:$LD_LIBRARY_PATH +export CPATH=${INSTALL_DIR}/include:$CPATH +export PATH=${INSTALL_DIR}/bin:$PATH +export PKG_CONFIG_PATH=${INSTALL_DIR}/lib/pkgconfig:$PKG_CONFIG_PATH +export NIXL_PLUGIN_DIR=${INSTALL_DIR}/lib/$ARCH-linux-gnu/plugins +export NIXL_PREFIX=${INSTALL_DIR} +export NIXL_DEBUG_LOGGING=yes + +# The PR image installs the CUDA-versioned bindings (nixl_ep_cu*) under +# ${INSTALL_DIR}/lib/python3/dist-packages via meson's --prefix, which is not +# on Python's default sys.path. Expose it so the nixl_ep dispatcher can load +# nixl_ep_cu13 at runtime. +export PYTHONPATH="${INSTALL_DIR}/lib/python3/dist-packages${PYTHONPATH:+:$PYTHONPATH}" + +# Install the nixl meta wheel (provides the nixl_ep dispatcher package that +# re-exports from nixl_ep_cu13). The wheel is staged in dist/ by the PR image +# build; without this install `import nixl_ep` fails with ModuleNotFoundError. +if [ -n "$VIRTUAL_ENV" ] && grep -q '^uv =' "$VIRTUAL_ENV/pyvenv.cfg" 2>/dev/null; then + pip3="uv pip" +else + pip3="python3 -m pip" +fi +$pip3 install --break-system-packages dist/nixl-*none-any.whl + +echo "==== Show system info ====" +env +nvidia-smi topo -m || true +ibv_devinfo || true +uname -a || true +cat /sys/devices/virtual/dmi/id/product_name || true + +echo "==== NVIDIA Peermem check ====" +if ! lsmod | grep -q nvidia_peermem; then + echo "nvidia_peermem module not loaded" +fi + +if [ -f /sys/kernel/mm/memory_peers/nv_mem/version ]; then + cat /sys/kernel/mm/memory_peers/nv_mem/version +else + echo "/sys/kernel/mm/memory_peers/nv_mem/version not found " +fi + +if [ -f /sys/module/nvidia_peermem/version ]; then + cat /sys/module/nvidia_peermem/version +else + echo "/sys/module/nvidia_peermem/version not found" +fi + +if [ -f /sys/module/nv_peer_mem/version ]; then + cat /sys/module/nv_peer_mem/version +else + echo "/sys/module/nv_peer_mem/version not found" +fi + +echo "==== Running elastic EP tests ====" +EP_SRC_DIR="examples/device/ep" +NIXL_BUILD_DIR=${NIXL_BUILD_DIR:-nixl_build} + +run_elastic_test() { + local plan_file=$1 + local extra_flags=${2:-} + echo "---- elastic: plan=$(basename "$plan_file") flags=[$extra_flags] ----" + ( + unset NIXL_ETCD_ENDPOINTS NIXL_ETCD_PEER_URLS NIXL_ETCD_NAMESPACE + unset UCX_NET_DEVICES # let UCX auto-select GPU-capable transport + # Force NVLink-only transports. + if [[ "$extra_flags" != *--disable-ll-nvlink* ]]; then + export UCX_TLS=cuda_copy,cuda_ipc,sm,self + fi + PYTHONPATH="${NIXL_BUILD_DIR}/${EP_SRC_DIR}:${EP_SRC_DIR}/tests:${EP_SRC_DIR}/tests/elastic${PYTHONPATH:+:$PYTHONPATH}" \ + timeout 300 python3 ${EP_SRC_DIR}/tests/elastic/elastic.py \ + --plan "$plan_file" \ + --num-processes 4 \ + --num-experts-per-rank 32 \ + --num-topk 8 \ + --num-tokens 256 \ + --timeout-ms 10000 \ + --validate-phase-failures $extra_flags + ) +} + +# NVLink (default) +run_elastic_test "${EP_SRC_DIR}/tests/elastic/no_expansion.json" +run_elastic_test "${EP_SRC_DIR}/tests/elastic/expansion_fault_contraction.json" + +# Only run the --disable-ll-nvlink (RDMA) elastic tests when all four CX-7 +# NICs (mlx5_0..mlx5_3) report PORT_ACTIVE. +all_rdma_nics_active() { + local hca + for hca in mlx5_0 mlx5_1 mlx5_2 mlx5_3; do + if ! ibv_devinfo -d "$hca" 2>/dev/null | grep -q "state:.*PORT_ACTIVE"; then + return 1 + fi + done + return 0 +} + +# RDMA (--disable-ll-nvlink) +if all_rdma_nics_active; then + run_elastic_test "${EP_SRC_DIR}/tests/elastic/no_expansion.json" "--disable-ll-nvlink" + run_elastic_test "${EP_SRC_DIR}/tests/elastic/expansion_fault_contraction.json" "--disable-ll-nvlink" +else + echo "Skipping RDMA elastic tests: not all of mlx5_0..mlx5_3 are PORT_ACTIVE on $(hostname)" +fi + +echo "==== nixl_ep elastic tests done ====" diff --git a/examples/device/ep/meson.build b/examples/device/ep/meson.build index 198df39bc7..c912e25967 100644 --- a/examples/device/ep/meson.build +++ b/examples/device/ep/meson.build @@ -118,10 +118,12 @@ nixl_ep_rpath += ':' + nixl_lib_dir nixl_ep_rpath += ':' + join_paths(nixl_lib_dir, 'core') nixl_ep_rpath += ':' + join_paths(nixl_lib_dir, 'plugins') -# For now, nixl ep cannot be built with -G due to register usage limits +# For now, nixl ep cannot be built with -G due to register usage limits; +# build the EP target as release even in a global debug build (same flags +# we ship in release wheels), and add -g back so we keep debug symbols. nixl_ep_override_options = [] if get_option('buildtype') == 'debug' - nixl_ep_override_options = ['optimization=3'] + nixl_ep_override_options = ['buildtype=release'] nixl_ep_cpp_args += ['-g'] endif diff --git a/examples/device/ep/tests/elastic/elastic.py b/examples/device/ep/tests/elastic/elastic.py index 5f57ce4d26..21825588f3 100644 --- a/examples/device/ep/tests/elastic/elastic.py +++ b/examples/device/ep/tests/elastic/elastic.py @@ -577,16 +577,27 @@ def worker(torch_rank: int, args: argparse.Namespace): kineto=args.kineto, fault_tolerance_test=kill_rank, ) - # Query mask buffer to detect any unexpected rank failures and clean them up + # Query mask buffer to detect rank failures and clean them up buffer.query_mask_buffer(mask_status) newly_failed_ranks = set() for r in range(current_num_ranks): if mask_status[r].item() != 0 and r in remote_ranks: newly_failed_ranks.add(r) + if args.validate_phase_failures: + expected_failed_ranks = set(ranks_to_kill) & remote_ranks + unexpected_failures = newly_failed_ranks - expected_failed_ranks + assert ( + not unexpected_failures + ), f"rank {global_rank}, local_rank={local_rank} phase {plan.get_phase()}: unexpected failures {unexpected_failures}" + missing_failures = expected_failed_ranks - newly_failed_ranks + assert ( + not missing_failures + ), f"rank {global_rank}, local_rank={local_rank} phase {plan.get_phase()}: missing expected failures {missing_failures}" + if len(newly_failed_ranks) > 0: print( - f"global_rank={global_rank}, local_rank={local_rank} -> detected unexpected rank failures: {newly_failed_ranks}, cleaning up...", + f"global_rank={global_rank}, local_rank={local_rank} -> detected rank failures: {newly_failed_ranks}, cleaning up...", flush=True, ) remote_ranks.difference_update(newly_failed_ranks) @@ -645,6 +656,11 @@ def main(): default=DEFAULT_TIMEOUT_MS, help="GPU timeout in milliseconds (non-negative integer)", ) + parser.add_argument( + "--validate-phase-failures", + action="store_true", + help="Enable strict phase-local validation of observed rank failures against the plan", + ) args = parser.parse_args() diff --git a/examples/device/ep/tests/elastic/expansion_fault_contraction.json b/examples/device/ep/tests/elastic/expansion_fault_contraction.json new file mode 100644 index 0000000000..89f15933ab --- /dev/null +++ b/examples/device/ep/tests/elastic/expansion_fault_contraction.json @@ -0,0 +1,6 @@ +[ + [0, 1], + [0, 1, 2, 3], + [0, 1, -2, 3], + [0, 3] +] diff --git a/src/bindings/python/nixl-meta/nixl_ep/__init__.py b/src/bindings/python/nixl-meta/nixl_ep/__init__.py index 0ac6b2b087..65ace49c9c 100644 --- a/src/bindings/python/nixl-meta/nixl_ep/__init__.py +++ b/src/bindings/python/nixl-meta/nixl_ep/__init__.py @@ -75,5 +75,8 @@ def _load_ep_module() -> str: setattr(sys.modules[__name__], attr, getattr(_pkg, attr)) if TYPE_CHECKING: + import torch from nixl_ep.buffer import Buffer # noqa: F401 from nixl_ep.utils import EventOverlap # noqa: F401 + + topk_idx_t: torch.dtype