ci: Add nixl ep to new dl ci pipeline

.ci/cidemo-init.sh

@@ -13,6 +13,7 @@ CI_FILES=(
 BUILD_MATRIX_YAML=".ci/jenkins/lib/build-matrix.yaml"
 TEST_MATRIX_YAML=".ci/jenkins/lib/test-matrix.yaml"
 TEST_DL_MATRIX_YAML=".ci/jenkins/lib/test-dl-matrix.yaml"
+TEST_DL_EP_MATRIX_YAML=".ci/jenkins/lib/test-dl-ep-matrix.yaml"
 SANITIZER_MATRIX_YAML=".ci/jenkins/lib/test-sanitizer-matrix.yaml"
 
 # Function to extract CI_IMAGE_TAG from a YAML file
@@ -61,22 +62,26 @@ echo "CI files were modified. Checking if CI_IMAGE_TAG was increased..."
 current_build_image_tag=$(get_ci_image_tag "$BUILD_MATRIX_YAML" "")
 current_test_image_tag=$(get_ci_image_tag "$TEST_MATRIX_YAML" "")
 current_test_dl_image_tag=$(get_ci_image_tag "$TEST_DL_MATRIX_YAML" "")
+current_test_dl_ep_image_tag=$(get_ci_image_tag "$TEST_DL_EP_MATRIX_YAML" "")
 current_sanitizer_image_tag=$(get_ci_image_tag "$SANITIZER_MATRIX_YAML" "")
 
 previous_build_image_tag=$(get_ci_image_tag "$BUILD_MATRIX_YAML" "HEAD~1")
 previous_test_image_tag=$(get_ci_image_tag "$TEST_MATRIX_YAML" "HEAD~1")
 previous_test_dl_image_tag=$(get_ci_image_tag "$TEST_DL_MATRIX_YAML" "HEAD~1")
+previous_test_dl_ep_image_tag=$(get_ci_image_tag "$TEST_DL_EP_MATRIX_YAML" "HEAD~1")
 previous_sanitizer_image_tag=$(get_ci_image_tag "$SANITIZER_MATRIX_YAML" "HEAD~1")
 
 echo "Build Matrix CI_IMAGE_TAG:     $previous_build_image_tag -> $current_build_image_tag"
 echo "Test Matrix CI_IMAGE_TAG:      $previous_test_image_tag -> $current_test_image_tag"
 echo "Test DL Matrix CI_IMAGE_TAG:   $previous_test_dl_image_tag -> $current_test_dl_image_tag"
+echo "Test DL EP Matrix CI_IMAGE_TAG: $previous_test_dl_ep_image_tag -> $current_test_dl_ep_image_tag"
 echo "Sanitizer Matrix CI_IMAGE_TAG: $previous_sanitizer_image_tag -> $current_sanitizer_image_tag"
 
 # Check if CI_IMAGE_TAG was changed in all files
 build_tag_changed=false
 test_tag_changed=false
 test_dl_tag_changed=false
+test_dl_ep_tag_changed=false
 sanitizer_tag_changed=false
 
 if [ "$current_build_image_tag" != "$previous_build_image_tag" ]; then
@@ -94,12 +99,17 @@ if [ "$current_test_dl_image_tag" != "$previous_test_dl_image_tag" ]; then
     test_dl_tag_changed=true
 fi
 
+if [ "$current_test_dl_ep_image_tag" != "$previous_test_dl_ep_image_tag" ]; then
+    echo "✓ CI_IMAGE_TAG in test-dl-ep-matrix.yaml was updated"
+    test_dl_ep_tag_changed=true
+fi
+
 if [ "$current_sanitizer_image_tag" != "$previous_sanitizer_image_tag" ]; then
     echo "✓ CI_IMAGE_TAG in test-sanitizer-matrix.yaml was updated"
     sanitizer_tag_changed=true
 fi
 
-if [ "$build_tag_changed" = false ] || [ "$test_tag_changed" = false ] || [ "$test_dl_tag_changed" = false ] || [ "$sanitizer_tag_changed" = false ]; then
+if [ "$build_tag_changed" = false ] || [ "$test_tag_changed" = false ] || [ "$test_dl_tag_changed" = false ] || [ "$test_dl_ep_tag_changed" = false ] || [ "$sanitizer_tag_changed" = false ]; then
     echo ""
     echo "❌ ERROR: You have changed CI files but forgot to increase CI_IMAGE_TAG!"
     echo ""
@@ -116,6 +126,7 @@ if [ "$build_tag_changed" = false ] || [ "$test_tag_changed" = false ] || [ "$te
     echo "  - $BUILD_MATRIX_YAML (line 46)"
     echo "  - $TEST_MATRIX_YAML (line 53)"
     echo "  - $TEST_DL_MATRIX_YAML (line 52)"
+    echo "  - $TEST_DL_EP_MATRIX_YAML"
     echo "  - $SANITIZER_MATRIX_YAML"
     echo ""
     exit 1

.ci/dockerfiles/Dockerfile.gpu-test

@@ -20,6 +20,7 @@ ARG PRE_INSTALLED_NIXL_ENV
 ARG PRE_INSTALLED_UCX_ENV
 ARG UCX_VERSION=v1.21.x
 ARG HAS_GPU=true
+ARG BUILD_NIXL_EP=false
 ARG WORKSPACE=/workspace/nixl
 
 

.ci/jenkins/lib/build-matrix.yaml

@@ -44,7 +44,7 @@ env:
   TEST_TIMEOUT: 30
   UCX_TLS: "^shm"
   STORAGE_DRIVER: 'overlay'
-  CI_IMAGE_TAG: "20260607-1"
+  CI_IMAGE_TAG: "20260625-1"
 
 runs_on_dockers:
   - {

.ci/jenkins/lib/test-dl-ep-matrix.yaml

@@ -0,0 +1,168 @@
+---
+#
+# DLCluster GPU EP Test Matrix Configuration for dlcluster.nvidia.com
+#
+# Runs the nixl_ep elastic tests on the DL cluster. This job is split out from
+# nixl-ci-dl-gpu so the EP-specific image build (BUILD_NIXL_EP=true) and EP
+# Slurm reservation do not affect the existing DL test flow.
+#
+# Key Components:
+# - Job Configuration: Defines timeout, failure behavior, and server resources
+# - Docker Images: Dedicated EP base + build_helper (nixl-ci-dl-gpu-ep-base-... and
+#                  build_helper_dl_ep)
+# - Matrix Axes: aarch64, UCX master + v1.21.x
+# - Run Steps: build PR image with BUILD_NIXL_EP=true, allocate Slurm, run
+#              .gitlab/test_ep.sh (elastic.py NVLink + RDMA on 4 GPUs)
+#
+# When Modified:
+# - Adding/removing Docker images: Affects available test environments
+# - Modifying matrix axes: Changes test variations
+# - Adjusting resource limits: Impacts test performance and resource allocation
+# - Adding/removing steps: Changes the test pipeline sequence
+#
+# Note: Changes to this file are tested as part of the PR CI flow no need to test them manually.
+
+
+job: nixl-ci-dl-gpu-ep
+
+# Fail job if one of the steps fails or continue
+failFast: false
+
+timeout_minutes: 240
+
+registry_host: artifactory.nvidia.com
+registry_auth: svc-nixl-new-artifactory-token
+registry_path: /sw-nbu-swx-nixl-docker-local/ci
+
+kubernetes:
+  cloud: il-ipp-blossom-prod
+  namespace: nbu-swx-nixl
+  imagePullSecrets: "['artifactory-pull-secret']"
+  limits: "{memory: 16Gi, cpu: 16000m}"
+  requests: "{memory: 8Gi, cpu: 8000m}"
+  privileged: true
+
+credentials:
+  - {credentialsId: 'svc-nixl-new-artifactory-token', usernameVariable: 'REPO_USER', passwordVariable: 'REPO_PASS'}
+
+env:
+  ARTIFACTORY_PATH: /sw-nbu-swx-nixl-docker-local/ci
+  NIXL_INSTALL_DIR: /opt/nixl
+  NIXL_BUILD_DIR: nixl_build
+  SLURM_NODES: 1
+  SLURM_PARTITION: gb200nvl72_cx8
+  SLURM_HEAD_NODE: dlcluster.nvidia.com
+  SLURM_HEAD_USER: svc-nixl
+  SLURM_ACCOUNT: 'blackwell'
+  SLURM_JOB_TIMEOUT: '01:30:00'
+  SLURM_IMMEDIATE_TIMEOUT: 3600
+  SSH_CREDENTIALS_ID: 'svc-nixl-ssh_key'
+  JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
+  TEST_TIMEOUT: 50
+  STORAGE_DRIVER: overlay
+  CI_IMAGE_TAG: "20260625-1"
+
+empty_volumes:
+  - {mountPath: /var/lib/containers/storage, memory: false}
+
+pvc_volumes:
+  - {claimName: nbu-swx-nixl-pvc, mountPath: /mnt/pvc, readOnly: false}
+
+# Docker images for DL EP testing.
+# Dedicated EP base + build_helper images (separate artifactory paths from the
+# nixl-ci-dl-gpu job).
+runs_on_dockers:
+  - {
+     file: '.ci/dockerfiles/Dockerfile.base',
+     name: 'nixl-ci-dl-gpu-ep-base-25.10-cuda13.0-ubuntu24.04',
+     tag: "${CI_IMAGE_TAG}",
+     arch: "aarch64",
+     build_args: '--build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} --build-arg BASE_IMAGE=nvcr.io/nvidia/cuda-dl-base:25.10-cuda13.0-devel-ubuntu24.04 --build-arg PRE_INSTALLED_UCX_ENV=true --build-arg PRE_INSTALLED_NIXL_ENV=true --build-arg ARCH=${arch} --pull --no-cache'
+    }
+
+  - {
+     file: '.ci/dockerfiles/Dockerfile.build_helper',
+     name: 'build_helper_dl_ep',
+     arch: "aarch64",
+     tag: "${CI_IMAGE_TAG}",
+     build_args: '--build-arg BASE_IMAGE=dockerhub.nvidia.com/ubuntu:24.04 --build-arg ARCH=${arch}'
+    }
+
+matrix:
+  axes:
+    arch:
+      - aarch64
+    ucx_version:
+      - master
+      - v1.21.x
+
+taskName: "${name}/${arch}/ucx-${ucx_version}/${axis_index}"
+
+steps:
+  - name: Compiling NIXL EP Docker Image for DL
+    containerSelector: "{name: 'build_helper_dl_ep'}"
+    credentialsId: "svc-nixl-new-artifactory-token"
+    parallel: false
+    run: |
+      set -x
+      export PR_IMAGE=${registry_host}${registry_path}/pr/${arch}/nixl-ci-dl-gpu-ep-test-${ucx_version}:${BUILD_NUMBER}
+      rm -rf /etc/containers/storage.conf && rm -f /usr/share/containers/storage.conf
+      podman build --network host --creds ${REPO_USER}:${REPO_PASS} \
+      --build-arg UCX_VERSION=${ucx_version} \
+      --build-arg PRE_INSTALLED_ENV="true" \
+      --build-arg NIXL_INSTALL_DIR=${NIXL_INSTALL_DIR} \
+      --build-arg NIXL_BUILD_DIR=${NIXL_BUILD_DIR} \
+      --build-arg HAS_GPU=true \
+      --build-arg BUILD_NIXL_EP=true \
+      --build-arg BASE_IMAGE=${registry_host}${registry_path}/${arch}/nixl-ci-dl-gpu-ep-base-25.10-cuda13.0-ubuntu24.04:${CI_IMAGE_TAG} \
+      --tag ${PR_IMAGE} \
+      -f .ci/dockerfiles/Dockerfile.gpu-test .
+      podman push --creds ${REPO_USER}:${REPO_PASS} ${PR_IMAGE}
+
+  - name: Allocate DL EP Environment
+    containerSelector: "{name: 'build_helper_dl_ep'}"
+    parallel: false
+    shell: action
+    module: slurmCI
+    run: allocation
+    args:
+      partition: "${SLURM_PARTITION}"
+      headNode: "${SLURM_HEAD_NODE}"
+      headUser: "${SLURM_HEAD_USER}"
+      nodes: "${SLURM_NODES}"
+      jobTimeout: "${SLURM_JOB_TIMEOUT}"
+      immediateTimeout: "${SLURM_IMMEDIATE_TIMEOUT}"
+      jobName: "nixl-ci-ep-${ucx_version}-${BUILD_NUMBER}"
+      jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_ep_${ucx_version}_${BUILD_NUMBER}.txt"
+      credentialsId: "${SSH_CREDENTIALS_ID}"
+      extraArgs: [
+        "--account=${SLURM_ACCOUNT}"
+      ]
+
+  - name: Run DL EP elastic tests
+    containerSelector: "{name: 'build_helper_dl_ep'}"
+    timeout: "${TEST_TIMEOUT}"
+    parallel: false
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_ep_${ucx_version}_${BUILD_NUMBER}.txt"
+      testScript: ".gitlab/test_ep.sh ${NIXL_INSTALL_DIR}"
+      headNode: "${SLURM_HEAD_NODE}"
+      headUser: "${SLURM_HEAD_USER}"
+      dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-dl-gpu-ep-test-${ucx_version}:${BUILD_NUMBER}"
+      credentialsId: "${SSH_CREDENTIALS_ID}"
+      containerName: "nixl-ci-ep-${ucx_version}-${BUILD_NUMBER}"
+
+pipeline_stop:
+  containerSelector: "{name: 'build_helper_dl_ep'}"
+  parallel: false
+  shell: action
+  module: slurmCI
+  run: stopAllForBuild
+  args:
+    credentialsId: "${SSH_CREDENTIALS_ID}"
+    headNode: "${SLURM_HEAD_NODE}"
+    headUser: "${SLURM_HEAD_USER}"
+    jobIdDir: "${JOB_ID_FILE_ROOT}"

.ci/jenkins/lib/test-dl-matrix.yaml

@@ -54,7 +54,7 @@ env:
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
   TEST_TIMEOUT: 50
   STORAGE_DRIVER: overlay
-  CI_IMAGE_TAG: "20260607-1"
+  CI_IMAGE_TAG: "20260625-1"
 
 empty_volumes:
   - {mountPath: /var/lib/containers/storage, memory: false}

.ci/jenkins/lib/test-matrix.yaml

@@ -52,7 +52,7 @@ env:
   SCCTL_CREDENTIALS_ID: 'svc-nixl-scctl'
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
   STORAGE_DRIVER: overlay
-  CI_IMAGE_TAG: "20260607-1"
+  CI_IMAGE_TAG: "20260625-1"
 
 empty_volumes:
   - {mountPath: /root, memory: false}

.ci/jenkins/lib/test-sanitizer-matrix.yaml

@@ -42,7 +42,7 @@ env:
   TEST_TIMEOUT: 90
   UCX_TLS: "^shm"
   STORAGE_DRIVER: 'overlay'
-  CI_IMAGE_TAG: "20260607-1"
+  CI_IMAGE_TAG: "20260625-1"
 
 runs_on_dockers:
   # TSan-instrumented dependency image for the tsan build (DEPS_SANITIZE=thread

.ci/jenkins/pipeline/proj-jjb.yaml

@@ -125,6 +125,12 @@
                     string(name: 'sha1', value: githubHelper.getMergedSHA()),
                     string(name: 'githubData', value: VARIABLE_FROM_POST)
                 ], propagate: false, wait: true
+            }}, 'dl-gpu-ep': {{
+                def jobName = 'nixl-ci-dl-gpu-ep'
+                build job: jobName, parameters: [
+                    string(name: 'sha1', value: githubHelper.getMergedSHA()),
+                    string(name: 'githubData', value: VARIABLE_FROM_POST)
+                ], propagate: false, wait: true
             }}, wheel: {{
                 def buildJob = 'nixl-ci-build-wheel'
                 build job: buildJob, parameters: [
@@ -324,6 +330,67 @@
                     parent-credentials: true
         script-path: "{jjb_jenkinsfile}"  # Path to Jenkinsfile that defines the build steps
 
+# Template for the DLCluster GPU EP test job that runs nixl_ep elastic tests on dlcluster.nvidia.com
+- job-template:
+    name: "{jjb_proj}-dl-gpu-ep"      # Will be expanded to 'nixl-ci-dl-gpu-ep'
+    project-type: pipeline
+    disabled: false
+    properties:
+        # Similar properties as dispatcher job
+        - github:
+            url: "{jjb_gh_url}"
+        - build-discarder:
+            days-to-keep: 14
+            num-to-keep: 1000
+        - inject:
+            keep-system-variables: true
+            properties-content: |
+              jjb_proj={jjb_proj}-dl-gpu-ep
+    description: Do NOT edit this job through the Web GUI !
+    concurrent: true
+    sandbox: true
+    # Test job parameters
+    parameters:
+        - string:
+            name: "sha1"
+            default: "{jjb_branch}"    # Default to 'main' branch
+            description: "Commit to be checked, usually set by PR"
+        - string:
+            name: "githubData"
+            default: ""
+            description: "Variables from post"
+        - string:
+            name: "conf_file"
+            default: ".ci/jenkins/lib/test-dl-ep-matrix.yaml"  # DLCluster EP test matrix configuration
+            description: "Job config file"
+        - bool:
+            name: "build_dockers"
+            default: false
+            description: "Force rebuild docker containers"
+        - string:
+            name: "DEBUG"
+            default: 0
+            description: "Enable debug prints and traces, valid values are 0-9"
+    # SCM configuration for the build job
+    pipeline-scm:
+        scm:
+            - git:
+                url: "{jjb_git}"
+                branches: ['$sha1']
+                shallow-clone: false
+                do-not-fetch-tags: false
+                # Configure refspec to handle branches, PRs, and tags
+                refspec: "{jjb_gh_refspec}"
+                browser: githubweb
+                browser-url: "{jjb_git}"
+                # Handle git submodules
+                submodule:
+                    disable: false
+                    recursive: true
+                    tracking: true
+                    parent-credentials: true
+        script-path: "{jjb_jenkinsfile}"  # Path to Jenkinsfile that defines the build steps
+
 # Template for the DLCluster GPU test job that runs on dlcluster.nvidia.com
 - job-template:
     name: "{jjb_proj}-dl-gpu"      # Will be expanded to 'nixl-ci-dl-gpu'
@@ -561,4 +628,5 @@
         - "{jjb_proj}-build-container"  # Create container builder job
         - "{jjb_proj}-gpu"        # Create gpu job
         - "{jjb_proj}-dl-gpu"   # Create dl-gpu job for dlcluster.nvidia.com
+        - "{jjb_proj}-dl-gpu-ep"  # Create dl-gpu-ep job for nixl_ep elastic tests on dlcluster.nvidia.com
         - "{jjb_proj}-build-wheel"  # Create build wheel job

.gitlab/build.sh

@@ -406,6 +406,9 @@ export UCX_TLS=^cuda_ipc
 if [ -n "$PRE_INSTALLED_NIXL_ENV" ]; then
     echo "PRE_INSTALLED_NIXL_ENV is set, skipping compilation"
 else
+    if [ "${BUILD_NIXL_EP}" = "true" ]; then
+        EXTRA_BUILD_ARGS="${EXTRA_BUILD_ARGS} -Dbuild_nixl_ep=true"
+    fi
     # shellcheck disable=SC2086
     meson setup ${NIXL_BUILD_DIR} --prefix=${INSTALL_DIR} -Ducx_path=${UCX_INSTALL_DIR} -Dbuild_docs=true -Drust=false ${EXTRA_BUILD_ARGS} -Dlibfabric_path="${LIBFABRIC_INSTALL_DIR}" --buildtype=debug
     ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} && ninja -j"$NPROC" -C ${NIXL_BUILD_DIR} install