ci: import the Slurm container image before the timed test stages

.ci/docs/ci-overview.md

@@ -251,6 +251,14 @@ Jobs submitted via the `slurmCI` module are named `${JOB_BASE_NAME}-${BUILD_NUMB
 
 Use `squeue --name <pattern>` or `squeue -u <user>` to identify which pipeline owns a running job.
 
+## Slurm container image sync
+
+Every Slurm pipeline has a `Sync Container Image` step right after its allocation step. The first `srun --container-image=...` on a node makes pyxis pull and unpack the whole CI image, which can take many minutes on a slow registry or node. Doing that inside the first test stage would spend that stage's timeout before any test runs.
+
+The sync step runs a cheap environment check (`hostname`, `nvidia-smi -L`, `df -h /`) with the same `dockerImage` and `containerName` the test steps use, under its own 30 min timeout. Later steps reuse the named container, so their timeouts only cover the tests, and a failed or slow import shows up as its own stage. The GPU check does not fail the step when no GPU is visible; it only prints `no-gpu-visible`.
+
+When adding a Slurm pipeline, add this step after the allocation and keep its `dockerImage`/`containerName` identical to the test steps'.
+
 ## How to trigger CI manually
 
 - **Full Jenkins pipeline for a PR:** comment `/build` on the PR (requires authorization — see `Authorization` step in `blossom-ci.yml`).

.ci/jenkins/lib/build-wheel-matrix.yaml

@@ -228,7 +228,7 @@ env:
   SLURM_HEAD_NODE: dlcluster.nvidia.com
   SLURM_HEAD_USER: svc-nixl
   SLURM_ACCOUNT: oberon-gb-ci
-  SLURM_JOB_TIMEOUT: '01:30:00'
+  SLURM_JOB_TIMEOUT: '02:00:00'
   SLURM_IMMEDIATE_TIMEOUT: "3600"
   SSH_CREDENTIALS_ID: 'svc-nixl-ssh_key'
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
@@ -362,6 +362,24 @@ steps:
         "--account=${SLURM_ACCOUNT}"
       ]
 
+  - name: Sync Container Image
+    # Import the image onto the node before the timed test stages, so a slow
+    # registry pull is not counted against (and does not time out) a test stage.
+    containerSelector: "{name: 'build_helper_(vllm|sglang)'}"
+    timeout: 30
+    parallel: false
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_${fw}_${BUILD_NUMBER}.txt"
+      testScript: "hostname; nvidia-smi -L || echo no-gpu-visible; df -h /"
+      headNode: "${SLURM_HEAD_NODE}"
+      headUser: "${SLURM_HEAD_USER}"
+      dockerImage: "${registry_host}#${registry_path}/pr/${arch}/${fw}-nixl:${BUILD_NUMBER}"
+      credentialsId: "${SSH_CREDENTIALS_ID}"
+      containerName: "nixl-vs-${fw}-${BUILD_NUMBER}"
+
   # The two run steps are identical modulo ${fw} as well, but are kept as separate,
   # explicitly-named steps so the test that failed is obvious at a glance in the UI.
   - name: Run vLLM sanity

.ci/jenkins/lib/test-dl-ep-matrix.yaml

@@ -58,7 +58,7 @@ env:
   SLURM_HEAD_NODE: dlcluster.nvidia.com
   SLURM_HEAD_USER: svc-nixl
   SLURM_ACCOUNT: 'blackwell'
-  SLURM_JOB_TIMEOUT: '02:00:00'
+  SLURM_JOB_TIMEOUT: '02:30:00'
   SLURM_IMMEDIATE_TIMEOUT: 3600
   SSH_CREDENTIALS_ID: 'svc-nixl-ssh_key'
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
@@ -164,6 +164,24 @@ steps:
         "--account=${SLURM_ACCOUNT}"
       ]
 
+  - name: Sync Container Image
+    # Import the image onto the node before the timed test stages, so a slow
+    # registry pull is not counted against (and does not time out) a test stage.
+    containerSelector: "{name: 'build_helper_dl_ep'}"
+    timeout: 30
+    parallel: false
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_ep_${BUILD_NUMBER}.txt"
+      testScript: "hostname; nvidia-smi -L || echo no-gpu-visible; df -h /"
+      headNode: "${SLURM_HEAD_NODE}"
+      headUser: "${SLURM_HEAD_USER}"
+      dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-dl-gpu-ep-test:${BUILD_NUMBER}"
+      credentialsId: "${SSH_CREDENTIALS_ID}"
+      containerName: "nixl-ci-ep-${BUILD_NUMBER}"
+
   - name: Run DL NIXL EP tests
     containerSelector: "{name: 'build_helper_dl_ep'}"
     timeout: "${TEST_TIMEOUT}"

.ci/jenkins/lib/test-dl-matrix.yaml

@@ -152,6 +152,24 @@ steps:
         "--account=${SLURM_ACCOUNT}"
       ]
 
+  - name: Sync Container Image
+    # Import the image onto the node before the timed test stages, so a slow
+    # registry pull is not counted against (and does not time out) a test stage.
+    containerSelector: "{name: 'build_helper_dl'}"
+    timeout: 30
+    parallel: false
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_${BUILD_NUMBER}.txt"
+      testScript: "hostname; nvidia-smi -L || echo no-gpu-visible; df -h /"
+      headNode: "${SLURM_HEAD_NODE}"
+      headUser: "${SLURM_HEAD_USER}"
+      dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-dl-gpu-test:${BUILD_NUMBER}"
+      credentialsId: "${SSH_CREDENTIALS_ID}"
+      containerName: "nixl-ci-${BUILD_NUMBER}"
+
   - name: Run DL Python tests
     containerSelector: "{name: 'build_helper_dl'}"
     timeout: "${TEST_TIMEOUT}"

.ci/jenkins/lib/test-llm-container-matrix.yaml

@@ -155,6 +155,26 @@ steps:
         "--exclude=${SLURM_EXCLUDE}"
       ]
 
+  - name: Sync Container Image
+    # Import the image onto the node before the timed test stages, so a slow
+    # registry pull is not counted against (and does not time out) a test stage.
+    containerSelector: "{name: 'build_helper_llm'}"
+    timeout: 30
+    parallel: false
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_${BUILD_NUMBER}.txt"
+      testScript: "hostname; nvidia-smi -L || echo no-gpu-visible; df -h /"
+      headNode: "${SLURM_HEAD_NODE}"
+      dockerImage: "${DOCKER_IMAGE}"
+      credentialsId: "${SCCTL_CREDENTIALS_ID}"
+      containerName: "nixl-llm-${FRAMEWORK}-${BUILD_NUMBER}"
+      extraArgs: [
+        "--container-mount-home"
+      ]
+
   - name: Copy Tests
     containerSelector: "{name: 'build_helper_llm'}"
     timeout: 30

.ci/jenkins/lib/test-matrix.yaml

@@ -21,8 +21,8 @@ job: nixl-ci-gpu
 failFast: false
 
 # Job cap. Axes run in parallel, so this bounds the longest single axis:
-# docker build (~11) + allocation wait (up to immediateTimeout 60) + run steps (110) + stop (~5).
-timeout_minutes: 200
+# docker build (~11) + allocation wait (up to immediateTimeout 60) + run steps (150, incl. the 30 image sync) + stop (~5).
+timeout_minutes: 240
 
 registry_host: artifactory.nvidia.com
 registry_auth: svc-nixl-new-artifactory-token
@@ -52,7 +52,7 @@ env:
   SLURM_GRES: gpu:4
   SLURM_MEM: 128G
   SLURM_MINCPUS: 24
-  SLURM_JOB_TIMEOUT: '02:20:00'
+  SLURM_JOB_TIMEOUT: '02:50:00'
   SLURM_IMMEDIATE_TIMEOUT: "3600"
   SCCTL_CREDENTIALS_ID: 'svc-nixl-scctl'
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
@@ -155,6 +155,24 @@ steps:
         "--mem=${SLURM_MEM}"
       ]
 
+  - name: Sync Container Image
+    # Import the image onto the node before the timed test stages, so a slow
+    # registry pull is not counted against (and does not time out) a test stage.
+    containerSelector: "{name: 'build_helper'}"
+    timeout: 30
+    parallel: false
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_${BUILD_NUMBER}.txt"
+      testScript: "hostname; nvidia-smi -L || echo no-gpu-visible; df -h /"
+      headNode: "${SLURM_HEAD_NODE}"
+      dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-gpu-test:${BUILD_NUMBER}"
+      credentialsId: "${SCCTL_CREDENTIALS_ID}"
+      containerName: "nixl-ci-${BUILD_NUMBER}"
+      exportEnroot: "true"  # on failure, export the container to ${ENROOT_MOUNT_PATH} for crash debugging
+
   - name: Run CPP tests
     containerSelector: "{name: 'build_helper'}"
     timeout: 60

.ci/jenkins/lib/test-vr-matrix.yaml

@@ -66,7 +66,7 @@ env:
   SLURM_MINCPUS: 24
   # Nodes whose configuration differs from the rest of the partition
   SLURM_EXCLUDE: 'vr-nvl72-ts3-l06-comp137,vr-nvl72-ts3-l06-comp139,vr-nvl72-ts3-l06-comp141,vr-nvl72-ts2b-l10-comp102'
-  SLURM_JOB_TIMEOUT: '01:30:00'
+  SLURM_JOB_TIMEOUT: '02:00:00'
   SLURM_IMMEDIATE_TIMEOUT: 3600
   SSH_CREDENTIALS_ID: 'svc-nixl-ssh_key'
   JOB_ID_FILE_ROOT: "/mnt/pvc/${JOB_BASE_NAME}"
@@ -170,6 +170,24 @@ steps:
         "--exclude=${SLURM_EXCLUDE}"
       ]
 
+  - name: Sync Container Image
+    # Import the image onto the node before the timed test stages, so a slow
+    # registry pull is not counted against (and does not time out) a test stage.
+    containerSelector: "{name: 'build_helper_dl'}"
+    timeout: 30
+    parallel: false
+    shell: action
+    module: slurmCI
+    run: run
+    args:
+      jobIdFile: "${JOB_ID_FILE_ROOT}/job_id_${BUILD_NUMBER}.txt"
+      testScript: "hostname; nvidia-smi -L || echo no-gpu-visible; df -h /"
+      headNode: "${SLURM_HEAD_NODE}"
+      headUser: "${SLURM_HEAD_USER}"
+      dockerImage: "${registry_host}#${ARTIFACTORY_PATH}/pr/${arch}/nixl-ci-gpu-vr-test:${BUILD_NUMBER}"
+      credentialsId: "${SSH_CREDENTIALS_ID}"
+      containerName: "nixl-ci-vr-${BUILD_NUMBER}"
+
   - name: Run Python tests
     containerSelector: "{name: 'build_helper_dl'}"
     timeout: 15