CI: Jenkins integration testing (do not merge)

.ci/dockerfiles/Dockerfile.build_helper

@@ -31,7 +31,7 @@ RUN apt update && \
 
 # Install scctl for slurm client management (x86_64 only, not available for ARM64)
 RUN if [ "$(uname -m)" = "x86_64" ]; then \
-        curl -s https://urm.nvidia.com/artifactory/sw-swx-lab-infra-generic/scctl/get-app.sh | bash -s -- --output-dir /usr/local/bin || echo "INFO: scctl installation failed, continuing..."; \
+        curl -fsS https://artifactory.nvidia.com/artifactory/sw-swx-lab-infra-generic-local/scctl/get-app.sh | bash -s -- --output-dir /usr/local/bin; \
     else \
         echo "INFO: Skipping scctl installation on $(uname -m) architecture (not available)"; \
     fi

.ci/dockerfiles/Dockerfile.dl-base

@@ -1,13 +1,13 @@
 # UCX's aarch64 CUDA 13 + MOFED 24.10 release image plus what the DL cluster job
-# needs on top: gdrcopy, which the cluster has no environment module for, and
-# the Python packaging module test_ucx_tls.py imports.
+# needs on top: gdrcopy, which the cluster has no environment module for, the
+# Python packaging module test_ucx_tls.py imports, and ip for the io_demo test.
 
 FROM harbor.mellanox.com/ucx/aarch64/ubuntu24.04-mofed24.10-cuda13:1
 
 ARG GDRCOPY_VERSION=2.5.1
 
 RUN apt-get update && \
-    apt-get install -y --no-install-recommends python3-setuptools && \
+    apt-get install -y --no-install-recommends python3-setuptools iproute2 && \
     rm -rf /var/lib/apt/lists/*
 
 # configure finds gdrcopy under /usr/local on its own.

.ci/jenkins/Jenkinsfile.dispatcher

@@ -13,58 +13,148 @@ withCredentials([usernamePassword(credentialsId: 'github-token', passwordVariabl
   githubHelper = GithubHelper.getInstance("${GIT_PASSWORD}", VARIABLE_FROM_POST)
 }
 
-// Abort previous dispatcher runs for the same PR
+// Abort previous dispatcher runs for the same PR and their leaf jobs
 def currentPrNumber = githubHelper.getPRNumber()?.toString()
 if (currentPrNumber) {
-  def slurper = new groovy.json.JsonSlurper()
   try {
-    Jenkins.instance.getItemByFullName(env.JOB_NAME).builds.findAll {
+    def staleBuilds = Jenkins.instance.getItemByFullName(env.JOB_NAME).builds.findAll {
       it.isBuilding() && it.number < currentBuild.number
-    }.each { b ->
+    }.findAll { b ->
       def gd = b.getAction(hudson.model.ParametersAction)?.getParameters()?.find { it.name == 'VARIABLE_FROM_POST' }?.value
-      def otherPr = gd ? slurper.parseText(gd)?.issue?.number?.toString() : null
-      if (otherPr == currentPrNumber) {
-        echo "Aborting dispatcher #${b.number} for same PR #${currentPrNumber}"
-        b.doStop()
+      // JsonSlurper is created per call: it is not Serializable, and holding it in
+      // a local across the sleep below fails the pipeline state checkpoint.
+      def otherPr = gd ? new groovy.json.JsonSlurper().parseText(gd)?.issue?.number?.toString() : null
+      otherPr == currentPrNumber
+    }
+
+    if (staleBuilds) {
+      echo "PR #${currentPrNumber}: found ${staleBuilds.size()} stale dispatcher(s) to abort"
+    }
+
+    // Targets to kill as [jobFullName, buildNumber] pairs - holding Run objects
+    // across the sleep below breaks the pipeline state checkpoint (WorkflowRun is
+    // not serializable). Leaf builds are collected from each stale dispatcher's
+    // console log and listed first, so they are killed before their dispatcher.
+    // Leaf jobs are logged as "UCX-OSS » <job>".
+    def targets = []
+    staleBuilds.each { b ->
+      b.getLog(1000).each { line ->
+        def m = (line =~ /Starting building: (.+) #(\d+)/)
+        if (m) {
+          targets.add([m[0][1].replace(' » ', '/'), m[0][2] as int])
+        }
+      }
+    }
+    targets.addAll(staleBuilds.collect { [env.JOB_NAME, it.number] })
+    staleBuilds = null
+
+    // Stop gracefully first so pipeline_stop can release Slurm allocations.
+    targets.each { t ->
+      if (Jenkins.instance.getItemByFullName(t[0])?.getBuildByNumber(t[1])?.isBuilding()) {
+        echo "Stopping ${t[0]} #${t[1]}"
+        Jenkins.instance.getItemByFullName(t[0])?.getBuildByNumber(t[1])?.doStop()
+      }
+    }
+    for (int i = 0; i < 30; i++) {
+      def stopping = targets.findAll { t ->
+        Jenkins.instance.getItemByFullName(t[0])?.getBuildByNumber(t[1])?.isBuilding()
+      }
+      if (!stopping) break
+      sleep 10
+    }
+
+    for (int attempt = 1; attempt <= 3; attempt++) {
+      def alive = targets.findAll { t ->
+        Jenkins.instance.getItemByFullName(t[0])?.getBuildByNumber(t[1])?.isBuilding()
+      }
+      if (!alive) break
+      alive.each { t ->
+        echo "${attempt > 1 ? 'Retry ' + attempt + ': ' : ''}Killing ${t[0]} #${t[1]}"
+        Jenkins.instance.getItemByFullName(t[0])?.getBuildByNumber(t[1])?.doKill()
       }
+      if (attempt < 3) sleep 5
     }
   } catch(Exception e) {
-    echo "Could not abort previous dispatchers: ${e.message}"
+    echo "Could not abort previous builds: ${e.message}"
   }
 }
 
-def sha = githubHelper.getMergedSHA()
 def pipelineOverviewUrl = "${env.BUILD_URL}pipeline-overview/"
 
-githubHelper.updateCommitStatus(pipelineOverviewUrl, "Blossom CI started", GitHubCommitState.PENDING)
+githubHelper.updateCommitStatus(pipelineOverviewUrl, "UCX CI started", GitHubCommitState.PENDING)
 currentBuild.description = githubHelper.getBuildDescription()
 
-def jobs = [
-  'ucx-codestyle-oss',
-  'ucx-static-checks-oss',
-  'ucx-coverity-oss',
-  'ucx-build-oss',
-  'ucx-build-static-oss',
-  'ucx-test-oss',
-  'ucx-test-gpu-oss',
-  'ucx-test-dl-gpu-oss',
+// Basic compile gates every leaf but codestyle, as on Azure: a PR that does
+// not build must not take GPU, BlueField or Slurm slots.
+def gateJob = 'UCX-OSS/ucx-basic-compile-oss'
+
+// Leaf jobs to dispatch, keyed by parallel-branch name
+def gateStageJobs = [
+  'codestyle':      'UCX-OSS/ucx-codestyle-oss',
+]
+def leafJobs = [
+  'static-checks':  'UCX-OSS/ucx-static-checks-oss',
+  'coverity':       'UCX-OSS/ucx-coverity-oss',
+  'build':          'UCX-OSS/ucx-build-oss',
+  'build-static':   'UCX-OSS/ucx-build-static-oss',
+  'test':           'UCX-OSS/ucx-test-oss',
+  'gpu':            'UCX-OSS/ucx-test-gpu-oss',
+  'cuda':           'UCX-OSS/ucx-test-cuda-oss',
+  'dl-gpu':         'UCX-OSS/ucx-test-dl-gpu-oss',
+  'bindings':       'UCX-OSS/ucx-test-bindings-oss',
+  'bindings-gpu':   'UCX-OSS/ucx-test-bindings-gpu-oss',
+  'bindings-arm':   'UCX-OSS/ucx-test-bindings-arm-oss',
+  'ucxx':           'UCX-OSS/ucx-test-ucxx-oss',
+  'ucxx-gpu':       'UCX-OSS/ucx-test-ucxx-gpu-oss',
+  'namespace':      'UCX-OSS/ucx-test-namespace-oss',
+  'wirecompat':     'UCX-OSS/ucx-test-wirecompat-oss',
+  'wirecompat-gpu': 'UCX-OSS/ucx-test-wirecompat-gpu-oss',
+  'efa':            'UCX-OSS/ucx-test-efa-oss',
+  'mad':            'UCX-OSS/ucx-test-mad-oss',
+  'bf':             'UCX-OSS/ucx-test-bf-oss',
 ]
 
 try {
-  def fanout = [:]
-  for (def j in jobs) {
-    def jobName = "UCX-OSS/${j}"
-    fanout[jobName] = {
-      build job: jobName, parameters: [
-        string(name: 'sha1', value: sha),
-        string(name: 'githubData', value: VARIABLE_FROM_POST),
-      ], propagate: false, wait: true
+  // Compute the merged SHA once so all leaf jobs test the same commit
+  def mergedSHA = githubHelper.getMergedSHA()
+  def leafParams = [
+    string(name: 'sha1', value: mergedSHA),
+    string(name: 'githubData', value: VARIABLE_FROM_POST)
+  ]
+
+  // Leaf jobs report their own GitHub statuses; the dispatcher build stays
+  // SUCCESS unless dispatching itself fails. catchError paints a failed
+  // leaf's branch red without failing the build, and all branches run to
+  // completion regardless of sibling failures.
+  def leafBranches = { jobs ->
+    jobs.collectEntries { branchName, jobName ->
+      [(branchName): {
+        catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', catchInterruptions: false) {
+          build job: jobName, parameters: leafParams, wait: true
+        }
+      }]
     }
   }
-  parallel fanout
-  githubHelper.updateCommitStatus(pipelineOverviewUrl, "Blossom CI passed", GitHubCommitState.SUCCESS)
+
+  // Any non-SUCCESS gate result, an abort included, skips the leaf jobs.
+  def gate = null
+  parallel(leafBranches(gateStageJobs) + ['basic-compile': {
+    catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE', catchInterruptions: false) {
+      gate = build job: gateJob, parameters: leafParams, wait: true, propagate: false
+      if (gate.result != 'SUCCESS') {
+        error("Basic compile ${gate.result}: ${gate.absoluteUrl}")
+      }
+    }
+  }])
+  if (gate?.result != 'SUCCESS') {
+    error("Basic compile did not pass, skipping the other jobs: ${gate?.absoluteUrl}")
+  }
+
+  parallel(leafBranches(leafJobs))
+
+  githubHelper.updateCommitStatus(pipelineOverviewUrl, "UCX CI ended", GitHubCommitState.SUCCESS)
 } catch(Exception ex) {
   println ex
-  githubHelper.updateCommitStatus(pipelineOverviewUrl, "Blossom CI failed", GitHubCommitState.FAILURE)
+  githubHelper.updateCommitStatus(pipelineOverviewUrl, "UCX CI ended", GitHubCommitState.FAILURE)
   throw ex
 }

.ci/jenkins/proj_jjb_oss.yaml

@@ -49,11 +49,10 @@
         - git:
             url: "https://github.com/openucx/ucx.git"
             branches: [ 'master' ]
-            shallow-clone: true
-            depth: 100
             refspec: "+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*"
             browser: githubweb
             browser-url: "https://github.com/openucx/ucx"
+            wipe-workspace: false
       script-path: ".ci/jenkins/Jenkinsfile.dispatcher"
 
 - job-template:
@@ -100,11 +99,10 @@
         - git:
             url: "https://github.com/openucx/ucx.git"
             branches: [ '$sha1' ]
-            shallow-clone: true
-            depth: 100
             refspec: "+refs/heads/*:refs/remotes/origin/* +refs/pull/*:refs/remotes/origin/pr/*"
             browser: githubweb
             browser-url: "https://github.com/openucx/ucx"
+            wipe-workspace: false
       script-path: ".ci/jenkins/Jenkinsfile.github"
 
 - project:
@@ -121,6 +119,14 @@
     jobs:
       - "{jjb_proj}"
 
+- project:
+    name: ucx-basic-compile
+    jjb_proj: "ucx-basic-compile-oss"
+    jjb_conf_file: ".ci/pipeline/basic_compile_matrix.yaml"
+    jjb_description: "UCX Basic compile gate (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
 - project:
     name: ucx-static-checks
     jjb_proj: "ucx-static-checks-oss"
@@ -169,10 +175,114 @@
     jobs:
       - "{jjb_proj}"
 
+- project:
+    name: ucx-test-cuda
+    jjb_proj: "ucx-test-cuda-oss"
+    jjb_conf_file: ".ci/pipeline/test_cuda_matrix.yaml"
+    jjb_description: "UCX CUDA memory hooks and SONAME suffix tests on funk (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
 - project:
     name: ucx-test-dl-gpu
     jjb_proj: "ucx-test-dl-gpu-oss"
     jjb_conf_file: ".ci/pipeline/test_dl_matrix.yaml"
     jjb_description: "UCX GTest on DL cluster (GB200/aarch64). Do NOT edit via Web GUI!"
     jobs:
       - "{jjb_proj}"
+
+- project:
+    name: ucx-test-bindings
+    jjb_proj: "ucx-test-bindings-oss"
+    jjb_conf_file: ".ci/pipeline/test_bindings_matrix.yaml"
+    jjb_description: "UCX Go and JUCX bindings tests (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-bindings-gpu
+    jjb_proj: "ucx-test-bindings-gpu-oss"
+    jjb_conf_file: ".ci/pipeline/test_bindings_gpu_matrix.yaml"
+    jjb_description: "UCX Go and JUCX GPU bindings tests on funk (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-bindings-arm
+    jjb_proj: "ucx-test-bindings-arm-oss"
+    jjb_conf_file: ".ci/pipeline/test_bindings_arm_matrix.yaml"
+    jjb_description: "UCX JUCX tests on DL cluster (GB200/aarch64) (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-ucxx
+    jjb_proj: "ucx-test-ucxx-oss"
+    jjb_conf_file: ".ci/pipeline/test_ucxx_matrix.yaml"
+    jjb_description: "UCXX builds and CPU tests against UCX (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-ucxx-gpu
+    jjb_proj: "ucx-test-ucxx-gpu-oss"
+    jjb_conf_file: ".ci/pipeline/test_ucxx_gpu_matrix.yaml"
+    jjb_description: "UCXX wheel and GPU tests against UCX on funk (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-namespace
+    jjb_proj: "ucx-test-namespace-oss"
+    jjb_conf_file: ".ci/pipeline/test_namespace_matrix.yaml"
+    jjb_description: "UCX namespace tests (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-namespace-gpu
+    jjb_proj: "ucx-test-namespace-gpu-oss"
+    jjb_conf_file: ".ci/pipeline/test_namespace_gpu_matrix.yaml"
+    jjb_description: "UCX GPU namespace tests on funk (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-wirecompat
+    jjb_proj: "ucx-test-wirecompat-oss"
+    jjb_conf_file: ".ci/pipeline/test_wirecompat_matrix.yaml"
+    jjb_description: "UCX wire compatibility tests (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-wirecompat-gpu
+    jjb_proj: "ucx-test-wirecompat-gpu-oss"
+    jjb_conf_file: ".ci/pipeline/test_wirecompat_gpu_matrix.yaml"
+    jjb_description: "UCX GPU wire compatibility tests on funk (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-efa
+    jjb_proj: "ucx-test-efa-oss"
+    jjb_conf_file: ".ci/pipeline/test_efa_matrix.yaml"
+    jjb_description: "UCX EFA tests on AWS (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-mad
+    jjb_proj: "ucx-test-mad-oss"
+    jjb_conf_file: ".ci/pipeline/test_mad_matrix.yaml"
+    jjb_description: "UCX MAD tests (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"
+
+- project:
+    name: ucx-test-bf
+    jjb_proj: "ucx-test-bf-oss"
+    jjb_conf_file: ".ci/pipeline/test_bf_matrix.yaml"
+    jjb_description: "UCX GTest on BlueField (GitHub Blossom). Do NOT edit via Web GUI!"
+    jobs:
+      - "{jjb_proj}"