feat(rccl): score the madengine gate against a same-node baseline

.github/workflows/therock-rccl-test-madengine.yml

@@ -101,19 +101,26 @@ jobs:
             exit 1
           fi
 
-      - name: Set up Python
-        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
-        with:
-          python-version: '3.12'
-
-      # A venv on top of it: these runners are shared and long-lived, so anything
-      # installed straight into the interpreter would persist into the next job.
-      # This one lives under WORK_DIR and the cleanup step deletes it.
-      - name: Create Python virtual environment
+      # Conda-forge Python under WORK_DIR: the compute nodes (another distro, older
+      # glibc) run madengine from it via PATH, where setup-python's build won't start.
+      - name: Create Python environment
+        env:
+          MINIFORGE_VERSION: "26.7.2-0"
+          MINIFORGE_SHA256: "281b0ac7d550802efc81af633225a5e6116d29ae72f3ab4eae7168c3931a4c05"
         run: |
+          # conda must write to $HOME, a small shared quota; redirect it into the
+          # conda prefix for this step only, so the nodes keep the real home.
+          export HOME="${WORK_DIR}/conda/home"
+          echo "PIP_CACHE_DIR=${HOME}/.cache/pip" >> "$GITHUB_ENV"
           mkdir -p "${WORK_DIR}"
-          python3 -m venv "${WORK_DIR}/venv"
-          echo "${WORK_DIR}/venv/bin" >> "$GITHUB_PATH"
+          installer="${WORK_DIR}/Miniforge3.sh"
+          curl -fsSL -o "${installer}" \
+            "https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERSION}/Miniforge3-${MINIFORGE_VERSION}-Linux-x86_64.sh"
+          echo "${MINIFORGE_SHA256}  ${installer}" | sha256sum -c -
+          bash "${installer}" -b -p "${WORK_DIR}/conda"
+          rm -f "${installer}"
+          "${WORK_DIR}/conda/bin/conda" create -y -q -p "${WORK_DIR}/env" python=3.12
+          echo "${WORK_DIR}/env/bin" >> "$GITHUB_PATH"
 
           # The datastore is only appended to at the very end of the workload, so
           # an unwritable RESULTS_DIR would surface three hours in. A directory
@@ -163,13 +170,13 @@ jobs:
             --output-dir="${OUTPUT_ARTIFACTS_DIR}" \
             --rccl
 
+      # Both A/B phases in one allocation, plus its queue wait.
       - name: Run MADEngine workload
-        timeout-minutes: 210
+        timeout-minutes: 360
         env:
           NCCL_DEBUG: WARN
           MAD_SECRETS_HFTOKEN: ${{ secrets.HF_TOKEN || '' }}
         run: |
-          source "${WORK_DIR}/venv/bin/activate"
           export PYTHONPATH="rocm-systems/projects/rccl/ci/scripts:${PYTHONPATH}"
 
           NOTIFY_ARGS=""
@@ -189,6 +196,33 @@ jobs:
             --work-dir "${WORK_DIR}" \
             $NOTIFY_ARGS
 
+      # Cancelling a run does not signal the driver: the runner kills this
+      # job's process tree outright and then reports "Terminate orphan
+      # process" for madengine and srun. No Python handler gets to run, and
+      # salloc holds the allocation in a session of its own, so the nodes stay
+      # taken until its --time expires -- four of them, for five hours, in the
+      # 405B case. Measured on run 40879865705: the job reached `cancelled`
+      # while SLURM job 104884 kept running with 4:48 left.
+      #
+      # The runner starts this step itself, which is what makes it the one
+      # piece of cleanup a cancellation cannot skip.
+      - name: Release the SLURM allocation
+        if: cancelled()
+        run: |
+          recorded="${WORK_DIR}/allocation.txt"
+          if [ ! -s "${recorded}" ]; then
+            echo "No allocation was granted; nothing to release."
+            exit 0
+          fi
+          job="$(head -n1 "${recorded}" | tr -d '[:space:]')"
+          case "${job}" in
+            ''|*[!0-9]*)
+              echo "::warning::no usable job id in ${recorded}, got '${job}'"
+              exit 0 ;;
+          esac
+          echo "Releasing SLURM job ${job}"
+          scancel "${job}" || echo "::warning::scancel ${job} failed"
+
       - name: Upload results
         if: always()
         uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
@@ -198,22 +232,32 @@ jobs:
           # nodes can only be read by someone with cluster access: the job log
           # shows that SLURM failed, not why.
           path: |
-            ${{ env.WORK_DIR }}/perf.csv
-            ${{ env.WORK_DIR }}/perf_entry_super.csv
-            ${{ env.WORK_DIR }}/perf_entry_super.json
-            ${{ env.WORK_DIR }}/perf_super.csv
             ${{ env.WORK_DIR }}/madengine_summary.txt
-            ${{ env.WORK_DIR }}/manifest.json
-            ${{ env.WORK_DIR }}/slurm_output/*.out
-            ${{ env.WORK_DIR }}/slurm_output/*.err
+            ${{ env.WORK_DIR }}/ab/*/perf.csv
+            ${{ env.WORK_DIR }}/ab/*/perf_entry_super.csv
+            ${{ env.WORK_DIR }}/ab/*/perf_entry_super.json
+            ${{ env.WORK_DIR }}/ab/*/perf_super.csv
+            ${{ env.WORK_DIR }}/ab/*/manifest.json
+            ${{ env.WORK_DIR }}/ab/*/slurm_output/*.out
+            ${{ env.WORK_DIR }}/ab/*/slurm_output/*.err
           retention-days: 30
 
       - name: Cleanup work directory
         if: always()
         run: |
-          # node_venv is the per-node bootstrap: on a single-node run the template
-          # puts WORKSPACE on the shared filesystem, so it lands here rather than
-          # in node-local storage and would otherwise outlive the run.
-          rm -rf "${WORK_DIR}/madengine" "${WORK_DIR}/MAD" "${WORK_DIR}/venv" \
-                 "${WORK_DIR}/node_venv" "${WORK_DIR}/overlay_ctx" \
-                 "${WORK_DIR}/docker_builds"
+          # `ab` holds the per-node logs, the only part that grows with the run;
+          # the upload step above has already kept them as an artifact.
+          rm -rf "${WORK_DIR}/madengine" "${WORK_DIR}/MAD" "${WORK_DIR}/conda" \
+                 "${WORK_DIR}/env" "${WORK_DIR}/overlay_ctx" \
+                 "${WORK_DIR}/docker_builds" "${WORK_DIR}/ab"
+
+          # Remove per-run tags and bases left by earlier versions of this job on
+          # the shared Docker disk; no -f, so images a container still uses stay.
+          docker system df || true
+          {
+            docker images --format '{{.Repository}}:{{.Tag}}' \
+              | grep -E -- '-(rccl-gfx[0-9a-z]+|primus-v[0-9.]+|mad-[0-9a-f]{8})-' || true
+            python3 -c 'import sys; sys.path.insert(0, sys.argv[1]); import test_madengine as t; print(*sorted({c["base_image"] for c in t.WORKLOAD_CONFIGS.values()}), sep="\n")' \
+              rocm-systems/projects/rccl/ci/scripts
+          } | sort -u | xargs -r docker image rm 2>&1 | grep -v 'No such image' || true
+          docker system df || true