From c916a03322cc865495f54e6951218fa4d80f0c11 Mon Sep 17 00:00:00 2001
From: Ivan Ganev <ivan.ganev@amd.com>
Date: Thu, 13 Aug 2026 10:59:07 -0700
Subject: [PATCH 1/3] fix(ci): isolate and clean up SPUR CI sessions

Scope remote worktrees by GitHub run and attempt, while retaining the SHA-scoped image tarball used to hand artifacts between hardware-test stages.

Make worktree and tarball cleanup best-effort so NFS .nfs files cannot replace a successful test result with a cleanup failure.

Record each active Slurm job and isolated remote process group in per-stage control files. On interruption, reconnect as the CI user to cancel the exact job and terminate its poller; use a cancellation sentinel to cover startup races.

Add a numeric cleanup-only workflow dispatch for recovering jobs orphaned before automatic cancellation tracking was available.
---
 .github/scripts/spur-ci-common.sh             | 99 +++++++++++++++++++
 .github/scripts/spur-dist-build.sh            | 51 +++++++---
 .github/scripts/spur-smoke-test.sh            | 53 +++++++---
 .github/scripts/spur-tiny-test.sh             | 58 +++++++----
 .github/workflows/aic-amd-dist-build-fast.yml | 48 ++++++++-
 .slurm/run-build-distribute.sh                | 16 +++
 6 files changed, 278 insertions(+), 47 deletions(-)
 create mode 100644 .github/scripts/spur-ci-common.sh

diff --git a/.github/scripts/spur-ci-common.sh b/.github/scripts/spur-ci-common.sh
new file mode 100644
index 00000000..fd5ed089
--- /dev/null
+++ b/.github/scripts/spur-ci-common.sh
@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Shared lifecycle helpers for CI wrappers that run work on the SPUR head node.
+# Call aic_ci_session_init before starting SSH, then use aic_ci_ssh_bash for the
+# remote heredoc.  If the local wrapper is interrupted, its EXIT trap reconnects
+# and cancels only the Slurm job/process group recorded for this run and stage.
+
+aic_ci_session_init() {
+    local short_sha="${1:?short SHA is required}"
+    local stage="${2:?CI stage is required}"
+    local run_id="${GITHUB_RUN_ID:-manual-${BASHPID}}"
+    local run_attempt="${GITHUB_RUN_ATTEMPT:-1}"
+
+    AIC_CI_RUN_KEY="${run_id}-${run_attempt}"
+    AIC_CI_STAGE="${stage}"
+    case "${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}.${short_sha}" in
+        *[!A-Za-z0-9._-]*)
+            echo "ERROR: unsafe CI session identifier" >&2
+            exit 2
+            ;;
+    esac
+
+    AIC_CI_SHORT_SHA="${short_sha}"
+    trap 'exit 130' INT
+    trap 'exit 143' TERM
+    trap 'exit 129' HUP
+    trap _aic_ci_session_exit EXIT
+}
+
+aic_ci_ssh_bash() {
+    ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+        AIC_CI_RUN_KEY="${AIC_CI_RUN_KEY}" \
+        AIC_CI_STAGE="${AIC_CI_STAGE}" \
+        "$@" \
+        setsid --fork --wait bash
+}
+
+_aic_ci_session_exit() {
+    local rc=$?
+    trap - EXIT INT TERM HUP
+
+    if (( rc != 0 )); then
+        echo "=== CI wrapper interrupted/failed; cancelling its remote session ===" >&2
+        ssh -o ConnectTimeout=15 -o ServerAliveInterval=10 -o ServerAliveCountMax=2 \
+            "${AIC_SPUR_HOST}" env \
+            AIC_CI_RUN_KEY="${AIC_CI_RUN_KEY}" \
+            AIC_CI_STAGE="${AIC_CI_STAGE}" \
+            AIC_CI_SHORT_SHA="${AIC_CI_SHORT_SHA}" \
+            AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT}" \
+            AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
+            bash <<'REMOTE_CANCEL' || true
+set -u
+
+CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-$HOME/Projects/rocm-aic-ci}"
+CONTROL_PREFIX="${CI_STORAGE_ROOT}/control/${AIC_CI_SHORT_SHA}.${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}"
+JOB_FILE="${CONTROL_PREFIX}.job"
+PID_FILE="${CONTROL_PREFIX}.pid"
+CANCEL_FILE="${CONTROL_PREFIX}.cancel"
+remote_session_found=0
+
+# Close the small race where cancellation arrives before the remote shell has
+# written its PID.  A shell that starts later sees this sentinel and exits
+# before cloning or submitting work.
+mkdir -p "${CI_STORAGE_ROOT}/control"
+: > "${CANCEL_FILE}"
+
+if [[ -s "${JOB_FILE}" ]]; then
+    read -r job_id < "${JOB_FILE}" || job_id=""
+    if [[ "${job_id}" =~ ^[0-9]+$ ]]; then
+        echo "Cancelling Slurm job ${job_id}"
+        scancel --controller="${AIC_SPUR_CONTROLLER}" "${job_id}" || true
+    fi
+fi
+
+if [[ -s "${PID_FILE}" ]]; then
+    read -r remote_pid < "${PID_FILE}" || remote_pid=""
+    if [[ "${remote_pid}" =~ ^[0-9]+$ ]]; then
+        remote_session_found=1
+        if kill -0 -- "-${remote_pid}" 2>/dev/null; then
+            echo "Terminating remote process group ${remote_pid}"
+            kill -TERM -- "-${remote_pid}" 2>/dev/null || true
+            for _ in 1 2 3 4 5 6 7 8 9 10; do
+                kill -0 -- "-${remote_pid}" 2>/dev/null || break
+                sleep 0.2
+            done
+            kill -KILL -- "-${remote_pid}" 2>/dev/null || true
+        fi
+    fi
+fi
+
+rm -f "${JOB_FILE}" "${PID_FILE}" 2>/dev/null || true
+if (( remote_session_found == 1 )); then
+    rm -f "${CANCEL_FILE}" 2>/dev/null || true
+fi
+REMOTE_CANCEL
+    fi
+
+    exit "${rc}"
+}
diff --git a/.github/scripts/spur-dist-build.sh b/.github/scripts/spur-dist-build.sh
index 3b652e33..1e5f0934 100755
--- a/.github/scripts/spur-dist-build.sh
+++ b/.github/scripts/spur-dist-build.sh
@@ -3,10 +3,8 @@ set -euo pipefail
 
 # Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), clones the repo at
 # the current SHA, and runs the requested dist-build target with a CI-scoped
-# image name and tarball path. The clone and tarball are left in place for
-# spur-smoke-test.sh to use; spur-smoke-test.sh owns the final cleanup.
-#
-# On failure, cleans up immediately so no stale state is left behind.
+# image name and tarball path.  The tarball is left in place for
+# spur-smoke-test.sh; the run-attempt-scoped clone is always removed.
 
 SHA="${1:?usage: $0 <full-sha> [dist-build|dist-build-fast]}"
 AIC_DIST_BUILD_TARGET="${2:-dist-build}"
@@ -25,8 +23,12 @@ AIC_SPUR_HOST="${AIC_SPUR_HOST//[$'\t\r\n ']}"
 AIC_SHARED_NFS="${AIC_SHARED_NFS:?AIC_SHARED_NFS must be set (e.g. via GitHub repo variable)}"
 AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g. via GitHub repo variable)}"
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=spur-ci-common.sh
+source "${SCRIPT_DIR}/spur-ci-common.sh"
+aic_ci_session_init "${SHORT}" "dist-build"
 
-ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+aic_ci_ssh_bash \
     SHA="${SHA}" \
     REPO="${REPO}" \
     AIC_IMAGE_NAME="${AIC_IMAGE_NAME}" \
@@ -34,21 +36,46 @@ ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
     AIC_SHARED_NFS="${AIC_SHARED_NFS}" \
     AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT}" \
     AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
-    SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" \
-    bash << 'REMOTE'
+    SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" << 'REMOTE'
 set -euo pipefail
 
 SHORT="${SHA:0:7}"
-WORKDIR="$HOME/Projects/rocm-aic.${SHORT}"
+WORKDIR="$HOME/Projects/rocm-aic.${SHORT}.${AIC_CI_RUN_KEY}"
 CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-$HOME/Projects/rocm-aic-ci}"
 TARBALL_DIR="${CI_STORAGE_ROOT}/images/aic-ci-${SHORT}"
 CACHE_DIR="${CI_STORAGE_ROOT}/buildcache"
+CONTROL_PREFIX="${CI_STORAGE_ROOT}/control/${SHORT}.${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}"
+PID_FILE="${CONTROL_PREFIX}.pid"
+JOB_FILE="${CONTROL_PREFIX}.job"
+CANCEL_FILE="${CONTROL_PREFIX}.cancel"
 
-cleanup_on_fail() {
-    echo "=== Build failed — cleaning up ==="
-    rm -rf "${WORKDIR}" "${TARBALL_DIR}"
+mkdir -p "${CI_STORAGE_ROOT}/control"
+printf '%s\n' "${BASHPID}" > "${PID_FILE}"
+if [[ -e "${CANCEL_FILE}" ]]; then
+    echo "CI session was cancelled before remote startup completed" >&2
+    rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    exit 143
+fi
+export AIC_CI_ACTIVE_JOB_FILE="${JOB_FILE}"
+
+_best_effort_remove() {
+    rm -rf "$@" || echo "WARNING: cleanup could not fully remove: $*" >&2
+}
+_cleanup() {
+    local rc=$?
+    trap - EXIT
+    echo "=== Cleaning up run-attempt worktree ==="
+    _best_effort_remove "${WORKDIR}"
+    if (( rc != 0 )); then
+        echo "=== Build failed — removing staged image ==="
+        _best_effort_remove "${TARBALL_DIR}"
+    fi
+    if (( rc == 0 )); then
+        rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    fi
+    exit "${rc}"
 }
-trap cleanup_on_fail ERR
+trap _cleanup EXIT
 
 echo "=== Cloning ${REPO} at ${SHA} into ${WORKDIR} ==="
 rm -rf "${WORKDIR}"
diff --git a/.github/scripts/spur-smoke-test.sh b/.github/scripts/spur-smoke-test.sh
index a9d76ec2..45d71d28 100755
--- a/.github/scripts/spur-smoke-test.sh
+++ b/.github/scripts/spur-smoke-test.sh
@@ -3,9 +3,8 @@ set -euo pipefail
 
 # Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and
 # runs the requested smoke-test target against the tarball produced by
-# spur-dist-build.sh for the same SHA. The clone and tarball are left in place
-# for spur-tiny-test.sh (the next stage) to use; spur-tiny-test.sh owns the final
-# cleanup. On failure, cleans up immediately so no stale state is left behind.
+# spur-dist-build.sh for the same SHA. The tarball is left in place for
+# spur-tiny-test.sh; the run-attempt-scoped clone is always removed.
 
 SHA="${1:?usage: $0 <full-sha> [smoke-test|smoke-test-fast]}"
 AIC_SMOKE_TEST_TARGET="${2:-smoke-test}"
@@ -24,8 +23,12 @@ AIC_SHARED_NFS="${AIC_SHARED_NFS:?AIC_SHARED_NFS must be set (e.g. via GitHub re
 AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g. via GitHub repo variable)}"
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 REPO="https://github.com/ROCm/rocm-aic.git"
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=spur-ci-common.sh
+source "${SCRIPT_DIR}/spur-ci-common.sh"
+aic_ci_session_init "${SHORT}" "smoke-test"
 
-ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+aic_ci_ssh_bash \
     SHA="${SHA}" \
     REPO="${REPO}" \
     AIC_IMAGE_NAME="${AIC_IMAGE_NAME}" \
@@ -33,23 +36,45 @@ ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
     AIC_SHARED_NFS="${AIC_SHARED_NFS}" \
     AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT}" \
     AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
-    SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" \
-    bash << 'REMOTE'
+    SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" << 'REMOTE'
 set -euo pipefail
 
 SHORT="${SHA:0:7}"
-WORKDIR="$HOME/Projects/rocm-aic.${SHORT}"
+WORKDIR="$HOME/Projects/rocm-aic.${SHORT}.${AIC_CI_RUN_KEY}"
 CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-$HOME/Projects/rocm-aic-ci}"
 TARBALL_DIR="${CI_STORAGE_ROOT}/images/aic-ci-${SHORT}"
+CONTROL_PREFIX="${CI_STORAGE_ROOT}/control/${SHORT}.${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}"
+PID_FILE="${CONTROL_PREFIX}.pid"
+JOB_FILE="${CONTROL_PREFIX}.job"
+CANCEL_FILE="${CONTROL_PREFIX}.cancel"
 
-cleanup_on_fail() {
-    echo "=== Smoke test failed — cleaning up ==="
-    rm -rf "${TARBALL_DIR}"
-    find "${WORKDIR}" -mindepth 1 -maxdepth 1 -not -name logs -exec rm -rf {} +
+mkdir -p "${CI_STORAGE_ROOT}/control"
+printf '%s\n' "${BASHPID}" > "${PID_FILE}"
+if [[ -e "${CANCEL_FILE}" ]]; then
+    echo "CI session was cancelled before remote startup completed" >&2
+    rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    exit 143
+fi
+export AIC_CI_ACTIVE_JOB_FILE="${JOB_FILE}"
+
+_best_effort_remove() {
+    rm -rf "$@" || echo "WARNING: cleanup could not fully remove: $*" >&2
+}
+_cleanup() {
+    local rc=$?
+    trap - EXIT
+    echo "=== Cleaning up run-attempt worktree ==="
+    _best_effort_remove "${WORKDIR}"
+    if (( rc != 0 )); then
+        echo "=== Smoke test failed — removing staged image ==="
+        _best_effort_remove "${TARBALL_DIR}"
+    fi
+    if (( rc == 0 )); then
+        rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    fi
+    exit "${rc}"
 }
-# On success: preserve TARBALL_DIR so the following tiny-test stage can use it.
-# On failure: clean up immediately so no stale state is left behind.
-trap cleanup_on_fail ERR
+trap _cleanup EXIT
 
 # Re-clone if WORKDIR is missing or checked out at the wrong SHA.
 ACTUAL_SHA="$(git -C "${WORKDIR}" rev-parse HEAD 2>/dev/null || true)"
diff --git a/.github/scripts/spur-tiny-test.sh b/.github/scripts/spur-tiny-test.sh
index f1b2dd7d..fd20c1fa 100755
--- a/.github/scripts/spur-tiny-test.sh
+++ b/.github/scripts/spur-tiny-test.sh
@@ -7,12 +7,12 @@ set -euo pipefail
 # (standalone lmcache server + vLLM LMCacheMPConnector) with a tiny model and
 # asserts one non-empty chat completion.
 #
-# Cleanup ownership depends on whether a cliff stage follows:
+# Tarball cleanup ownership depends on whether a cliff stage follows:
 #   * PR flow (dist-build -> smoke-test -> tiny-test): tiny-test is terminal, so
-#     it owns the final cleanup (removes the clone + tarball on exit).
+#     it removes the tarball on exit.
 #   * Nightly (dist-build -> smoke -> tiny -> cliff): cliff runs next and needs
-#     the artifacts, so the nightly tiny-test step sets KEEP_ARTIFACTS=1 and this
-#     script only cleans up on failure (spur-cliff.sh does the final cleanup).
+#     the tarball, so the nightly tiny-test step sets KEEP_ARTIFACTS=1.
+# The run-attempt-scoped clone is always removed best-effort.
 # The tiny model uses the cluster-wide HF cache so it is downloaded once and
 # reused across CI workflows and SPUR accounts.
 
@@ -34,8 +34,12 @@ AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 KEEP_ARTIFACTS="${KEEP_ARTIFACTS:-0}"
 REPO="https://github.com/ROCm/rocm-aic.git"
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=spur-ci-common.sh
+source "${SCRIPT_DIR}/spur-ci-common.sh"
+aic_ci_session_init "${SHORT}" "tiny-test"
 
-ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+aic_ci_ssh_bash \
     SHA="${SHA}" \
     REPO="${REPO}" \
     AIC_IMAGE_NAME="${AIC_IMAGE_NAME}" \
@@ -45,27 +49,45 @@ ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
     KEEP_ARTIFACTS="${KEEP_ARTIFACTS}" \
     AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
     SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" \
-    HF_TOKEN="${HF_TOKEN:-}" \
-    bash << 'REMOTE'
+    HF_TOKEN="${HF_TOKEN:-}" << 'REMOTE'
 set -euo pipefail
 
 SHORT="${SHA:0:7}"
-WORKDIR="$HOME/Projects/rocm-aic.${SHORT}"
+WORKDIR="$HOME/Projects/rocm-aic.${SHORT}.${AIC_CI_RUN_KEY}"
 CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-$HOME/Projects/rocm-aic-ci}"
 TARBALL_DIR="${CI_STORAGE_ROOT}/images/aic-ci-${SHORT}"
+CONTROL_PREFIX="${CI_STORAGE_ROOT}/control/${SHORT}.${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}"
+PID_FILE="${CONTROL_PREFIX}.pid"
+JOB_FILE="${CONTROL_PREFIX}.job"
+CANCEL_FILE="${CONTROL_PREFIX}.cancel"
+
+mkdir -p "${CI_STORAGE_ROOT}/control"
+printf '%s\n' "${BASHPID}" > "${PID_FILE}"
+if [[ -e "${CANCEL_FILE}" ]]; then
+    echo "CI session was cancelled before remote startup completed" >&2
+    rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    exit 143
+fi
+export AIC_CI_ACTIVE_JOB_FILE="${JOB_FILE}"
 
+_best_effort_remove() {
+    rm -rf "$@" || echo "WARNING: cleanup could not fully remove: $*" >&2
+}
 _cleanup() {
-    echo "=== Cleaning up ==="
-    rm -rf "${WORKDIR}" "${TARBALL_DIR}"
+    local rc=$?
+    trap - EXIT
+    echo "=== Cleaning up run-attempt worktree ==="
+    _best_effort_remove "${WORKDIR}"
+    if (( rc != 0 )) || [[ "${KEEP_ARTIFACTS}" != "1" ]]; then
+        echo "=== Removing staged image ==="
+        _best_effort_remove "${TARBALL_DIR}"
+    fi
+    if (( rc == 0 )); then
+        rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    fi
+    exit "${rc}"
 }
-if [[ "${KEEP_ARTIFACTS}" == "1" ]]; then
-    # A cliff stage follows and reuses the artifacts; only clean up on failure.
-    cleanup_on_fail() { echo "=== Tiny test failed — cleaning up ==="; _cleanup; }
-    trap cleanup_on_fail ERR
-else
-    # Terminal stage: always clean up.
-    trap _cleanup EXIT
-fi
+trap _cleanup EXIT
 
 # Re-clone if WORKDIR is missing or checked out at the wrong SHA (e.g. stale
 # leftover from a prior failed run at a different commit with the same prefix).
diff --git a/.github/workflows/aic-amd-dist-build-fast.yml b/.github/workflows/aic-amd-dist-build-fast.yml
index 4607ccdc..6165b61d 100644
--- a/.github/workflows/aic-amd-dist-build-fast.yml
+++ b/.github/workflows/aic-amd-dist-build-fast.yml
@@ -2,6 +2,11 @@ name: AIC Hardware Test - AMD Dist Build (fast)
 
 on:
   workflow_dispatch:
+    inputs:
+      cancel_slurm_job:
+        description: "Cancel one orphaned SPUR Slurm job instead of running CI"
+        required: false
+        type: string
   issue_comment:
     types: [created]
 
@@ -14,13 +19,50 @@ concurrency:
   cancel-in-progress: false
 
 jobs:
+  cleanup-orphan:
+    if: github.event_name == 'workflow_dispatch' && inputs.cancel_slurm_job != ''
+    runs-on: [self-hosted, rocm-aic-cicd]
+    timeout-minutes: 5
+    env:
+      AIC_SPUR_HOST: ${{ secrets.AIC_SPUR_HOST }}
+      AIC_SPUR_CONTROLLER: ${{ secrets.AIC_SPUR_CONTROLLER }}
+      JOB_ID: ${{ inputs.cancel_slurm_job }}
+
+    steps:
+      - name: Cancel orphaned SPUR job
+        shell: bash
+        run: |
+          set -euo pipefail
+          [[ "${JOB_ID}" =~ ^[0-9]+$ ]] || {
+            echo "ERROR: Slurm job ID must be numeric" >&2
+            exit 2
+          }
+          ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+            JOB_ID="${JOB_ID}" \
+            AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
+            bash <<'REMOTE'
+          set -euo pipefail
+          scancel --controller="${AIC_SPUR_CONTROLLER}" "${JOB_ID}"
+          for _ in 1 2 3 4 5 6 7 8 9 10; do
+              if ! squeue --controller="${AIC_SPUR_CONTROLLER}" -h 2>/dev/null |
+                  awk -v id="${JOB_ID}" '$1 == id { found = 1 } END { exit found ? 0 : 1 }'; then
+                  echo "Slurm job ${JOB_ID} is no longer queued"
+                  exit 0
+              fi
+              sleep 1
+          done
+          echo "ERROR: Slurm job ${JOB_ID} is still queued after scancel" >&2
+          exit 1
+          REMOTE
+
   # Validate /run-ci-fast slash command on PRs; skipped for manual triggers.
   authorize:
-    if: >
-      github.event_name != 'issue_comment' || (
+    if: >-
+      (github.event_name != 'workflow_dispatch' || inputs.cancel_slurm_job == '') &&
+      (github.event_name != 'issue_comment' || (
         github.event.issue.pull_request != null &&
         contains(github.event.comment.body, '/run-ci-fast')
-      )
+      ))
     runs-on: ubuntu-latest
     permissions:
       pull-requests: write
diff --git a/.slurm/run-build-distribute.sh b/.slurm/run-build-distribute.sh
index 66ea7ea7..d8ba4806 100755
--- a/.slurm/run-build-distribute.sh
+++ b/.slurm/run-build-distribute.sh
@@ -326,6 +326,19 @@ PROLOGUE
 )"
 
     local jobid="" logfile="" rc=0
+    local active_job_file="${AIC_CI_ACTIVE_JOB_FILE:-}"
+
+    _record_active_job() {
+        [[ -n "${active_job_file}" ]] || return 0
+        mkdir -p "$(dirname "${active_job_file}")"
+        printf '%s\n' "${jobid}" > "${active_job_file}.tmp.${BASHPID}"
+        mv -f "${active_job_file}.tmp.${BASHPID}" "${active_job_file}"
+    }
+
+    _clear_active_job() {
+        [[ -n "${active_job_file}" ]] || return 0
+        rm -f "${active_job_file}" 2>/dev/null || true
+    }
 
     if [[ "${AIC_SPUR_CLUSTER}" == "1" ]]; then
         # SPUR sbatch does not support --parsable, --wait, or reading the script
@@ -349,6 +362,7 @@ PROLOGUE
 
         jobid="$(printf '%s\n' "${submit_out}" | grep -oE '[0-9]+$' | tail -1)"
         [[ -n "${jobid}" ]] || die "could not parse job id from sbatch output: ${submit_out}"
+        _record_active_job
         logfile="${AIC_DAY_DIR}/logs/${jobid}/${logname}.out"
         log "submitted ${jobname} as job ${jobid} (partition ${AIC_BUILD_PARTITION})"
         log "log: ${logfile}"
@@ -446,6 +460,7 @@ PROLOGUE
             sleep 0.2; tries=$((tries + 1))
         done
         jobid="$(head -n1 "${idfile}" 2>/dev/null | tr -d '[:space:]' | cut -d';' -f1)"
+        [[ -z "${jobid}" ]] || _record_active_job
 
         logfile="${AIC_DAY_DIR}/logs/${jobid:-unknown}/${logname}.out"
         if [[ -n "${jobid}" ]]; then
@@ -469,6 +484,7 @@ PROLOGUE
         rm -f "${idfile}" 2>/dev/null || true
     fi
 
+    _clear_active_job
     return "${rc}"
 }
 

From 3cba75085dffae2702f248bc21c734c5d98369c2 Mon Sep 17 00:00:00 2001
From: Ivan Ganev <ivan.ganev@amd.com>
Date: Thu, 13 Aug 2026 11:10:37 -0700
Subject: [PATCH 2/3] fix(ci): resolve SPUR helper during shellcheck

Point ShellCheck's source annotations at the repository-relative path for spur-ci-common.sh in all three SPUR test wrappers. Keep runtime sourcing based on SCRIPT_DIR so installed CI scripts remain location-independent.
---
 .github/scripts/spur-dist-build.sh | 2 +-
 .github/scripts/spur-smoke-test.sh | 2 +-
 .github/scripts/spur-tiny-test.sh  | 2 +-
 3 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/.github/scripts/spur-dist-build.sh b/.github/scripts/spur-dist-build.sh
index 1e5f0934..220c1ae2 100755
--- a/.github/scripts/spur-dist-build.sh
+++ b/.github/scripts/spur-dist-build.sh
@@ -24,7 +24,7 @@ AIC_SHARED_NFS="${AIC_SHARED_NFS:?AIC_SHARED_NFS must be set (e.g. via GitHub re
 AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g. via GitHub repo variable)}"
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=spur-ci-common.sh
+# shellcheck source=.github/scripts/spur-ci-common.sh
 source "${SCRIPT_DIR}/spur-ci-common.sh"
 aic_ci_session_init "${SHORT}" "dist-build"
 
diff --git a/.github/scripts/spur-smoke-test.sh b/.github/scripts/spur-smoke-test.sh
index 45d71d28..8054c5d4 100755
--- a/.github/scripts/spur-smoke-test.sh
+++ b/.github/scripts/spur-smoke-test.sh
@@ -24,7 +24,7 @@ AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 REPO="https://github.com/ROCm/rocm-aic.git"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=spur-ci-common.sh
+# shellcheck source=.github/scripts/spur-ci-common.sh
 source "${SCRIPT_DIR}/spur-ci-common.sh"
 aic_ci_session_init "${SHORT}" "smoke-test"
 
diff --git a/.github/scripts/spur-tiny-test.sh b/.github/scripts/spur-tiny-test.sh
index fd20c1fa..c8612de1 100755
--- a/.github/scripts/spur-tiny-test.sh
+++ b/.github/scripts/spur-tiny-test.sh
@@ -35,7 +35,7 @@ AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 KEEP_ARTIFACTS="${KEEP_ARTIFACTS:-0}"
 REPO="https://github.com/ROCm/rocm-aic.git"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=spur-ci-common.sh
+# shellcheck source=.github/scripts/spur-ci-common.sh
 source "${SCRIPT_DIR}/spur-ci-common.sh"
 aic_ci_session_init "${SHORT}" "tiny-test"
 

From 0027f515904556f01284b875b211ad5e48dbd4c9 Mon Sep 17 00:00:00 2001
From: Ivan Ganev <ivan.ganev@amd.com>
Date: Thu, 13 Aug 2026 13:23:59 -0700
Subject: [PATCH 3/3] refactor(ci): separate runner and workflow scripts

Move scripts installed under /usr/local/lib/aic-ci into .github/scripts/runners and limit install-ci-scripts to that directory.

Move checkout-invoked helpers into .github/scripts/workflows, update their workflow call sites, and keep ShellCheck source annotations aligned with the new runner path.
---
 .github/scripts/{ => runners}/spur-ci-common.sh   |  2 +-
 .github/scripts/{ => runners}/spur-cliff.sh       |  2 +-
 .github/scripts/{ => runners}/spur-dist-build.sh  |  4 ++--
 .github/scripts/{ => runners}/spur-smoke-test.sh  |  4 ++--
 .github/scripts/{ => runners}/spur-tiny-test.sh   |  4 ++--
 .../{ => workflows}/render-release-notes.sh       |  1 +
 .../scripts/{ => workflows}/spur-cliff-harvest.sh |  4 ++--
 .../{ => workflows}/spur-monitoring-cpu-smoke.sh  |  2 +-
 .github/workflows/aic-amd-nightly-cliff.yml       |  2 +-
 .github/workflows/aic-export.yml                  |  2 +-
 .github/workflows/aic-monitoring-cpu-smoke.yml    |  2 +-
 .github/workflows/aic-release.yml                 |  2 +-
 Makefile                                          | 15 ++++++++-------
 13 files changed, 24 insertions(+), 22 deletions(-)
 rename .github/scripts/{ => runners}/spur-ci-common.sh (97%)
 rename .github/scripts/{ => runners}/spur-cliff.sh (97%)
 rename .github/scripts/{ => runners}/spur-dist-build.sh (95%)
 rename .github/scripts/{ => runners}/spur-smoke-test.sh (95%)
 rename .github/scripts/{ => runners}/spur-tiny-test.sh (96%)
 rename .github/scripts/{ => workflows}/render-release-notes.sh (96%)
 rename .github/scripts/{ => workflows}/spur-cliff-harvest.sh (97%)
 rename .github/scripts/{ => workflows}/spur-monitoring-cpu-smoke.sh (99%)

diff --git a/.github/scripts/spur-ci-common.sh b/.github/scripts/runners/spur-ci-common.sh
similarity index 97%
rename from .github/scripts/spur-ci-common.sh
rename to .github/scripts/runners/spur-ci-common.sh
index fd5ed089..0e67b417 100644
--- a/.github/scripts/spur-ci-common.sh
+++ b/.github/scripts/runners/spur-ci-common.sh
@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-# Shared lifecycle helpers for CI wrappers that run work on the SPUR head node.
+# Shared lifecycle helpers for installed CI wrappers that run work on the SPUR head node.
 # Call aic_ci_session_init before starting SSH, then use aic_ci_ssh_bash for the
 # remote heredoc.  If the local wrapper is interrupted, its EXIT trap reconnects
 # and cancels only the Slurm job/process group recorded for this run and stage.
diff --git a/.github/scripts/spur-cliff.sh b/.github/scripts/runners/spur-cliff.sh
similarity index 97%
rename from .github/scripts/spur-cliff.sh
rename to .github/scripts/runners/spur-cliff.sh
index 312610ab..f3525042 100755
--- a/.github/scripts/spur-cliff.sh
+++ b/.github/scripts/runners/spur-cliff.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), uses the clone and
+# Installed on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), uses the clone and
 # tarball left by spur-dist-build.sh, and runs a cliff benchmark.
 #
 # Usage: spur-cliff.sh <full-sha> <target>
diff --git a/.github/scripts/spur-dist-build.sh b/.github/scripts/runners/spur-dist-build.sh
similarity index 95%
rename from .github/scripts/spur-dist-build.sh
rename to .github/scripts/runners/spur-dist-build.sh
index 220c1ae2..e623ad79 100755
--- a/.github/scripts/spur-dist-build.sh
+++ b/.github/scripts/runners/spur-dist-build.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), clones the repo at
+# Installed on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), clones the repo at
 # the current SHA, and runs the requested dist-build target with a CI-scoped
 # image name and tarball path.  The tarball is left in place for
 # spur-smoke-test.sh; the run-attempt-scoped clone is always removed.
@@ -24,7 +24,7 @@ AIC_SHARED_NFS="${AIC_SHARED_NFS:?AIC_SHARED_NFS must be set (e.g. via GitHub re
 AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g. via GitHub repo variable)}"
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=.github/scripts/spur-ci-common.sh
+# shellcheck source=.github/scripts/runners/spur-ci-common.sh
 source "${SCRIPT_DIR}/spur-ci-common.sh"
 aic_ci_session_init "${SHORT}" "dist-build"
 
diff --git a/.github/scripts/spur-smoke-test.sh b/.github/scripts/runners/spur-smoke-test.sh
similarity index 95%
rename from .github/scripts/spur-smoke-test.sh
rename to .github/scripts/runners/spur-smoke-test.sh
index 8054c5d4..28b2c5eb 100755
--- a/.github/scripts/spur-smoke-test.sh
+++ b/.github/scripts/runners/spur-smoke-test.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and
+# Installed on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and
 # runs the requested smoke-test target against the tarball produced by
 # spur-dist-build.sh for the same SHA. The tarball is left in place for
 # spur-tiny-test.sh; the run-attempt-scoped clone is always removed.
@@ -24,7 +24,7 @@ AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 REPO="https://github.com/ROCm/rocm-aic.git"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=.github/scripts/spur-ci-common.sh
+# shellcheck source=.github/scripts/runners/spur-ci-common.sh
 source "${SCRIPT_DIR}/spur-ci-common.sh"
 aic_ci_session_init "${SHORT}" "smoke-test"
 
diff --git a/.github/scripts/spur-tiny-test.sh b/.github/scripts/runners/spur-tiny-test.sh
similarity index 96%
rename from .github/scripts/spur-tiny-test.sh
rename to .github/scripts/runners/spur-tiny-test.sh
index c8612de1..a0ac6fff 100755
--- a/.github/scripts/spur-tiny-test.sh
+++ b/.github/scripts/runners/spur-tiny-test.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and runs tiny-test
+# Installed on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and runs tiny-test
 # against the tarball produced by spur-dist-build.sh for the same SHA (the stage
 # after spur-smoke-test.sh).  tiny-test brings up the compose MP stack
 # (standalone lmcache server + vLLM LMCacheMPConnector) with a tiny model and
@@ -35,7 +35,7 @@ AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 KEEP_ARTIFACTS="${KEEP_ARTIFACTS:-0}"
 REPO="https://github.com/ROCm/rocm-aic.git"
 SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
-# shellcheck source=.github/scripts/spur-ci-common.sh
+# shellcheck source=.github/scripts/runners/spur-ci-common.sh
 source "${SCRIPT_DIR}/spur-ci-common.sh"
 aic_ci_session_init "${SHORT}" "tiny-test"
 
diff --git a/.github/scripts/render-release-notes.sh b/.github/scripts/workflows/render-release-notes.sh
similarity index 96%
rename from .github/scripts/render-release-notes.sh
rename to .github/scripts/workflows/render-release-notes.sh
index 0fbec63c..70e744dc 100755
--- a/.github/scripts/render-release-notes.sh
+++ b/.github/scripts/workflows/render-release-notes.sh
@@ -5,6 +5,7 @@ set -euo pipefail
 #
 # SPDX-License-Identifier: MIT
 
+# This script runs directly from a workflow checkout; it is not installed on a runner.
 if [[ $# -ne 6 ]]; then
     echo "usage: $0 <Dockerfile> <tag> <source-sha> <gpu-arch> <image-name> <repository>" >&2
     exit 2
diff --git a/.github/scripts/spur-cliff-harvest.sh b/.github/scripts/workflows/spur-cliff-harvest.sh
similarity index 97%
rename from .github/scripts/spur-cliff-harvest.sh
rename to .github/scripts/workflows/spur-cliff-harvest.sh
index d4e29405..db102f60 100755
--- a/.github/scripts/spur-cliff-harvest.sh
+++ b/.github/scripts/workflows/spur-cliff-harvest.sh
@@ -5,7 +5,7 @@ set -euo pipefail
 #
 # SPDX-License-Identifier: MIT
 #
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST),
+# Runs from a workflow checkout on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST),
 # submits the full cliff-submit job via `make cliff-submit`, waits for it to
 # finish, then:
 #   1. Copies CSVs + plots to a stable NFS staging dir (survives workdir cleanup)
@@ -21,7 +21,7 @@ set -euo pipefail
 #   secrets.AIC_SPUR_CONTROLLER  — SPUR controller address
 #
 # Usage:
-#   bash .github/scripts/spur-cliff-harvest.sh <full-sha> <run-date-ISO>
+#   bash .github/scripts/workflows/spur-cliff-harvest.sh <full-sha> <run-date-ISO>
 #   # run-date-ISO defaults to $(date +%Y-%m-%d) if omitted
 
 SHA="${1:?usage: $0 <full-sha> [run-date]}"
diff --git a/.github/scripts/spur-monitoring-cpu-smoke.sh b/.github/scripts/workflows/spur-monitoring-cpu-smoke.sh
similarity index 99%
rename from .github/scripts/spur-monitoring-cpu-smoke.sh
rename to .github/scripts/workflows/spur-monitoring-cpu-smoke.sh
index d5edf4fa..34c0af36 100755
--- a/.github/scripts/spur-monitoring-cpu-smoke.sh
+++ b/.github/scripts/workflows/spur-monitoring-cpu-smoke.sh
@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST),
+# Runs from a workflow checkout on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST),
 # submits a CPU-only srun job (no --gres=gpu), and:
 #   1. Launches node-exporter, nvme-exporter, rdma-exporter, and Prometheus
 #   2. Runs the vLLM emulator (backed by the existing rocm-aic image)
diff --git a/.github/workflows/aic-amd-nightly-cliff.yml b/.github/workflows/aic-amd-nightly-cliff.yml
index aff8f884..c208f96c 100644
--- a/.github/workflows/aic-amd-nightly-cliff.yml
+++ b/.github/workflows/aic-amd-nightly-cliff.yml
@@ -40,7 +40,7 @@ jobs:
 
       - name: Run cliff benchmark + harvest results from SPUR
         run: |
-          bash .github/scripts/spur-cliff-harvest.sh \
+          bash .github/scripts/workflows/spur-cliff-harvest.sh \
             "${{ github.event.workflow_run.head_sha }}" \
             "$(date -u +%Y-%m-%d)"
 
diff --git a/.github/workflows/aic-export.yml b/.github/workflows/aic-export.yml
index 78db3396..e901219c 100644
--- a/.github/workflows/aic-export.yml
+++ b/.github/workflows/aic-export.yml
@@ -82,7 +82,7 @@ jobs:
       - name: Validate release notes renderer
         run: |
           set -euo pipefail
-          renderer=.github/scripts/render-release-notes.sh
+          renderer=.github/scripts/workflows/render-release-notes.sh
           args=(docker/Dockerfile v-test deadbeef gfx942 rocm-aic ROCm/rocm-aic)
           fixtures="$(mktemp -d)"
           trap 'rm -rf "${fixtures}"' EXIT
diff --git a/.github/workflows/aic-monitoring-cpu-smoke.yml b/.github/workflows/aic-monitoring-cpu-smoke.yml
index 94eb3e2c..d4592f78 100644
--- a/.github/workflows/aic-monitoring-cpu-smoke.yml
+++ b/.github/workflows/aic-monitoring-cpu-smoke.yml
@@ -54,7 +54,7 @@ jobs:
       - name: Run CPU monitoring smoke test on SPUR
         run: |
           SHA="${{ github.event.workflow_run.head_sha || github.sha }}"
-          bash .github/scripts/spur-monitoring-cpu-smoke.sh "${SHA}"
+          bash .github/scripts/workflows/spur-monitoring-cpu-smoke.sh "${SHA}"
 
       - name: Upload metrics reference page artifact
         uses: actions/upload-artifact@v4
diff --git a/.github/workflows/aic-release.yml b/.github/workflows/aic-release.yml
index a5520e60..de83a81d 100644
--- a/.github/workflows/aic-release.yml
+++ b/.github/workflows/aic-release.yml
@@ -148,7 +148,7 @@ jobs:
       - name: Compose release notes
         run: |
           set -euo pipefail
-          bash .github/scripts/render-release-notes.sh \
+          bash .github/scripts/workflows/render-release-notes.sh \
             docker/Dockerfile \
             "${{ github.ref_name }}" \
             "${{ github.sha }}" \
diff --git a/Makefile b/Makefile
index b8ee9bdf..6ad6a511 100644
--- a/Makefile
+++ b/Makefile
@@ -138,10 +138,11 @@ DIST := $(CURDIR)/.slurm/run-build-distribute.sh
 # The hardware-CI workflows call helper scripts from AIC_CI_LIB_DIR on the
 # self-hosted runner (spur-dist-build.sh / spur-smoke-test.sh / spur-tiny-test.sh
 # / spur-cliff.sh).  `make install-ci-scripts` deploys the source copies from
-# .github/scripts there.  Writing under /usr/local usually needs root, so the
-# target uses sudo when the destination is not writable by the current user.
+# .github/scripts/runners there.  Scripts invoked directly from checked-out
+# workflows live separately under .github/scripts/workflows.  Writing under
+# /usr/local usually needs root, so the target uses sudo when needed.
 AIC_CI_LIB_DIR    ?= /usr/local/lib/aic-ci
-AIC_CI_SCRIPT_DIR := $(CURDIR)/.github/scripts
+AIC_CI_SCRIPT_DIR := $(CURDIR)/.github/scripts/runners
 
 AIC_FAST_ARCH ?= gfx950
 
@@ -298,7 +299,7 @@ help:
 	@echo "  make smoke-test-fast   Smoke-test the single-arch dev image (AIC_FAST_ARCH=$(AIC_FAST_ARCH))"
 	@echo "  make tiny-test         End-to-end serve check (MP stack + tiny model, one completion)"
 	@echo "  make tiny-test-fast    Fast variant of tiny-test"
-	@echo "  make install-ci-scripts  Deploy .github/scripts/spur-*.sh to $(AIC_CI_LIB_DIR) (sudo if needed)"
+	@echo "  make install-ci-scripts  Deploy .github/scripts/runners/*.sh to $(AIC_CI_LIB_DIR) (sudo if needed)"
 	@echo "  make cliff-submit      sbatch the full 3-arm cliff sweep -> logs/<job-id>/"
 	@echo "  make cliff-kvd         sbatch focused KVD cliff: shared prefix, sparse c ladder (1,8,32,64,128,250)"
 	@echo "  make cliff-spur-l2     sbatch SPUR-tuned L2 cliff: per_client prefix, util=0.40, 8GB DRAM L1, c=1/8/32 (vram+nvme)"
@@ -651,14 +652,14 @@ tiny-test-fast:
 	@$(MAKE) --no-print-directory tiny-test \
 	    AIC_ROCM_ARCH='$(AIC_FAST_ARCH)'
 
-install-ci-scripts:            # Deploy .github/scripts/spur-*.sh to the runner's AIC_CI_LIB_DIR
+install-ci-scripts:            # Deploy .github/scripts/runners/*.sh to the runner's AIC_CI_LIB_DIR
 	@set -e; \
 	src="$(AIC_CI_SCRIPT_DIR)"; dst="$(AIC_CI_LIB_DIR)"; \
-	ls "$$src"/spur-*.sh >/dev/null 2>&1 || { echo "ERROR: no spur-*.sh under $$src" >&2; exit 1; }; \
+	ls "$$src"/*.sh >/dev/null 2>&1 || { echo "ERROR: no runner scripts under $$src" >&2; exit 1; }; \
 	if [ -w "$$(dirname "$$dst")" ] || [ -w "$$dst" ]; then SUDO=; else SUDO="sudo"; \
 		echo "$$dst not writable; using sudo"; fi; \
 	$$SUDO install -d -m 0755 "$$dst"; \
-	for f in "$$src"/spur-*.sh; do \
+	for f in "$$src"/*.sh; do \
 		$$SUDO install -m 0755 "$$f" "$$dst/$$(basename "$$f")"; \
 		echo "installed $$(basename "$$f") -> $$dst/"; \
 	done; \