refactor(ci): isolate SPUR runner sessions

.github/scripts/runners/spur-ci-common.sh

@@ -0,0 +1,99 @@
+#!/usr/bin/env bash
+
+# Shared lifecycle helpers for installed CI wrappers that run work on the SPUR head node.
+# Call aic_ci_session_init before starting SSH, then use aic_ci_ssh_bash for the
+# remote heredoc.  If the local wrapper is interrupted, its EXIT trap reconnects
+# and cancels only the Slurm job/process group recorded for this run and stage.
+
+aic_ci_session_init() {
+    local short_sha="${1:?short SHA is required}"
+    local stage="${2:?CI stage is required}"
+    local run_id="${GITHUB_RUN_ID:-manual-${BASHPID}}"
+    local run_attempt="${GITHUB_RUN_ATTEMPT:-1}"
+
+    AIC_CI_RUN_KEY="${run_id}-${run_attempt}"
+    AIC_CI_STAGE="${stage}"
+    case "${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}.${short_sha}" in
+        *[!A-Za-z0-9._-]*)
+            echo "ERROR: unsafe CI session identifier" >&2
+            exit 2
+            ;;
+    esac
+
+    AIC_CI_SHORT_SHA="${short_sha}"
+    trap 'exit 130' INT
+    trap 'exit 143' TERM
+    trap 'exit 129' HUP
+    trap _aic_ci_session_exit EXIT
+}
+
+aic_ci_ssh_bash() {
+    ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+        AIC_CI_RUN_KEY="${AIC_CI_RUN_KEY}" \
+        AIC_CI_STAGE="${AIC_CI_STAGE}" \
+        "$@" \
+        setsid --fork --wait bash
+}
+
+_aic_ci_session_exit() {
+    local rc=$?
+    trap - EXIT INT TERM HUP
+
+    if (( rc != 0 )); then
+        echo "=== CI wrapper interrupted/failed; cancelling its remote session ===" >&2
+        ssh -o ConnectTimeout=15 -o ServerAliveInterval=10 -o ServerAliveCountMax=2 \
+            "${AIC_SPUR_HOST}" env \
+            AIC_CI_RUN_KEY="${AIC_CI_RUN_KEY}" \
+            AIC_CI_STAGE="${AIC_CI_STAGE}" \
+            AIC_CI_SHORT_SHA="${AIC_CI_SHORT_SHA}" \
+            AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT}" \
+            AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
+            bash <<'REMOTE_CANCEL' || true
+set -u
+
+CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-$HOME/Projects/rocm-aic-ci}"
+CONTROL_PREFIX="${CI_STORAGE_ROOT}/control/${AIC_CI_SHORT_SHA}.${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}"
+JOB_FILE="${CONTROL_PREFIX}.job"
+PID_FILE="${CONTROL_PREFIX}.pid"
+CANCEL_FILE="${CONTROL_PREFIX}.cancel"
+remote_session_found=0
+
+# Close the small race where cancellation arrives before the remote shell has
+# written its PID.  A shell that starts later sees this sentinel and exits
+# before cloning or submitting work.
+mkdir -p "${CI_STORAGE_ROOT}/control"
+: > "${CANCEL_FILE}"
+
+if [[ -s "${JOB_FILE}" ]]; then
+    read -r job_id < "${JOB_FILE}" || job_id=""
+    if [[ "${job_id}" =~ ^[0-9]+$ ]]; then
+        echo "Cancelling Slurm job ${job_id}"
+        scancel --controller="${AIC_SPUR_CONTROLLER}" "${job_id}" || true
+    fi
+fi
+
+if [[ -s "${PID_FILE}" ]]; then
+    read -r remote_pid < "${PID_FILE}" || remote_pid=""
+    if [[ "${remote_pid}" =~ ^[0-9]+$ ]]; then
+        remote_session_found=1
+        if kill -0 -- "-${remote_pid}" 2>/dev/null; then
+            echo "Terminating remote process group ${remote_pid}"
+            kill -TERM -- "-${remote_pid}" 2>/dev/null || true
+            for _ in 1 2 3 4 5 6 7 8 9 10; do
+                kill -0 -- "-${remote_pid}" 2>/dev/null || break
+                sleep 0.2
+            done
+            kill -KILL -- "-${remote_pid}" 2>/dev/null || true
+        fi
+    fi
+fi
+
+rm -f "${JOB_FILE}" "${PID_FILE}" 2>/dev/null || true
+if (( remote_session_found == 1 )); then
+    rm -f "${CANCEL_FILE}" 2>/dev/null || true
+fi
+REMOTE_CANCEL
+    fi
+
+    exit "${rc}"
+}

.github/scripts/spur-cliff.sh → .github/scripts/runners/spur-cliff.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), uses the clone and
+# Installed on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), uses the clone and
 # tarball left by spur-dist-build.sh, and runs a cliff benchmark.
 #
 # Usage: spur-cliff.sh <full-sha> <target>

.github/scripts/spur-dist-build.sh → .github/scripts/runners/spur-dist-build.sh

@@ -1,12 +1,10 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), clones the repo at
+# Installed on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST), clones the repo at
 # the current SHA, and runs the requested dist-build target with a CI-scoped
-# image name and tarball path. The clone and tarball are left in place for
-# spur-smoke-test.sh to use; spur-smoke-test.sh owns the final cleanup.
-#
-# On failure, cleans up immediately so no stale state is left behind.
+# image name and tarball path.  The tarball is left in place for
+# spur-smoke-test.sh; the run-attempt-scoped clone is always removed.
 
 SHA="${1:?usage: $0 <full-sha> [dist-build|dist-build-fast]}"
 AIC_DIST_BUILD_TARGET="${2:-dist-build}"
@@ -25,30 +23,59 @@ AIC_SPUR_HOST="${AIC_SPUR_HOST//[$'\t\r\n ']}"
 AIC_SHARED_NFS="${AIC_SHARED_NFS:?AIC_SHARED_NFS must be set (e.g. via GitHub repo variable)}"
 AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g. via GitHub repo variable)}"
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/runners/spur-ci-common.sh
+source "${SCRIPT_DIR}/spur-ci-common.sh"
+aic_ci_session_init "${SHORT}" "dist-build"
 
-ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+aic_ci_ssh_bash \
     SHA="${SHA}" \
     REPO="${REPO}" \
     AIC_IMAGE_NAME="${AIC_IMAGE_NAME}" \
     AIC_DIST_BUILD_TARGET="${AIC_DIST_BUILD_TARGET}" \
     AIC_SHARED_NFS="${AIC_SHARED_NFS}" \
     AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT}" \
     AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
-    SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" \
-    bash << 'REMOTE'
+    SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" << 'REMOTE'
 set -euo pipefail
 
 SHORT="${SHA:0:7}"
-WORKDIR="$HOME/Projects/rocm-aic.${SHORT}"
+WORKDIR="$HOME/Projects/rocm-aic.${SHORT}.${AIC_CI_RUN_KEY}"
 CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-$HOME/Projects/rocm-aic-ci}"
 TARBALL_DIR="${CI_STORAGE_ROOT}/images/aic-ci-${SHORT}"
 CACHE_DIR="${CI_STORAGE_ROOT}/buildcache"
+CONTROL_PREFIX="${CI_STORAGE_ROOT}/control/${SHORT}.${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}"
+PID_FILE="${CONTROL_PREFIX}.pid"
+JOB_FILE="${CONTROL_PREFIX}.job"
+CANCEL_FILE="${CONTROL_PREFIX}.cancel"
 
-cleanup_on_fail() {
-    echo "=== Build failed — cleaning up ==="
-    rm -rf "${WORKDIR}" "${TARBALL_DIR}"
+mkdir -p "${CI_STORAGE_ROOT}/control"
+printf '%s\n' "${BASHPID}" > "${PID_FILE}"
+if [[ -e "${CANCEL_FILE}" ]]; then
+    echo "CI session was cancelled before remote startup completed" >&2
+    rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    exit 143
+fi
+export AIC_CI_ACTIVE_JOB_FILE="${JOB_FILE}"
+
+_best_effort_remove() {
+    rm -rf "$@" || echo "WARNING: cleanup could not fully remove: $*" >&2
+}
+_cleanup() {
+    local rc=$?
+    trap - EXIT
+    echo "=== Cleaning up run-attempt worktree ==="
+    _best_effort_remove "${WORKDIR}"
+    if (( rc != 0 )); then
+        echo "=== Build failed — removing staged image ==="
+        _best_effort_remove "${TARBALL_DIR}"
+    fi
+    if (( rc == 0 )); then
+        rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    fi
+    exit "${rc}"
 }
-trap cleanup_on_fail ERR
+trap _cleanup EXIT
 
 echo "=== Cloning ${REPO} at ${SHA} into ${WORKDIR} ==="
 rm -rf "${WORKDIR}"

.github/scripts/spur-smoke-test.sh → .github/scripts/runners/spur-smoke-test.sh

@@ -1,11 +1,10 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and
+# Installed on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and
 # runs the requested smoke-test target against the tarball produced by
-# spur-dist-build.sh for the same SHA. The clone and tarball are left in place
-# for spur-tiny-test.sh (the next stage) to use; spur-tiny-test.sh owns the final
-# cleanup. On failure, cleans up immediately so no stale state is left behind.
+# spur-dist-build.sh for the same SHA. The tarball is left in place for
+# spur-tiny-test.sh; the run-attempt-scoped clone is always removed.
 
 SHA="${1:?usage: $0 <full-sha> [smoke-test|smoke-test-fast]}"
 AIC_SMOKE_TEST_TARGET="${2:-smoke-test}"
@@ -24,32 +23,58 @@ AIC_SHARED_NFS="${AIC_SHARED_NFS:?AIC_SHARED_NFS must be set (e.g. via GitHub re
 AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g. via GitHub repo variable)}"
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 REPO="https://github.com/ROCm/rocm-aic.git"
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/runners/spur-ci-common.sh
+source "${SCRIPT_DIR}/spur-ci-common.sh"
+aic_ci_session_init "${SHORT}" "smoke-test"
 
-ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+aic_ci_ssh_bash \
     SHA="${SHA}" \
     REPO="${REPO}" \
     AIC_IMAGE_NAME="${AIC_IMAGE_NAME}" \
     AIC_SMOKE_TEST_TARGET="${AIC_SMOKE_TEST_TARGET}" \
     AIC_SHARED_NFS="${AIC_SHARED_NFS}" \
     AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT}" \
     AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
-    SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" \
-    bash << 'REMOTE'
+    SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" << 'REMOTE'
 set -euo pipefail
 
 SHORT="${SHA:0:7}"
-WORKDIR="$HOME/Projects/rocm-aic.${SHORT}"
+WORKDIR="$HOME/Projects/rocm-aic.${SHORT}.${AIC_CI_RUN_KEY}"
 CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-$HOME/Projects/rocm-aic-ci}"
 TARBALL_DIR="${CI_STORAGE_ROOT}/images/aic-ci-${SHORT}"
+CONTROL_PREFIX="${CI_STORAGE_ROOT}/control/${SHORT}.${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}"
+PID_FILE="${CONTROL_PREFIX}.pid"
+JOB_FILE="${CONTROL_PREFIX}.job"
+CANCEL_FILE="${CONTROL_PREFIX}.cancel"
 
-cleanup_on_fail() {
-    echo "=== Smoke test failed — cleaning up ==="
-    rm -rf "${TARBALL_DIR}"
-    find "${WORKDIR}" -mindepth 1 -maxdepth 1 -not -name logs -exec rm -rf {} +
+mkdir -p "${CI_STORAGE_ROOT}/control"
+printf '%s\n' "${BASHPID}" > "${PID_FILE}"
+if [[ -e "${CANCEL_FILE}" ]]; then
+    echo "CI session was cancelled before remote startup completed" >&2
+    rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    exit 143
+fi
+export AIC_CI_ACTIVE_JOB_FILE="${JOB_FILE}"
+
+_best_effort_remove() {
+    rm -rf "$@" || echo "WARNING: cleanup could not fully remove: $*" >&2
+}
+_cleanup() {
+    local rc=$?
+    trap - EXIT
+    echo "=== Cleaning up run-attempt worktree ==="
+    _best_effort_remove "${WORKDIR}"
+    if (( rc != 0 )); then
+        echo "=== Smoke test failed — removing staged image ==="
+        _best_effort_remove "${TARBALL_DIR}"
+    fi
+    if (( rc == 0 )); then
+        rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    fi
+    exit "${rc}"
 }
-# On success: preserve TARBALL_DIR so the following tiny-test stage can use it.
-# On failure: clean up immediately so no stale state is left behind.
-trap cleanup_on_fail ERR
+trap _cleanup EXIT
 
 # Re-clone if WORKDIR is missing or checked out at the wrong SHA.
 ACTUAL_SHA="$(git -C "${WORKDIR}" rev-parse HEAD 2>/dev/null || true)"

.github/scripts/spur-tiny-test.sh → .github/scripts/runners/spur-tiny-test.sh

@@ -1,18 +1,18 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-# Runs on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and runs tiny-test
+# Installed on the self-hosted runner; SSHes to the SPUR head node (AIC_SPUR_HOST) and runs tiny-test
 # against the tarball produced by spur-dist-build.sh for the same SHA (the stage
 # after spur-smoke-test.sh).  tiny-test brings up the compose MP stack
 # (standalone lmcache server + vLLM LMCacheMPConnector) with a tiny model and
 # asserts one non-empty chat completion.
 #
-# Cleanup ownership depends on whether a cliff stage follows:
+# Tarball cleanup ownership depends on whether a cliff stage follows:
 #   * PR flow (dist-build -> smoke-test -> tiny-test): tiny-test is terminal, so
-#     it owns the final cleanup (removes the clone + tarball on exit).
+#     it removes the tarball on exit.
 #   * Nightly (dist-build -> smoke -> tiny -> cliff): cliff runs next and needs
-#     the artifacts, so the nightly tiny-test step sets KEEP_ARTIFACTS=1 and this
-#     script only cleans up on failure (spur-cliff.sh does the final cleanup).
+#     the tarball, so the nightly tiny-test step sets KEEP_ARTIFACTS=1.
+# The run-attempt-scoped clone is always removed best-effort.
 # The tiny model uses the cluster-wide HF cache so it is downloaded once and
 # reused across CI workflows and SPUR accounts.
 
@@ -34,8 +34,12 @@ AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER:?AIC_SPUR_CONTROLLER must be set (e.g
 AIC_CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-}"
 KEEP_ARTIFACTS="${KEEP_ARTIFACTS:-0}"
 REPO="https://github.com/ROCm/rocm-aic.git"
+SCRIPT_DIR="$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" && pwd)"
+# shellcheck source=.github/scripts/runners/spur-ci-common.sh
+source "${SCRIPT_DIR}/spur-ci-common.sh"
+aic_ci_session_init "${SHORT}" "tiny-test"
 
-ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
+aic_ci_ssh_bash \
     SHA="${SHA}" \
     REPO="${REPO}" \
     AIC_IMAGE_NAME="${AIC_IMAGE_NAME}" \
@@ -45,27 +49,45 @@ ssh -o ServerAliveInterval=30 -o ServerAliveCountMax=4 "${AIC_SPUR_HOST}" env \
     KEEP_ARTIFACTS="${KEEP_ARTIFACTS}" \
     AIC_SPUR_CONTROLLER="${AIC_SPUR_CONTROLLER}" \
     SPUR_CONTROLLER_ADDR="${AIC_SPUR_CONTROLLER}" \
-    HF_TOKEN="${HF_TOKEN:-}" \
-    bash << 'REMOTE'
+    HF_TOKEN="${HF_TOKEN:-}" << 'REMOTE'
 set -euo pipefail
 
 SHORT="${SHA:0:7}"
-WORKDIR="$HOME/Projects/rocm-aic.${SHORT}"
+WORKDIR="$HOME/Projects/rocm-aic.${SHORT}.${AIC_CI_RUN_KEY}"
 CI_STORAGE_ROOT="${AIC_CI_STORAGE_ROOT:-$HOME/Projects/rocm-aic-ci}"
 TARBALL_DIR="${CI_STORAGE_ROOT}/images/aic-ci-${SHORT}"
+CONTROL_PREFIX="${CI_STORAGE_ROOT}/control/${SHORT}.${AIC_CI_RUN_KEY}.${AIC_CI_STAGE}"
+PID_FILE="${CONTROL_PREFIX}.pid"
+JOB_FILE="${CONTROL_PREFIX}.job"
+CANCEL_FILE="${CONTROL_PREFIX}.cancel"
+
+mkdir -p "${CI_STORAGE_ROOT}/control"
+printf '%s\n' "${BASHPID}" > "${PID_FILE}"
+if [[ -e "${CANCEL_FILE}" ]]; then
+    echo "CI session was cancelled before remote startup completed" >&2
+    rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    exit 143
+fi
+export AIC_CI_ACTIVE_JOB_FILE="${JOB_FILE}"
 
+_best_effort_remove() {
+    rm -rf "$@" || echo "WARNING: cleanup could not fully remove: $*" >&2
+}
 _cleanup() {
-    echo "=== Cleaning up ==="
-    rm -rf "${WORKDIR}" "${TARBALL_DIR}"
+    local rc=$?
+    trap - EXIT
+    echo "=== Cleaning up run-attempt worktree ==="
+    _best_effort_remove "${WORKDIR}"
+    if (( rc != 0 )) || [[ "${KEEP_ARTIFACTS}" != "1" ]]; then
+        echo "=== Removing staged image ==="
+        _best_effort_remove "${TARBALL_DIR}"
+    fi
+    if (( rc == 0 )); then
+        rm -f "${PID_FILE}" "${JOB_FILE}" "${CANCEL_FILE}" 2>/dev/null || true
+    fi
+    exit "${rc}"
 }
-if [[ "${KEEP_ARTIFACTS}" == "1" ]]; then
-    # A cliff stage follows and reuses the artifacts; only clean up on failure.
-    cleanup_on_fail() { echo "=== Tiny test failed — cleaning up ==="; _cleanup; }
-    trap cleanup_on_fail ERR
-else
-    # Terminal stage: always clean up.
-    trap _cleanup EXIT
-fi
+trap _cleanup EXIT
 
 # Re-clone if WORKDIR is missing or checked out at the wrong SHA (e.g. stale
 # leftover from a prior failed run at a different commit with the same prefix).

.github/scripts/render-release-notes.sh → .github/scripts/workflows/render-release-notes.sh

@@ -5,6 +5,7 @@ set -euo pipefail
 #
 # SPDX-License-Identifier: MIT
 
+# This script runs directly from a workflow checkout; it is not installed on a runner.
 if [[ $# -ne 6 ]]; then
     echo "usage: $0 <Dockerfile> <tag> <source-sha> <gpu-arch> <image-name> <repository>" >&2
     exit 2