From ee58c52cbce9c31ef8d96ce3c2aafd0ed1514e4c Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 3 Oct 2026 09:05:04 -0700 Subject: [PATCH 1/4] fix(ci): isolate canonical build roots per runner Concurrent self-hosted runners must not remove each other\s canonical checkout, DerivedData, or compilation CAS. Derive the root from RUNNER_NAME and include non-default roots in cache fingerprints. Co-Authored-By: OpenAI --- .github/workflows/ci-macos.yml | 26 ++++++++++----- .github/workflows/nightly.yml | 16 +++++---- scripts/ci/canonical-build-root.sh | 26 +++++++++++++-- scripts/ci/compile-app-host-test-product.sh | 14 ++++---- tests/test_ci_canonical_build_root.py | 35 ++++++++++++++++++++ tests/test_ci_test_compilation_cache_seed.sh | 9 ++--- 6 files changed, 98 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index e1ada3396d8c..1bfc2f243282 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -534,7 +534,7 @@ jobs: # compilation cache made at another root. Consumers read the root # from the product's receipt (restore-app-host-test-product.sh). Its # kept build state sits in its own directory beside the first slot's. - root="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" + root="$(scripts/ci/canonical-build-root.sh --print-root)" case "$root" in /private/tmp/cmux-ci) ;; /private/tmp/cmux-ci-[0-9]|/private/tmp/cmux-ci-[0-9][0-9]) @@ -546,20 +546,30 @@ jobs: echo "CMUX_OWNED_PACKAGE_STORE=$CMUX_OWNED_STATE_ROOT" >> "$GITHUB_ENV" echo "CMUX_OWNED_STATE_ROOT=$CMUX_OWNED_STATE_ROOT/${root##*/}" >> "$GITHUB_ENV" ;; + /private/tmp/cmux-ci-*) + [ "${RUNNER_ENVIRONMENT:-}" = self-hosted ] || { + echo "::error::unexpected self-hosted canonical root $root"; exit 1; + } + ;; *) echo "::error::unexpected CMUX_CI_CANONICAL_ROOT $root"; exit 1 ;; esac + echo "CMUX_CI_CANONICAL_ROOT=$root" >> "$GITHUB_ENV" echo "root=$root" >> "$GITHUB_OUTPUT" - name: Prepare isolated admission DerivedData run: | set -euo pipefail - # Fixed paths survive across jobs on persistent runners. Clear them - # before any trusted product/cache restoration. - root="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" + # A self-hosted Mac can have several runners. Derive one root per + # RUNNER_NAME before cleanup so one admission cannot remove another's + # source tree, DerivedData, or compilation CAS. + root="$(scripts/ci/canonical-build-root.sh --print-root)" scripts/ci/clear-dirs.sh "$root/derived-data-compile-admission" "$root/compile-admission-cas" - echo "CMUX_COMPILE_ADMISSION_DERIVED_DATA=${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/derived-data-compile-admission" >> "$GITHUB_ENV" - # Outside DerivedData: the resolve step below recreates that directory. - echo "CMUX_COMPILE_ADMISSION_CAS=${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/compile-admission-cas" >> "$GITHUB_ENV" + { + echo "CMUX_CI_CANONICAL_ROOT=$root" + echo "CMUX_COMPILE_ADMISSION_DERIVED_DATA=$root/derived-data-compile-admission" + # Outside DerivedData: the resolve step below recreates that directory. + echo "CMUX_COMPILE_ADMISSION_CAS=$root/compile-admission-cas" + } >> "$GITHUB_ENV" - name: Select Xcode run: ./scripts/select-ci-xcode.sh @@ -1077,7 +1087,7 @@ jobs: # Fresh manifests embed the canonical checkout. Stamp that producer # before relocating into this job's workspace for reuse and consumers. if [ "${{ steps.reuse-products.outputs.hit }}" != true ]; then - (cd "${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/src" && python3 scripts/ci/app_host_test_products.py stamp "$CMUX_COMPILE_ADMISSION_DERIVED_DATA") + (cd "$CMUX_CI_CANONICAL_ROOT/src" && python3 scripts/ci/app_host_test_products.py stamp "$CMUX_COMPILE_ADMISSION_DERIVED_DATA") python3 scripts/ci/app_host_test_products.py restore "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" fi python3 scripts/ci/app_host_test_products.py stamp "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 4e6e7869b3b3..9dbe2f65253d 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -678,12 +678,16 @@ jobs: set -euo pipefail # The same paths as ci.yml `macos-compile-admission`: they are part of # every cache entry this job writes. - # Fixed paths survive across jobs on persistent runners. Clear them - # before any trusted product/cache restoration. - root="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" + # A self-hosted Mac can have several runners. Derive one root per + # RUNNER_NAME before cleanup so one seed build cannot remove another + # runner's source tree, DerivedData, or compilation CAS. + root="$(scripts/ci/canonical-build-root.sh --print-root)" scripts/ci/clear-dirs.sh "$root/derived-data-compile-admission" "$root/compile-admission-cas" - echo "CMUX_COMPILE_ADMISSION_DERIVED_DATA=${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/derived-data-compile-admission" >> "$GITHUB_ENV" - echo "CMUX_COMPILE_ADMISSION_CAS=${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/compile-admission-cas" >> "$GITHUB_ENV" + { + echo "CMUX_CI_CANONICAL_ROOT=$root" + echo "CMUX_COMPILE_ADMISSION_DERIVED_DATA=$root/derived-data-compile-admission" + echo "CMUX_COMPILE_ADMISSION_CAS=$root/compile-admission-cas" + } >> "$GITHUB_ENV" - name: Select Xcode run: ./scripts/select-ci-xcode.sh @@ -802,7 +806,7 @@ jobs: if: steps.compilation-cache-restore.outputs.cache-hit != 'true' && steps.swift-packages-restore.outputs.cache-hit != 'true' run: | set -euo pipefail - resolved="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/src/.ci-source-packages" + resolved="$CMUX_CI_CANONICAL_ROOT/src/.ci-source-packages" test -d "$resolved/.package-cache" rsync -a --delete "$resolved/" .ci-source-packages/ diff --git a/scripts/ci/canonical-build-root.sh b/scripts/ci/canonical-build-root.sh index 8c0034d455c8..5cc0545efb5f 100755 --- a/scripts/ci/canonical-build-root.sh +++ b/scripts/ci/canonical-build-root.sh @@ -1,5 +1,6 @@ #!/usr/bin/env bash # canonical-build-root.sh [workspace] +# canonical-build-root.sh --print-root # # Put the build somewhere every macOS runner pool can name identically. # @@ -12,8 +13,11 @@ # instead of a wrong hit. # # This removes the disagreement at the source: the build runs from -# $CMUX_CI_CANONICAL_ROOT/src, a constant, so the key can drop the paths and -# one seed serves every pool. +# $CMUX_CI_CANONICAL_ROOT/src, a stable path for the runner. Self-hosted +# runners derive that root from RUNNER_NAME so concurrent runners on one Mac do +# not delete each other's source tree, DerivedData, or compilation CAS. The +# cache fingerprint includes a non-default root, so a seed from another +# runner's absolute path is never adopted. # # A symlink will not do. The compiler records the path it actually opens, and # a link back into the workspace resolves to the pool-specific path again, so @@ -23,7 +27,23 @@ # compile measured at ~18 minutes. set -euo pipefail -root="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" +default_root=/private/tmp/cmux-ci +if [ "${RUNNER_ENVIRONMENT:-}" = self-hosted ] \ + && [ -n "${RUNNER_NAME:-}" ] \ + && [ "${CMUX_CI_CANONICAL_ROOT:-$default_root}" = "$default_root" ]; then + runner_key="$(printf '%s' "$RUNNER_NAME" | tr -c 'A-Za-z0-9_.-' '_')" + root="$default_root-$runner_key" +elif [ -n "${CMUX_CI_CANONICAL_ROOT:-}" ]; then + root="$CMUX_CI_CANONICAL_ROOT" +else + root="$default_root" +fi + +if [ "${1:-}" = --print-root ]; then + printf '%s\n' "$root" + exit 0 +fi + src="$root/src" runtime_source=false if [ "${1:-}" = --runtime-source ]; then diff --git a/scripts/ci/compile-app-host-test-product.sh b/scripts/ci/compile-app-host-test-product.sh index 0c70defded7c..0b9238dcfb6d 100755 --- a/scripts/ci/compile-app-host-test-product.sh +++ b/scripts/ci/compile-app-host-test-product.sh @@ -18,11 +18,11 @@ # built on one pool could never hit on another. # # scripts/ci/canonical-build-root.sh removes that disagreement by building from -# a fixed location every pool can reproduce. When the build runs there the key -# drops the paths, because they are now a constant, and one seed serves every -# pool. A build anywhere else keeps the old path-scoped key and its own private -# cache, so an unconverted lane degrades to a miss rather than downloading a -# seed whose entries cannot hit. +# a stable per-runner location every job on that runner can reproduce. A +# self-hosted runner derives its root from RUNNER_NAME, so another runner on +# the same Mac cannot remove its source tree or DerivedData. The root is part +# of the fingerprint when it is not the historical default, so a build never +# adopts a cache whose absolute paths belong to another runner. set -euo pipefail SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" @@ -38,8 +38,8 @@ usage() { # Same limit as the Release seed in nightly.yml. cache_limit_bytes=3221225472 -# Keep in sync with scripts/ci/canonical-build-root.sh. -CANONICAL_BUILD_ROOT="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" +# Keep root selection in sync with scripts/ci/canonical-build-root.sh. +CANONICAL_BUILD_ROOT="$("$SCRIPT_DIR/canonical-build-root.sh" --print-root)" # How Swift Build's llbuild decides a file changed. The default, # device-agnostic, compares modification times, which cannot survive a move to diff --git a/tests/test_ci_canonical_build_root.py b/tests/test_ci_canonical_build_root.py index 4fb1847628ad..7e1fc4691fde 100644 --- a/tests/test_ci_canonical_build_root.py +++ b/tests/test_ci_canonical_build_root.py @@ -14,6 +14,7 @@ ROOT = Path(__file__).resolve().parents[1] SCRIPT = ROOT / "scripts" / "ci" / "compile-app-host-test-product.sh" +ROOT_SCRIPT = ROOT / "scripts" / "ci" / "canonical-build-root.sh" # The two layouts observed in CI. One underscore is the whole difference, and # it was enough to give the nightly seed and pull-request admission different @@ -114,6 +115,40 @@ def test_toolchain_still_invalidates_the_canonical_key(self): new = self._fp(f"{root}/src", f"{root}/derived-data-compile-admission", xcode="Xcode 27.0", root=root) self.assertNotEqual(old, new) + def test_self_hosted_runners_get_distinct_roots(self): + roots = [] + for name in ("aws-m4pro-9-glaeda", "aws-m4pro-9-glaeda-1"): + result = subprocess.run( + [str(ROOT_SCRIPT), "--print-root"], + env={"PATH": "/usr/bin:/bin", "RUNNER_ENVIRONMENT": "self-hosted", "RUNNER_NAME": name}, + text=True, + capture_output=True, + check=True, + ) + roots.append(result.stdout.strip()) + self.assertEqual(roots, [ + "/private/tmp/cmux-ci-aws-m4pro-9-glaeda", + "/private/tmp/cmux-ci-aws-m4pro-9-glaeda-1", + ]) + + def test_hosted_runner_keeps_the_shared_default_and_override_wins(self): + hosted = subprocess.run( + [str(ROOT_SCRIPT), "--print-root"], + env={"PATH": "/usr/bin:/bin", "RUNNER_ENVIRONMENT": "github", "RUNNER_NAME": "GitHub Actions 42"}, + text=True, + capture_output=True, + check=True, + ) + self.assertEqual(hosted.stdout.strip(), "/private/tmp/cmux-ci") + override = subprocess.run( + [str(ROOT_SCRIPT), "--print-root"], + env={"PATH": "/usr/bin:/bin", "RUNNER_ENVIRONMENT": "self-hosted", "RUNNER_NAME": "aws-m4pro-9-glaeda-1", "CMUX_CI_CANONICAL_ROOT": "/private/tmp/custom"}, + text=True, + capture_output=True, + check=True, + ) + self.assertEqual(override.stdout.strip(), "/private/tmp/custom") + class CanonicalRootMaterializationTests(unittest.TestCase): """The root has to be a real directory, exact, and refuse unsafe inputs.""" diff --git a/tests/test_ci_test_compilation_cache_seed.sh b/tests/test_ci_test_compilation_cache_seed.sh index e943858a24a3..c8a578ae6d25 100755 --- a/tests/test_ci_test_compilation_cache_seed.sh +++ b/tests/test_ci_test_compilation_cache_seed.sh @@ -45,11 +45,12 @@ echo "PASS: admission and the seeder build the app-host test product through one # Pools may differ: the executable canonical recipe test checks absolute paths. -# The build paths are part of every cache entry, so both jobs must use the -# same ones. +# The build paths are part of every cache entry, so both jobs must derive the +# same per-runner root before cleanup and export it for later steps. for line in \ - 'CMUX_COMPILE_ADMISSION_DERIVED_DATA=${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/derived-data-compile-admission' \ - 'CMUX_COMPILE_ADMISSION_CAS=${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/compile-admission-cas'; do + 'root="$(scripts/ci/canonical-build-root.sh --print-root)"' \ + 'CMUX_COMPILE_ADMISSION_DERIVED_DATA=$root/derived-data-compile-admission' \ + 'CMUX_COMPILE_ADMISSION_CAS=$root/compile-admission-cas'; do if ! grep -Fq "$line" <<<"$ADMISSION" || ! grep -Fq "$line" <<<"$SEEDER"; then echo "FAIL: admission and the seeder must both set $line" exit 1 From 428279da1ba309816d5b9c660c0975b5c9daac36 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 3 Oct 2026 09:19:17 -0700 Subject: [PATCH 2/4] fix(ci): isolate remaining macOS workflow roots Co-Authored-By: OpenAI --- .github/workflows/ci-macos.yml | 2 +- .github/workflows/main-compile-probe.yml | 4 +++- .github/workflows/seed-derived-data.yml | 13 +++++++++--- .github/workflows/test-e2e.yml | 26 ++++++++++++++++-------- tests/test_ci_e2e_compilation_cache.py | 2 +- 5 files changed, 32 insertions(+), 15 deletions(-) diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 1bfc2f243282..c9aa7568795b 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -1438,7 +1438,7 @@ jobs: continue-on-error: true run: | python3 scripts/ci/owned_build_state.py save "$CMUX_OWNED_STATE_ROOT" \ - "${CMUX_CI_CANONICAL_SRC:-/private/tmp/cmux-ci/src}/.ci-source-packages" "$PWD" "${CMUX_OWNED_PACKAGE_STORE:-}" + "$CMUX_CI_CANONICAL_SRC/.ci-source-packages" "$PWD" "${CMUX_OWNED_PACKAGE_STORE:-}" # On an owned Mac the suites share its one console session with every # other GUI job, so they need glaeda's gui token, which glaeda never diff --git a/.github/workflows/main-compile-probe.yml b/.github/workflows/main-compile-probe.yml index 178a5faf69ac..0378de8219cc 100644 --- a/.github/workflows/main-compile-probe.yml +++ b/.github/workflows/main-compile-probe.yml @@ -83,7 +83,9 @@ jobs: # The paths compile admission and the seeds build at: part of the # seed key. On an owned Mac hold the root the hook exported (or root # 1) for the job, as the seed job does. - root="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" + root="$(scripts/ci/canonical-build-root.sh --print-root)" + export CMUX_CI_CANONICAL_ROOT="$root" + echo "CMUX_CI_CANONICAL_ROOT=$root" >> "$GITHUB_ENV" helper=/Users/Shared/cmux-build-fleet/bin/glaeda-canonical-root if [ -x "$helper" ]; then status=0 diff --git a/.github/workflows/seed-derived-data.yml b/.github/workflows/seed-derived-data.yml index 76e1dde1e96d..6b310da46aff 100644 --- a/.github/workflows/seed-derived-data.yml +++ b/.github/workflows/seed-derived-data.yml @@ -212,7 +212,11 @@ jobs: # compile slot does. glaeda pins this job (one it does not know) to # root 1, and the trusted runner runs nothing else, so the root is # this job's alone. - root="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" + if [ -x scripts/ci/canonical-build-root.sh ]; then + root="$(scripts/ci/canonical-build-root.sh --print-root)" + else + root="${CMUX_CI_CANONICAL_ROOT:?canonical root resolver is unavailable}" + fi if [ -n "$CMUX_SEED_ROOT" ]; then case "$CMUX_SEED_ROOT:$MATRIX_POOL" in [2-9]:"$TRUSTED_POOL") ;; @@ -248,6 +252,9 @@ jobs: scripts/ci/clear-dirs.sh "$root/derived-data-compile-admission" "$root/compile-admission-cas" echo "CMUX_COMPILE_ADMISSION_DERIVED_DATA=$root/derived-data-compile-admission" >> "$GITHUB_ENV" echo "CMUX_COMPILE_ADMISSION_CAS=$root/compile-admission-cas" >> "$GITHUB_ENV" + if [ -z "$CMUX_SEED_ROOT" ]; then + echo "CMUX_CI_CANONICAL_ROOT=$root" >> "$GITHUB_ENV" + fi - name: Select Xcode run: ./scripts/select-ci-xcode.sh @@ -371,7 +378,7 @@ jobs: continue-on-error: true run: | set -euo pipefail - resolved="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/src/.ci-source-packages" + resolved="$CMUX_CI_CANONICAL_ROOT/src/.ci-source-packages" test -d "$resolved/.package-cache" rsync -a --delete "$resolved/" .ci-source-packages/ python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages @@ -532,7 +539,7 @@ jobs: set -euo pipefail # Fresh manifests embed the canonical checkout. Stamp that producer # before relocating into this job's workspace, as admission does. - (cd "${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/src" && python3 scripts/ci/app_host_test_products.py stamp "$CMUX_COMPILE_ADMISSION_DERIVED_DATA") + (cd "$CMUX_CI_CANONICAL_ROOT/src" && python3 scripts/ci/app_host_test_products.py stamp "$CMUX_COMPILE_ADMISSION_DERIVED_DATA") python3 scripts/ci/app_host_test_products.py restore "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" python3 scripts/ci/app_host_test_products.py stamp "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" python3 scripts/ci/reuse_app_host_products.py seal "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" diff --git a/.github/workflows/test-e2e.yml b/.github/workflows/test-e2e.yml index f68c1944dad8..b989a7d2ec71 100644 --- a/.github/workflows/test-e2e.yml +++ b/.github/workflows/test-e2e.yml @@ -570,10 +570,17 @@ jobs: # admission uses, so it can adopt the DerivedData seeds main # publishes for admission (see "Adopt the DerivedData seed"). workspace_path="$(cd "$GITHUB_WORKSPACE" && pwd -P)" + if [ -x scripts/ci/canonical-build-root.sh ]; then + root="$(scripts/ci/canonical-build-root.sh --print-root)" + else + root="${CMUX_CI_CANONICAL_ROOT:?canonical root resolver is unavailable}" + fi + export CMUX_CI_CANONICAL_ROOT="$root" + echo "CMUX_CI_CANONICAL_ROOT=$root" >> "$GITHUB_ENV" # The CAS path is a compiler argument, so it must match admission's # too: any other value invalidates every compile the seed carries. - DERIVED_DATA_PATH="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/derived-data-compile-admission" - CACHE_PATH="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/compile-admission-cas" + DERIVED_DATA_PATH="$root/derived-data-compile-admission" + CACHE_PATH="$root/compile-admission-cas" # A leftover indexer can refill the tree while `rm -rf` walks it # ("Directory not empty", run 36317930241). Rename first, as # scripts/ci/clear-dirs.sh does; inline, because the tested tree @@ -691,10 +698,10 @@ jobs: # build-slot lays it out; packages hold no build paths, so every # root shares the Mac's. packages=/Users/Shared/cmux-build-fleet/ci - root="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}" + root="$CMUX_CI_CANONICAL_ROOT" case "$root" in /private/tmp/cmux-ci) store="$packages" ;; - /private/tmp/cmux-ci-[0-9]|/private/tmp/cmux-ci-[0-9][0-9]) store="$packages/${root##*/}" ;; + /private/tmp/cmux-ci-*) store="$packages/${root##*/}" ;; *) echo "::error::unexpected CMUX_CI_CANONICAL_ROOT $root"; exit 1 ;; esac tools="$RUNNER_TEMP/owned-state-tools" @@ -745,7 +752,7 @@ jobs: if: ${{ steps.reuse.outputs.hit != 'true' && steps.compilation-cache-restore.outcome == 'failure' }} run: | set -euo pipefail - if [ "$CMUX_E2E_COMPILATION_CACHE" != "${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/compile-admission-cas" ]; then + if [ "$CMUX_E2E_COMPILATION_CACHE" != "$CMUX_CI_CANONICAL_ROOT/compile-admission-cas" ]; then echo "::error::refusing to clear an unowned compilation cache" exit 1 fi @@ -885,7 +892,7 @@ jobs: run: | set -euo pipefail case "$CMUX_DERIVED_DATA_PATH" in - /private/tmp/cmux-ci/derived-data-compile-admission|/private/tmp/cmux-ci-[0-9]/derived-data-compile-admission|/private/tmp/cmux-ci-[0-9][0-9]/derived-data-compile-admission) ;; + /private/tmp/cmux-ci/derived-data-compile-admission|/private/tmp/cmux-ci-*/derived-data-compile-admission) ;; *) echo "::error::refusing to clear an unowned DerivedData path: $CMUX_DERIVED_DATA_PATH"; exit 1 ;; esac rm -rf -- "$CMUX_DERIVED_DATA_PATH" @@ -925,7 +932,7 @@ jobs: # ci-macos.yml compile admission. Stamp that producer before # relocating into this job's workspace for consumers. if [ "${{ steps.reuse.outputs.hit }}" != true ]; then - (cd "${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/src" && python3 scripts/ci/app_host_test_products.py stamp "$CMUX_DERIVED_DATA_PATH") + (cd "$CMUX_CI_CANONICAL_ROOT/src" && python3 scripts/ci/app_host_test_products.py stamp "$CMUX_DERIVED_DATA_PATH") python3 scripts/ci/app_host_test_products.py restore "$CMUX_DERIVED_DATA_PATH" fi python3 scripts/ci/app_host_test_products.py stamp "$CMUX_DERIVED_DATA_PATH" @@ -1078,12 +1085,13 @@ jobs: set -euo pipefail derived_data_path="${CMUX_DERIVED_DATA_PATH:-}" cache_path="${CMUX_E2E_COMPILATION_CACHE:-}" + root="${CMUX_CI_CANONICAL_ROOT:-}" case "$cache_path" in - "${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/compile-admission-cas"|"") ;; + "$root/compile-admission-cas"|"") ;; *) echo "::error::refusing to remove unowned compilation cache: $cache_path"; exit 1 ;; esac case "$derived_data_path" in - "${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/derived-data-compile-admission") + "$root/derived-data-compile-admission") # Rename first, like the prepare step, so a leftover indexer # cannot turn a finished run red with "Directory not empty". for dir in "$derived_data_path" ${cache_path:+"$cache_path"}; do diff --git a/tests/test_ci_e2e_compilation_cache.py b/tests/test_ci_e2e_compilation_cache.py index 3acf89897db3..7f716994f322 100644 --- a/tests/test_ci_e2e_compilation_cache.py +++ b/tests/test_ci_e2e_compilation_cache.py @@ -399,7 +399,7 @@ def test_the_build_adopts_the_admission_seed_at_the_canonical_root(self): str(self.root / 'canonical' / 'derived-data-compile-admission')) # The CAS path is a compiler argument: a different one than admission # passes invalidates every compile the seed carries. - self.assertIn('CMUX_COMPILE_ADMISSION_CAS=${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/compile-admission-cas', admission) + self.assertIn('CMUX_COMPILE_ADMISSION_CAS=$root/compile-admission-cas', admission) self.assertEqual(values['CMUX_E2E_COMPILATION_CACHE'], str(self.root / 'canonical' / 'compile-admission-cas')) From 892577acc717e8dcc7b7441b9f092bc89a1eb311 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 3 Oct 2026 09:21:58 -0700 Subject: [PATCH 3/4] fix(ci): verify probe canonical root placement Co-Authored-By: OpenAI --- .github/workflows/main-compile-probe.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/.github/workflows/main-compile-probe.yml b/.github/workflows/main-compile-probe.yml index 0378de8219cc..8e70faabb95e 100644 --- a/.github/workflows/main-compile-probe.yml +++ b/.github/workflows/main-compile-probe.yml @@ -90,8 +90,15 @@ jobs: if [ -x "$helper" ]; then status=0 "$helper" take "$root" --wait 900 >/dev/null || status=$? + placed="$(cat "${RUNNER_TEMP:-/nonexistent}/glaeda-canonical-root" 2>/dev/null || true)" case "$status" in - 0|2) ;; + 0) ;; + 2) + [ "$placed" = "$root" ] || { + echo "::error::glaeda placed this job in ${placed:-no root}, not $root"; + exit 1; + } + ;; *) echo "::error::could not hold $root for this job (glaeda-canonical-root exit $status)"; exit 1 ;; esac fi From b110febbed9882fbb7ef8612eb7e0ead2ad03e40 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sat, 3 Oct 2026 09:28:58 -0700 Subject: [PATCH 4/4] test(ci): follow per-runner prefix mapping Co-Authored-By: OpenAI --- tests/test_ci_test_compilation_cache_seed.sh | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/test_ci_test_compilation_cache_seed.sh b/tests/test_ci_test_compilation_cache_seed.sh index c8a578ae6d25..ec76e510a064 100755 --- a/tests/test_ci_test_compilation_cache_seed.sh +++ b/tests/test_ci_test_compilation_cache_seed.sh @@ -194,6 +194,10 @@ fi echo "PASS: the fingerprint follows the build path and the toolchain" run_script build "$TMP_DIR/derived" "$TMP_DIR/packages" "$TMP_DIR/cas" "$TMP_DIR/build.log" >/dev/null +canonical_root="$("$ROOT_DIR/scripts/ci/canonical-build-root.sh" --print-root)" +printf -v prefix_map_flags \ + 'OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -file-prefix-map -Xfrontend %s=/private/tmp/cmux-test-source -Xfrontend -debug-prefix-map -Xfrontend %s=/private/tmp/cmux-test-source $(CMUX_CI_SWIFT_FLAGS_$(TARGET_NAME))' \ + "$canonical_root" "$canonical_root" for expected in \ cmux \ cmux-unit \ @@ -204,7 +208,7 @@ for expected in \ CMUX_CI_COMPILATION_CACHE_cmuxTests=NO \ 'SWIFT_USE_INTEGRATED_DRIVER=$(CMUX_CI_INTEGRATED_DRIVER_$(TARGET_NAME):default=YES)' \ CMUX_CI_INTEGRATED_DRIVER_cmuxTests=NO \ - 'OTHER_SWIFT_FLAGS=$(inherited) -Xfrontend -file-prefix-map -Xfrontend /private/tmp/cmux-ci=/private/tmp/cmux-test-source -Xfrontend -debug-prefix-map -Xfrontend /private/tmp/cmux-ci=/private/tmp/cmux-test-source $(CMUX_CI_SWIFT_FLAGS_$(TARGET_NAME))' \ + "$prefix_map_flags" \ CMUX_CI_SWIFT_FLAGS_cmuxTests=-no-emit-module-separately \ 'SWIFT_INSTALL_MODULE=$(CMUX_CI_INSTALL_MODULE_$(TARGET_NAME):default=YES)' \ CMUX_CI_INSTALL_MODULE_cmuxTests=NO \