From a386692ab83d6f43782ccc0737280962aaa0fe88 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 01:46:56 +0000 Subject: [PATCH 1/2] ci(cmux-tui): owned minis skip the Go and branch Rust caches that never restore On owned Mac minis, the macOS job's setup-go cache never restored: untarring it over the mini's own read-only ~/go/pkg/mod failed with Permission denied (8 of 8 sampled jobs), after a ~63 s download and before another save attempt. The rust-cache restored 0 of 8; its branch saves cost ~155 s each and filled most of the 10 GB GitHub cache store, evicting other entries. Owned minis now run setup-go without the Actions cache, and save the cargo cache only from feat-cmux-next. Blacksmith VMs keep both. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-a7306ef145c005ad24ab --- .github/workflows/ci-guards.yml | 4 ++ .github/workflows/cmux-tui.yml | 17 ++++++ tests/test-execution.toml | 4 ++ tests/test_cmux_tui_owned_mini_caches.py | 67 ++++++++++++++++++++++++ 4 files changed, 92 insertions(+) create mode 100644 tests/test_cmux_tui_owned_mini_caches.py diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 0dedb254c86b..9ba5cbacf49b 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -1050,6 +1050,10 @@ jobs: if: ${{ !cancelled() && matrix.group == 'ci' }} run: python3 tests/test_ci_rust_cache_keeps_runner_bin.py + - name: Validate cmux-tui's macOS job skips caches owned minis never restore + if: ${{ !cancelled() && matrix.group == 'ci' }} + run: python3 tests/test_cmux_tui_owned_mini_caches.py + - name: Validate every job that can start acpmux gives it a home of the job's own if: ${{ !cancelled() && matrix.group == 'ci' }} run: python3 tests/test_ci_acpmux_job_home.py diff --git a/.github/workflows/cmux-tui.yml b/.github/workflows/cmux-tui.yml index 14df17701088..fe4f41b26276 100644 --- a/.github/workflows/cmux-tui.yml +++ b/.github/workflows/cmux-tui.yml @@ -713,12 +713,28 @@ jobs: # cmux-tailnet's build script compiles libtailscale with `go build`. The # owned Mac minis have no Go, so clippy died there before any test ran. # go-version matches the pinned libtailscale go.mod (see its build.rs). + # + # No Actions cache for Go on an owned mini. The mini keeps its own + # ~/go/pkg/mod between jobs, and Go makes that tree read-only, so + # untarring the ~590 MB cache over it failed with "Can't unlink + # already-existing object: Permission denied" on 8 of 8 sampled jobs + # (2026-10-29). Each of those jobs still downloaded the tarball (about 63 s) + # and then tried to save it again (about 48 s). Blacksmith VMs start empty, + # so they keep the cache. - name: Set up pinned Go (libtailscale) uses: actions/setup-go@4a3601121dd01d1626a1e23e37211e3254c1c06c # v6.4.0 with: go-version-file: cmux-tui/crates/cmux-tailnet/go-version + cache: ${{ !contains(runner.name, '-glaeda') }} cache-dependency-path: cmux-tui/crates/cmux-tailnet/build.rs + # On an owned mini, only feat-cmux-next saves. The cache key carries a hash + # of each mini's Rust environment, and a dispatch on a branch can read only + # that branch's entries and the default branch's. So a branch save almost + # never comes back: 0 of 8 sampled jobs restored one (2026-10-29). Each + # branch save took about 155 s and 0.6-0.9 GB of the repository's 10 GB + # GitHub cache store. These saves filled about 70% of the store, and the + # store evicts by least-recent use, so other caches were pushed out. - name: Cache cargo build id: cargo-cache uses: Swatinem/rust-cache@6323deb102c322ba6fcbdcafc7e3dddab59af2b6 # v2.9.2 @@ -726,6 +742,7 @@ jobs: cache-bin: false workspaces: cmux-tui -> target shared-key: cmux-tui-macos + save-if: ${{ !contains(runner.name, '-glaeda') || github.ref == 'refs/heads/feat-cmux-next' }} - name: Report cargo cache hit rate if: always() diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 617fbc01e290..1f688a7ecd1c 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -345,6 +345,10 @@ lane = "linux-guard" path = "tests/test_ci_rust_cache_keeps_runner_bin.py" lane = "linux-guard" +[[test]] +path = "tests/test_cmux_tui_owned_mini_caches.py" +lane = "linux-guard" + [[test]] path = "tests/test_ci_acpmux_job_home.py" lane = "linux-guard" diff --git a/tests/test_cmux_tui_owned_mini_caches.py b/tests/test_cmux_tui_owned_mini_caches.py new file mode 100644 index 000000000000..aa5bdb4dae48 --- /dev/null +++ b/tests/test_cmux_tui_owned_mini_caches.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +"""cmux-tui's macOS job keeps owned minis out of caches that never come back. + +Measured 2026-10-29 on eight sampled `macOS lint and tests` jobs on owned minis: + +* setup-go's Actions cache never restored (8 of 8). The mini keeps its own + read-only ~/go/pkg/mod, so untarring the ~590 MB cache over it failed with + "Permission denied". Each job still downloaded the tarball (about 63 s) and + then tried to save it again. +* rust-cache restored 0 of 8. Each branch save cost about 155 s and 0.6-0.9 GB + of the 10 GB GitHub cache store, and those saves filled most of the store. + +So on an owned mini (a runner name containing `-glaeda`), setup-go gets no +Actions cache, and rust-cache saves only from feat-cmux-next. Blacksmith VMs +start empty, so they keep both caches. +""" + +from __future__ import annotations + +import sys +import unittest +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +WORKFLOW = ROOT / ".github/workflows/cmux-tui.yml" +OWNED = "contains(runner.name, '-glaeda')" + + +def macos_steps() -> list[dict]: + workflow = yaml.safe_load(WORKFLOW.read_text(encoding="utf-8")) + return list(workflow["jobs"]["macos"]["steps"]) + + +def step_using(steps: list[dict], action: str) -> dict: + found = [step for step in steps if str(step.get("uses", "")).lower().startswith(action)] + if len(found) != 1: + raise AssertionError(f"expected one {action} step in the macOS job, found {len(found)}") + return found[0] + + +def squash(value: object) -> str: + return " ".join(str(value).split()) + + +class OwnedMiniCaches(unittest.TestCase): + def setUp(self) -> None: + self.steps = macos_steps() + + def test_setup_go_has_no_actions_cache_on_an_owned_mini(self) -> None: + step = step_using(self.steps, "actions/setup-go@") + cache = squash((step.get("with") or {}).get("cache", "")) + self.assertEqual(cache, "${{ !" + OWNED + " }}", f"setup-go cache is {cache!r}") + + def test_rust_cache_saves_from_an_owned_mini_only_on_feat_cmux_next(self) -> None: + step = step_using(self.steps, "swatinem/rust-cache@") + save_if = squash((step.get("with") or {}).get("save-if", "")) + self.assertEqual( + save_if, + "${{ !" + OWNED + " || github.ref == 'refs/heads/feat-cmux-next' }}", + f"rust-cache save-if is {save_if!r}", + ) + + +if __name__ == "__main__": + sys.exit(0 if unittest.main(exit=False).result.wasSuccessful() else 1) From 9d9e4d6291690377ca949497a4e3a2c21208df3a Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 10:21:45 +0000 Subject: [PATCH 2/2] ci: keep cache regression out of guard workflow Agent-Session: https://cmux.com/deeplink/agent?id=aid-4777d4f75452817a742b --- .github/workflows/ci-guards.yml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 9ba5cbacf49b..0dedb254c86b 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -1050,10 +1050,6 @@ jobs: if: ${{ !cancelled() && matrix.group == 'ci' }} run: python3 tests/test_ci_rust_cache_keeps_runner_bin.py - - name: Validate cmux-tui's macOS job skips caches owned minis never restore - if: ${{ !cancelled() && matrix.group == 'ci' }} - run: python3 tests/test_cmux_tui_owned_mini_caches.py - - name: Validate every job that can start acpmux gives it a home of the job's own if: ${{ !cancelled() && matrix.group == 'ci' }} run: python3 tests/test_ci_acpmux_job_home.py