From 03aaac3c4ff20643a7d13df8869125ffb443359c Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 25 Sep 2026 09:19:40 -0400 Subject: [PATCH 1/2] ci: seed the Swift package cache from main pushes seed-derived-data.yml already restores the `spm-` cache from R2 and runs canonical-resolve on every main push. On an exact-key miss it now copies the resolved packages from the canonical root back into the workspace, sanitizes them and saves them to R2 under the exact key, right after resolve and without failing the seed. A Package.resolved change gets a correct package seed from the next main push instead of the next nightly. The read-only guard allows this one writer and pins its conditions: main ref, exact-key miss, the canonical resolved copy as the source, the same store and key as nightly.yml, and placement directly after resolve. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/seed-derived-data.yml | 33 +++++++++++++++++++ ...st_ci_pull_request_caches_are_read_only.py | 30 +++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/.github/workflows/seed-derived-data.yml b/.github/workflows/seed-derived-data.yml index c313aa1a9017..bf02461f05b2 100644 --- a/.github/workflows/seed-derived-data.yml +++ b/.github/workflows/seed-derived-data.yml @@ -238,6 +238,7 @@ jobs: run: python3 scripts/ci/reuse_app_host_products.py key "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" - name: Cache Swift packages + id: swift-package-cache uses: ./.github/actions/cache-restore with: backend: ${{ vars.CI_CACHE_BACKEND || 'r2' }} @@ -282,6 +283,38 @@ jobs: scripts/ci/compile-app-host-test-product.sh canonical-resolve \ "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" "$PWD/.ci-source-packages" + # Seed the `spm-` package cache from the first main push after a + # Package.resolved change, instead of waiting up to a day for nightly.yml. + # canonical-resolve resolved a copy in the canonical root, so the + # workspace still holds only what was restored (on a miss, the previous + # lockfile's packages); save the resolved copy, as nightly.yml does. + # Right after resolve, so a later cancel or failure cannot lose it. R2 + # saves are write-once per key, so matrix legs racing here are harmless. + - name: Collect resolved Swift packages + id: swift-package-collect + if: steps.swift-package-cache.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' + continue-on-error: true + run: | + set -euo pipefail + resolved="${CMUX_CI_CANONICAL_ROOT:-/private/tmp/cmux-ci}/src/.ci-source-packages" + test -d "$resolved/.package-cache" + rsync -a --delete "$resolved/" .ci-source-packages/ + python3 scripts/ci/sanitize-xcode-source-packages-cache.py .ci-source-packages + + - name: Save Swift packages + if: steps.swift-package-collect.outcome == 'success' + continue-on-error: true + uses: ./.github/actions/cache-save + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CI_CACHE_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_CACHE_R2_SECRET_ACCESS_KEY }} + CI_CACHE_R2_ENDPOINT: ${{ format('https://{0}.r2.cloudflarestorage.com', secrets.CI_CACHE_R2_ACCOUNT_ID) }} + CI_CACHE_R2_BUCKET: ${{ vars.CI_CACHE_R2_BUCKET }} + with: + backend: ${{ vars.CI_CACHE_BACKEND || 'r2' }} + path: .ci-source-packages + key: spm-${{ hashFiles('cmux.xcodeproj/project.xcworkspace/xcshareddata/swiftpm/Package.resolved', 'scripts/ci/swiftpm-cache-layout') }} + - name: Stage SwiftPM manifest cache id: swiftpm-manifest-stage if: steps.swiftpm-manifest-restore.outputs.cache-hit != 'true' && github.ref == 'refs/heads/main' diff --git a/tests/test_ci_pull_request_caches_are_read_only.py b/tests/test_ci_pull_request_caches_are_read_only.py index 2d4b8dacb2d5..40a1b87a8ae6 100644 --- a/tests/test_ci_pull_request_caches_are_read_only.py +++ b/tests/test_ci_pull_request_caches_are_read_only.py @@ -60,6 +60,7 @@ def main() -> int: # Package.resolved under the new exact key, and every exact hit then fails # its offline resolve (test-e2e.yml, run 36134675453). Every other reader # restores the store nightly.yml writes, through cache-restore. + # seed-derived-data.yml is the one other writer, checked below. for path in sorted((ROOT / ".github/workflows").glob("*.yml")): if path.name == "nightly.yml": continue @@ -68,6 +69,8 @@ def main() -> int: key, cache_path = step["with"]["key"], step["with"]["path"] if not (key.startswith("spm-") and cache_path == ".ci-source-packages"): continue + if (path.name, job_name, step.get("name")) == ("seed-derived-data.yml", "seed", "Save Swift packages"): + continue if not step["uses"].startswith(RESTORE): failures.append(f"{path.name} {job_name}: '{step.get('name')}' saves '{cache_path}' under an `spm-` key; restore only and let nightly.yml seed it") elif (key, cache_path) not in seeded: @@ -80,6 +83,33 @@ def main() -> int: if not any("CI_CACHE_R2_PUBLIC_URL" in (scope or {}) for scope in scopes): failures.append(f"{path.name} {job_name}: '{step.get('name')}' has no CI_CACHE_R2_PUBLIC_URL, so its R2 restore always misses") + # seed-derived-data.yml seeds the package cache from the first main push + # after a lockfile change. It may save only on main, only on an exact-key + # miss, only the copy canonical-resolve resolved, and never fail the seed. + seed_steps = yaml.safe_load((ROOT / ".github/workflows/seed-derived-data.yml").read_text(encoding="utf-8"))["jobs"]["seed"]["steps"] + by_name = {step.get("name"): step for step in seed_steps} + names = [step.get("name") for step in seed_steps] + restore, collect, save = (by_name.get(n) for n in ("Cache Swift packages", "Collect resolved Swift packages", "Save Swift packages")) + if not (restore and collect and save): + failures.append("seed-derived-data.yml seed: the package cache restore, collect and save steps must exist") + else: + if save["with"]["key"] != restore["with"]["key"] or save["with"]["path"] != ".ci-source-packages" or (save["with"]["key"], save["with"]["path"]) not in seeded: + failures.append("seed-derived-data.yml seed: 'Save Swift packages' must save nightly.yml's exact `spm-` key and path") + if not save["uses"].startswith("./.github/actions/cache-save") or save["with"].get("backend") != restore["with"].get("backend"): + failures.append("seed-derived-data.yml seed: 'Save Swift packages' must save to the store it restores from") + if save.get("if") != f"steps.{collect.get('id')}.outcome == 'success'": + failures.append("seed-derived-data.yml seed: 'Save Swift packages' must run only after a successful collect") + condition = str(collect.get("if", "")) + if f"steps.{restore.get('id')}.outputs.cache-hit != 'true'" not in condition or "github.ref == 'refs/heads/main'" not in condition or restore.get("id") is None: + failures.append("seed-derived-data.yml seed: collecting packages must require an exact-key miss and the main ref") + run = collect.get("run", "") + if '/src/.ci-source-packages"' not in run or 'rsync -a --delete "$resolved/" .ci-source-packages/' not in run or "sanitize-xcode-source-packages-cache.py .ci-source-packages" not in run: + failures.append("seed-derived-data.yml seed: collect must copy the canonical resolved packages into the workspace and sanitize them") + if not (collect.get("continue-on-error") is True and save.get("continue-on-error") is True): + failures.append("seed-derived-data.yml seed: the package seed must never fail the DerivedData seed") + if not (names.index("Resolve Swift packages") + 1 == names.index("Collect resolved Swift packages") and names.index("Collect resolved Swift packages") + 1 == names.index("Save Swift packages")): + failures.append("seed-derived-data.yml seed: collect and save must directly follow resolve, before any step a cancel can cut off") + # The wrappers pick one store per call. Exactly one branch may run, the # provider branch only on its own runners, and upstream actions stay pinned. warp = "inputs.backend == 'warp' && startsWith(runner.name, 'warp-')" From 674e19ad0b3ea4f08d9cfee2fc928259f8338a81 Mon Sep 17 00:00:00 2001 From: Leo Date: Fri, 25 Sep 2026 09:21:49 -0400 Subject: [PATCH 2/2] test: report a renamed resolve step instead of raising Co-Authored-By: Claude Opus 5.5 --- tests/test_ci_pull_request_caches_are_read_only.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_ci_pull_request_caches_are_read_only.py b/tests/test_ci_pull_request_caches_are_read_only.py index 40a1b87a8ae6..b311f1dafeac 100644 --- a/tests/test_ci_pull_request_caches_are_read_only.py +++ b/tests/test_ci_pull_request_caches_are_read_only.py @@ -107,7 +107,7 @@ def main() -> int: failures.append("seed-derived-data.yml seed: collect must copy the canonical resolved packages into the workspace and sanitize them") if not (collect.get("continue-on-error") is True and save.get("continue-on-error") is True): failures.append("seed-derived-data.yml seed: the package seed must never fail the DerivedData seed") - if not (names.index("Resolve Swift packages") + 1 == names.index("Collect resolved Swift packages") and names.index("Collect resolved Swift packages") + 1 == names.index("Save Swift packages")): + if "Resolve Swift packages" not in names or not (names.index("Resolve Swift packages") + 1 == names.index("Collect resolved Swift packages") and names.index("Collect resolved Swift packages") + 1 == names.index("Save Swift packages")): failures.append("seed-derived-data.yml seed: collect and save must directly follow resolve, before any step a cancel can cut off") # The wrappers pick one store per call. Exactly one branch may run, the