From 2925daddeb0116730e2395e73ba04d618bad2413 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 18:31:57 -0700 Subject: [PATCH 1/5] ci: prune expired R2 cache archives, never one a latest pointer names Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-guards.yml | 4 + .github/workflows/r2-cache-prune.yml | 50 +++++++ scripts/ci/detect_ci_change_areas.py | 1 + scripts/ci/r2_cache_prune.py | 215 +++++++++++++++++++++++++++ tests/test-execution.toml | 4 + tests/test_ci_change_areas.py | 1 + tests/test_r2_cache_prune.py | 144 ++++++++++++++++++ 7 files changed, 419 insertions(+) create mode 100644 .github/workflows/r2-cache-prune.yml create mode 100644 scripts/ci/r2_cache_prune.py create mode 100644 tests/test_r2_cache_prune.py diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 26886018529b..d2036eed907c 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -403,6 +403,10 @@ jobs: if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_r2_cache_census.py + - name: Validate the R2 cache pruner + if: ${{ matrix.group == 'ci' }} + run: python3 tests/test_r2_cache_prune.py + - name: Validate nightly prune Python compatibility if: ${{ matrix.group == 'release-tooling' }} run: | diff --git a/.github/workflows/r2-cache-prune.yml b/.github/workflows/r2-cache-prune.yml new file mode 100644 index 000000000000..69676136259a --- /dev/null +++ b/.github/workflows/r2-cache-prune.yml @@ -0,0 +1,50 @@ +name: Prune R2 cache + +# scripts/ci/r2-cache.sh saves archives and never deletes, so the CI cache +# bucket grows by every seed ever written. This deletes archives past their +# family's retention (see scripts/ci/r2_cache_prune.py) and never one a +# `latest/` pointer names. It is a dry run unless a dispatch asks to delete or +# the repository variable CI_R2_CACHE_PRUNE_DELETE is 1. +on: + schedule: + - cron: "23 5 * * *" + workflow_dispatch: + inputs: + delete: + description: Send deletes. Unchecked reports what would be deleted. + required: false + default: false + type: boolean + +permissions: {} + +concurrency: + group: r2-cache-prune + cancel-in-progress: false + +jobs: + prune: + name: Prune expired cache archives + # The bucket credentials only ever run main's copy of this script. + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + timeout-minutes: 20 + permissions: + contents: read + steps: + - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + persist-credentials: false + + - name: Prune + env: + AWS_ACCESS_KEY_ID: ${{ secrets.CI_CACHE_R2_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_CACHE_R2_SECRET_ACCESS_KEY }} + R2_ENDPOINT: ${{ format('https://{0}.r2.cloudflarestorage.com', secrets.CI_CACHE_R2_ACCOUNT_ID) }} + R2_BUCKET: ${{ vars.CI_CACHE_R2_BUCKET }} + PRUNE_DELETE: ${{ (inputs.delete || vars.CI_R2_CACHE_PRUNE_DELETE == '1') && 'true' || 'false' }} + run: | + set -euo pipefail + args=(--endpoint-url "$R2_ENDPOINT" --bucket "$R2_BUCKET") + if [ "$PRUNE_DELETE" = true ]; then args+=(--delete); fi + python3 scripts/ci/r2_cache_prune.py "${args[@]}" diff --git a/scripts/ci/detect_ci_change_areas.py b/scripts/ci/detect_ci_change_areas.py index 9b71c1fa6acb..6d95d37652bc 100644 --- a/scripts/ci/detect_ci_change_areas.py +++ b/scripts/ci/detect_ci_change_areas.py @@ -118,6 +118,7 @@ def is_other_workflow_config(path: str) -> bool: "scripts/ci/queue_janitor.py", "scripts/ci/r2-canary-cloudflare.py", "scripts/ci/r2_cache_census.py", + "scripts/ci/r2_cache_prune.py", "scripts/ci/swift_incremental_diagnostics.py", "scripts/ci/triage-radar.py", "scripts/ci/validate_test_execution_registry.py", diff --git a/scripts/ci/r2_cache_prune.py b/scripts/ci/r2_cache_prune.py new file mode 100644 index 000000000000..2054ae05adbe --- /dev/null +++ b/scripts/ci/r2_cache_prune.py @@ -0,0 +1,215 @@ +#!/usr/bin/env python3 +"""Delete CI cache archives nothing will restore again. + + r2_cache_prune.py --endpoint-url URL --bucket NAME [--delete] + +`r2-cache.sh` saves `v1/-/objects/.tar.zst|.tar.gz` plus one +`v1/-/latest/` pointer per dash-terminated prefix of the key, +and nothing ever deletes. This removes archives older than their family's +retention, with one rule that is never relaxed: an archive any `latest/` +pointer names is kept, whatever its age, because prefix restores read it. + +Without `--delete` this is a dry run: it lists, reads every pointer, and +reports what it would delete, but sends no DELETE. Only `objects/` archives are +candidates. Pointers, and anything outside `v1/*/objects/`, are never touched. + +Retention, by the first matching key prefix: + + admission-derived-data- 3 days. One snapshot per main commit, and every + consumer adopts the newest by prefix; an exact key + only matters for a pull request on a base that old. + xcode-compilation- 7 days. Seeds every 6 hours, restored newest by + prefix; a cache from an older main mostly misses. + everything else 30 days. Keyed by content (a Package.resolved or + toolchain hash), so an old key stays exact for a + pull request whose base still has that input. + +Credentials come from AWS_ACCESS_KEY_ID / AWS_SECRET_ACCESS_KEY. +""" +from __future__ import annotations + +import argparse +import datetime as dt +import hashlib +import hmac +import json +import os +from pathlib import Path +import re +import sys +import urllib.parse +import urllib.request + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import r2_cache_census as census # noqa: E402 + +RETENTION_DAYS = ( + ("admission-derived-data-", 3), + ("xcode-compilation-", 7), + ("", 30), +) +ARCHIVE = re.compile(r"^(v1/[^/]+)/objects/(?P[A-Za-z0-9._-]+)\.(?:tar\.zst|tar\.gz)$") +POINTER = re.compile(r"^(v1/[^/]+)/latest/[A-Za-z0-9._-]+$") +VALID_NAME = re.compile(r"^[A-Za-z0-9._-]+$") +# One run never deletes more than this; a larger backlog drains over days. +MAX_DELETES = 2000 + + +def retention_days(name: str) -> int: + return next(days for prefix, days in RETENTION_DAYS if name.startswith(prefix)) + + +def signed(method: str, endpoint_url: str, bucket: str, key: str, *, timeout: int = 30) -> bytes: + """One SigV4 request against a single object, in census.list_page's style.""" + access_key = os.environ.get("AWS_ACCESS_KEY_ID", "") + secret_key = os.environ.get("AWS_SECRET_ACCESS_KEY", "") + region = os.environ.get("AWS_DEFAULT_REGION", "auto") + if not access_key or not secret_key: + raise SystemExit("AWS_ACCESS_KEY_ID and AWS_SECRET_ACCESS_KEY are required") + parsed = urllib.parse.urlsplit(endpoint_url.rstrip("/")) + if parsed.scheme != "https" or not parsed.netloc: + raise SystemExit("R2 endpoint URL must use HTTPS and include a host") + canonical_uri = "/" + urllib.parse.quote(bucket, safe="~") + "/" + urllib.parse.quote(key, safe="~/") + amz_date = dt.datetime.now(dt.timezone.utc).strftime("%Y%m%dT%H%M%SZ") + date_stamp = amz_date[:8] + payload_hash = hashlib.sha256(b"").hexdigest() + canonical_request = "\n".join([ + method, canonical_uri, "", + f"host:{parsed.netloc}\nx-amz-content-sha256:{payload_hash}\nx-amz-date:{amz_date}\n", + "host;x-amz-content-sha256;x-amz-date", payload_hash, + ]) + scope = f"{date_stamp}/{region}/s3/aws4_request" + to_sign = "\n".join(["AWS4-HMAC-SHA256", amz_date, scope, + hashlib.sha256(canonical_request.encode()).hexdigest()]) + signature = hmac.new(census._signing_key(secret_key, date_stamp, region), + to_sign.encode(), hashlib.sha256).hexdigest() + request = urllib.request.Request( + urllib.parse.urlunsplit((parsed.scheme, parsed.netloc, canonical_uri, "", "")), method=method) + request.add_header("x-amz-content-sha256", payload_hash) + request.add_header("x-amz-date", amz_date) + request.add_header("Authorization", + f"AWS4-HMAC-SHA256 Credential={access_key}/{scope}, " + "SignedHeaders=host;x-amz-content-sha256;x-amz-date, " + f"Signature={signature}") + with urllib.request.build_opener(census._RejectRedirects()).open(request, timeout=timeout) as response: + return response.read() + + +class Bucket: + """The three operations pruning needs. Tests replace it with a fake.""" + + def __init__(self, endpoint_url: str, bucket: str): + self.endpoint_url, self.bucket = endpoint_url, bucket + + def list(self) -> tuple[list[dict], bool]: + return census.collect(self.endpoint_url, self.bucket) + + def read(self, key: str) -> str: + return signed("GET", self.endpoint_url, self.bucket, key)[:512].decode("utf-8", "replace") + + def delete(self, key: str) -> None: + signed("DELETE", self.endpoint_url, self.bucket, key) + + +def protected(bucket, objects: list[dict]) -> set[str]: + """Every archive a pointer names. Any unreadable pointer aborts the run.""" + keep: set[str] = set() + for item in objects: + match = POINTER.match(item["key"]) + if not match: + continue + target = bucket.read(item["key"]).strip() + if not VALID_NAME.match(target): + raise RuntimeError(f"pointer {item['key']} names an invalid key; refusing to prune") + keep.update(f"{match.group(1)}/objects/{target}.{ext}" for ext in ("tar.zst", "tar.gz")) + return keep + + +def plan(objects: list[dict], keep: set[str], now: dt.datetime) -> tuple[list[dict], dict]: + candidates, families = [], {} + for item in objects: + match = ARCHIVE.match(item["key"]) + if not match: + continue + name = match.group("name") + days = retention_days(name) + family = next(prefix for prefix, limit in RETENTION_DAYS if name.startswith(prefix)) or "other" + row = families.setdefault(family, {"retention_days": days, "archives": 0, "bytes": 0, + "delete": 0, "delete_bytes": 0, "kept_by_pointer": 0}) + row["archives"] += 1 + row["bytes"] += int(item["size"]) + age = census.age_days(item.get("last_modified", ""), now) + if age is None or age <= days: + continue + if item["key"] in keep: + row["kept_by_pointer"] += 1 + continue + row["delete"] += 1 + row["delete_bytes"] += int(item["size"]) + candidates.append(item) + candidates.sort(key=lambda item: item.get("last_modified", "")) + return candidates, families + + +def prune(bucket, now: dt.datetime, delete: bool) -> dict: + objects, complete = bucket.list() + if not complete: + # A partial listing can miss pointers, and a missed pointer would let + # its target look unreferenced. Delete nothing. + return {"complete": False, "deleted": 0, "families": {}} + candidates, families = plan(objects, protected(bucket, objects), now) + deleted = 0 + if delete and candidates: + # A save of an existing key re-points `latest/` at an old archive, and + # can land between the first pointer read and a delete. Read every + # pointer again right before deleting. + keep = protected(bucket, objects) + for item in candidates[:MAX_DELETES]: + if item["key"] in keep: + continue + bucket.delete(item["key"]) + deleted += 1 + return { + "complete": True, + "dry_run": not delete, + "would_delete": len(candidates), + "would_delete_bytes": sum(int(item["size"]) for item in candidates), + "deleted": deleted, + "families": families, + } + + +def render(summary: dict) -> str: + gib = 1024**3 + if not summary["complete"]: + return "R2 cache prune: listing incomplete; deleted nothing" + verb = "would delete" if summary["dry_run"] else "deleted" + count = summary["would_delete"] if summary["dry_run"] else summary["deleted"] + lines = [f"R2 cache prune ({'dry run' if summary['dry_run'] else 'delete'}): {verb} {count} archives, " + f"{summary['would_delete_bytes'] / gib:.1f} GiB eligible", ""] + lines.append("| family | retention | archives | GiB | eligible | eligible GiB | kept by pointer |") + lines.append("|---|---:|---:|---:|---:|---:|---:|") + for family, row in sorted(summary["families"].items()): + lines.append(f"| `{family}` | {row['retention_days']} d | {row['archives']} | {row['bytes'] / gib:.1f} | " + f"{row['delete']} | {row['delete_bytes'] / gib:.1f} | {row['kept_by_pointer']} |") + return "\n".join(lines) + + +def main(argv: list[str] | None = None) -> int: + parser = argparse.ArgumentParser(description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter) + parser.add_argument("--endpoint-url", required=True) + parser.add_argument("--bucket", required=True) + parser.add_argument("--delete", action="store_true", help="send deletes; the default is a dry run") + args = parser.parse_args(argv) + summary = prune(Bucket(args.endpoint_url, args.bucket), dt.datetime.now(dt.timezone.utc), args.delete) + print(json.dumps(summary, sort_keys=True)) + text = render(summary) + print(text) + if os.environ.get("GITHUB_STEP_SUMMARY"): + with open(os.environ["GITHUB_STEP_SUMMARY"], "a") as handle: + handle.write(text + "\n") + return 0 if summary["complete"] else 1 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 6fc8255e0376..cc32dcca390b 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -1139,3 +1139,7 @@ lane = "linux-guard" [[test]] path = "tests/test_web_complexity_source_enumeration.py" lane = "linux-guard" + +[[test]] +path = "tests/test_r2_cache_prune.py" +lane = "linux-guard" diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 1a4584729f00..151b8a5b5e01 100755 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -591,6 +591,7 @@ def test_operational_ci_helpers_skip_product_areas() -> None: "scripts/ci/triage-radar.py", "scripts/ci/notify-indexnow.py", "scripts/ci/r2_cache_census.py", + "scripts/ci/r2_cache_prune.py", "scripts/ci/verify-r2-canary.py", "scripts/ci/build_graph_health.py", "scripts/ci/validate_test_execution_registry.py", diff --git a/tests/test_r2_cache_prune.py b/tests/test_r2_cache_prune.py new file mode 100644 index 000000000000..8487bae0dd8c --- /dev/null +++ b/tests/test_r2_cache_prune.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Pruning the CI cache bucket must never delete what a prefix restore reads.""" +import datetime as dt +from pathlib import Path +import sys +import unittest + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts/ci")) +import r2_cache_prune as prune # noqa: E402 + +NOW = dt.datetime(2026, 9, 24, tzinfo=dt.timezone.utc) +NS = "v1/macOS-ARM64" + + +def stamp(days): + return (NOW - dt.timedelta(days=days)).strftime("%Y-%m-%dT%H:%M:%S.000Z") + + +class FakeBucket: + def __init__(self, archives, pointers, complete=True): + self.objects = [{"key": key, "size": size, "last_modified": stamp(age)} for key, size, age in archives] + self.pointers = dict(pointers) + self.objects += [{"key": key, "size": 64, "last_modified": stamp(90)} for key in self.pointers] + self.complete = complete + self.deleted = [] + self.on_first_delete = None + + def list(self): + return list(self.objects), self.complete + + def read(self, key): + return self.pointers[key] + "\n" + + def delete(self, key): + if self.on_first_delete and not self.deleted: + self.on_first_delete(self) + self.deleted.append(key) + + +def archive(name, age, size=10, ext="tar.zst"): + return (f"{NS}/objects/{name}.{ext}", size, age) + + +class Prune(unittest.TestCase): + def test_dry_run_reports_and_deletes_nothing(self): + bucket = FakeBucket([archive("spm-old", 40), archive("spm-new", 1)], {}) + summary = prune.prune(bucket, NOW, delete=False) + self.assertEqual((summary["would_delete"], summary["would_delete_bytes"]), (1, 10)) + self.assertEqual(bucket.deleted, []) + + def test_each_family_ages_out_on_its_own_retention(self): + bucket = FakeBucket([ + archive("admission-derived-data-v1-macOS-ARM64-fp-a", 4), + archive("admission-derived-data-v1-macOS-ARM64-fp-b", 2), + archive("xcode-compilation-test-macOS-ARM64-fp-a", 8), + archive("xcode-compilation-test-macOS-ARM64-fp-b", 6), + archive("spm-a", 31), + archive("spm-b", 29), + ], {}) + prune.prune(bucket, NOW, delete=True) + self.assertEqual(sorted(bucket.deleted), sorted([ + f"{NS}/objects/admission-derived-data-v1-macOS-ARM64-fp-a.tar.zst", + f"{NS}/objects/xcode-compilation-test-macOS-ARM64-fp-a.tar.zst", + f"{NS}/objects/spm-a.tar.zst", + ])) + + def test_an_archive_a_pointer_names_is_kept_at_any_age_in_either_format(self): + bucket = FakeBucket( + [archive("spm-pinned", 400), archive("zig-pinned", 400, ext="tar.gz"), archive("spm-orphan", 400)], + {f"{NS}/latest/spm-": "spm-pinned", f"{NS}/latest/zig-": "zig-pinned"}, + ) + summary = prune.prune(bucket, NOW, delete=True) + self.assertEqual(bucket.deleted, [f"{NS}/objects/spm-orphan.tar.zst"]) + self.assertEqual(summary["families"]["other"]["kept_by_pointer"], 2) + + def test_pointers_and_foreign_keys_are_never_candidates(self): + bucket = FakeBucket([("github/manaflow-ai/cmux/1/product.zip", 10, 400), + (f"{NS}/objects/nested/escape.tar.zst", 10, 400)], + {f"{NS}/latest/spm-": "spm-gone"}) + prune.prune(bucket, NOW, delete=True) + self.assertEqual(bucket.deleted, []) + + def test_a_pointer_moved_onto_an_old_archive_mid_run_protects_it(self): + bucket = FakeBucket([archive("spm-a", 40), archive("spm-b", 41)], {f"{NS}/latest/spm-": "spm-new"}) + # r2-cache.sh re-points latest/ at an existing key without re-uploading. + bucket.on_first_delete = None + original = bucket.read + reads = [] + + def read(key): + reads.append(key) + if len(reads) > 1: + return "spm-a\n" + return original(key) + + bucket.read = read + prune.prune(bucket, NOW, delete=True) + self.assertEqual(bucket.deleted, [f"{NS}/objects/spm-b.tar.zst"]) + + def test_an_incomplete_listing_or_bad_pointer_deletes_nothing(self): + bucket = FakeBucket([archive("spm-old", 400)], {}, complete=False) + self.assertFalse(prune.prune(bucket, NOW, delete=True)["complete"]) + self.assertEqual(bucket.deleted, []) + bucket = FakeBucket([archive("spm-old", 400)], {f"{NS}/latest/spm-": "../../etc"}) + with self.assertRaises(RuntimeError): + prune.prune(bucket, NOW, delete=True) + self.assertEqual(bucket.deleted, []) + + def test_one_run_is_bounded(self): + limit = prune.MAX_DELETES + try: + prune.MAX_DELETES = 2 + bucket = FakeBucket([archive(f"spm-{i}", 40 + i) for i in range(5)], {}) + prune.prune(bucket, NOW, delete=True) + # Oldest first. + self.assertEqual(bucket.deleted, [f"{NS}/objects/spm-4.tar.zst", f"{NS}/objects/spm-3.tar.zst"]) + finally: + prune.MAX_DELETES = limit + + +class Workflow(unittest.TestCase): + def test_only_main_prunes_and_only_on_an_explicit_opt_in(self): + workflow = yaml.safe_load((ROOT / ".github/workflows/r2-cache-prune.yml").read_text()) + triggers = workflow.get("on", workflow.get(True)) + self.assertEqual(set(triggers), {"schedule", "workflow_dispatch"}) + self.assertEqual(workflow["permissions"], {}) + job = workflow["jobs"]["prune"] + self.assertIn("github.ref == 'refs/heads/main'", job["if"]) + self.assertEqual(job["permissions"], {"contents": "read"}) + self.assertNotIn("macos", str(job["runs-on"]).lower()) + run = next(step for step in job["steps"] if "r2_cache_prune.py" in str(step.get("run", ""))) + # Deletes need a dispatch that says so, or the repository variable. + self.assertIn("--delete", run["run"]) + self.assertIn("PRUNE_DELETE", run["run"]) + condition = run["env"]["PRUNE_DELETE"] + self.assertIn("vars.CI_R2_CACHE_PRUNE_DELETE == '1'", condition) + self.assertIn("inputs.delete", condition) + + +if __name__ == "__main__": + unittest.main() From 82c1c229457fe7aca07dc4fd0dd64dc741fea07e Mon Sep 17 00:00:00 2001 From: Leo Li Date: Wed, 23 Sep 2026 18:34:02 -0700 Subject: [PATCH 2/5] ci: keep compilation caches three days, not seven Co-Authored-By: Claude Opus 5.5 --- scripts/ci/r2_cache_prune.py | 9 ++++++--- tests/test_r2_cache_prune.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/scripts/ci/r2_cache_prune.py b/scripts/ci/r2_cache_prune.py index 2054ae05adbe..94ffd76c99c9 100644 --- a/scripts/ci/r2_cache_prune.py +++ b/scripts/ci/r2_cache_prune.py @@ -18,8 +18,11 @@ admission-derived-data- 3 days. One snapshot per main commit, and every consumer adopts the newest by prefix; an exact key only matters for a pull request on a base that old. - xcode-compilation- 7 days. Seeds every 6 hours, restored newest by - prefix; a cache from an older main mostly misses. + xcode-compilation- 3 days. Restored newest by prefix, and a cache from + an older main misses for every changed module + (#14015: 3 of 2,665 app jobs hit). On 2026-09-24 + these were 141 GiB of a 143 GiB bucket after five + days, about 35 GiB a day. everything else 30 days. Keyed by content (a Package.resolved or toolchain hash), so an old key stays exact for a pull request whose base still has that input. @@ -45,7 +48,7 @@ RETENTION_DAYS = ( ("admission-derived-data-", 3), - ("xcode-compilation-", 7), + ("xcode-compilation-", 3), ("", 30), ) ARCHIVE = re.compile(r"^(v1/[^/]+)/objects/(?P[A-Za-z0-9._-]+)\.(?:tar\.zst|tar\.gz)$") diff --git a/tests/test_r2_cache_prune.py b/tests/test_r2_cache_prune.py index 8487bae0dd8c..325e6c969948 100644 --- a/tests/test_r2_cache_prune.py +++ b/tests/test_r2_cache_prune.py @@ -55,8 +55,8 @@ def test_each_family_ages_out_on_its_own_retention(self): bucket = FakeBucket([ archive("admission-derived-data-v1-macOS-ARM64-fp-a", 4), archive("admission-derived-data-v1-macOS-ARM64-fp-b", 2), - archive("xcode-compilation-test-macOS-ARM64-fp-a", 8), - archive("xcode-compilation-test-macOS-ARM64-fp-b", 6), + archive("xcode-compilation-test-macOS-ARM64-fp-a", 4), + archive("xcode-compilation-test-macOS-ARM64-fp-b", 2), archive("spm-a", 31), archive("spm-b", 29), ], {}) From 90fc5283956603b731b7469760316fd9c9dedc4c Mon Sep 17 00:00:00 2001 From: teamleaderleo Date: Thu, 24 Sep 2026 03:46:55 -0400 Subject: [PATCH 3/5] ci: run the R2 prune job in the main-only ci-cache-writer environment The guard from #14147 requires every job holding the R2 write credentials to declare it. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/r2-cache-prune.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/r2-cache-prune.yml b/.github/workflows/r2-cache-prune.yml index 69676136259a..fab7c85a3688 100644 --- a/.github/workflows/r2-cache-prune.yml +++ b/.github/workflows/r2-cache-prune.yml @@ -29,6 +29,8 @@ jobs: if: ${{ github.ref == 'refs/heads/main' }} runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 20 + # Only main may use the R2 write credentials (see #14147). + environment: ${{ github.ref == 'refs/heads/main' && 'ci-cache-writer' || '' }} permissions: contents: read steps: From 3b7844ccdcbc63103183257b5f64fccc140260ff Mon Sep 17 00:00:00 2001 From: teamleaderleo Date: Thu, 24 Sep 2026 03:50:52 -0400 Subject: [PATCH 4/5] ci: route the R2 prune job off Blacksmith outside manaflow-ai Required by the fork runner routing guard (#14066). Also correct the pointer re-read comment: with run-ID generations (#14174) a re-save no longer moves a pointer backwards; a new prefix's first pointer is the remaining race. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/r2-cache-prune.yml | 2 +- scripts/ci/r2_cache_prune.py | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/r2-cache-prune.yml b/.github/workflows/r2-cache-prune.yml index fab7c85a3688..7953456e6d47 100644 --- a/.github/workflows/r2-cache-prune.yml +++ b/.github/workflows/r2-cache-prune.yml @@ -27,7 +27,7 @@ jobs: name: Prune expired cache archives # The bucket credentials only ever run main's copy of this script. if: ${{ github.ref == 'refs/heads/main' }} - runs-on: ${{ vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} timeout-minutes: 20 # Only main may use the R2 write credentials (see #14147). environment: ${{ github.ref == 'refs/heads/main' && 'ci-cache-writer' || '' }} diff --git a/scripts/ci/r2_cache_prune.py b/scripts/ci/r2_cache_prune.py index 94ffd76c99c9..bf6749c0006c 100644 --- a/scripts/ci/r2_cache_prune.py +++ b/scripts/ci/r2_cache_prune.py @@ -163,9 +163,9 @@ def prune(bucket, now: dt.datetime, delete: bool) -> dict: candidates, families = plan(objects, protected(bucket, objects), now) deleted = 0 if delete and candidates: - # A save of an existing key re-points `latest/` at an old archive, and - # can land between the first pointer read and a delete. Read every - # pointer again right before deleting. + # A save can publish a pointer between the first read and a delete + # (a prefix's first pointer is written regardless of generation). + # Read every pointer again right before deleting. keep = protected(bucket, objects) for item in candidates[:MAX_DELETES]: if item["key"] in keep: From 7180091db79e068c70e8c98d52a1dc562206b323 Mon Sep 17 00:00:00 2001 From: teamleaderleo Date: Thu, 24 Sep 2026 03:54:18 -0400 Subject: [PATCH 5/5] ci: keep per-commit R2 caches one day, not three Both families try the pull request's exact base first, then the newest by prefix. On 2026-09-24, 92 of the 100 most recently updated open pull requests had a base under a day old, and each day of retention costs about 35 GiB. Co-Authored-By: Claude Opus 5.5 --- scripts/ci/r2_cache_prune.py | 12 +++++++----- tests/test_r2_cache_prune.py | 9 +++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/scripts/ci/r2_cache_prune.py b/scripts/ci/r2_cache_prune.py index bf6749c0006c..57bfe8c7350f 100644 --- a/scripts/ci/r2_cache_prune.py +++ b/scripts/ci/r2_cache_prune.py @@ -15,10 +15,12 @@ Retention, by the first matching key prefix: - admission-derived-data- 3 days. One snapshot per main commit, and every + admission-derived-data- 1 day. One snapshot per main commit, and every consumer adopts the newest by prefix; an exact key - only matters for a pull request on a base that old. - xcode-compilation- 3 days. Restored newest by prefix, and a cache from + only matters for a pull request on a base that old + (2026-09-24: 92 of the 100 most recently updated + open pull requests had a base under a day old). + xcode-compilation- 1 day. Restored newest by prefix, and a cache from an older main misses for every changed module (#14015: 3 of 2,665 app jobs hit). On 2026-09-24 these were 141 GiB of a 143 GiB bucket after five @@ -47,8 +49,8 @@ import r2_cache_census as census # noqa: E402 RETENTION_DAYS = ( - ("admission-derived-data-", 3), - ("xcode-compilation-", 3), + ("admission-derived-data-", 1), + ("xcode-compilation-", 1), ("", 30), ) ARCHIVE = re.compile(r"^(v1/[^/]+)/objects/(?P[A-Za-z0-9._-]+)\.(?:tar\.zst|tar\.gz)$") diff --git a/tests/test_r2_cache_prune.py b/tests/test_r2_cache_prune.py index 325e6c969948..868e2d9072bf 100644 --- a/tests/test_r2_cache_prune.py +++ b/tests/test_r2_cache_prune.py @@ -53,12 +53,13 @@ def test_dry_run_reports_and_deletes_nothing(self): def test_each_family_ages_out_on_its_own_retention(self): bucket = FakeBucket([ - archive("admission-derived-data-v1-macOS-ARM64-fp-a", 4), - archive("admission-derived-data-v1-macOS-ARM64-fp-b", 2), - archive("xcode-compilation-test-macOS-ARM64-fp-a", 4), - archive("xcode-compilation-test-macOS-ARM64-fp-b", 2), + archive("admission-derived-data-v1-macOS-ARM64-fp-a", 2), + archive("admission-derived-data-v1-macOS-ARM64-fp-b", 0.5), + archive("xcode-compilation-test-macOS-ARM64-fp-a", 2), + archive("xcode-compilation-test-macOS-ARM64-fp-b", 0.5), archive("spm-a", 31), archive("spm-b", 29), + archive("spm-c", 2), ], {}) prune.prune(bucket, NOW, delete=True) self.assertEqual(sorted(bucket.deleted), sorted([