From 4cd451eaa3773abcc5cd355354cdfebc5010d8a8 Mon Sep 17 00:00:00 2001 From: teamleaderleo Date: Sat, 26 Sep 2026 09:03:29 -0400 Subject: [PATCH 1/2] ci: build the nightly app on the trusted owned minis first, Blacksmith as fallback build-nightly-app was hard-coded to blacksmith-12vcpu-macos-26, so a push to main never considered an owned mini (run 36239331473). Attempt 1 of a push or schedule run on main now asks for vars.CI_SEED_TRUSTED_POOL, the trusted owned pool (no pull request runners; the glaeda hook admits only main's push and schedule jobs), because the job holds the ci-cache-writer R2 keys and the Sentry token and its app is signed and shipped. Forks, rc/**, dispatches, fast dogfood and re-runs keep Blacksmith. - owned_pool_rescue.py / ci-owned-pool-rescue.yml: watch nightly.yml runs like a picker-less side lane (NIGHTLY_WORKFLOW_PATH). Recognise glaeda-trusted-* labels. Allow one queue round behind a DerivedData seed. A stuck or refused build is re-run on Blacksmith, unless a newer nightly run on main is still pending. - nightly.yml: an owned runner folds its workspace path into the release compile-cache key, so the mini and Blacksmith lineages never evict each other (Blacksmith's key is unchanged). Clear build outputs a persistent runner kept. - runner_label_policy.py: CI_SEED_TRUSTED_POOL may only name a glaeda-trusted-* label; the health report and variable check read it. - Signing and notarization stay on Blacksmith (unproven on owned Macs; #6264). Docs list every macOS job's route. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-health-report.yml | 1 + .github/workflows/ci-owned-pool-rescue.yml | 10 +- .github/workflows/ci-repo-variables.yml | 1 + .github/workflows/nightly.yml | 42 +++++- docs/ci-runners.md | 28 +++- docs/ci/mac-fleet.md | 6 +- scripts/ci/owned_pool_rescue.py | 81 +++++++++++- scripts/ci/runner_label_policy.py | 27 ++++ tests/test_ci_owned_pool_rescue.py | 145 ++++++++++++++++++++- tests/test_nightly_universal_build.sh | 2 +- tests/test_runner_label_policy.py | 23 ++++ 11 files changed, 341 insertions(+), 25 deletions(-) diff --git a/.github/workflows/ci-health-report.yml b/.github/workflows/ci-health-report.yml index 0ead8e62e368..1401320e206b 100644 --- a/.github/workflows/ci-health-report.yml +++ b/.github/workflows/ci-health-report.yml @@ -88,6 +88,7 @@ jobs: # CI_PR_POOL_ORDER is the one list that may also name owned pools. CMUX_CI_RUNNER_VARIABLES: | CI_PR_POOL_ORDER=${{ vars.CI_PR_POOL_ORDER }} + CI_SEED_TRUSTED_POOL=${{ vars.CI_SEED_TRUSTED_POOL }} CI_SIDE_LANE_RUNNER=${{ vars.CI_SIDE_LANE_RUNNER }} CMUX_CI_RUNNER_FLEET=${{ vars.CMUX_CI_RUNNER_FLEET }} CMUX_CI_RUNNER_OVERRIDES=${{ vars.CMUX_CI_RUNNER_OVERRIDES }} diff --git a/.github/workflows/ci-owned-pool-rescue.yml b/.github/workflows/ci-owned-pool-rescue.yml index 5f1c25d4530e..8d7d4f94e178 100644 --- a/.github/workflows/ci-owned-pool-rescue.yml +++ b/.github/workflows/ci-owned-pool-rescue.yml @@ -37,6 +37,10 @@ run-name: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow # terminal-hang-diagnostics) have no picker: while vars.CI_SIDE_LANE_RUNNER # names a glaeda-side-* label, every same-repository attempt-1 pull request # run of theirs is on the fleet, so a dispatch would save no run. +# - nightly.yml: build-nightly-app has no picker either. While +# vars.CI_SEED_TRUSTED_POOL names a glaeda-trusted-* pool, attempt 1 of +# every push or schedule run on main asks for it, so every such run is +# watched (owned_pool_rescue.NIGHTLY_WORKFLOW_PATH). # It needs actions: write, and its code comes from main. It runs whenever # owned pools are on (CI_PR_POOL_OWNED is 1), because a run on an owned pool # has no other way off it; CI_OWNED_POOL_RESCUE=0 turns it off. The switch @@ -58,6 +62,9 @@ on: - IROH v2 - Relay TLS system trust - Terminal hang diagnostics + # nightly.yml: its app build takes the trusted owned pool on attempt 1 + # of main's push and schedule runs (owned_pool_rescue.NIGHTLY_WORKFLOW_PATH). + - Nightly macOS build types: [requested] workflow_dispatch: inputs: @@ -89,6 +96,7 @@ env: .github/workflows/cloud-machine-tests.yml .github/workflows/cloud-task-local-tests.yml .github/workflows/iroh-v2.yml .github/workflows/relay-tls.yml .github/workflows/terminal-hang-diagnostics.yml + .github/workflows/nightly.yml concurrency: group: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && !inputs.run_id) && 'owned-pool-sweeper' || format('owned-pool-rescue-{0}-{1}', inputs.run_id || github.event.workflow_run.id, inputs.run_attempt || github.event.workflow_run.run_attempt) }} @@ -98,7 +106,7 @@ concurrency: jobs: rescue: name: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && !inputs.run_id) && 'Sweep runs on persistent pools' || 'Rescue a run stuck on a persistent pool' }} - if: ${{ vars.CI_PR_POOL_OWNED == '1' && (vars.CI_OWNED_POOL_RESCUE || '1') != '0' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event.workflow_run.event == 'pull_request' && startsWith(vars.CI_SIDE_LANE_RUNNER, 'glaeda-side-') || github.event.workflow_run.path == '.github/workflows/ios-screenshots.yml' && github.event.workflow_run.event == 'workflow_dispatch') && github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.run_attempt == 1) }} + if: ${{ vars.CI_PR_POOL_OWNED == '1' && (vars.CI_OWNED_POOL_RESCUE || '1') != '0' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event.workflow_run.event == 'pull_request' && startsWith(vars.CI_SIDE_LANE_RUNNER, 'glaeda-side-') || github.event.workflow_run.path == '.github/workflows/ios-screenshots.yml' && github.event.workflow_run.event == 'workflow_dispatch' || github.event.workflow_run.path == '.github/workflows/nightly.yml' && (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule') && github.event.workflow_run.head_branch == 'main' && startsWith(vars.CI_SEED_TRUSTED_POOL, 'glaeda-trusted-')) && github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.run_attempt == 1) }} runs-on: ubuntu-24.04 # github-hosted-required: polls the Actions API; keeps CI's Linux pool free # A sweeper adopts runs for 300 minutes and gives a rescue 25 more to # settle (SWEEP_SECONDS, RESCUE_GRACE_SECONDS). A single run's watch is diff --git a/.github/workflows/ci-repo-variables.yml b/.github/workflows/ci-repo-variables.yml index f5382cdcdd65..699218e55d9f 100644 --- a/.github/workflows/ci-repo-variables.yml +++ b/.github/workflows/ci-repo-variables.yml @@ -54,6 +54,7 @@ jobs: CMUX_CI_XCODE_APP_PR: ${{ vars.CMUX_CI_XCODE_APP_PR }} CMUX_CI_RUNNER_VARIABLES: | CI_PR_POOL_ORDER=${{ vars.CI_PR_POOL_ORDER }} + CI_SEED_TRUSTED_POOL=${{ vars.CI_SEED_TRUSTED_POOL }} CI_SIDE_LANE_RUNNER=${{ vars.CI_SIDE_LANE_RUNNER }} CMUX_CI_RUNNER_FLEET=${{ vars.CMUX_CI_RUNNER_FLEET }} CMUX_CI_RUNNER_OVERRIDES=${{ vars.CMUX_CI_RUNNER_OVERRIDES }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 167187bf68a5..6538b9c1da40 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -903,13 +903,28 @@ jobs: daemon_build: ${{ steps.remote_daemon.outputs.build }} daemon_version: ${{ steps.remote_daemon.outputs.version }} if: needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') - # Full runs share the cache warmer's and stable release lane's image and - # toolchain, which is what the compilation cache is keyed on — OS, arch and - # toolchain, never instance size, which is deliberately larger here. Fast + # Owned minis first. Attempt 1 of a push or schedule run on main takes the + # trusted owned pool (vars.CI_SEED_TRUSTED_POOL, + # glaeda-trusted--xcode-) while CI_PR_POOL_OWNED is 1: + # minis with no pull request runners, whose job-started hook admits only + # main's own push and schedule jobs. Not the pull request pool: this job + # holds the ci-cache-writer R2 keys and the Sentry token, and its app is + # what build-sign-notarize-nightly signs and ships. It is the pool + # seed-derived-data.yml already seeds from with the same credentials, on + # the Xcode 26.6 that CMUX_CI_XCODE_APP_MACOS_26 pins. + # ci-owned-pool-rescue.yml watches the run (owned_pool_rescue.py, + # NIGHTLY_WORKFLOW_PATH): when no trusted mini takes the job within its + # budget, or the mini refuses it at job start, the failed jobs are re-run, + # and every later attempt takes Blacksmith below. Signing stays on + # Blacksmith (build-sign-notarize-nightly). + # Blacksmith runs share the cache warmer's and stable release lane's image + # and toolchain, which is what the compilation cache is keyed on — OS, arch + # and toolchain, never instance size, which is deliberately larger here. + # An owned mini keys its own lineage (see the cache key step). Fast # branch dogfood always uses the dedicated Blacksmith image. A # repository-wide runner override may point at a slower shared builder, # which defeats the purpose of the one-architecture path. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-12vcpu-macos-26' || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26_LARGE || 'blacksmith-12vcpu-macos-26') }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-12vcpu-macos-26' || github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/main' && vars.CI_PR_POOL_OWNED == '1' && vars.CI_SEED_TRUSTED_POOL || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26_LARGE || 'blacksmith-12vcpu-macos-26') }} environment: ${{ github.ref == 'refs/heads/main' && 'ci-cache-writer' || '' }} # The Blacksmith cache is scoped per branch and also drops main's own # entry (runs 35179030871 and 35182663752 restored nothing and were @@ -941,11 +956,28 @@ jobs: - name: Select Xcode run: ./scripts/select-ci-xcode.sh + # A persistent owned mini keeps the workspace between jobs. Checkout's + # clean already drops untracked output; clear the build tree by name as + # well, so a stale product or dSYM from an earlier job can never ship. + - name: Clear build outputs a persistent runner kept + run: scripts/ci/clear-dirs.sh build-universal remote-daemon-assets + - name: Compute Xcode compilation cache key id: compilation-cache-key run: | set -euo pipefail - echo "toolchain=$(xcodebuild -version | shasum -a 256 | awk '{print $1}')" >> "$GITHUB_OUTPUT" + # The key is OS, arch and toolchain. An owned mini (runner + # -glaeda[-K]) runs the same Xcode build as Blacksmith but + # compiles in another workspace path, so its cache entries would + # mostly miss there and push Blacksmith's own entry out of the + # prefix restore. Fold its workspace path into the toolchain hash: + # each lineage restores only its own, and Blacksmith's key is + # unchanged. + identity="$(xcodebuild -version)" + case "${RUNNER_NAME:-}" in + *-glaeda | *-glaeda-[0-9]*) identity="$identity"$'\n'"owned-workspace:$GITHUB_WORKSPACE" ;; + esac + echo "toolchain=$(printf '%s\n' "$identity" | shasum -a 256 | awk '{print $1}')" >> "$GITHUB_OUTPUT" - name: Restore Xcode compilation cache id: compilation-cache-restore diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 668befcde8c0..6c122bd174d1 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -644,11 +644,22 @@ root) stay on Blacksmith. Clear the variable to send every side lane back. Owned minis run macOS 26 with Xcode 26.6 only, run same-repository pull request code, and keep their home directory and caches between jobs. So a job -stays on Blacksmith when it signs, notarizes, uploads or publishes (anything -with signing, store or release secrets, or whose output ships or seeds a -shared cache), when it runs fork code, or when it needs an OS or Xcode the -minis lack. Everything else routes through a picker, with Blacksmith as the -overflow and ci-owned-pool-rescue.yml as the way off a busy or refusing mini. +stays off the pull request pools when it signs, notarizes, uploads or +publishes (anything with signing, store or release secrets, or whose output +ships or seeds a shared cache), when it runs fork code, or when it needs an OS +or Xcode the minis lack. Everything else routes through a picker, with +Blacksmith as the overflow and ci-owned-pool-rescue.yml as the way off a busy +or refusing mini. + +The trusted pool (`vars.CI_SEED_TRUSTED_POOL`, +`glaeda-trusted--xcode-`) is the owned home for main's own +cache writers and builds: minis with no pull request runners, whose +job-started hook admits only a push or schedule run on main. The DerivedData +seed and the nightly app compile take it first; Blacksmith is the fallback. +`runner_label_policy.py` refuses any other value for the variable. Signing and +notarization are not on it: no signing run on an owned Mac has been proven, +and the retired self-hosted fleet failed `codesign` with +`errSecInternalComponent` (#6264). | Jobs | Route | Why | | --- | --- | --- | @@ -664,8 +675,11 @@ overflow and ci-owned-pool-rescue.yml as the way off a busy or refusing mini. | low-volume dispatches: `test-macos-suite`, `tmux-corpus`, `perf-activation`, command palette benchmarks, `iroh-release-gate` version skew | Blacksmith or the caller's runner input | a few runs a week; benchmarks want a quiet machine | | `relay-tls` `system-keychain` | Blacksmith | edits the System keychain trust store | | `plain-paste-worker`, `ci-macos-compat`, `seed-swiftpm-manifests`, release and nightly Ghostty helpers | Blacksmith macOS 15 / 14 | an OS or SDK the minis lack | -| `release.yml`, nightly sign/notarize, `ios-testflight`, `ios-app-store`, `ios-appstore-upload` | Blacksmith | signing and store secrets | -| nightly app and compilation caches, `seed-derived-data` Blacksmith pools, `build-ghosttykit`, `cmux-tui-build-package` (artifacts, nightly, release), `relay-publish-npm` | Blacksmith | publish, or write a cache other runs trust, with R2 or release secrets | +| `release.yml`, nightly sign/notarize, `ios-testflight`, `ios-app-store`, `ios-appstore-upload` | Blacksmith | signing and store secrets; signing on an owned Mac is unproven | +| `nightly.yml` `build-nightly-app` | trusted owned pool (`CI_SEED_TRUSTED_POOL`) on attempt 1 of main's push and schedule runs; Blacksmith 12 vCPU otherwise, for `rc/**`, dispatches and fast dogfood, and on every re-run | ci-owned-pool-rescue.yml watches it (`NIGHTLY_WORKFLOW_PATH`): stuck one queue round past `CI_OWNED_POOL_RESCUE_SECONDS`, or refused, its failed jobs re-run on Blacksmith. Its compilation cache keys its own lineage (the mini's workspace path) | +| `seed-derived-data` trusted pool | trusted owned pool, push to main | the minis' own j14 seed | +| `nightly.yml` `refresh-compilation-cache`, `refresh-test-compilation-cache`, `seed-derived-data` Blacksmith pools | Blacksmith | they seed Blacksmith's own lanes: the release cache the nightly fallback restores, and the pull request admission seeds for each Blacksmith pool | +| `build-ghosttykit`, `cmux-tui-build-package` (artifacts, nightly, release), `relay-publish-npm` | Blacksmith | publish with R2 or release secrets | | `ios-streamed-validate`, `iroh-release-gate` simulator E2E | Blacksmith | secrets in the job, fixed ports, GUI session changes | ## Retired: Tart VM fleet diff --git a/docs/ci/mac-fleet.md b/docs/ci/mac-fleet.md index 4b8506d9ac72..9f1859b097de 100644 --- a/docs/ci/mac-fleet.md +++ b/docs/ci/mac-fleet.md @@ -216,8 +216,10 @@ them (section 5). job gets a fresh VM and an Aqua login session. A shared mini cannot give it either. (The isolated Tart pool that once offered this was retired on 2026-09-25; see `ci-runners.md`.) -4. **Nightly app compile** - nightlies run on Blacksmith until Glaeda routing - (glaeda#1174) sends every job std > light > Blacksmith > GitHub-hosted. +4. **Nightly app compile** - `build-nightly-app` takes the trusted owned pool + (`CI_SEED_TRUSTED_POOL`) first on main's push and schedule runs, with + Blacksmith as the fallback through ci-owned-pool-rescue.yml + (`docs/ci-runners.md`). Signing and notarization stay on Blacksmith. 5. **`release-build`, signing, notarization, TestFlight** - never. Unchanged from `ci-runners.md`. diff --git a/scripts/ci/owned_pool_rescue.py b/scripts/ci/owned_pool_rescue.py index 452c2d267d59..e22494c6e58f 100644 --- a/scripts/ci/owned_pool_rescue.py +++ b/scripts/ci/owned_pool_rescue.py @@ -118,6 +118,21 @@ not followed. A stuck run that finished some other way (a newer push cancelled it) is not re-run. Its watch lasts SIDE_WATCH_LIMIT_SECONDS. +Nightly builds (NIGHTLY_WORKFLOW_PATH) are watched like a side lane: there is +no picker, and attempt 1 of a push or schedule run on main puts +build-nightly-app on vars.CI_SEED_TRUSTED_POOL, the trusted owned pool +(glaeda-trusted--xcode-, TRUSTED_LABEL), while every later +attempt takes Blacksmith. The trusted minis also seed DerivedData on every +push, so the job may wait behind a seed: its budget adds one QUEUE_ROUND_SECONDS +round. A stuck job gets the run cancelled and its failed and cancelled jobs +re-run; a refused one waits for the run to finish (the signing job is skipped +behind it) and then gets its failed jobs re-run. The run is not a pull request, +so in place of a head it checks for a newer nightly run on main that has not +finished: nightly.yml's concurrency group holds that run pending behind this +one, and a re-run would join the group and cancel it. With one, a stuck run is +cancelled and not re-run (so the newer run starts), and a refused one is left +as it is; the newer run builds main's newer HEAD. + A job's wait is measured from the later of its `created_at` and the first time the watcher saw it queued, so a job record created before its `needs` were met can never count as already past the budget. @@ -175,6 +190,7 @@ import http.client import json import os +import re import sys import threading import time @@ -201,6 +217,15 @@ # Side-lane workflows: no picker job. Their light macOS jobs take # vars.CI_SIDE_LANE_RUNNER (a glaeda-side-* label) on attempt 1 of a same-repo # pull request run, and every later attempt takes their Blacksmith default. +# nightly.yml: no picker job either. build-nightly-app takes the trusted owned +# pool (vars.CI_SEED_TRUSTED_POOL) on attempt 1 of a push or schedule run on +# main, and Blacksmith on every later attempt. +NIGHTLY_WORKFLOW_PATH = ".github/workflows/nightly.yml" +NIGHTLY_EVENTS = frozenset({"push", "schedule"}) +# The trusted owned pool's labels: minis with no pull request runners, whose +# job-started hook admits only main's push and schedule jobs. Only nightly.yml's +# app build asks for one through this watch. +TRUSTED_LABEL = re.compile(r"glaeda-(?:root-)?trusted-(?:xl|std|light)-xcode-[0-9]+(?:\.[0-9]+)*") SIDE_WORKFLOW_PATHS = frozenset({ ".github/workflows/auth-refresh-tests.yml", ".github/workflows/cloud-command-deadlines.yml", @@ -311,9 +336,9 @@ def parse_time(value: object) -> dt.datetime | None: def job_pool(job: Mapping[str, Any]) -> str | None: - """The owned pool a job asked for, if any.""" + """The owned pool a job asked for, if any: a pull request pool or the trusted one.""" for label in job.get("labels") or []: - if persistent(str(label)): + if persistent(str(label)) or TRUSTED_LABEL.fullmatch(str(label)): return str(label) return None @@ -504,6 +529,14 @@ def has_artifact(self, run_id: int, prefix: str, pages: int = 5) -> bool: def pull(self, number: int) -> Mapping[str, Any]: return self.request("GET", f"/pulls/{number}") + def newer_unfinished_runs(self, path: str, run_id: int, branch: str) -> list[int]: + """Ids of `path`'s runs on `branch` newer than `run_id` that have not finished (one request).""" + workflow = path.rsplit("/", 1)[-1] + data = self.request("GET", f"/actions/workflows/{workflow}/runs?branch={branch}&per_page=20") + return sorted(int(run.get("id") or 0) for run in (data or {}).get("workflow_runs") or [] + if isinstance(run, Mapping) and int(run.get("id") or 0) > run_id + and run.get("status") != "completed") + def branch_head(self, branch: str) -> str: return str(((self.request("GET", f"/branches/{branch}") or {}).get("commit") or {}).get("sha") or "") @@ -533,6 +566,9 @@ class Target: full_rerun: bool = False side: bool = False # a side-lane workflow (SIDE_WORKFLOW_PATHS): no picker job main: bool = False # main's full-suite dispatch of ci.yml: no pull request, main's HEAD instead + # A push or schedule run of nightly.yml on main (watched as a side lane, + # against main's HEAD). + nightly: bool = False # ci.yml started this watch because late-placement may move jobs onto owned # root runners after compile admission (LATE_PLACEMENT=1); the picker placed none. late: bool = False @@ -549,13 +585,15 @@ def watch_limit(self) -> int: def target_from_event(event: Mapping[str, Any], repository: str) -> Target | str: - """The CI, E2E or iOS run to watch, or why this event is not one.""" + """The CI, E2E, iOS or nightly run to watch, or why this event is not one.""" run = event.get("workflow_run") or {} path = run.get("path") + if path == NIGHTLY_WORKFLOW_PATH: + return nightly_target(run, repository) side = path in SIDE_WORKFLOW_PATHS if path != CI_WORKFLOW_PATH and path not in DISPATCH_WORKFLOW_PATHS and not side: - return (f"started by {path or 'an unknown workflow'}, not {CI_WORKFLOW_PATH}, a side-lane workflow " - f"or one of {', '.join(DISPATCH_WORKFLOW_PATHS)}") + return (f"started by {path or 'an unknown workflow'}, not {CI_WORKFLOW_PATH}, {NIGHTLY_WORKFLOW_PATH}, " + f"a side-lane workflow or one of {', '.join(DISPATCH_WORKFLOW_PATHS)}") e2e = path in DISPATCH_WORKFLOW_PATHS on_main = (path == CI_WORKFLOW_PATH and run.get("event") == "workflow_dispatch" and run.get("head_branch") == MAIN_BRANCH) @@ -583,6 +621,22 @@ def target_from_event(event: Mapping[str, Any], repository: str) -> Target | str e2e=e2e, side=side, path=str(path)) +def nightly_target(run: Mapping[str, Any], repository: str) -> Target | str: + """A nightly.yml run to watch, or why not: only attempt 1 of main's own push or schedule run.""" + if run.get("event") not in NIGHTLY_EVENTS: + return f"a {run.get('event') or 'unknown'} run of {NIGHTLY_WORKFLOW_PATH}, not a push or schedule" + if run.get("head_branch") != MAIN_BRANCH: + return f"a run of {NIGHTLY_WORKFLOW_PATH} on {run.get('head_branch') or 'an unknown branch'}, not {MAIN_BRANCH}" + head = (run.get("head_repository") or {}).get("full_name") or "" + if head.casefold() != repository.casefold(): + return "a fork head; forks never take a persistent pool" + attempt = int(run.get("run_attempt") or 0) + if attempt != 1: + return f"attempt {attempt}; its first attempt's watch follows it" + return Target(int(run["id"]), attempt, str(run.get("head_sha") or ""), 0, path=NIGHTLY_WORKFLOW_PATH, + side=True, nightly=True) + + def marker_name(target: Target) -> str: """The marker's name up to its jobs and pool, which only the janitor reads.""" return f"{MARKER_PREFIX}-{target.run_id}-{target.attempt}-" @@ -744,6 +798,14 @@ def pull_moved(api: GitHub, target: Target, sleep: Callable[[float], None], """Why the pull request (or main) no longer wants this run, or "" when it still does.""" if target.e2e and not target.pr_number: return "" # a dispatch has no head to move; a newer one cancels it by concurrency + if target.nightly: + # nightly.yml's concurrency group never cancels in progress: a newer + # run waits behind this one, and a re-run of this one would join the + # group and cancel that pending run. Leave the revision to it. + newer = read(lambda: api.newer_unfinished_runs(target.path, target.run_id, MAIN_BRANCH), sleep, log) + if newer: + return f"a newer nightly run on {MAIN_BRANCH} ({newer[0]}) has not finished and builds instead" + return "" if target.main: head = read(lambda: api.branch_head(MAIN_BRANCH), sleep, log) if head != target.head_sha: @@ -773,7 +835,7 @@ def rescue(api: GitHub, target: Target, *, now: Callable[[], dt.datetime], sleep # refusal is the fleet's, and a red run would open main's red-CI issue. keep_main = target.main and failed_only moved = "" if keep_main else pull_moved(api, target, sleep, log) - if moved and target.main: + if moved and (target.main or target.nightly): # Main's stuck run holds its concurrency group, so nothing newer can # start until it finishes: cancel it, and its completion dispatches # the new HEAD. @@ -924,12 +986,17 @@ def capped(deadline: dt.datetime) -> dt.datetime: subject = (f"pull request #{target.pr_number}'s {target.path}" if target.pr_number else "an E2E dispatch" if target.path == E2E_WORKFLOW_PATH else f"a dispatch of {target.path}") \ if target.e2e else f"main's full-suite dispatch at {target.head_sha[:12]}" if target.main \ + else f"main's nightly build at {target.head_sha[:12]}" if target.nightly \ else f"pull request #{target.pr_number}" - if target.side: + if target.side and not target.nightly: subject += " (side lane)" # ci.yml's, test-ios.yml's and test-e2e.yml's pickers queue on purpose, within the queue # rounds: their owned jobs may wait up to the pool's expected wait (see the docstring). queue_extra = queue_seconds(queue_rounds) if target.path in QUEUEING_WORKFLOW_PATHS else 0 + if target.nightly: + # The trusted minis seed DerivedData on every push to main, as this run + # starts: let the app build wait one round behind a seed. + queue_extra = QUEUE_ROUND_SECONDS log(f"watching run {target.run_id} of {subject} (budget {seconds + queue_extra}s" + (f": {seconds}s past the {queue_extra}s an owned job may expect to wait)" if queue_extra else ")")) # A watch deadline for attempt 1, and a fresh one (capped by the job's diff --git a/scripts/ci/runner_label_policy.py b/scripts/ci/runner_label_policy.py index 4f3c7f8b9faa..a639c5788057 100644 --- a/scripts/ci/runner_label_policy.py +++ b/scripts/ci/runner_label_policy.py @@ -155,6 +155,31 @@ def side_lane_reason(label: str) -> str | None: return forbidden_reason(label) +TRUSTED_POOL_VARIABLE = "CI_SEED_TRUSTED_POOL" +TRUSTED_POOL_PREFIX = "glaeda-trusted-" + + +def trusted_pool_reason(label: str) -> str | None: + """Why CI_SEED_TRUSTED_POOL is not allowed, or None when it is fine. + + seed-derived-data.yml seeds on this pool and nightly.yml's app build reads + it as its attempt-1 runs-on, both with the ci-cache-writer R2 keys, and the + app build's product is signed and shipped. So it may name only the trusted + owned pool, glaeda-trusted--xcode-: minis with no pull + request runners, whose hook admits only main's push and schedule jobs. + A pull request pool label (glaeda-std-...) would put those credentials and + that build on machines that run pull request code. Empty is fine (off). + """ + if not label: + return None + if label.startswith(TRUSTED_POOL_PREFIX) and _owned_pattern().fullmatch( + "glaeda-" + label[len(TRUSTED_POOL_PREFIX):] + ): + return None + return (f"`{label}` is not a trusted owned pool label " + f"({TRUSTED_POOL_PREFIX}-xcode-)") + + def forbidden_reason(label: str) -> str | None: """Why this runner label is not allowed, or None when it is fine. @@ -196,6 +221,8 @@ def drifted_runner_variables( reason = pool_order_reason(value.strip()) elif name == SIDE_LANE_VARIABLE: reason = side_lane_reason(value.strip()) + elif name == TRUSTED_POOL_VARIABLE: + reason = trusted_pool_reason(value.strip()) elif "RUNNER" not in name: continue else: diff --git a/tests/test_ci_owned_pool_rescue.py b/tests/test_ci_owned_pool_rescue.py index 6af9965d557c..1165741d37c8 100644 --- a/tests/test_ci_owned_pool_rescue.py +++ b/tests/test_ci_owned_pool_rescue.py @@ -987,6 +987,141 @@ def jobs(seconds): self.assertIn("refused", summary) +TRUSTED = "glaeda-trusted-std-xcode-26.6" + + +def nightly_event(**overrides): + return event(**{"path": ".github/workflows/nightly.yml", "event": "push", "head_branch": "main", + "pull_requests": [], **overrides}) + + +def nightly_run(*, queued_at=15, started_at=None, refused_at=None): + """nightly.yml: decide, then the app build on the trusted pool beside a Blacksmith helper build.""" + def jobs(seconds): + found = [job("decide", status="completed", labels=[BLACKSMITH])] + if seconds < queued_at: + return found + if refused_at is not None and seconds >= refused_at: + app = refused_job("build-nightly-app", labels=(TRUSTED,)) + else: + started = started_at is not None and seconds >= started_at + app = job("build-nightly-app", labels=[TRUSTED], created=queued_at, + status="in_progress" if started else "queued", runner="cmux15-glaeda" if started else "") + if started: + app["started_at"] = stamp(started_at) + return found + [app, job("build-nightly-ghostty-cli-helper", labels=["blacksmith-6vcpu-macos-15"], + status="in_progress", runner="bs")] + return jobs + + +class NightlyAPI(FakeAPI): + def __init__(self, *args, newer=(), **kwargs): + super().__init__(*args, **kwargs) + self.newer = list(newer) + + def newer_unfinished_runs(self, path, run_id, branch): + self.calls.append(f"newer:{branch}") + return self.newer + + +class Nightly(unittest.TestCase): + """nightly.yml's app build takes the trusted owned pool on attempt 1 of main's push and schedule runs.""" + + def test_only_attempt_1_of_main_s_own_push_or_schedule_run(self): + for kind in ("push", "schedule"): + target = rescue.target_from_event(nightly_event(event=kind), "manaflow-ai/cmux") + self.assertEqual((target.nightly, target.side, target.main, target.e2e, target.pr_number), + (True, True, False, False, 0), kind) + cases = { + "dispatch": nightly_event(event="workflow_dispatch"), + "release candidate branch": nightly_event(head_branch="rc/1.2"), + "fork head": nightly_event(head_repository={"full_name": "someone/cmux"}), + "attempt 2": nightly_event(run_attempt=2), + } + for why, payload in cases.items(): + self.assertIsInstance(rescue.target_from_event(payload, "manaflow-ai/cmux"), str, why) + + def test_the_trusted_pool_is_an_owned_label_only_here(self): + self.assertEqual(rescue.job_pool({"labels": [TRUSTED]}), TRUSTED) + self.assertEqual(rescue.job_pool({"labels": ["glaeda-root-trusted-std-xcode-26.6"]}), + "glaeda-root-trusted-std-xcode-26.6") + self.assertIsNone(rescue.job_pool({"labels": ["glaeda-trusted"]})) + # The pickers never hand it out: it is not a pull request pool. + self.assertFalse(rescue.persistent(TRUSTED)) + + def test_newer_unfinished_runs_reads_one_page_of_main_s_nightly_runs(self): + api = rescue.GitHub("token", "manaflow-ai/cmux") + seen = [] + runs = [{"id": RUN_ID + 2, "status": "queued"}, {"id": RUN_ID + 1, "status": "completed"}, + {"id": RUN_ID, "status": "in_progress"}, {"id": RUN_ID - 1, "status": "in_progress"}, + {"id": RUN_ID + 3, "status": "in_progress"}] + api.request = lambda method, path, **_: seen.append((method, path)) or {"workflow_runs": runs} + self.assertEqual(api.newer_unfinished_runs(rescue.NIGHTLY_WORKFLOW_PATH, RUN_ID, "main"), + [RUN_ID + 2, RUN_ID + 3]) + self.assertEqual(seen, [("GET", "/actions/workflows/nightly.yml/runs?branch=main&per_page=20")]) + + def test_a_run_with_no_trusted_job_stops_when_it_finishes(self): + clock = Clock() + api = NightlyAPI(clock, lambda s: [job("decide", status="completed", labels=[BLACKSMITH])]) + code, summary = run_main(api, clock, payload=nightly_event()) + self.assertEqual(code, 0) + self.assertIn("no job of the run asked for a persistent pool", summary) + + def test_the_watch_ends_once_a_trusted_mini_takes_the_build(self): + clock = Clock() + api = NightlyAPI(clock, nightly_run(started_at=30)) + code, summary = run_main(api, clock, payload=nightly_event()) + self.assertEqual(code, 0) + self.assertIn("main's nightly build", summary) + self.assertIn("the fleet accepted", summary) + self.assertNotIn("cancel", api.calls) + + def test_the_build_may_wait_one_round_behind_a_seed(self): + clock = Clock() + api = NightlyAPI(clock, nightly_run(started_at=15 + 600)) + code, summary = run_main(api, clock, payload=nightly_event(), env_extra={"RESCUE_SECONDS": "30"}) + self.assertEqual(code, 0) + self.assertIn(f"budget {30 + rescue.QUEUE_ROUND_SECONDS}s", summary) + self.assertNotIn("cancel", api.calls) + + def test_a_stuck_build_moves_to_blacksmith_keeping_what_passed(self): + clock = Clock() + api = NightlyAPI(clock, nightly_run()) + code, summary = run_main(api, clock, payload=nightly_event(), env_extra={"RESCUE_SECONDS": "30"}) + self.assertEqual(code, 0) + self.assertIn("cancel", api.calls) + self.assertEqual(api.calls[-1], "rerun-failed") + self.assertNotIn("rerun", api.calls) + self.assertNotIn("pull", api.calls) + self.assertIn(f"queued on {TRUSTED}", summary) + + def test_a_stuck_build_behind_which_a_newer_nightly_waits_is_cancelled_not_rerun(self): + # nightly.yml never cancels in progress: the newer run is pending behind + # this one, and a re-run would join the group and cancel it. + clock = Clock() + api = NightlyAPI(clock, nightly_run(), newer=[RUN_ID + 7]) + _, summary = run_main(api, clock, payload=nightly_event(), env_extra={"RESCUE_SECONDS": "30"}) + self.assertIn("cancel", api.calls) + self.assertNotIn("rerun-failed", api.calls) + self.assertIn("a newer nightly run on main", summary) + + def test_a_refused_build_is_rerun_once_the_run_finishes(self): + clock = Clock() + api = NightlyAPI(clock, nightly_run(refused_at=60), finished=lambda s: s >= 200) + code, summary = run_main(api, clock, payload=nightly_event()) + self.assertEqual(code, 0) + self.assertIn("waiting for the rest of the run to finish", summary) + self.assertNotIn("cancel", api.calls) + self.assertEqual(api.calls[-1], "rerun-failed") + self.assertGreaterEqual(clock.seconds, 200) + # A newer nightly run builds main's newer HEAD instead: left as it is. + clock = Clock() + api = NightlyAPI(clock, nightly_run(refused_at=60), finished=lambda s: s >= 200, newer=[RUN_ID + 1]) + _, summary = run_main(api, clock, payload=nightly_event()) + self.assertNotIn("rerun-failed", api.calls) + self.assertIn("not rescued", summary) + + IOS_SIM = "glaeda-ios-sim" @@ -1361,14 +1496,20 @@ def test_runs_when_dispatched_or_for_a_screenshots_or_side_lane_run(self): self.assertEqual(triggers["workflow_run"]["workflows"][0], "iOS App Store screenshots") self.assertNotIn("CI", triggers["workflow_run"]["workflows"]) paths = self.doc["env"]["SOURCE_WORKFLOW_PATHS"].split() - self.assertEqual(set(paths), {rescue.IOS_SCREENSHOTS_WORKFLOW_PATH, *rescue.SIDE_WORKFLOW_PATHS}) + self.assertEqual(set(paths), {rescue.IOS_SCREENSHOTS_WORKFLOW_PATH, *rescue.SIDE_WORKFLOW_PATHS, + rescue.NIGHTLY_WORKFLOW_PATH}) + self.assertIn("Nightly macOS build", triggers["workflow_run"]["workflows"]) condition = self.doc["jobs"]["rescue"]["if"] for part in ("vars.CI_PR_POOL_OWNED == '1'", "(vars.CI_OWNED_POOL_RESCUE || '1') != '0'", "(github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || " "(github.event.workflow_run.event == 'pull_request' && " "startsWith(vars.CI_SIDE_LANE_RUNNER, 'glaeda-side-') || " "github.event.workflow_run.path == '.github/workflows/ios-screenshots.yml' && " - "github.event.workflow_run.event == 'workflow_dispatch') && " + "github.event.workflow_run.event == 'workflow_dispatch' || " + "github.event.workflow_run.path == '.github/workflows/nightly.yml' && " + "(github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule') && " + "github.event.workflow_run.head_branch == 'main' && " + "startsWith(vars.CI_SEED_TRUSTED_POOL, 'glaeda-trusted-')) && " "github.event.workflow_run.head_repository.full_name == github.repository && " "github.event.workflow_run.run_attempt == 1)"): self.assertIn(part, condition) diff --git a/tests/test_nightly_universal_build.sh b/tests/test_nightly_universal_build.sh index d0b0b2142161..01235e47a0be 100644 --- a/tests/test_nightly_universal_build.sh +++ b/tests/test_nightly_universal_build.sh @@ -174,7 +174,7 @@ if ! awk ' fi if ! awk -v helper_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-15' || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-6vcpu-macos-15' || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}" \ - -v app_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-12vcpu-macos-26' || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26_LARGE || 'blacksmith-12vcpu-macos-26') }}" ' + -v app_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-12vcpu-macos-26' || github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/main' && vars.CI_PR_POOL_OWNED == '1' && vars.CI_SEED_TRUSTED_POOL || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26_LARGE || 'blacksmith-12vcpu-macos-26') }}" ' /^ build-nightly-ghostty-cli-helper:/ { job="helper"; next } /^ build-nightly-app:/ { job="app"; next } /^ build-sign-notarize-nightly:/ { job="publish"; next } diff --git a/tests/test_runner_label_policy.py b/tests/test_runner_label_policy.py index cf34693274a3..d55849f465d1 100644 --- a/tests/test_runner_label_policy.py +++ b/tests/test_runner_label_policy.py @@ -237,6 +237,29 @@ def test_only_an_owned_side_label_is_allowed_beyond_the_policy(self) -> None: self.assertTrue(drifted_runner_variables({"MACOS_RUNNER_PR": "glaeda-side-std-xcode-26.6"})) +class TrustedPoolVariable(unittest.TestCase): + def test_only_a_trusted_owned_label_is_allowed(self) -> None: + # CI_SEED_TRUSTED_POOL carries the ci-cache-writer seeds and nightly.yml's + # app build (attempt 1 of main's push and schedule runs): only the + # trusted pool, never a pull request pool. + self.assertEqual(drifted_runner_variables({"CI_SEED_TRUSTED_POOL": "glaeda-trusted-std-xcode-26.6"}), []) + self.assertEqual(drifted_runner_variables({"CI_SEED_TRUSTED_POOL": ""}), []) + for label in ("glaeda-std-xcode-26.6", "glaeda-side-std-xcode-26.6", "glaeda-trusted-nonsense", + "blacksmith-6vcpu-macos-26"): + with self.subTest(label=label): + self.assertEqual( + [name for name, _, _ in drifted_runner_variables({"CI_SEED_TRUSTED_POOL": label})], + ["CI_SEED_TRUSTED_POOL"], + ) + # Other runner variables still may not name it. + self.assertTrue(drifted_runner_variables({"MACOS_RUNNER_26_LARGE": "glaeda-trusted-std-xcode-26.6"})) + + def test_the_reports_check_it(self) -> None: + for workflow in ("ci-health-report.yml", "ci-repo-variables.yml"): + text = (ROOT / ".github" / "workflows" / workflow).read_text(encoding="utf-8") + self.assertIn("CI_SEED_TRUSTED_POOL=${{ vars.CI_SEED_TRUSTED_POOL }}", text, workflow) + + class OwnedPoolLabels(unittest.TestCase): OWNED = ("glaeda-std-xcode-26.6", "glaeda-light-xcode-26.6", "glaeda-xl-xcode-26") From aee716b1880c32436034325d7d4c7c30409dbebb Mon Sep 17 00:00:00 2001 From: teamleaderleo Date: Sat, 26 Sep 2026 09:41:22 -0400 Subject: [PATCH 2/2] ci: send the nightly app build to cmux15's trusted runner only Both trusted minis carry glaeda-trusted-std-xcode-26.6, but cmuxs-mac-mini-6 also runs the team dev-build worker as the same user, so the shipped build should never land there. Seeding keeps both, so CI_SEED_TRUSTED_POOL is unchanged. build-nightly-app now asks for ["", ""]: the trusted pool and one runner's own glaeda-runner- label (glaeda-cmux-runner's runner_label()). Either variable empty means Blacksmith, so this is inert until cmux15's runner is re-registered with that label and the variable is set. - runner_label_policy.py: CI_NIGHTLY_TRUSTED_RUNNER may only be a lowercase glaeda-runner-* label (also safe inside the JSON runs-on). - ci-health-report.yml / ci-repo-variables.yml: report it. - ci-owned-pool-rescue.yml: watch nightly runs only while both are set. - Docs and tests follow. Co-Authored-By: Claude Opus 5.5 --- .github/workflows/ci-health-report.yml | 1 + .github/workflows/ci-owned-pool-rescue.yml | 9 +++--- .github/workflows/ci-repo-variables.yml | 1 + .github/workflows/nightly.yml | 32 ++++++++++++---------- docs/ci-runners.md | 12 ++++++-- docs/ci/mac-fleet.md | 4 +-- scripts/ci/runner_label_policy.py | 30 ++++++++++++++++++-- tests/test_ci_owned_pool_rescue.py | 5 +++- tests/test_nightly_universal_build.sh | 2 +- tests/test_runner_label_policy.py | 22 +++++++++++++++ 10 files changed, 90 insertions(+), 28 deletions(-) diff --git a/.github/workflows/ci-health-report.yml b/.github/workflows/ci-health-report.yml index 1401320e206b..5c42844d4573 100644 --- a/.github/workflows/ci-health-report.yml +++ b/.github/workflows/ci-health-report.yml @@ -87,6 +87,7 @@ jobs: # every read; `macos-15` is an approved label either way. # CI_PR_POOL_ORDER is the one list that may also name owned pools. CMUX_CI_RUNNER_VARIABLES: | + CI_NIGHTLY_TRUSTED_RUNNER=${{ vars.CI_NIGHTLY_TRUSTED_RUNNER }} CI_PR_POOL_ORDER=${{ vars.CI_PR_POOL_ORDER }} CI_SEED_TRUSTED_POOL=${{ vars.CI_SEED_TRUSTED_POOL }} CI_SIDE_LANE_RUNNER=${{ vars.CI_SIDE_LANE_RUNNER }} diff --git a/.github/workflows/ci-owned-pool-rescue.yml b/.github/workflows/ci-owned-pool-rescue.yml index 8d7d4f94e178..4c7f415924ce 100644 --- a/.github/workflows/ci-owned-pool-rescue.yml +++ b/.github/workflows/ci-owned-pool-rescue.yml @@ -38,9 +38,10 @@ run-name: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow # names a glaeda-side-* label, every same-repository attempt-1 pull request # run of theirs is on the fleet, so a dispatch would save no run. # - nightly.yml: build-nightly-app has no picker either. While -# vars.CI_SEED_TRUSTED_POOL names a glaeda-trusted-* pool, attempt 1 of -# every push or schedule run on main asks for it, so every such run is -# watched (owned_pool_rescue.NIGHTLY_WORKFLOW_PATH). +# vars.CI_SEED_TRUSTED_POOL names a glaeda-trusted-* pool and +# vars.CI_NIGHTLY_TRUSTED_RUNNER a glaeda-runner-* label, attempt 1 of every +# push or schedule run on main asks for that one trusted mini, so every such +# run is watched (owned_pool_rescue.NIGHTLY_WORKFLOW_PATH). # It needs actions: write, and its code comes from main. It runs whenever # owned pools are on (CI_PR_POOL_OWNED is 1), because a run on an owned pool # has no other way off it; CI_OWNED_POOL_RESCUE=0 turns it off. The switch @@ -106,7 +107,7 @@ concurrency: jobs: rescue: name: ${{ (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' && !inputs.run_id) && 'Sweep runs on persistent pools' || 'Rescue a run stuck on a persistent pool' }} - if: ${{ vars.CI_PR_POOL_OWNED == '1' && (vars.CI_OWNED_POOL_RESCUE || '1') != '0' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event.workflow_run.event == 'pull_request' && startsWith(vars.CI_SIDE_LANE_RUNNER, 'glaeda-side-') || github.event.workflow_run.path == '.github/workflows/ios-screenshots.yml' && github.event.workflow_run.event == 'workflow_dispatch' || github.event.workflow_run.path == '.github/workflows/nightly.yml' && (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule') && github.event.workflow_run.head_branch == 'main' && startsWith(vars.CI_SEED_TRUSTED_POOL, 'glaeda-trusted-')) && github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.run_attempt == 1) }} + if: ${{ vars.CI_PR_POOL_OWNED == '1' && (vars.CI_OWNED_POOL_RESCUE || '1') != '0' && (github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' || (github.event.workflow_run.event == 'pull_request' && startsWith(vars.CI_SIDE_LANE_RUNNER, 'glaeda-side-') || github.event.workflow_run.path == '.github/workflows/ios-screenshots.yml' && github.event.workflow_run.event == 'workflow_dispatch' || github.event.workflow_run.path == '.github/workflows/nightly.yml' && (github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule') && github.event.workflow_run.head_branch == 'main' && startsWith(vars.CI_SEED_TRUSTED_POOL, 'glaeda-trusted-') && startsWith(vars.CI_NIGHTLY_TRUSTED_RUNNER, 'glaeda-runner-')) && github.event.workflow_run.head_repository.full_name == github.repository && github.event.workflow_run.run_attempt == 1) }} runs-on: ubuntu-24.04 # github-hosted-required: polls the Actions API; keeps CI's Linux pool free # A sweeper adopts runs for 300 minutes and gives a rescue 25 more to # settle (SWEEP_SECONDS, RESCUE_GRACE_SECONDS). A single run's watch is diff --git a/.github/workflows/ci-repo-variables.yml b/.github/workflows/ci-repo-variables.yml index 699218e55d9f..fccd842e48db 100644 --- a/.github/workflows/ci-repo-variables.yml +++ b/.github/workflows/ci-repo-variables.yml @@ -53,6 +53,7 @@ jobs: CI_OWNED_POOL_SLOTS: ${{ vars.CI_OWNED_POOL_SLOTS }} CMUX_CI_XCODE_APP_PR: ${{ vars.CMUX_CI_XCODE_APP_PR }} CMUX_CI_RUNNER_VARIABLES: | + CI_NIGHTLY_TRUSTED_RUNNER=${{ vars.CI_NIGHTLY_TRUSTED_RUNNER }} CI_PR_POOL_ORDER=${{ vars.CI_PR_POOL_ORDER }} CI_SEED_TRUSTED_POOL=${{ vars.CI_SEED_TRUSTED_POOL }} CI_SIDE_LANE_RUNNER=${{ vars.CI_SIDE_LANE_RUNNER }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6538b9c1da40..0951b862098c 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -903,20 +903,24 @@ jobs: daemon_build: ${{ steps.remote_daemon.outputs.build }} daemon_version: ${{ steps.remote_daemon.outputs.version }} if: needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') - # Owned minis first. Attempt 1 of a push or schedule run on main takes the - # trusted owned pool (vars.CI_SEED_TRUSTED_POOL, - # glaeda-trusted--xcode-) while CI_PR_POOL_OWNED is 1: - # minis with no pull request runners, whose job-started hook admits only - # main's own push and schedule jobs. Not the pull request pool: this job - # holds the ci-cache-writer R2 keys and the Sentry token, and its app is - # what build-sign-notarize-nightly signs and ships. It is the pool - # seed-derived-data.yml already seeds from with the same credentials, on - # the Xcode 26.6 that CMUX_CI_XCODE_APP_MACOS_26 pins. + # Owned minis first. Attempt 1 of a push or schedule run on main takes one + # trusted owned mini while CI_PR_POOL_OWNED is 1 and both variables below + # are set: runs-on asks for the trusted pool label (vars.CI_SEED_TRUSTED_POOL, + # glaeda-trusted--xcode-) and that runner's own label + # (vars.CI_NIGHTLY_TRUSTED_RUNNER, glaeda-runner-), so only a + # runner carrying both can take it. Today that is cmux15-glaeda: the + # trusted mini with no pull request runners and no dev-build worker. The + # other trusted mini (cmuxs-mac-mini-6) builds team dev builds as the same + # user, so it keeps seeding but never builds the shipped app. Its + # job-started hook admits only main's own push and schedule jobs. Not the + # pull request pool: this job holds the ci-cache-writer R2 keys and the + # Sentry token, and its app is what build-sign-notarize-nightly signs and + # ships. runner_label_policy.py holds both variables to those shapes. # ci-owned-pool-rescue.yml watches the run (owned_pool_rescue.py, - # NIGHTLY_WORKFLOW_PATH): when no trusted mini takes the job within its - # budget, or the mini refuses it at job start, the failed jobs are re-run, - # and every later attempt takes Blacksmith below. Signing stays on - # Blacksmith (build-sign-notarize-nightly). + # NIGHTLY_WORKFLOW_PATH): when the mini does not take the job within its + # budget, or refuses it at job start, the failed jobs are re-run, and + # every later attempt takes Blacksmith below. Either variable empty is + # Blacksmith. Signing stays on Blacksmith (build-sign-notarize-nightly). # Blacksmith runs share the cache warmer's and stable release lane's image # and toolchain, which is what the compilation cache is keyed on — OS, arch # and toolchain, never instance size, which is deliberately larger here. @@ -924,7 +928,7 @@ jobs: # branch dogfood always uses the dedicated Blacksmith image. A # repository-wide runner override may point at a slower shared builder, # which defeats the purpose of the one-architecture path. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-12vcpu-macos-26' || github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/main' && vars.CI_PR_POOL_OWNED == '1' && vars.CI_SEED_TRUSTED_POOL || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26_LARGE || 'blacksmith-12vcpu-macos-26') }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || needs.decide.outputs.fast_build != 'true' && github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/main' && vars.CI_PR_POOL_OWNED == '1' && vars.CI_SEED_TRUSTED_POOL != '' && vars.CI_NIGHTLY_TRUSTED_RUNNER != '' && fromJSON(format('["{0}", "{1}"]', vars.CI_SEED_TRUSTED_POOL, vars.CI_NIGHTLY_TRUSTED_RUNNER)) || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-12vcpu-macos-26' || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26_LARGE || 'blacksmith-12vcpu-macos-26') }} environment: ${{ github.ref == 'refs/heads/main' && 'ci-cache-writer' || '' }} # The Blacksmith cache is scoped per branch and also drops main's own # entry (runs 35179030871 and 35182663752 restored nothing and were diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 6c122bd174d1..c2d316238d3a 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -655,8 +655,14 @@ The trusted pool (`vars.CI_SEED_TRUSTED_POOL`, `glaeda-trusted--xcode-`) is the owned home for main's own cache writers and builds: minis with no pull request runners, whose job-started hook admits only a push or schedule run on main. The DerivedData -seed and the nightly app compile take it first; Blacksmith is the fallback. -`runner_label_policy.py` refuses any other value for the variable. Signing and +seed takes it on every main push. The nightly app compile takes one runner of +it first, `vars.CI_NIGHTLY_TRUSTED_RUNNER` (`glaeda-runner-cmux15-glaeda`): +runs-on asks for the pool label and that runner's own label together, so it +never lands on cmuxs-mac-mini-6, whose dev-build worker builds team code as the +same user. Either variable empty sends it to Blacksmith, which is also its +fallback. `runner_label_policy.py` refuses any other shape for either +variable. glaeda classes the job `isolated` (teamleaderleo/glaeda#1287), so it +never holds the canonical root a seed on the same mini waits for. Signing and notarization are not on it: no signing run on an owned Mac has been proven, and the retired self-hosted fleet failed `codesign` with `errSecInternalComponent` (#6264). @@ -676,7 +682,7 @@ and the retired self-hosted fleet failed `codesign` with | `relay-tls` `system-keychain` | Blacksmith | edits the System keychain trust store | | `plain-paste-worker`, `ci-macos-compat`, `seed-swiftpm-manifests`, release and nightly Ghostty helpers | Blacksmith macOS 15 / 14 | an OS or SDK the minis lack | | `release.yml`, nightly sign/notarize, `ios-testflight`, `ios-app-store`, `ios-appstore-upload` | Blacksmith | signing and store secrets; signing on an owned Mac is unproven | -| `nightly.yml` `build-nightly-app` | trusted owned pool (`CI_SEED_TRUSTED_POOL`) on attempt 1 of main's push and schedule runs; Blacksmith 12 vCPU otherwise, for `rc/**`, dispatches and fast dogfood, and on every re-run | ci-owned-pool-rescue.yml watches it (`NIGHTLY_WORKFLOW_PATH`): stuck one queue round past `CI_OWNED_POOL_RESCUE_SECONDS`, or refused, its failed jobs re-run on Blacksmith. Its compilation cache keys its own lineage (the mini's workspace path) | +| `nightly.yml` `build-nightly-app` | one trusted runner (`CI_SEED_TRUSTED_POOL` plus `CI_NIGHTLY_TRUSTED_RUNNER`, cmux15) on attempt 1 of main's push and schedule runs; Blacksmith 12 vCPU otherwise, for `rc/**`, dispatches and fast dogfood, and on every re-run | ci-owned-pool-rescue.yml watches it (`NIGHTLY_WORKFLOW_PATH`): stuck one queue round past `CI_OWNED_POOL_RESCUE_SECONDS`, or refused, its failed jobs re-run on Blacksmith. Its compilation cache keys its own lineage (the mini's workspace path) | | `seed-derived-data` trusted pool | trusted owned pool, push to main | the minis' own j14 seed | | `nightly.yml` `refresh-compilation-cache`, `refresh-test-compilation-cache`, `seed-derived-data` Blacksmith pools | Blacksmith | they seed Blacksmith's own lanes: the release cache the nightly fallback restores, and the pull request admission seeds for each Blacksmith pool | | `build-ghosttykit`, `cmux-tui-build-package` (artifacts, nightly, release), `relay-publish-npm` | Blacksmith | publish with R2 or release secrets | diff --git a/docs/ci/mac-fleet.md b/docs/ci/mac-fleet.md index 9f1859b097de..fc6758ed576f 100644 --- a/docs/ci/mac-fleet.md +++ b/docs/ci/mac-fleet.md @@ -216,8 +216,8 @@ them (section 5). job gets a fresh VM and an Aqua login session. A shared mini cannot give it either. (The isolated Tart pool that once offered this was retired on 2026-09-25; see `ci-runners.md`.) -4. **Nightly app compile** - `build-nightly-app` takes the trusted owned pool - (`CI_SEED_TRUSTED_POOL`) first on main's push and schedule runs, with +4. **Nightly app compile** - `build-nightly-app` takes the trusted runner on cmux15 + (`CI_SEED_TRUSTED_POOL` plus `CI_NIGHTLY_TRUSTED_RUNNER`) first on main's push and schedule runs, with Blacksmith as the fallback through ci-owned-pool-rescue.yml (`docs/ci-runners.md`). Signing and notarization stay on Blacksmith. 5. **`release-build`, signing, notarization, TestFlight** - never. diff --git a/scripts/ci/runner_label_policy.py b/scripts/ci/runner_label_policy.py index a639c5788057..7af468b54f72 100644 --- a/scripts/ci/runner_label_policy.py +++ b/scripts/ci/runner_label_policy.py @@ -162,9 +162,9 @@ def side_lane_reason(label: str) -> str | None: def trusted_pool_reason(label: str) -> str | None: """Why CI_SEED_TRUSTED_POOL is not allowed, or None when it is fine. - seed-derived-data.yml seeds on this pool and nightly.yml's app build reads - it as its attempt-1 runs-on, both with the ci-cache-writer R2 keys, and the - app build's product is signed and shipped. So it may name only the trusted + seed-derived-data.yml seeds on this pool and nightly.yml's app build asks + for it beside CI_NIGHTLY_TRUSTED_RUNNER on attempt 1, both with the + ci-cache-writer R2 keys, and the app build's product is signed and shipped. So it may name only the trusted owned pool, glaeda-trusted--xcode-: minis with no pull request runners, whose hook admits only main's push and schedule jobs. A pull request pool label (glaeda-std-...) would put those credentials and @@ -180,6 +180,28 @@ def trusted_pool_reason(label: str) -> str | None: f"({TRUSTED_POOL_PREFIX}-xcode-)") +NIGHTLY_RUNNER_VARIABLE = "CI_NIGHTLY_TRUSTED_RUNNER" +# glaeda-cmux-runner's runner_label(): the static label only the runner +# registered under that name carries (lowercase, [a-z0-9._-]). +RUNNER_NAME_LABEL = re.compile(r"glaeda-runner-[a-z0-9][a-z0-9._-]*") + + +def nightly_runner_reason(label: str) -> str | None: + """Why CI_NIGHTLY_TRUSTED_RUNNER is not allowed, or None when it is fine. + + nightly.yml's app build asks for this label together with + CI_SEED_TRUSTED_POOL (`["", ""]`), so it lands only on + the one trusted runner both name (cmux15-glaeda): a runner's own + glaeda-runner- label. A pull request runner's name label can match + no runner here, since those runners never carry the trusted pool label. + The shape also keeps the value safe inside the JSON runs-on array. Empty + is fine (Blacksmith). + """ + if not label or RUNNER_NAME_LABEL.fullmatch(label): + return None + return f"`{label}` is not a runner name label (glaeda-runner-, lowercase)" + + def forbidden_reason(label: str) -> str | None: """Why this runner label is not allowed, or None when it is fine. @@ -223,6 +245,8 @@ def drifted_runner_variables( reason = side_lane_reason(value.strip()) elif name == TRUSTED_POOL_VARIABLE: reason = trusted_pool_reason(value.strip()) + elif name == NIGHTLY_RUNNER_VARIABLE: + reason = nightly_runner_reason(value.strip()) elif "RUNNER" not in name: continue else: diff --git a/tests/test_ci_owned_pool_rescue.py b/tests/test_ci_owned_pool_rescue.py index 1165741d37c8..5a25ec828bf1 100644 --- a/tests/test_ci_owned_pool_rescue.py +++ b/tests/test_ci_owned_pool_rescue.py @@ -1043,6 +1043,8 @@ def test_only_attempt_1_of_main_s_own_push_or_schedule_run(self): def test_the_trusted_pool_is_an_owned_label_only_here(self): self.assertEqual(rescue.job_pool({"labels": [TRUSTED]}), TRUSTED) + # nightly.yml asks for the pool and one runner's own label together. + self.assertEqual(rescue.job_pool({"labels": [TRUSTED, "glaeda-runner-cmux15-glaeda"]}), TRUSTED) self.assertEqual(rescue.job_pool({"labels": ["glaeda-root-trusted-std-xcode-26.6"]}), "glaeda-root-trusted-std-xcode-26.6") self.assertIsNone(rescue.job_pool({"labels": ["glaeda-trusted"]})) @@ -1509,7 +1511,8 @@ def test_runs_when_dispatched_or_for_a_screenshots_or_side_lane_run(self): "github.event.workflow_run.path == '.github/workflows/nightly.yml' && " "(github.event.workflow_run.event == 'push' || github.event.workflow_run.event == 'schedule') && " "github.event.workflow_run.head_branch == 'main' && " - "startsWith(vars.CI_SEED_TRUSTED_POOL, 'glaeda-trusted-')) && " + "startsWith(vars.CI_SEED_TRUSTED_POOL, 'glaeda-trusted-') && " + "startsWith(vars.CI_NIGHTLY_TRUSTED_RUNNER, 'glaeda-runner-')) && " "github.event.workflow_run.head_repository.full_name == github.repository && " "github.event.workflow_run.run_attempt == 1)"): self.assertIn(part, condition) diff --git a/tests/test_nightly_universal_build.sh b/tests/test_nightly_universal_build.sh index 01235e47a0be..1696ea241181 100644 --- a/tests/test_nightly_universal_build.sh +++ b/tests/test_nightly_universal_build.sh @@ -174,7 +174,7 @@ if ! awk ' fi if ! awk -v helper_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-15' || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-6vcpu-macos-15' || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}" \ - -v app_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-12vcpu-macos-26' || github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/main' && vars.CI_PR_POOL_OWNED == '1' && vars.CI_SEED_TRUSTED_POOL || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26_LARGE || 'blacksmith-12vcpu-macos-26') }}" ' + -v app_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || needs.decide.outputs.fast_build != 'true' && github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'schedule') && github.ref == 'refs/heads/main' && vars.CI_PR_POOL_OWNED == '1' && vars.CI_SEED_TRUSTED_POOL != '' && vars.CI_NIGHTLY_TRUSTED_RUNNER != '' && fromJSON(format('[\"{0}\", \"{1}\"]', vars.CI_SEED_TRUSTED_POOL, vars.CI_NIGHTLY_TRUSTED_RUNNER)) || (needs.decide.outputs.fast_build == 'true' && 'blacksmith-12vcpu-macos-26' || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_26_LARGE || 'blacksmith-12vcpu-macos-26') }}" ' /^ build-nightly-ghostty-cli-helper:/ { job="helper"; next } /^ build-nightly-app:/ { job="app"; next } /^ build-sign-notarize-nightly:/ { job="publish"; next } diff --git a/tests/test_runner_label_policy.py b/tests/test_runner_label_policy.py index d55849f465d1..6927bf792413 100644 --- a/tests/test_runner_label_policy.py +++ b/tests/test_runner_label_policy.py @@ -258,6 +258,28 @@ def test_the_reports_check_it(self) -> None: for workflow in ("ci-health-report.yml", "ci-repo-variables.yml"): text = (ROOT / ".github" / "workflows" / workflow).read_text(encoding="utf-8") self.assertIn("CI_SEED_TRUSTED_POOL=${{ vars.CI_SEED_TRUSTED_POOL }}", text, workflow) + self.assertIn("CI_NIGHTLY_TRUSTED_RUNNER=${{ vars.CI_NIGHTLY_TRUSTED_RUNNER }}", text, workflow) + + +class NightlyRunnerVariable(unittest.TestCase): + def test_only_a_runner_name_label_is_allowed(self) -> None: + # nightly.yml asks for ["", ""]: one trusted runner. + self.assertEqual(drifted_runner_variables({"CI_NIGHTLY_TRUSTED_RUNNER": "glaeda-runner-cmux15-glaeda"}), []) + self.assertEqual(drifted_runner_variables({"CI_NIGHTLY_TRUSTED_RUNNER": ""}), []) + for label in ("glaeda-trusted-std-xcode-26.6", "glaeda-runner-", "glaeda-runner-CMUX15", + 'glaeda-runner-x", "self-hosted', "blacksmith-12vcpu-macos-26", "cmux15-glaeda"): + with self.subTest(label=label): + self.assertEqual( + [name for name, _, _ in drifted_runner_variables({"CI_NIGHTLY_TRUSTED_RUNNER": label})], + ["CI_NIGHTLY_TRUSTED_RUNNER"], + ) + self.assertTrue(drifted_runner_variables({"MACOS_RUNNER_26_LARGE": "glaeda-runner-cmux15-glaeda"})) + + def test_nightly_asks_for_the_trusted_pool_and_the_runner_together(self) -> None: + text = (ROOT / ".github" / "workflows" / "nightly.yml").read_text(encoding="utf-8") + self.assertIn("fromJSON(format('[\"{0}\", \"{1}\"]', vars.CI_SEED_TRUSTED_POOL, " + "vars.CI_NIGHTLY_TRUSTED_RUNNER))", text) + self.assertIn("vars.CI_SEED_TRUSTED_POOL != '' && vars.CI_NIGHTLY_TRUSTED_RUNNER != ''", text) class OwnedPoolLabels(unittest.TestCase):