diff --git a/.github/workflows/ci-cloud-overflow-probe.yml b/.github/workflows/ci-cloud-overflow-probe.yml index c9f3ef055abe..9fd97c09ac4a 100644 --- a/.github/workflows/ci-cloud-overflow-probe.yml +++ b/.github/workflows/ci-cloud-overflow-probe.yml @@ -5,18 +5,19 @@ name: CI cloud overflow probe # 150 runs queued for two hours), the runner variables that send overflow # there (LINUX_RUNNER, CI_TRUSTED_RUNNER, MACOS_RUNNER_*, CI_PAID_MACOS_OVERFLOW) are pointed at # the owned pools and another Blacksmith Linux size, and pointed back once it serves -# again. One probe, one switch: `probe` asks for a Blacksmith label and only -# prints a line; `watch`, on another Blacksmith pool, waits for it. +# again. One switch: `probe` asks for each Blacksmith Linux size compiles run +# on and only prints a line; `watch`, on Blacksmith macOS, waits for them. # scripts/ci/cloud_overflow_switch.py has the rules: # -# - The probe starts within CI_CLOUD_PROBE_MINUTES (default 5) of being +# - Every probe starts within CI_CLOUD_PROBE_MINUTES (default 5) of being # queued: if this switch had turned overflow off, every variable it changed # is restored from the CI_CLOUD_OVERFLOW_SAVED record (unless someone # changed it since) and the record is deleted. -# - It does not: the record is written (each variable's value before, its +# - None does: the record is written (each variable's value before, its # failover, when and why), the variables are pointed at their failovers, # runs already stuck on a Blacksmith label are force-cancelled and re-run, -# and this run force-cancels itself, since its probe would stay queued. +# and this run force-cancels itself, since its probes would stay queued. +# - Some start and some do not: nothing changes either way. # # The lane switches (CI_PR_POOL_OWNED, CI_E2E_OWNED_UI, CI_IOS_OWNED, ...) are # never touched: minis-first routing does not change, only where overflow @@ -52,12 +53,18 @@ concurrency: jobs: probe: - name: Blacksmith probe + # One probe per Blacksmith Linux size compiles run on + # (cloud_overflow_switch.PROBE_LABELS). Overflow goes off only when none of + # them starts, and back on only when all start: a single probed label is a pin, and on 2026-10-28 the 4 vCPU + # label alone stalling flipped the whole fleet four times. A drill probes + # only its own label. + name: Blacksmith probe (${{ matrix.label }}) if: github.repository_owner == 'manaflow-ai' - # The Blacksmith label overflow uses, never the variable's failover: - # while overflow is off LINUX_RUNNER names the failover size, so the record - # keeps the label to probe (cloud_overflow_switch.probe_label()). - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'workflow_dispatch' && inputs.probe_label != '' && inputs.probe_label || startsWith(fromJSON(vars.CI_CLOUD_OVERFLOW_SAVED || '{"probe":""}').probe, 'blacksmith-') && fromJSON(vars.CI_CLOUD_OVERFLOW_SAVED).probe || startsWith(vars.LINUX_RUNNER, 'blacksmith-') && vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + strategy: + fail-fast: false + matrix: + label: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.probe_label != '' && format('["{0}"]', inputs.probe_label) || '["blacksmith-32vcpu-ubuntu-2404", "blacksmith-16vcpu-ubuntu-2404"]') }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || matrix.label }} timeout-minutes: 2 permissions: {} steps: @@ -72,13 +79,10 @@ jobs: # (19:22Z to 20:56Z) on hosted ubuntu-24.04, which the switch's own # failover had filled; this run held the concurrency group meanwhile, so # every later scheduled run was evicted with no jobs and nothing decided. - # While overflow is on, the watch takes a Blacksmith Linux size the probe - # does not ask for (the switch's Linux failover). While it is off, Linux - # overflow is on that size, so the watch takes Blacksmith macOS: it can - # still put overflow back however the Linux sizes fare. The same holds - # whenever the probe itself asks for 16 vCPU (LINUX_RUNNER or a drill - # naming it), so the watch never queues behind its own probe. - runs-on: ${{ (vars.CI_CLOUD_OVERFLOW_SAVED || startsWith(vars.LINUX_RUNNER, 'blacksmith-16vcpu-') || startsWith(inputs.probe_label, 'blacksmith-16vcpu-')) && 'blacksmith-6vcpu-macos-26' || 'blacksmith-16vcpu-ubuntu-2404' }} + # The probes ask for the Blacksmith Linux sizes, so the watch sits on + # Blacksmith macOS: it never queues behind its own probes and can put + # overflow back however the Linux sizes fare. + runs-on: blacksmith-6vcpu-macos-26 timeout-minutes: 25 permissions: actions: write @@ -134,9 +138,9 @@ jobs: run: python3 scripts/ci/cloud_overflow_switch.py # Last, in its own step so the summary above is already saved. If the - # switch did not observe a started probe, cancel this run even when the - # switch failed before it could write its output; otherwise a queued - # probe can hold the concurrency group forever. + # switch did not observe every probe start (one may still be queued), + # cancel this run even when the switch failed before it could write its + # output; otherwise a queued probe can hold the concurrency group forever. - name: End this run's stuck probe if: always() && steps.switch.outputs.outcome != 'started' env: diff --git a/docs/ci-runners.md b/docs/ci-runners.md index a841ebe5fb02..0ff4a1b2c93c 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -983,22 +983,27 @@ on a dead pool needs force-cancel and then rerun. Blacksmith is overflow, so when it stops starting jobs the fix is to stop sending overflow there, not to reroute a lane. `ci-cloud-overflow-probe.yml` does that automatically (`scripts/ci/cloud_overflow_switch.py`). Every 10 -minutes its `probe` job asks for one Blacksmith label and prints a line, and -its `watch` job waits for the probe. The watch never runs on GitHub-hosted -Ubuntu: while overflow is on it takes `blacksmith-16vcpu-ubuntu-2404` (a size -the probe does not ask for), and while it is off it takes -`blacksmith-6vcpu-macos-26`, so it can always put overflow back. On +minutes its `probe` jobs ask for each Blacksmith Linux size compiles run on +(`PROBE_LABELS`: `blacksmith-32vcpu-ubuntu-2404` and +`blacksmith-16vcpu-ubuntu-2404`, one matrix job each) and print a line, and +its `watch` job waits for them. Never one label: on 2026-10-28 the 4 vCPU +label alone stalled four times while the other sizes started jobs, and each +stall flipped the whole fleet. The watch never runs on GitHub-hosted Ubuntu +or on a size the probes ask for: it takes `blacksmith-6vcpu-macos-26`, so it +can always put overflow back. On 2026-10-27 a watch on hosted Ubuntu queued 94 minutes behind the switch's own failover and held the workflow's concurrency group, so every later run was evicted with nothing decided. -- **The probe waits `CI_CLOUD_PROBE_MINUTES` (default 5, 2 to 20) with no +- **Every probe waits `CI_CLOUD_PROBE_MINUTES` (default 5, 2 to 20) with no runner.** Overflow goes off. The switch first writes the `CI_CLOUD_OVERFLOW_SAVED` record (each variable's value before, its failover, when and which run), then points the variables at their failovers: `LINUX_RUNNER` to `blacksmith-16vcpu-ubuntu-2404` and `CI_TRUSTED_RUNNER` to `blacksmith-2vcpu-ubuntu-2404` (another Blacksmith - size; never GitHub-hosted Ubuntu), `MACOS_RUNNER_15`, + size; never GitHub-hosted Ubuntu, and only a size whose probe started in + that run, so with every size stalled the Linux variables are left alone), + `MACOS_RUNNER_15`, `_26_LARGE`, `_PR` and `_DUAL_XCODE` to the std owned pool of the lane's Xcode pin (`glaeda-std-xcode-` from `CMUX_CI_XCODE_APP_PR`), `MACOS_RUNNER_DISPLAY` to its gui label, `MACOS_RUNNER_IOS` to @@ -1014,18 +1019,34 @@ evicted with nothing decided. is unset or names a `blacksmith-*` label changes; one someone pointed elsewhere is left alone. Then up to 25 runs holding a job queued 5 minutes or more on a `blacksmith-*` label are force-cancelled (a plain cancel did - nothing on 2026-09-29) and re-run, so they pick their runners again; merge + nothing on 2026-09-29) and re-run, so they pick their runners again (a job on + a Blacksmith Linux label only when some Linux size started, since otherwise + its re-run would land on the same label); merge queue, release and publish runs, and runs already on attempt 3, are only listed. If a variable write fails after the record is saved, the next stalled probe retries entries still at their recorded before-value and leaves hand edits alone. The run force-cancels itself whenever the watcher - did not observe a started probe, including a switch failure before it could - publish its outcome, since a queued probe would otherwise hold the - concurrency group. -- **The probe starts.** If the record exists, each variable it changed is put + did not observe a started probe, or another probe is still queued, including + a switch failure before it could publish its outcome, since a queued probe + would otherwise hold the concurrency group. +- **Every probe starts.** If the record exists, each variable it changed is put back (a variable changed by hand since is left alone and named) and the - record is deleted. While the record exists the probe asks for the label it - names, not `LINUX_RUNNER`, which names the failover size then. + record is deleted. The probes ask for the same sizes whatever the record + says. A Linux variable goes back only onto a size whose probe started; one + whose value before is a label nobody probed (an older record's + `blacksmith-4vcpu-ubuntu-2404`, or unset) stays where it is, unless it is on + GitHub-hosted Ubuntu (a record from before 2026-10-27): then `LINUX_RUNNER` + moves to a probed size that started and `CI_TRUSTED_RUNNER` is unset, so no + hosted value outlives the record. A GitHub-hosted value before is never put + back. +- **Some start and some do not.** The switch changes nothing: overflow does + not flip and a record is not cleared. Moving work off one stalled size is + the `RUNNER_POOLS` reconciler's job, which reads the health below. + +Every deciding run that holds the switch token writes `CI_CLOUD_LABEL_HEALTH`: +`{"checked", "run", "minutes", "labels": {label: "started" | "stalled"}}`. The +switch never reads it back; it is per-label health for routing that picks a +size per label (the `RUNNER_POOLS` reconciler). The lane switches (`CI_PR_POOL_OWNED`, `CI_E2E_OWNED_UI`, `CI_IOS_OWNED` and the other `CI_OWNED_*`) are never touched: minis first does not change, only @@ -1043,7 +1064,7 @@ variable `GLAEDA_ROUTE_APP_ID` and its key in the repository secret `GLAEDA_ROUTE_APP_KEY`. Without it the watch still probes, names each value to set by hand, and fails. A manual dispatch is a dry run by default. -One Linux probe decides for macOS too: on 2026-09-29 both went at once. A +The Linux probes decide for macOS too: on 2026-09-29 both went at once. A probe can start on a pool that is only partly back; the next probe, 10 minutes later, turns overflow off again if that pool still starts nothing. diff --git a/scripts/ci/check_repo_variables.py b/scripts/ci/check_repo_variables.py index c2424d7dbd9e..87822c5b7838 100755 --- a/scripts/ci/check_repo_variables.py +++ b/scripts/ci/check_repo_variables.py @@ -127,7 +127,15 @@ def problems(env: dict[str, str]) -> list[str]: "that gate the nightly-next promotion queue behind pull request side lanes. " "Fix: point it at a pool pull requests do not fill, or clear it for the promote pool") found.extend(pin_pool_findings(env.get(PIN_POOL_VARIABLE, "").strip(), daemon_pool, variables)[0]) - for name, value, reason in drifted_runner_variables(variables): + drifted = list(drifted_runner_variables(variables)) + reported = {name for name, _, _ in drifted} + for name, value in sorted(variables.items()): + # Checked on their own: the drift policy accepts some labels (an older provider's, + # say) that no signing job may run on. + if name not in reported and not allowed_value(name, value): + found.append(f"{name}={value!r}: signing jobs run on it, so it may only name blacksmith-* " + "or ubuntu-* (scripts/ci/signing_runner_variables.py)") + for name, value, reason in drifted: if (switched.get(name) == value and not HOSTED_MACOS_LABEL.search(value.strip().lower()) and allowed_value(name, value)): # The overflow switch's failover while Blacksmith is down; never a hosted Mac, and a diff --git a/scripts/ci/cloud_overflow_switch.py b/scripts/ci/cloud_overflow_switch.py index 1d5849abe19c..94cac73a02dd 100644 --- a/scripts/ci/cloud_overflow_switch.py +++ b/scripts/ci/cloud_overflow_switch.py @@ -8,22 +8,24 @@ 10:15Z; about 150 runs sat queued for two hours until someone pointed those variables at the owned pools and GitHub-hosted Linux by hand. -ci-cloud-overflow-probe.yml runs this every 10 minutes. Its `probe` job asks -for one Blacksmith label (the record's `probe`, else LINUX_RUNNER when it is -a Blacksmith label, else the Blacksmith Linux fallback every workflow bakes -in) and only prints a line. This script, in the `watch` job on a -GitHub-hosted runner, waits for it: +ci-cloud-overflow-probe.yml runs this every 10 minutes. Its `probe` jobs ask +for each Blacksmith Linux size in PROBE_LABELS (32 and 16 vCPU), one job per +label, and only print a line. This script, in the `watch` job on Blacksmith +macOS, waits for them: -- The probe starts within PROBE_MINUTES of being queued: Blacksmith serves. +- Every probe starts within PROBE_MINUTES of being queued: Blacksmith serves. If this switch turned overflow off (the CI_CLOUD_OVERFLOW_SAVED record exists), every variable it changed is put back, unless someone changed that variable since, and the record is deleted. -- It does not: Blacksmith is not serving. Unless overflow is already off, +- None does: Blacksmith is not serving. Unless overflow is already off, the record is written first (what each variable held, what it gets, when and why), then each variable is pointed at its failover. Then, if it is cheap, runs already stuck on a Blacksmith label are force-cancelled and re-run so they pick their runners again. The workflow then force-cancels - its own run, whose probe would otherwise stay queued. + its own run, whose probes would otherwise stay queued. +- Some start and some do not: nothing changes. One stalled size never flips + the fleet, and a record is not cleared while a size is still stalled. A + size stalling alone is for that size's own failover, not this switch. One probe, one switch: the only state is the variables and the record. The failover of each variable (failover_values()) is derived from the lane's @@ -85,8 +87,13 @@ OVERRIDES_VARIABLE = "CI_CLOUD_FAILOVER" PAID_OVERFLOW_VARIABLE = "CI_PAID_MACOS_OVERFLOW" CLOUD_PREFIX = pr_runner_pool.EPHEMERAL_PREFIX # "blacksmith-" -# Every workflow's fallback when LINUX_RUNNER is unset (docs/ci-runners.md). -DEFAULT_PROBE_LABEL = "blacksmith-4vcpu-ubuntu-2404" +# What the probe asks for: the Blacksmith Linux sizes compiles run on, every +# one of them (the workflow's probe matrix restates this). Overflow goes off +# only when none of them starts its probe; one that starts means Blacksmith +# serves. A single probed label is a pin: on 2026-10-28 the 4 vCPU label +# alone queued past 5 minutes at 21:59Z, 22:13Z, 22:53Z and 23:30Z while the +# other sizes started jobs, and each time the whole fleet failed over. +PROBE_LABELS = ("blacksmith-32vcpu-ubuntu-2404", "blacksmith-16vcpu-ubuntu-2404") # Where Linux overflow goes while the probed label stalls: another Blacksmith # Linux size, never GitHub-hosted Ubuntu. CI_TRUSTED_RUNNER's readers accept # only ubuntu-24.04 and the 2 and 4 vCPU labels (docs/ci-runners.md), so it @@ -176,14 +183,9 @@ def linux_cloud_label(label: str | None) -> bool: return bool(re.fullmatch(r"blacksmith-\d+vcpu-ubuntu-\d+", str(label or ""))) -def probe_label(record: Mapping[str, Any] | None, linux_runner: str | None) -> str: - """The Blacksmith label a probe asks for; the workflow's runs-on restates this.""" - saved = (record or {}).get("probe") - if cloud_label(saved): - return str(saved) - if cloud_label(linux_runner): - return str(linux_runner) - return DEFAULT_PROBE_LABEL +def probe_job_name(label: str) -> str: + """The name of the probe job for one label (the workflow's matrix job name).""" + return f"{PROBE_JOB} ({label})" def parse_record(raw: str | None) -> dict[str, Any] | None: @@ -297,23 +299,52 @@ class Resume: skipped: str = "" # why it is left alone -def plan_on(record: Mapping[str, Any], current: Mapping[str, str | None]) -> list[Restore]: - """What turning overflow back on writes: each recorded variable's value before, unless it moved since.""" +def plan_on(record: Mapping[str, Any], current: Mapping[str, str | None], + started: Iterable[str] | None = None) -> list[Restore]: + """What turning overflow back on writes: each recorded variable's value before, unless it moved since. + + With `started` (the labels whose probes started in this run), a Linux + variable goes back only onto a label whose probe started: an older + record's before (blacksmith-4vcpu-ubuntu-2404 from when that one label was + the probe, or unset, whose readers fall back to an unprobed literal) may + still be stalled, and once the record is deleted nothing would see it. + Such a variable stays where it is. A GitHub-hosted label is never put back. + """ + healthy = None if started is None else set(started) restores: list[Restore] = [] for name, change in sorted((record.get("changed") or {}).items()): if not isinstance(change, Mapping): continue before, after = change.get("before"), change.get("after") now = (current.get(name) or "").strip() or None + hosted_now = name in LINUX_VARIABLES and now is not None and bool(HOSTED_LINUX_LABEL.match(now.lower())) + if hosted_now and healthy is not None and (before or "") not in healthy: + # A record from when Linux failed over to GitHub-hosted Ubuntu, whose before is not a + # size seen serving: the record is about to be deleted, so the hosted value must not + # outlive it. LINUX_RUNNER takes a probed size that started; CI_TRUSTED_RUNNER, whose + # readers accept no probed size, is unset (its readers fall back to Blacksmith). + live = [label for label in PROBE_LABELS if label in healthy] + target = live[0] if name == LINUX_VARIABLE and live else None + restores.append(Restore(name, target)) + continue if now != after and not (now is not None and now == change.get("was")): restores.append(Restore(name, before, f"now {now or 'unset'}, not the failover {after}; " "someone changed it, so it is left alone")) continue + if before and (HOSTED_LINUX_LABEL.match(before.lower()) or HOSTED_MACOS_LABEL.search(before.lower())): + restores.append(Restore(name, before, f"stays {now or 'unset'}; its value before, {before}, is " + "GitHub-hosted, which no job runs on")) + continue + if healthy is not None and name in LINUX_VARIABLES and (before or "") not in healthy: + restores.append(Restore(name, before, f"stays {now or 'unset'}; its value before, {before or 'unset'}, " + "is not a size whose probe started")) + continue restores.append(Restore(name, before)) return restores -def plan_resume(record: Mapping[str, Any], current: Mapping[str, str | None]) -> list[Resume]: +def plan_resume(record: Mapping[str, Any], current: Mapping[str, str | None], + started: Iterable[str] | None = None) -> list[Resume]: """What a stalled probe should retry after a partial failover write. A value still at the recorded ``before`` value was never switched, so it @@ -338,6 +369,13 @@ def plan_resume(record: Mapping[str, Any], current: Mapping[str, str | None]) -> resumes.append(Resume(name, after, f"signing jobs run on {name}, and {after} is not an ephemeral " "runner; left alone")) continue + if name in LINUX_VARIABLES and HOSTED_LINUX_LABEL.match(after.lower()): + # No job runs on GitHub-hosted Ubuntu (Leo, 2026-10-27), whatever an older record says. + resumes.append(Resume(name, after, "a GitHub-hosted Ubuntu failover from an older record; left alone")) + continue + if name in LINUX_VARIABLES and started is not None and after not in set(started): + resumes.append(Resume(name, after, f"{after} did not start a probe in this run; left alone")) + continue if now == before or (now is not None and now == change.get("was")): resumes.append(Resume(name, after)) else: @@ -346,20 +384,45 @@ def plan_resume(record: Mapping[str, Any], current: Mapping[str, str | None]) -> return resumes -# Where each Linux variable may go, in order, when its failover is the stalled label. +# Where each Linux variable may go, in order. LINUX_FAILOVER_CHAIN = {LINUX_VARIABLE: (LINUX_FAILOVER, ALTERNATE_FAILOVERS[LINUX_FAILOVER]), TRUSTED_VARIABLE: (TRUSTED_FAILOVER, ALTERNATE_FAILOVERS[TRUSTED_FAILOVER])} -def away_from_probe(failover: Mapping[str, str], probe: str) -> dict[str, str]: - """A Linux failover, an override included, never names the label whose stall triggered it.""" +def toward_started(failover: Mapping[str, str], started: Iterable[str]) -> dict[str, str]: + """Linux failovers, overrides included, name only a Blacksmith label whose probe started. + + A Linux variable whose failover (and every other label in its chain) did + not start in this run is left alone: moving it onto a size not seen + serving moves nothing, and may move it onto one seen stalling. The macOS + failovers are owned pools and are not probed here. + """ + healthy = set(started) values = dict(failover) for name in LINUX_VARIABLES: - if values.get(name) == probe: - values[name] = next(label for label in LINUX_FAILOVER_CHAIN[name] if label != probe) + if name not in values or values[name] in healthy: + continue + other = next((label for label in LINUX_FAILOVER_CHAIN[name] if label in healthy), None) + if other is None: + values.pop(name) + else: + values[name] = other return values +# Per-label probe health, rewritten by every deciding run: {"checked", "run", "minutes", +# "labels": {label: "started" | "stalled"}}. The switch reads none of it back; it is +# there for whatever routes on per-label health (the RUNNER_POOLS reconciler). +HEALTH_VARIABLE = "CI_CLOUD_LABEL_HEALTH" + + +def label_health(labels: Sequence[str], stalled: Iterable[str], *, checked: str, run: str, + minutes: int) -> dict[str, Any]: + stalled_set = set(stalled) + return {"checked": checked, "run": run, "minutes": minutes, + "labels": {label: "stalled" if label in stalled_set else "started" for label in labels}} + + def migrate_hosted_linux(record: Mapping[str, Any], current: Mapping[str, str | None], failover: Mapping[str, str]) -> tuple[dict[str, Any], list[Resume]]: """Move a record written when Linux failed over to GitHub-hosted Ubuntu onto today's failover. @@ -389,7 +452,7 @@ def migrate_hosted_linux(record: Mapping[str, Any], current: Mapping[str, str | return {**record, "changed": changed}, moves -def failover_destinations(record: Mapping[str, Any] | None, probe: str) -> set[str]: +def failover_destinations(record: Mapping[str, Any] | None, probes: Iterable[str]) -> set[str]: """Linux labels the switch moved overflow onto: a job queued there has nowhere better to go. A label that was also some variable's value before (or the probed one) @@ -399,7 +462,7 @@ def failover_destinations(record: Mapping[str, Any] | None, probe: str) -> set[s if name in LINUX_VARIABLES and isinstance(change, Mapping)] afters = {str(change.get("after")) for change in changes if change.get("after")} befores = {str(change.get("before")) for change in changes if change.get("before")} - return afters - befores - {probe} + return afters - befores - set(probes) def job_started(job: Mapping[str, Any]) -> bool: @@ -433,18 +496,25 @@ def protected_reason(run: Mapping[str, Any]) -> str: def stuck_job(jobs: Iterable[Mapping[str, Any]], now: dt.datetime, minutes: int, - failovers: Iterable[str] = ()) -> Mapping[str, Any] | None: + failovers: Iterable[str] = (), linux_moves: bool = True) -> Mapping[str, Any] | None: """The oldest job queued with no runner on a Blacksmith label for `minutes` or more. A job queued on a failover label (another Blacksmith size) already went - where the switch sends overflow, so re-running it moves nothing. + where the switch sends overflow, so re-running it moves nothing. Without + `linux_moves` (no Linux size started, so no Linux variable has anywhere to + go), a job on a Blacksmith Linux label would come back on the same label, + so it is not one either. """ skip = {str(label) for label in failovers} + + def movable(label: str) -> bool: + return cloud_label(label) and label not in skip and (linux_moves or not linux_cloud_label(label)) + stuck = [] for job in jobs: if job.get("status") != "queued" or job.get("runner_name"): continue - if not any(cloud_label(str(label)) and str(label) not in skip for label in job.get("labels") or ()): + if not any(movable(str(label)) for label in job.get("labels") or ()): continue created = parse_time(job.get("created_at")) if created is not None and (now - created).total_seconds() >= minutes * 60: @@ -512,29 +582,52 @@ def delete_variable(self, name: str) -> None: raise -def wait_for_probe(api: GitHub, run_id: int, attempt: int, minutes: int, *, +def probe_set_outcome(jobs: Iterable[Mapping[str, Any]], labels: Sequence[str], now: dt.datetime, + minutes: int) -> tuple[str, list[str]]: + """The set's outcome once every probe has decided, and the labels that stalled. + + "started": every probe started, so Blacksmith serves every size and an + existing record is put back. "stalled": none started, so Blacksmith serves + no size and overflow goes off. "partial": some started and some stalled; + nothing changes either way, so one stalled size never flips the fleet and + a record is not cleared while a size jobs fall back from is still stalled. + "waiting": a probe has not decided yet. + """ + by_name = {str(job.get("name")): job for job in jobs} + outcomes = {label: probe_outcome(by_name.get(probe_job_name(label)), now, minutes) for label in labels} + stalled = [label for label, outcome in outcomes.items() if outcome == "stalled"] + if not outcomes or "waiting" in outcomes.values(): + return "waiting", stalled + if not stalled: + return "started", stalled + if len(stalled) == len(outcomes): + return "stalled", stalled + return "partial", stalled + + +def wait_for_probe(api: GitHub, run_id: int, attempt: int, minutes: int, labels: Sequence[str], *, now: Callable[[], dt.datetime], sleep: Callable[[float], None], - log: Callable[[str], None]) -> str: - """Watch this run's probe job until it starts or has waited `minutes`.""" + log: Callable[[str], None]) -> tuple[str, list[str]]: + """Watch this run's probe jobs until every one has started or waited `minutes`.""" deadline = now() + dt.timedelta(minutes=minutes + 5) while True: try: - probe = next((job for job in api.jobs(run_id, attempt) if job.get("name") == PROBE_JOB), None) + jobs = api.jobs(run_id, attempt) except RuntimeError as error: log(f"could not list this run's jobs ({error}); trying again") - probe = None - outcome = probe_outcome(probe, now(), minutes) + jobs = [] + outcome, stalled = probe_set_outcome(jobs, labels, now(), minutes) if outcome != "waiting": - return outcome + return outcome, stalled if now() >= deadline: - # Never saw the probe: nothing measured, so nothing is decided. - return "unknown" + # Never saw every probe decide: nothing measured, so nothing is decided. + return "unknown", stalled sleep(POLL_SECONDS) def rescue_stuck_runs(api: GitHub, *, own_run_id: int, minutes: int, now: Callable[[], dt.datetime], sleep: Callable[[float], None], log: Callable[[str], None], dry_run: bool, - failovers: Iterable[str] = ()) -> list[str]: + failovers: Iterable[str] = (), linux_moves: bool = True) -> list[str]: """Force-cancel and re-run runs holding a job queued on Blacksmith; one line per run touched.""" lines: list[str] = [] seen: dict[int, dict[str, Any]] = {} @@ -551,7 +644,7 @@ def rescue_stuck_runs(api: GitHub, *, own_run_id: int, minutes: int, now: Callab if len(chosen) >= MAX_RESCUES: break try: - job = stuck_job(api.jobs(int(run["id"])), now(), minutes, failovers) + job = stuck_job(api.jobs(int(run["id"])), now(), minutes, failovers, linux_moves) except RuntimeError as error: lines.append(f"run {run['id']}: could not list jobs ({error})") continue @@ -661,25 +754,40 @@ def log(message: str) -> None: log(f"::warning title=Cloud overflow switch::{problem}") names = sorted(set(SWITCHED_VARIABLES) | set(failover) | set((record or {}).get("changed") or {})) current = current_values(env, names) - label = probe_label(record, current.get(LINUX_VARIABLE)) + labels: tuple[str, ...] = PROBE_LABELS drill = (env.get("PROBE_LABEL_DRILL") or "").strip() if drill: # A drill (workflow_dispatch probe_label) probes another label and never writes. - label, args.dry_run = drill, True + labels, args.dry_run = (drill,), True log(f"drill: probing {drill} instead; a dry run whatever it finds") - failover = away_from_probe(failover, label) + label = ", ".join(labels) state = f"off since {record.get('since')}" if record else "on" - log(f"Blacksmith overflow is {state}; probing {label} (a probe queued {minutes} min with no runner is a stall)") + log(f"Blacksmith overflow is {state}; probing {label} (a stall is every probe queued {minutes} min with no " + "runner)") - outcome = wait_for_probe(api, run_id, attempt, minutes, now=now, sleep=sleep, log=log) + outcome, stalled = wait_for_probe(api, run_id, attempt, minutes, labels, now=now, sleep=sleep, log=log) stamp = iso(now()) code = 0 + started = [item for item in labels if item not in stalled] if outcome != "unknown" else [] + failover = toward_started(failover, started) + if outcome != "unknown" and not args.dry_run and switch is not None: + health = label_health(labels, stalled, checked=stamp, run=f"https://github.com/{repo}/actions/runs/{run_id}", + minutes=minutes) + try: + switch.set_variable(HEALTH_VARIABLE, json.dumps(health, sort_keys=True)) + except RuntimeError as error: + # Reporting only: the switch decides from this run's probes, never from this variable. + log(f"::warning title=Cloud overflow switch::could not write {HEALTH_VARIABLE} ({error})") if outcome == "unknown": - log("::warning title=Cloud overflow switch::never saw the probe job; nothing decided") + log("::warning title=Cloud overflow switch::never saw every probe job decide; nothing decided") + elif outcome == "partial": + log(f"{stamp}: {', '.join(stalled)} waited {minutes} min with no runner while the other sizes started; " + f"overflow stays {'off' if record else 'on'}; nothing changes (per-size routing reads " + f"{HEALTH_VARIABLE})") elif outcome == "started": - log(f"{stamp}: the probe started on {label}; Blacksmith is serving") + log(f"{stamp}: every probe started ({label}); Blacksmith is serving") if record: - restores = plan_on(record, current) + restores = plan_on(record, current, started) if args.dry_run: for item in restores: if item.skipped: @@ -702,11 +810,11 @@ def log(message: str) -> None: switch.delete_variable(RECORD_VARIABLE) log(f"overflow back on; {RECORD_VARIABLE} deleted (it was off since {record.get('since')})") else: - log(f"{stamp}: the probe waited {minutes} min on {label} with no runner; Blacksmith is not serving") + log(f"{stamp}: every probe waited {minutes} min on {label} with no runner; Blacksmith is not serving") if not record: changed = plan_off(current, failover) - new_record = {"since": stamp, "probe": label, "reason": f"probe on {label} queued {minutes} min " - "with no runner", "run": f"https://github.com/{repo}/actions/runs/{run_id}", + new_record = {"since": stamp, "probe": label, "reason": f"every probe on {label} queued {minutes} " + "min with no runner", "run": f"https://github.com/{repo}/actions/runs/{run_id}", "changed": changed} if args.dry_run: for name, change in changed.items(): @@ -743,7 +851,7 @@ def log(message: str) -> None: log(f"- {item.name}: moved from GitHub-hosted Ubuntu to {item.value}") record = migrated current = {**current, **{item.name: item.value for item in moves}} - resumes = plan_resume(migrated if args.dry_run else record, current) + resumes = plan_resume(migrated if args.dry_run else record, current, started) if args.dry_run: for item in resumes: if item.skipped: @@ -764,8 +872,9 @@ def log(message: str) -> None: if record or args.dry_run: for line in rescue_stuck_runs(api, own_run_id=run_id, minutes=minutes, now=now, sleep=sleep, log=log, dry_run=args.dry_run, + linux_moves=bool(started), failovers=failover_destinations( - record or {"changed": plan_off(current, failover)}, label)): + record or {"changed": plan_off(current, failover)}, labels)): log(f"- {line}") out = env.get("GITHUB_OUTPUT") if out: diff --git a/tests/test_ci_cloud_overflow_switch.py b/tests/test_ci_cloud_overflow_switch.py index 716f3d7fbc78..74c4b44262ad 100644 --- a/tests/test_ci_cloud_overflow_switch.py +++ b/tests/test_ci_cloud_overflow_switch.py @@ -9,6 +9,8 @@ import sys import tempfile import unittest + +import yaml from pathlib import Path ROOT = Path(__file__).resolve().parents[1] @@ -30,7 +32,7 @@ def ago(minutes: float) -> str: def probe_job(minutes: float, **extra: object) -> dict: - return {"name": switch.PROBE_JOB, "status": "queued", "created_at": ago(minutes), + return {"name": switch.probe_job_name(switch.PROBE_LABELS[0]), "status": "queued", "created_at": ago(minutes), # GitHub stamps started_at on a queued job too; only a runner means it started. "started_at": ago(minutes), "runner_name": None, "labels": ["blacksmith-4vcpu-ubuntu-2404"], **extra} @@ -59,7 +61,12 @@ def __init__(self, probe: dict | None, runs: list[dict] | None = None, def jobs(self, run_id: int, attempt: int | None = None) -> list[dict]: if run_id == 1: - return [self.probe] if self.probe else [] + # The same state for every probe in the set (and a drill's label); tests of one + # probe starting while another stalls pass a list. + if isinstance(self.probe, list): + return self.probe + labels = (*switch.PROBE_LABELS, DRILL) + return [{**self.probe, "name": switch.probe_job_name(label)} for label in labels] if self.probe else [] return self.run_jobs.get(run_id, []) def runs(self, status: str) -> list[dict]: @@ -81,14 +88,20 @@ class FakeSwitch: def __init__(self) -> None: self.writes: list[tuple[str, str | None]] = [] + self.health: list[dict] = [] # CI_CLOUD_LABEL_HEALTH, kept apart from the routing writes def set_variable(self, name: str, value: str) -> None: + if name == switch.HEALTH_VARIABLE: + self.health.append(json.loads(value)) + return self.writes.append((name, value)) def delete_variable(self, name: str) -> None: self.writes.append((name, None)) +DRILL = "blacksmith-0vcpu-no-such-label" + BLACKSMITH_STEADY = { "VAR_LINUX_RUNNER": "blacksmith-4vcpu-ubuntu-2404", "VAR_MACOS_RUNNER_PR": "blacksmith-6vcpu-macos-26", @@ -132,7 +145,7 @@ def test_failover_never_touches_a_lane_switch(self) -> None: for name, value in values.items(): if switch.cloud_label(value): self.assertIn(name, switch.LINUX_VARIABLES) - self.assertNotEqual(value, switch.DEFAULT_PROBE_LABEL) + self.assertNotEqual(value, "blacksmith-4vcpu-ubuntu-2404") def test_overrides_replace_drop_and_refuse(self) -> None: values, problems = switch.failover_values(PIN, json.dumps({ @@ -307,11 +320,9 @@ def test_probe_minutes_are_clamped(self) -> None: self.assertEqual(switch.probe_minutes("1"), 2) self.assertEqual(switch.probe_minutes("90"), 20) - def test_probe_label_prefers_the_record(self) -> None: - self.assertEqual(switch.probe_label({"probe": "blacksmith-8vcpu-ubuntu-2404"}, "ubuntu-24.04"), - "blacksmith-8vcpu-ubuntu-2404") - self.assertEqual(switch.probe_label(None, "ubuntu-24.04"), switch.DEFAULT_PROBE_LABEL) - self.assertEqual(switch.probe_label(None, "blacksmith-2vcpu-ubuntu-2404"), "blacksmith-2vcpu-ubuntu-2404") + def test_the_probe_set_is_the_compile_sizes(self) -> None: + # Never one label: one stalled size must not flip the fleet (Leo's no-pins order, 2026-10-28). + self.assertEqual(switch.PROBE_LABELS, ("blacksmith-32vcpu-ubuntu-2404", "blacksmith-16vcpu-ubuntu-2404")) def test_stuck_job_needs_a_blacksmith_label_and_the_wait(self) -> None: jobs = [ @@ -326,6 +337,10 @@ def test_stuck_job_needs_a_blacksmith_label_and_the_wait(self) -> None: # A job already on the failover size is where the switch sends it; re-running moves nothing. on_failover = [{"status": "queued", "labels": ["blacksmith-16vcpu-ubuntu-2404"], "created_at": ago(30)}] self.assertIsNone(switch.stuck_job(on_failover, NOW, 5, ["blacksmith-16vcpu-ubuntu-2404"])) + # With no Linux size serving, a Linux job has nowhere to go; a macOS one still moves to the owned pool. + self.assertIsNone(switch.stuck_job(on_failover, NOW, 5, linux_moves=False)) + on_mac = [{"status": "queued", "labels": ["blacksmith-6vcpu-macos-26"], "created_at": ago(30)}] + self.assertIsNotNone(switch.stuck_job(on_mac, NOW, 5, linux_moves=False)) self.assertIsNotNone(switch.stuck_job(on_failover, NOW, 5)) # Only a destination that was nobody's value before counts: LINUX_RUNNER on 2 vCPU moved to 16, # while CI_TRUSTED_RUNNER moved onto 2, so jobs queued on 2 vCPU still move. @@ -349,22 +364,30 @@ def test_a_stall_writes_the_record_first_then_the_failovers_then_moves_stuck_run "created_at": ago(40)} merge = {"id": 8, "status": "queued", "event": "merge_group", "name": "CI", "run_attempt": 1, "created_at": ago(30)} - queued = [{"status": "queued", "labels": ["blacksmith-4vcpu-ubuntu-2404"], "created_at": ago(35)}] - actions = FakeActions(probe_job(0), [stuck, merge], {7: queued, 8: queued}) + queued = [{"status": "queued", "labels": ["blacksmith-6vcpu-macos-26"], "created_at": ago(35)}] + linux = {"id": 9, "status": "queued", "event": "pull_request", "name": "CI", "run_attempt": 1, + "created_at": ago(35)} + on_linux = [{"status": "queued", "labels": ["blacksmith-4vcpu-ubuntu-2404"], "created_at": ago(35)}] + actions = FakeActions(probe_job(0), [stuck, merge, linux], {7: queued, 8: queued, 9: on_linux}) writer = FakeSwitch() code, out, summary = run_main(BLACKSMITH_STEADY, actions, writer) self.assertEqual(code, 0) self.assertIn("outcome=stalled", out) self.assertEqual(writer.writes[0][0], "CI_CLOUD_OVERFLOW_SAVED") record = json.loads(str(writer.writes[0][1])) - self.assertEqual(record["probe"], "blacksmith-4vcpu-ubuntu-2404") - self.assertEqual(record["changed"]["LINUX_RUNNER"]["before"], "blacksmith-4vcpu-ubuntu-2404") + self.assertEqual(record["probe"], ", ".join(switch.PROBE_LABELS)) + # Every Linux size it could move to stalled with the probes, so LINUX_RUNNER is left alone. + self.assertNotIn("LINUX_RUNNER", record["changed"]) + # No Linux size started, so neither Linux variable moves onto one. + self.assertNotIn("CI_TRUSTED_RUNNER", record["changed"]) written = dict(writer.writes[1:]) - self.assertEqual(written["LINUX_RUNNER"], "blacksmith-16vcpu-ubuntu-2404") + self.assertNotIn("LINUX_RUNNER", written) self.assertEqual(written["MACOS_RUNNER_PR"], "glaeda-std-xcode-26.6") self.assertEqual(written["CI_PAID_MACOS_OVERFLOW"], "1") for name in LANE_SWITCHES: self.assertNotIn(name, written) + # Run 9 waits on Blacksmith Linux, and no Linux size started: a re-run would land on the same + # label, so it is left alone instead of being cancelled on every scheduled run. self.assertEqual(actions.calls, [("force-cancel", 7), ("rerun", 7)]) self.assertIn("merge_group run", summary) @@ -380,10 +403,17 @@ def test_a_started_probe_restores_and_deletes_the_record(self) -> None: code, out, summary = run_main(env, actions, writer) self.assertEqual(code, 0) self.assertIn("outcome=started", out) - self.assertEqual(writer.writes, [("CI_PAID_MACOS_OVERFLOW", None), - ("LINUX_RUNNER", "blacksmith-4vcpu-ubuntu-2404"), - ("CI_CLOUD_OVERFLOW_SAVED", None)]) + # Review of this change: 4 vCPU is not probed and may still be stalled, so LINUX_RUNNER stays on + # the 16 vCPU size whose probe started rather than going back to a label nobody watches. + self.assertEqual(writer.writes, [("CI_PAID_MACOS_OVERFLOW", None), ("CI_CLOUD_OVERFLOW_SAVED", None)]) + self.assertIn("stays blacksmith-16vcpu-ubuntu-2404; its value before, blacksmith-4vcpu-ubuntu-2404", summary) self.assertIn("left alone", summary) + # A before that is a probed size goes back as before. + restores = switch.plan_on({"changed": {"LINUX_RUNNER": { + "before": "blacksmith-32vcpu-ubuntu-2404", "after": "blacksmith-16vcpu-ubuntu-2404"}}}, + {"LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404"}, switch.PROBE_LABELS) + self.assertEqual([(r.name, r.value, r.skipped) for r in restores], + [("LINUX_RUNNER", "blacksmith-32vcpu-ubuntu-2404", "")]) self.assertEqual(actions.calls, []) def test_a_stalled_record_finishes_a_partial_switch(self) -> None: @@ -392,12 +422,13 @@ def test_a_stalled_record_finishes_a_partial_switch(self) -> None: "MACOS_RUNNER_PR": {"before": "blacksmith-6vcpu-macos-26", "after": "glaeda-std-xcode-26.6"}}} env = {"VAR_CI_CLOUD_OVERFLOW_SAVED": json.dumps(record), "VAR_LINUX_RUNNER": "blacksmith-4vcpu-ubuntu-2404", - "VAR_MACOS_RUNNER_PR": "glaeda-std-xcode-26.6", "VAR_CMUX_CI_XCODE_APP_PR": PIN} + "VAR_MACOS_RUNNER_PR": "blacksmith-6vcpu-macos-26", "VAR_CMUX_CI_XCODE_APP_PR": PIN} writer = FakeSwitch() code, out, _ = run_main(env, FakeActions(probe_job(0)), writer) self.assertEqual(code, 0) self.assertIn("outcome=stalled", out) - self.assertEqual(writer.writes, [("LINUX_RUNNER", "blacksmith-16vcpu-ubuntu-2404")]) + # The owned macOS pool is retried; 16 vCPU stalled with the rest, so LINUX_RUNNER waits. + self.assertEqual(writer.writes, [("MACOS_RUNNER_PR", "glaeda-std-xcode-26.6")]) def test_a_record_on_hosted_ubuntu_moves_to_the_blacksmith_failover(self) -> None: # Written by the switch before 2026-10-27 (Linux failed over to GitHub-hosted Ubuntu). @@ -406,37 +437,47 @@ def test_a_record_on_hosted_ubuntu_moves_to_the_blacksmith_failover(self) -> Non "CI_TRUSTED_RUNNER": {"before": None, "after": "ubuntu-24.04"}, "MACOS_RUNNER_PR": {"before": "blacksmith-6vcpu-macos-26", "after": "glaeda-std-xcode-26.6"}}} # LINUX_RUNNER still at the old failover; CI_TRUSTED_RUNNER changed by hand since. - env = {"VAR_CI_CLOUD_OVERFLOW_SAVED": json.dumps(record), "VAR_LINUX_RUNNER": "ubuntu-24.04", - "CMUX_CI_RUNNER_VARIABLES": "CI_TRUSTED_RUNNER=blacksmith-4vcpu-ubuntu-2404\n", - "VAR_MACOS_RUNNER_PR": "glaeda-std-xcode-26.6", "VAR_CMUX_CI_XCODE_APP_PR": PIN} - writer = FakeSwitch() - code, out, _ = run_main(env, FakeActions(probe_job(0)), writer) - self.assertEqual(code, 0) - self.assertIn("outcome=stalled", out) - self.assertEqual(writer.writes[0][0], "CI_CLOUD_OVERFLOW_SAVED") - saved = json.loads(writer.writes[0][1])["changed"] + current = {"LINUX_RUNNER": "ubuntu-24.04", "CI_TRUSTED_RUNNER": "blacksmith-4vcpu-ubuntu-2404"} + failover = switch.toward_started({"LINUX_RUNNER": switch.LINUX_FAILOVER}, ["blacksmith-16vcpu-ubuntu-2404"]) + migrated, moves = switch.migrate_hosted_linux(record, current, failover) + saved = migrated["changed"] self.assertEqual(saved["LINUX_RUNNER"], {"before": "blacksmith-4vcpu-ubuntu-2404", "after": "blacksmith-16vcpu-ubuntu-2404", "was": "ubuntu-24.04"}) self.assertEqual(saved["CI_TRUSTED_RUNNER"], {"before": None, "after": "ubuntu-24.04"}) - self.assertEqual(writer.writes[1:], [("LINUX_RUNNER", "blacksmith-16vcpu-ubuntu-2404")]) + self.assertEqual([(m.name, m.value) for m in moves], [("LINUX_RUNNER", "blacksmith-16vcpu-ubuntu-2404")]) # Turning overflow back on then restores from the migrated record. restores = {item.name: item for item in switch.plan_on( {"changed": saved}, {"LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404"})} self.assertEqual(restores["LINUX_RUNNER"].value, "blacksmith-4vcpu-ubuntu-2404") self.assertFalse(restores["LINUX_RUNNER"].skipped) - def test_a_legacy_partial_switch_resumes_onto_blacksmith(self) -> None: + def test_a_hosted_ubuntu_record_never_moves_onto_a_stalled_size(self) -> None: + # Review of this change: with both 16 and 32 vCPU stalled, migrating onto 16 vCPU moves nothing. + record = {"since": ago(60), "probe": "blacksmith-4vcpu-ubuntu-2404", "changed": { + "LINUX_RUNNER": {"before": "blacksmith-4vcpu-ubuntu-2404", "after": "ubuntu-24.04"}}} + env = {"VAR_CI_CLOUD_OVERFLOW_SAVED": json.dumps(record), "VAR_LINUX_RUNNER": "ubuntu-24.04", + "VAR_CMUX_CI_XCODE_APP_PR": PIN} + writer = FakeSwitch() + code, out, _ = run_main(env, FakeActions(probe_job(0)), writer) + self.assertEqual(code, 0) + self.assertIn("outcome=stalled", out) + self.assertEqual(writer.writes, []) + # Without the switch token there is nothing to write either, so it does not fail. + code, _, _ = run_main(env, FakeActions(probe_job(0)), None) + self.assertEqual(code, 0) + + def test_a_legacy_partial_switch_never_resumes_onto_hosted_ubuntu(self) -> None: record = {"since": ago(60), "probe": "blacksmith-4vcpu-ubuntu-2404", "changed": { "LINUX_RUNNER": {"before": "blacksmith-4vcpu-ubuntu-2404", "after": "ubuntu-24.04"}}} env = {"VAR_CI_CLOUD_OVERFLOW_SAVED": json.dumps(record), "VAR_LINUX_RUNNER": "blacksmith-4vcpu-ubuntu-2404", "VAR_CMUX_CI_XCODE_APP_PR": PIN} writer = FakeSwitch() run_main(env, FakeActions(probe_job(0)), writer) - self.assertNotIn(("LINUX_RUNNER", "ubuntu-24.04"), writer.writes) - self.assertIn(("LINUX_RUNNER", "blacksmith-16vcpu-ubuntu-2404"), writer.writes) + self.assertEqual(writer.writes, []) - def test_an_interrupted_migration_resumes_and_still_restores(self) -> None: - # The record was rewritten (after -> 16 vCPU, was -> ubuntu-24.04) but the variable write failed. + def test_an_interrupted_migration_never_resumes_onto_a_stalled_size(self) -> None: + # The record was rewritten (after -> 16 vCPU, was -> ubuntu-24.04) but the variable write failed; + # 16 vCPU has since stalled, so the retry waits for a run where it starts. record = {"since": ago(60), "probe": "blacksmith-4vcpu-ubuntu-2404", "changed": { "LINUX_RUNNER": {"before": "blacksmith-4vcpu-ubuntu-2404", "after": "blacksmith-16vcpu-ubuntu-2404", "was": "ubuntu-24.04"}}} @@ -444,41 +485,157 @@ def test_an_interrupted_migration_resumes_and_still_restores(self) -> None: "VAR_CMUX_CI_XCODE_APP_PR": PIN} writer = FakeSwitch() run_main(env, FakeActions(probe_job(0)), writer) - self.assertEqual(writer.writes, [("LINUX_RUNNER", "blacksmith-16vcpu-ubuntu-2404")]) + self.assertEqual(writer.writes, []) + resumes = switch.plan_resume(record, {"LINUX_RUNNER": "ubuntu-24.04"}, ["blacksmith-16vcpu-ubuntu-2404"]) + self.assertEqual([(r.name, r.value, r.skipped) for r in resumes], + [("LINUX_RUNNER", "blacksmith-16vcpu-ubuntu-2404", "")]) restores = switch.plan_on(record, {"LINUX_RUNNER": "ubuntu-24.04"}) self.assertEqual([(r.name, r.value, r.skipped) for r in restores], [("LINUX_RUNNER", "blacksmith-4vcpu-ubuntu-2404", "")]) - def test_the_failover_is_never_the_stalled_probe_label(self) -> None: - # LINUX_RUNNER already names 16 vCPU, so the probe asks for 16 vCPU; its failover is the next size. - env = {"VAR_LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404", "VAR_CMUX_CI_XCODE_APP_PR": PIN} + def test_a_linux_failover_names_only_a_size_that_started(self) -> None: + # Nothing started: the Linux variables are left alone, the macOS ones still fail over. + self.assertEqual(switch.toward_started({"LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404", + "MACOS_RUNNER_PR": "glaeda-std-xcode-26.6"}, []), + {"MACOS_RUNNER_PR": "glaeda-std-xcode-26.6"}) + # Only 32 vCPU started: the chain moves on to it. + self.assertEqual(switch.toward_started({"LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404"}, + ["blacksmith-32vcpu-ubuntu-2404"]), + {"LINUX_RUNNER": "blacksmith-32vcpu-ubuntu-2404"}) + self.assertEqual(switch.toward_started({"LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404"}, + switch.PROBE_LABELS), + {"LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404"}) + # The live record's LINUX_RUNNER entry (32 vCPU) is not retried while 32 vCPU did not start. + record = {"changed": {"LINUX_RUNNER": {"before": "blacksmith-4vcpu-ubuntu-2404", + "after": "blacksmith-32vcpu-ubuntu-2404"}}} + resumes = switch.plan_resume(record, {"LINUX_RUNNER": "blacksmith-4vcpu-ubuntu-2404"}, + ["blacksmith-16vcpu-ubuntu-2404"]) + self.assertTrue(resumes[0].skipped) + + def test_one_stalled_size_changes_nothing(self) -> None: + stalled = probe_job(6) + started = {**probe_job(6, runner_name="bs-1", status="in_progress"), + "name": switch.probe_job_name(switch.PROBE_LABELS[1])} + self.assertEqual(switch.probe_set_outcome([stalled, started], switch.PROBE_LABELS, NOW, 5), + ("partial", [switch.PROBE_LABELS[0]])) + self.assertEqual(switch.probe_set_outcome([stalled], switch.PROBE_LABELS, NOW, 5), + ("waiting", [switch.PROBE_LABELS[0]])) + both = [{**stalled, "name": switch.probe_job_name(label)} for label in switch.PROBE_LABELS] + self.assertEqual(switch.probe_set_outcome(both, switch.PROBE_LABELS, NOW, 5), + ("stalled", list(switch.PROBE_LABELS))) + up = [{**started, "name": switch.probe_job_name(label)} for label in switch.PROBE_LABELS] + self.assertEqual(switch.probe_set_outcome(up, switch.PROBE_LABELS, NOW, 5), ("started", [])) + # Review of this change: 16 vCPU starting while 32 vCPU stalls must not clear the record, + # since fast-checks falls back to 32 vCPU without it. + record = {"since": ago(60), "probe": "blacksmith-4vcpu-ubuntu-2404", "changed": { + "MACOS_RUNNER_PR": {"before": "blacksmith-6vcpu-macos-26", "after": "glaeda-std-xcode-26.6"}}} + env = {"VAR_CI_CLOUD_OVERFLOW_SAVED": json.dumps(record), "VAR_MACOS_RUNNER_PR": "glaeda-std-xcode-26.6", + "VAR_CMUX_CI_XCODE_APP_PR": PIN} writer = FakeSwitch() - code, out, _ = run_main(env, FakeActions(probe_job(0)), writer) - self.assertEqual(code, 0) - self.assertIn("outcome=stalled", out) - saved = json.loads(writer.writes[0][1]) - self.assertEqual(saved["probe"], "blacksmith-16vcpu-ubuntu-2404") - self.assertEqual(saved["changed"]["LINUX_RUNNER"], {"before": "blacksmith-16vcpu-ubuntu-2404", - "after": "blacksmith-32vcpu-ubuntu-2404"}) - self.assertIn(("LINUX_RUNNER", "blacksmith-32vcpu-ubuntu-2404"), writer.writes) - self.assertEqual(switch.away_from_probe({"CI_TRUSTED_RUNNER": "blacksmith-2vcpu-ubuntu-2404"}, - "blacksmith-2vcpu-ubuntu-2404"), - {"CI_TRUSTED_RUNNER": "blacksmith-4vcpu-ubuntu-2404"}) - # An override naming the probed label is moved off it too. - self.assertEqual(switch.away_from_probe({"CI_TRUSTED_RUNNER": "blacksmith-4vcpu-ubuntu-2404", - "LINUX_RUNNER": "blacksmith-4vcpu-ubuntu-2404"}, - "blacksmith-4vcpu-ubuntu-2404"), - {"CI_TRUSTED_RUNNER": "blacksmith-2vcpu-ubuntu-2404", - "LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404"}) - - def test_a_legacy_record_without_the_switch_token_fails_loudly(self) -> None: + code, out, _ = run_main(env, FakeActions([stalled, started]), writer) + self.assertEqual((code, writer.writes), (0, [])) + self.assertIn("outcome=partial", out) + # Nor does it turn overflow off when there is no record. + writer = FakeSwitch() + code, out, _ = run_main(BLACKSMITH_STEADY, FakeActions([stalled, started]), writer) + self.assertEqual((code, writer.writes), (0, [])) + # Per-label health is written either way, for the pool reconciler to read. + self.assertEqual(writer.health[-1]["labels"], {switch.PROBE_LABELS[0]: "stalled", + switch.PROBE_LABELS[1]: "started"}) + self.assertEqual(writer.health[-1]["run"], "https://github.com/manaflow-ai/cmux/actions/runs/1") + # Every size started: the record is put back and deleted. + writer = FakeSwitch() + code, out, _ = run_main(env, FakeActions(up), writer) + self.assertIn("outcome=started", out) + self.assertEqual(writer.writes, [("MACOS_RUNNER_PR", "blacksmith-6vcpu-macos-26"), + ("CI_CLOUD_OVERFLOW_SAVED", None)]) + + def test_a_partial_result_writes_nothing_but_health(self) -> None: + # Review at 437df955 and edbbce33: per-size moves belong to the RUNNER_POOLS reconciler, which + # reads CI_CLOUD_LABEL_HEALTH. The switch neither flips nor writes a record on a partial result. + stalled = probe_job(6) + started = {**probe_job(6, runner_name="bs-1", status="in_progress"), + "name": switch.probe_job_name(switch.PROBE_LABELS[1])} + env = {"VAR_LINUX_RUNNER": "blacksmith-32vcpu-ubuntu-2404", "VAR_MACOS_RUNNER_PR": "blacksmith-6vcpu-macos-26", + "VAR_CMUX_CI_XCODE_APP_PR": PIN} + writer = FakeSwitch() + code, out, _ = run_main(env, FakeActions([stalled, started]), writer) + self.assertEqual((code, writer.writes), (0, [])) + self.assertIn("outcome=partial", out) + self.assertEqual(writer.health[-1]["labels"][switch.PROBE_LABELS[0]], "stalled") + + def test_a_restore_goes_back_only_onto_a_size_that_started(self) -> None: + # Review at 437df955: a legacy CI_TRUSTED_RUNNER record (before 4 vCPU, never probed). + record = {"changed": { + "CI_TRUSTED_RUNNER": {"before": "blacksmith-4vcpu-ubuntu-2404", "after": "blacksmith-2vcpu-ubuntu-2404"}, + "LINUX_RUNNER": {"before": None, "after": "blacksmith-16vcpu-ubuntu-2404"}, + "MACOS_RUNNER_PR": {"before": "blacksmith-6vcpu-macos-26", "after": "glaeda-std-xcode-26.6"}, + "MACOS_RUNNER_15": {"before": "macos-15", "after": "glaeda-std-xcode-26.6"}}} + current = {"CI_TRUSTED_RUNNER": "blacksmith-2vcpu-ubuntu-2404", "LINUX_RUNNER": "blacksmith-16vcpu-ubuntu-2404", + "MACOS_RUNNER_PR": "glaeda-std-xcode-26.6", "MACOS_RUNNER_15": "glaeda-std-xcode-26.6"} + restores = {r.name: r for r in switch.plan_on(record, current, switch.PROBE_LABELS)} + self.assertTrue(restores["CI_TRUSTED_RUNNER"].skipped) + self.assertTrue(restores["LINUX_RUNNER"].skipped) # unset falls back to an unprobed literal + self.assertFalse(restores["MACOS_RUNNER_PR"].skipped) + self.assertTrue(restores["MACOS_RUNNER_15"].skipped) # GitHub-hosted is never put back + + def test_a_recovered_legacy_hosted_record_never_leaves_linux_on_hosted(self) -> None: + # Review at cd797966: before was the unprobed 4 vCPU, after GitHub-hosted Ubuntu. Every size starts, + # the record is deleted, and LINUX_RUNNER must not stay on hosted Ubuntu. record = {"since": ago(60), "probe": "blacksmith-4vcpu-ubuntu-2404", "changed": { - "LINUX_RUNNER": {"before": "blacksmith-4vcpu-ubuntu-2404", "after": "ubuntu-24.04"}}} + "LINUX_RUNNER": {"before": "blacksmith-4vcpu-ubuntu-2404", "after": "ubuntu-24.04"}, + "CI_TRUSTED_RUNNER": {"before": None, "after": "ubuntu-24.04"}}} + started = probe_job(0, status="in_progress", runner_name="bs-1") + up = [{**started, "name": switch.probe_job_name(label)} for label in switch.PROBE_LABELS] env = {"VAR_CI_CLOUD_OVERFLOW_SAVED": json.dumps(record), "VAR_LINUX_RUNNER": "ubuntu-24.04", - "VAR_CMUX_CI_XCODE_APP_PR": PIN} - code, _, summary = run_main(env, FakeActions(probe_job(0)), None) - self.assertEqual(code, 1) - self.assertIn("LINUX_RUNNER -> blacksmith-16vcpu-ubuntu-2404", summary) + "VAR_CI_TRUSTED_RUNNER": "ubuntu-24.04", "VAR_CMUX_CI_XCODE_APP_PR": PIN} + writer = FakeSwitch() + code, out, _ = run_main(env, FakeActions(up), writer) + self.assertEqual(code, 0) + self.assertIn("outcome=started", out) + self.assertEqual(writer.writes, [("CI_TRUSTED_RUNNER", None), + ("LINUX_RUNNER", switch.PROBE_LABELS[0]), + ("CI_CLOUD_OVERFLOW_SAVED", None)]) + # Even an entry changed by hand to hosted Ubuntu since is moved, not left behind. + restores = {r.name: r for r in switch.plan_on( + {"changed": {"LINUX_RUNNER": {"before": None, "after": "blacksmith-16vcpu-ubuntu-2404"}}}, + {"LINUX_RUNNER": "ubuntu-24.04"}, switch.PROBE_LABELS)} + self.assertEqual((restores["LINUX_RUNNER"].value, restores["LINUX_RUNNER"].skipped), + (switch.PROBE_LABELS[0], "")) + + def test_no_code_path_writes_a_hosted_label(self) -> None: + # Leo, 2026-10-28: "most of what GitHub has is hilariously slow. We can't be using them." + self.assertFalse(hasattr(switch, "HOSTED_LINUX")) + hosted = re.compile(r"^(ubuntu-|macos-|windows-)") + stalled = probe_job(6) + started = {**probe_job(6, runner_name="bs-1", status="in_progress"), + "name": switch.probe_job_name(switch.PROBE_LABELS[1])} + up = [{**started, "name": switch.probe_job_name(label)} for label in switch.PROBE_LABELS] + legacy = {"since": ago(60), "probe": "blacksmith-4vcpu-ubuntu-2404", "changed": { + "LINUX_RUNNER": {"before": "ubuntu-24.04", "after": "ubuntu-24.04"}, + "CI_TRUSTED_RUNNER": {"before": "ubuntu-24.04", "after": "blacksmith-2vcpu-ubuntu-2404"}, + "MACOS_RUNNER_15": {"before": "macos-15", "after": "glaeda-std-xcode-26.6"}}} + envs = [BLACKSMITH_STEADY, {"VAR_CMUX_CI_XCODE_APP_PR": PIN}, + {**BLACKSMITH_STEADY, "VAR_CI_CLOUD_FAILOVER": json.dumps({"LINUX_RUNNER": "ubuntu-24.04", + "MACOS_RUNNER_15": "macos-15"})}, + {"VAR_CI_CLOUD_OVERFLOW_SAVED": json.dumps(legacy), "VAR_LINUX_RUNNER": "ubuntu-24.04", + "VAR_CI_TRUSTED_RUNNER": "blacksmith-2vcpu-ubuntu-2404", + "VAR_MACOS_RUNNER_15": "glaeda-std-xcode-26.6", "VAR_CMUX_CI_XCODE_APP_PR": PIN}, + {"VAR_LINUX_RUNNER": "blacksmith-32vcpu-ubuntu-2404", "VAR_CMUX_CI_XCODE_APP_PR": PIN}] + for env in envs: + for probes in (probe_job(0), [stalled, started], up): + writer = FakeSwitch() + run_main(env, FakeActions(probes), writer) + for name, value in writer.writes: + if name == switch.RECORD_VARIABLE or value is None: + continue + self.assertFalse(hosted.match(value.lower()), (env, name, value)) + + def test_a_drill_or_dry_run_writes_no_health(self) -> None: + writer = FakeSwitch() + run_main({**BLACKSMITH_STEADY, "PROBE_LABEL_DRILL": DRILL}, FakeActions(probe_job(0)), writer) + run_main(BLACKSMITH_STEADY, FakeActions(probe_job(0)), writer, argv=["--dry-run"]) + self.assertEqual(writer.health, []) def test_healthy_with_no_record_writes_nothing(self) -> None: writer = FakeSwitch() @@ -490,7 +647,8 @@ def test_no_switch_token_fails_loudly_and_writes_nothing(self) -> None: code, out, summary = run_main(BLACKSMITH_STEADY, actions, None) self.assertEqual(code, 1) self.assertIn("outcome=stalled", out) - self.assertIn("LINUX_RUNNER -> blacksmith-16vcpu-ubuntu-2404", summary) + self.assertIn("MACOS_RUNNER_PR -> glaeda-std-xcode-26.6", summary) + self.assertNotIn("LINUX_RUNNER ->", summary) self.assertEqual(actions.calls, []) def test_dry_run_changes_nothing(self) -> None: @@ -516,7 +674,9 @@ def test_a_drill_reports_failovers_without_the_app_token(self) -> None: code, out, summary = run_main(env, FakeActions(probe_job(0)), None) self.assertEqual(code, 0) self.assertIn("outcome=stalled", out) - self.assertIn("LINUX_RUNNER: would set blacksmith-16vcpu-ubuntu-2404", summary) + self.assertIn("MACOS_RUNNER_PR: would set glaeda-std-xcode-26.6", summary) + # The drill's label did not start, so no Linux size is known to serve. + self.assertNotIn("LINUX_RUNNER: would set", summary) self.assertNotIn("no switch token", summary) def test_runner_values_come_from_the_block(self) -> None: @@ -534,11 +694,18 @@ def test_an_unreadable_record_fails_without_writing(self) -> None: class WorkflowTests(unittest.TestCase): text = WORKFLOW.read_text(encoding="utf-8") - def test_probe_restates_probe_label(self) -> None: - runs_on = re.search(r"name: Blacksmith probe\n(?:.*\n)*?\s+runs-on: (.*)", self.text).group(1) - self.assertIn("vars.CI_CLOUD_OVERFLOW_SAVED", runs_on) - self.assertIn(f"'{switch.DEFAULT_PROBE_LABEL}'", runs_on) - self.assertIn(f"name: {switch.PROBE_JOB}\n", self.text) + def test_probe_matrix_restates_the_probe_set(self) -> None: + document = yaml.safe_load(self.text) + probe = document["jobs"]["probe"] + self.assertEqual(probe["name"], f"{switch.PROBE_JOB} (${{{{ matrix.label }}}})") + self.assertEqual(switch.probe_job_name("x"), f"{switch.PROBE_JOB} (x)") + self.assertIs(probe["strategy"]["fail-fast"], False) + matrix = probe["strategy"]["matrix"]["label"] + listed = json.loads(re.search(r"\|\| '(\[.*?\])'", matrix).group(1)) + self.assertEqual(tuple(listed), switch.PROBE_LABELS) + self.assertTrue(probe["runs-on"].endswith("|| matrix.label }}")) + # Unless every probe started, the run ends itself, so a queued probe frees the group. + self.assertIn("if: always() && steps.switch.outputs.outcome != 'started'", self.text) def test_watch_runs_off_blacksmith_and_passes_every_switched_variable(self) -> None: # Never GitHub-hosted Ubuntu: on 2026-10-27 the watch queued 94 minutes there behind the @@ -547,15 +714,8 @@ def test_watch_runs_off_blacksmith_and_passes_every_switched_variable(self) -> N self.assertNotIn("ubuntu-24.04", runs_on) self.assertNotIn("ubuntu-latest", runs_on) self.assertNotIn("github-hosted-required", runs_on) - # Overflow on: a Blacksmith Linux size the probe does not ask for. Overflow off: Blacksmith macOS, - # so the watch can put overflow back whatever the Linux sizes do. - # Whenever the probe itself may ask for that size, the watch takes macOS, so it never queues - # behind its own probe. - self.assertEqual(runs_on, "${{ (vars.CI_CLOUD_OVERFLOW_SAVED || startsWith(vars.LINUX_RUNNER, " - "'blacksmith-16vcpu-') || startsWith(inputs.probe_label, 'blacksmith-16vcpu-')) && " - f"'blacksmith-6vcpu-macos-26' || '{switch.LINUX_FAILOVER}' }}}}") - self.assertTrue(switch.LINUX_FAILOVER.startswith("blacksmith-16vcpu-")) - self.assertNotEqual(switch.LINUX_FAILOVER, switch.DEFAULT_PROBE_LABEL) + # Blacksmith macOS: never a size the probes ask for, so it never queues behind them. + self.assertEqual(runs_on, "blacksmith-6vcpu-macos-26") names = set(switch.SWITCHED_VARIABLES) | {switch.RECORD_VARIABLE, switch.OVERRIDES_VARIABLE, "CMUX_CI_XCODE_APP_PR"} for name in names: @@ -581,7 +741,7 @@ def test_every_overridable_variable_is_passed(self) -> None: self.assertIn(f" {name}=${{{{ vars.{name} }}}}\n", self.text) def test_a_drill_probes_its_label_and_is_always_a_dry_run(self) -> None: - self.assertIn("inputs.probe_label != '' && inputs.probe_label ||", self.text) + self.assertIn("inputs.probe_label != '' && format('[\"{0}\"]', inputs.probe_label) ||", self.text) self.assertIn("(inputs.dry_run || inputs.probe_label != '')", self.text) @@ -590,6 +750,13 @@ def env(self, **extra: str) -> dict[str, str]: return {"CMUX_CI_RUNNER_VARIABLES": "MACOS_RUNNER_PR=glaeda-std-xcode-26.6\nLINUX_RUNNER=ubuntu-24.04\n", "CI_OWNED_POOL_SLOTS": "", "CMUX_CI_XCODE_APP_PR": PIN, **extra} + def test_a_signing_variable_the_drift_policy_accepts_is_still_reported(self) -> None: + env = {"CMUX_CI_RUNNER_VARIABLES": "MACOS_RUNNER_26=warp-macos-15-arm64-6x\n", "CI_OWNED_POOL_SLOTS": "", + "CMUX_CI_XCODE_APP_PR": PIN} + self.assertTrue(any(p.startswith("MACOS_RUNNER_26=") and "signing jobs" in p for p in check.problems(env))) + env["CMUX_CI_RUNNER_VARIABLES"] = "MACOS_RUNNER_26=blacksmith-6vcpu-macos-26\n" + self.assertFalse(any(p.startswith("MACOS_RUNNER_26=") for p in check.problems(env))) + def test_a_trusted_runner_its_readers_reject_is_drift(self) -> None: def env(trusted: str) -> dict[str, str]: return {"CMUX_CI_RUNNER_VARIABLES": f"CI_TRUSTED_RUNNER={trusted}\n", "CI_OWNED_POOL_SLOTS": "",