ci(overflow): probe the 32 and 16 vCPU Linux sizes; trip only when none starts

.github/workflows/ci-cloud-overflow-probe.yml

@@ -5,18 +5,19 @@ name: CI cloud overflow probe
 # 150 runs queued for two hours), the runner variables that send overflow
 # there (LINUX_RUNNER, CI_TRUSTED_RUNNER, MACOS_RUNNER_*, CI_PAID_MACOS_OVERFLOW) are pointed at
 # the owned pools and another Blacksmith Linux size, and pointed back once it serves
-# again. One probe, one switch: `probe` asks for a Blacksmith label and only
-# prints a line; `watch`, on another Blacksmith pool, waits for it.
+# again. One switch: `probe` asks for each Blacksmith Linux size compiles run
+# on and only prints a line; `watch`, on Blacksmith macOS, waits for them.
 # scripts/ci/cloud_overflow_switch.py has the rules:
 #
-# - The probe starts within CI_CLOUD_PROBE_MINUTES (default 5) of being
+# - Every probe starts within CI_CLOUD_PROBE_MINUTES (default 5) of being
 #   queued: if this switch had turned overflow off, every variable it changed
 #   is restored from the CI_CLOUD_OVERFLOW_SAVED record (unless someone
 #   changed it since) and the record is deleted.
-# - It does not: the record is written (each variable's value before, its
+# - None does: the record is written (each variable's value before, its
 #   failover, when and why), the variables are pointed at their failovers,
 #   runs already stuck on a Blacksmith label are force-cancelled and re-run,
-#   and this run force-cancels itself, since its probe would stay queued.
+#   and this run force-cancels itself, since its probes would stay queued.
+# - Some start and some do not: nothing changes either way.
 #
 # The lane switches (CI_PR_POOL_OWNED, CI_E2E_OWNED_UI, CI_IOS_OWNED, ...) are
 # never touched: minis-first routing does not change, only where overflow
@@ -52,12 +53,18 @@ concurrency:
 
 jobs:
   probe:
-    name: Blacksmith probe
+    # One probe per Blacksmith Linux size compiles run on
+    # (cloud_overflow_switch.PROBE_LABELS). Overflow goes off only when none of
+    # them starts, and back on only when all start: a single probed label is a pin, and on 2026-10-28 the 4 vCPU
+    # label alone stalling flipped the whole fleet four times. A drill probes
+    # only its own label.
+    name: Blacksmith probe (${{ matrix.label }})
     if: github.repository_owner == 'manaflow-ai'
-    # The Blacksmith label overflow uses, never the variable's failover:
-    # while overflow is off LINUX_RUNNER names the failover size, so the record
-    # keeps the label to probe (cloud_overflow_switch.probe_label()).
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'workflow_dispatch' && inputs.probe_label != '' && inputs.probe_label || startsWith(fromJSON(vars.CI_CLOUD_OVERFLOW_SAVED || '{"probe":""}').probe, 'blacksmith-') && fromJSON(vars.CI_CLOUD_OVERFLOW_SAVED).probe || startsWith(vars.LINUX_RUNNER, 'blacksmith-') && vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }}
+    strategy:
+      fail-fast: false
+      matrix:
+        label: ${{ fromJSON(github.event_name == 'workflow_dispatch' && inputs.probe_label != '' && format('["{0}"]', inputs.probe_label) || '["blacksmith-32vcpu-ubuntu-2404", "blacksmith-16vcpu-ubuntu-2404"]') }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || matrix.label }}
     timeout-minutes: 2
     permissions: {}
     steps:
@@ -72,13 +79,10 @@ jobs:
     # (19:22Z to 20:56Z) on hosted ubuntu-24.04, which the switch's own
     # failover had filled; this run held the concurrency group meanwhile, so
     # every later scheduled run was evicted with no jobs and nothing decided.
-    # While overflow is on, the watch takes a Blacksmith Linux size the probe
-    # does not ask for (the switch's Linux failover). While it is off, Linux
-    # overflow is on that size, so the watch takes Blacksmith macOS: it can
-    # still put overflow back however the Linux sizes fare. The same holds
-    # whenever the probe itself asks for 16 vCPU (LINUX_RUNNER or a drill
-    # naming it), so the watch never queues behind its own probe.
-    runs-on: ${{ (vars.CI_CLOUD_OVERFLOW_SAVED || startsWith(vars.LINUX_RUNNER, 'blacksmith-16vcpu-') || startsWith(inputs.probe_label, 'blacksmith-16vcpu-')) && 'blacksmith-6vcpu-macos-26' || 'blacksmith-16vcpu-ubuntu-2404' }}
+    # The probes ask for the Blacksmith Linux sizes, so the watch sits on
+    # Blacksmith macOS: it never queues behind its own probes and can put
+    # overflow back however the Linux sizes fare.
+    runs-on: blacksmith-6vcpu-macos-26
     timeout-minutes: 25
     permissions:
       actions: write
@@ -134,9 +138,9 @@ jobs:
         run: python3 scripts/ci/cloud_overflow_switch.py
 
       # Last, in its own step so the summary above is already saved. If the
-      # switch did not observe a started probe, cancel this run even when the
-      # switch failed before it could write its output; otherwise a queued
-      # probe can hold the concurrency group forever.
+      # switch did not observe every probe start (one may still be queued),
+      # cancel this run even when the switch failed before it could write its
+      # output; otherwise a queued probe can hold the concurrency group forever.
       - name: End this run's stuck probe
         if: always() && steps.switch.outputs.outcome != 'started'
         env:

docs/ci-runners.md

@@ -983,22 +983,27 @@ on a dead pool needs force-cancel and then rerun.
 Blacksmith is overflow, so when it stops starting jobs the fix is to stop
 sending overflow there, not to reroute a lane. `ci-cloud-overflow-probe.yml`
 does that automatically (`scripts/ci/cloud_overflow_switch.py`). Every 10
-minutes its `probe` job asks for one Blacksmith label and prints a line, and
-its `watch` job waits for the probe. The watch never runs on GitHub-hosted
-Ubuntu: while overflow is on it takes `blacksmith-16vcpu-ubuntu-2404` (a size
-the probe does not ask for), and while it is off it takes
-`blacksmith-6vcpu-macos-26`, so it can always put overflow back. On
+minutes its `probe` jobs ask for each Blacksmith Linux size compiles run on
+(`PROBE_LABELS`: `blacksmith-32vcpu-ubuntu-2404` and
+`blacksmith-16vcpu-ubuntu-2404`, one matrix job each) and print a line, and
+its `watch` job waits for them. Never one label: on 2026-10-28 the 4 vCPU
+label alone stalled four times while the other sizes started jobs, and each
+stall flipped the whole fleet. The watch never runs on GitHub-hosted Ubuntu
+or on a size the probes ask for: it takes `blacksmith-6vcpu-macos-26`, so it
+can always put overflow back. On
 2026-10-27 a watch on hosted Ubuntu queued 94 minutes behind the switch's own
 failover and held the workflow's concurrency group, so every later run was
 evicted with nothing decided.
 
-- **The probe waits `CI_CLOUD_PROBE_MINUTES` (default 5, 2 to 20) with no
+- **Every probe waits `CI_CLOUD_PROBE_MINUTES` (default 5, 2 to 20) with no
   runner.** Overflow goes off. The switch first writes the
   `CI_CLOUD_OVERFLOW_SAVED` record (each variable's value before, its
   failover, when and which run), then points the variables at their
   failovers: `LINUX_RUNNER` to `blacksmith-16vcpu-ubuntu-2404` and
   `CI_TRUSTED_RUNNER` to `blacksmith-2vcpu-ubuntu-2404` (another Blacksmith
-  size; never GitHub-hosted Ubuntu), `MACOS_RUNNER_15`,
+  size; never GitHub-hosted Ubuntu, and only a size whose probe started in
+  that run, so with every size stalled the Linux variables are left alone),
+  `MACOS_RUNNER_15`,
   `_26_LARGE`, `_PR` and `_DUAL_XCODE` to the std owned pool of the lane's
   Xcode pin (`glaeda-std-xcode-<version>` from `CMUX_CI_XCODE_APP_PR`),
   `MACOS_RUNNER_DISPLAY` to its gui label, `MACOS_RUNNER_IOS` to
@@ -1014,18 +1019,34 @@ evicted with nothing decided.
   is unset or names a `blacksmith-*` label changes; one someone pointed
   elsewhere is left alone. Then up to 25 runs holding a job queued 5 minutes
   or more on a `blacksmith-*` label are force-cancelled (a plain cancel did
-  nothing on 2026-09-29) and re-run, so they pick their runners again; merge
+  nothing on 2026-09-29) and re-run, so they pick their runners again (a job on
+  a Blacksmith Linux label only when some Linux size started, since otherwise
+  its re-run would land on the same label); merge
   queue, release and publish runs, and runs already on attempt 3, are only
   listed. If a variable write fails after the record is saved, the next
   stalled probe retries entries still at their recorded before-value and
   leaves hand edits alone. The run force-cancels itself whenever the watcher
-  did not observe a started probe, including a switch failure before it could
-  publish its outcome, since a queued probe would otherwise hold the
-  concurrency group.
-- **The probe starts.** If the record exists, each variable it changed is put
+  did not observe a started probe, or another probe is still queued, including
+  a switch failure before it could publish its outcome, since a queued probe
+  would otherwise hold the concurrency group.
+- **Every probe starts.** If the record exists, each variable it changed is put
   back (a variable changed by hand since is left alone and named) and the
-  record is deleted. While the record exists the probe asks for the label it
-  names, not `LINUX_RUNNER`, which names the failover size then.
+  record is deleted. The probes ask for the same sizes whatever the record
+  says. A Linux variable goes back only onto a size whose probe started; one
+  whose value before is a label nobody probed (an older record's
+  `blacksmith-4vcpu-ubuntu-2404`, or unset) stays where it is, unless it is on
+  GitHub-hosted Ubuntu (a record from before 2026-10-27): then `LINUX_RUNNER`
+  moves to a probed size that started and `CI_TRUSTED_RUNNER` is unset, so no
+  hosted value outlives the record. A GitHub-hosted value before is never put
+  back.
+- **Some start and some do not.** The switch changes nothing: overflow does
+  not flip and a record is not cleared. Moving work off one stalled size is
+  the `RUNNER_POOLS` reconciler's job, which reads the health below.
+
+Every deciding run that holds the switch token writes `CI_CLOUD_LABEL_HEALTH`:
+`{"checked", "run", "minutes", "labels": {label: "started" | "stalled"}}`. The
+switch never reads it back; it is per-label health for routing that picks a
+size per label (the `RUNNER_POOLS` reconciler).
 
 The lane switches (`CI_PR_POOL_OWNED`, `CI_E2E_OWNED_UI`, `CI_IOS_OWNED` and
 the other `CI_OWNED_*`) are never touched: minis first does not change, only
@@ -1043,7 +1064,7 @@ variable `GLAEDA_ROUTE_APP_ID` and its key in the repository secret
 `GLAEDA_ROUTE_APP_KEY`. Without it the watch still probes, names each value to
 set by hand, and fails. A manual dispatch is a dry run by default.
 
-One Linux probe decides for macOS too: on 2026-09-29 both went at once. A
+The Linux probes decide for macOS too: on 2026-09-29 both went at once. A
 probe can start on a pool that is only partly back; the next probe, 10
 minutes later, turns overflow off again if that pool still starts nothing.
 

scripts/ci/check_repo_variables.py

@@ -127,7 +127,15 @@ def problems(env: dict[str, str]) -> list[str]:
                              "that gate the nightly-next promotion queue behind pull request side lanes. "
                              "Fix: point it at a pool pull requests do not fill, or clear it for the promote pool")
             found.extend(pin_pool_findings(env.get(PIN_POOL_VARIABLE, "").strip(), daemon_pool, variables)[0])
-            for name, value, reason in drifted_runner_variables(variables):
+            drifted = list(drifted_runner_variables(variables))
+            reported = {name for name, _, _ in drifted}
+            for name, value in sorted(variables.items()):
+                # Checked on their own: the drift policy accepts some labels (an older provider's,
+                # say) that no signing job may run on.
+                if name not in reported and not allowed_value(name, value):
+                    found.append(f"{name}={value!r}: signing jobs run on it, so it may only name blacksmith-* "
+                                 "or ubuntu-* (scripts/ci/signing_runner_variables.py)")
+            for name, value, reason in drifted:
                 if (switched.get(name) == value and not HOSTED_MACOS_LABEL.search(value.strip().lower())
                         and allowed_value(name, value)):
                     # The overflow switch's failover while Blacksmith is down; never a hosted Mac, and a