ci(nightly-next): the Ghostty helper takes an idle owned Mac while the app build holds the dedicated builder

.github/workflows/nightly.yml

@@ -1479,14 +1479,14 @@ jobs:
     # no macOS SDK, so only a reused helper (resolve-nightly-ghostty-cli-helper)
     # leaves this job on Linux, for the source archive. A fresh helper takes an
     # owned pool (main's trusted pool on attempt 1 of a push or schedule;
-    # nightly-next: pick-nightly-build-runner's owned pick when it was made for
-    # this attempt, as build-nightly-app reads it, never the dedicated builder,
-    # else its moving Blacksmith label); a re-run of failed jobs keeps the older
+    # nightly-next: pick-nightly-build-runner's `helper` pick made for this
+    # attempt, an idle owned Mac but never the dedicated builder, else its
+    # moving Blacksmith label); a re-run of failed jobs keeps the older
     # pick and takes Blacksmith macOS 26, so a rescue re-run leaves a busy mini. Not
     # CI_NIGHTLY_NEXT_BUILD_RUNNER: it holds every attempt on its label, and a
     # trusted NEXT runner admits only build-nightly-app. Never a hosted Mac past
     # the fork branch, never macOS 15. glaeda sizes it light at its measured cores.
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || inputs.nightly_next_unsigned_run != '' && (vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404') || needs.resolve-nightly-ghostty-cli-helper.outputs.hit == 'true' && (vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404') || github.ref == 'refs/heads/nightly-next' && (needs.pick-nightly-build-runner.outputs.attempt == github.run_attempt && needs.pick-nightly-build-runner.outputs.owned != needs.pick-nightly-build-runner.outputs.dedicated && needs.pick-nightly-build-runner.outputs.owned || needs.pick-nightly-build-runner.outputs.blacksmith || 'blacksmith-6vcpu-macos-26') || github.repository == 'manaflow-ai/cmux' && github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'schedule') && vars.CI_PR_POOL_OWNED == '1' && github.ref == 'refs/heads/main' && vars.CI_SEED_TRUSTED_POOL || 'blacksmith-6vcpu-macos-26' }}
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || inputs.nightly_next_unsigned_run != '' && (vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404') || needs.resolve-nightly-ghostty-cli-helper.outputs.hit == 'true' && (vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404') || github.ref == 'refs/heads/nightly-next' && (needs.pick-nightly-build-runner.outputs.attempt == github.run_attempt && needs.pick-nightly-build-runner.outputs.helper != needs.pick-nightly-build-runner.outputs.dedicated && needs.pick-nightly-build-runner.outputs.helper || needs.pick-nightly-build-runner.outputs.blacksmith || 'blacksmith-6vcpu-macos-26') || github.repository == 'manaflow-ai/cmux' && github.run_attempt == 1 && (github.event_name == 'push' || github.event_name == 'schedule') && vars.CI_PR_POOL_OWNED == '1' && github.ref == 'refs/heads/main' && vars.CI_SEED_TRUSTED_POOL || 'blacksmith-6vcpu-macos-26' }}
     # zig-fetch-all.sh may retry each source-archive fetch (about 8.5 min at most per tree).
     timeout-minutes: 30
     env:
@@ -1792,6 +1792,7 @@ jobs:
       # Outside attempt 1 of a push (the rescue's workflow_run watch), an owned pick stands only once
       # its owned-pool-watch marker is uploaded: with no marker nothing would move a refused or stuck job.
       owned: ${{ (github.event_name == 'push' && github.run_attempt == 1 || steps.upload-marker.outcome == 'success') && steps.pick.outputs.owned || '' }}
+      helper: ${{ (github.event_name == 'push' && github.run_attempt == 1 || steps.upload-marker.outcome == 'success') && steps.pick.outputs.helper || '' }}
       blacksmith: ${{ steps.pick.outputs.blacksmith }}
       # The attempt the pick was made for: a re-run of failed jobs keeps an older one.
       attempt: ${{ steps.pick.outputs.attempt }}

scripts/ci/nightly_build_runner.py

@@ -36,6 +36,12 @@
   past any stalled one). Never empty. EXCLUDE (a label another leg already
   waits on) is skipped while another candidate moves.
 
+- `helper`: the owned route for build-nightly-ghostty-cli-helper, which the dedicated
+  builder never admits: `owned` when that is not the dedicated builder, else a fresh
+  owned_route() read (an idle side or nightly-slot Mac), else empty. Until 2026-10-30 the
+  helper took `blacksmith` whenever the app build took the dedicated builder, and run
+  40990112293 (373ec0cfaa2f, the first green tip after a two-hour Release compile red) queued
+  its helper on blacksmith-6vcpu-macos-26 for 39 minutes until the run was cancelled unpublished.
 - `attempt`: GITHUB_RUN_ATTEMPT, the attempt this pick was made for.
 
 `owned` is `dedicated` when that is set. The workflow takes `owned` only while
@@ -307,11 +313,19 @@ def read() -> Sequence[Mapping[str, Any]] | None:
                                              f"blacksmith otherwise: {blacksmith} ({blacksmith_why})"])
         return 0
 
+    # The Ghostty helper never takes the dedicated builder (its hook admits only build-nightly-app),
+    # so while the app build holds it the helper gets its own owned pick, as the verify jobs do.
+    helper, helper_why = owned, owned_why
+    if dedicated:
+        helper_runners = read() if any(owned_label(label) for label in labels) else None
+        helper, helper_why = owned_route(labels, helper_runners)
     runner = owned or blacksmith
     write_output(env, {"dedicated": dedicated, "owned": owned, "blacksmith": blacksmith, "runner": runner,
-                       "attempt": attempt or "1"},
+                       "helper": helper, "attempt": attempt or "1"},
                  [f"dedicated: {dedicated or 'none'} ({dedicated_why})", f"owned: {owned or 'none'} ({owned_why})",
-                  f"blacksmith: {blacksmith} ({blacksmith_why})", f"route: {runner}", f"attempt: {attempt or '1'}"])
+                  f"blacksmith: {blacksmith} ({blacksmith_why})", f"route: {runner}",
+                  f"helper: {helper or blacksmith} ({helper_why if helper else 'no idle owned Mac besides the dedicated builder, so Blacksmith'})",
+                  f"attempt: {attempt or '1'}"])
     return 0
 
 

tests/test_nightly_build_runner.py

@@ -193,7 +193,8 @@ def run_main(self, env: dict, argv: tuple = (), **reads) -> dict:
 
     def test_idle_owned_first(self) -> None:
         out = self.run_main({}, read_runners=lambda t, r: [runner(SIDE)], read_snapshot=lambda t, r: snapshot({}))
-        self.assertEqual(out, {"dedicated": "", "owned": SIDE, "blacksmith": LARGE, "runner": SIDE, "attempt": "1"})
+        self.assertEqual(out, {"dedicated": "", "owned": SIDE, "blacksmith": LARGE, "runner": SIDE, "helper": SIDE,
+                               "attempt": "1"})
 
     def test_a_rerun_picks_like_attempt_one(self) -> None:
         # A re-run that runs the picker again (a full re-run, or one after the picker failed) picks
@@ -203,7 +204,7 @@ def test_a_rerun_picks_like_attempt_one(self) -> None:
                 out = self.run_main({"GITHUB_RUN_ATTEMPT": attempt}, read_runners=lambda t, r: [runner(SIDE)],
                                     read_snapshot=lambda t, r: snapshot({LARGE: 90}))
                 self.assertEqual(out, {"dedicated": "", "owned": SIDE, "blacksmith": SMALL, "runner": SIDE,
-                                       "attempt": attempt})
+                                       "helper": SIDE, "attempt": attempt})
                 out = self.run_main({"GITHUB_RUN_ATTEMPT": attempt, "DEDICATED_LABEL": DEDICATED},
                                     read_runners=lambda t, r: [runner(*BUILDER), runner(SIDE)],
                                     read_snapshot=lambda t, r: snapshot({}))
@@ -212,19 +213,39 @@ def test_a_rerun_picks_like_attempt_one(self) -> None:
     def test_busy_owned_goes_to_blacksmith(self) -> None:
         out = self.run_main({}, read_runners=lambda t, r: [runner(SIDE, busy=True)],
                             read_snapshot=lambda t, r: snapshot({LARGE: 90}))
-        self.assertEqual(out, {"dedicated": "", "owned": "", "blacksmith": SMALL, "runner": SMALL, "attempt": "1"})
+        self.assertEqual(out, {"dedicated": "", "owned": "", "blacksmith": SMALL, "runner": SMALL, "helper": "",
+                               "attempt": "1"})
 
     def test_failed_reads_never_fail_the_job(self) -> None:
         def broken(token: str, repo: str):
             raise OSError("HTTP 403")
         out = self.run_main({}, read_runners=broken, read_snapshot=broken)
-        self.assertEqual(out, {"dedicated": "", "owned": "", "blacksmith": LARGE, "runner": LARGE, "attempt": "1"})
+        self.assertEqual(out, {"dedicated": "", "owned": "", "blacksmith": LARGE, "runner": LARGE, "helper": "",
+                               "attempt": "1"})
 
     def test_the_dedicated_builder_comes_first(self) -> None:
         fleet = [runner(*BUILDER), runner(SIDE)]
         out = self.run_main({"DEDICATED_LABEL": DEDICATED}, read_runners=lambda t, r: fleet,
                             read_snapshot=lambda t, r: snapshot({}))
-        self.assertEqual(out, {"dedicated": DEDICATED, "owned": DEDICATED, "blacksmith": LARGE, "runner": DEDICATED, "attempt": "1"})
+        self.assertEqual(out, {"dedicated": DEDICATED, "owned": DEDICATED, "blacksmith": LARGE, "runner": DEDICATED,
+                               "helper": SIDE, "attempt": "1"})
+
+    def test_the_helper_takes_an_idle_owned_mac_while_the_app_build_holds_the_builder(self) -> None:
+        # Run 40990112293: the app build took the dedicated builder, so the helper took
+        # blacksmith-6vcpu-macos-26 and waited there 39 minutes while side Macs were idle.
+        out = self.run_main({"DEDICATED_LABEL": DEDICATED}, read_runners=lambda t, r: [runner(*BUILDER), runner(SIDE)],
+                            read_snapshot=lambda t, r: snapshot({}))
+        self.assertEqual((out["owned"], out["helper"]), (DEDICATED, SIDE))
+        # No idle owned Mac besides the builder: no owned helper pick, so the helper takes Blacksmith.
+        for fleet in ([runner(*BUILDER)], [runner(*BUILDER), runner(SIDE, busy=True)]):
+            with self.subTest(fleet=fleet):
+                out = self.run_main({"DEDICATED_LABEL": DEDICATED}, read_runners=lambda t, r: fleet,
+                                    read_snapshot=lambda t, r: snapshot({}))
+                self.assertEqual((out["owned"], out["helper"], out["blacksmith"]), (DEDICATED, "", LARGE))
+        # The helper pick is never the dedicated builder, even when only the builder carries a watched label.
+        out = self.run_main({"DEDICATED_LABEL": DEDICATED, "OWNED_LABELS": f"{DEV} {SIDE} {DEDICATED}"},
+                            read_runners=lambda t, r: [runner(*BUILDER)], read_snapshot=lambda t, r: snapshot({}))
+        self.assertNotEqual(out["helper"], DEDICATED)
 
     def test_only_a_push_is_offered_the_builder(self) -> None:
         # The builder admits only a push to nightly-next: run 40594111618, a dispatch, was refused at
@@ -235,7 +256,7 @@ def test_only_a_push_is_offered_the_builder(self) -> None:
                 out = self.run_main({"DEDICATED_LABEL": DEDICATED, "GITHUB_EVENT_NAME": event},
                                     read_runners=lambda t, r: fleet, read_snapshot=lambda t, r: snapshot({}))
                 self.assertEqual(out, {"dedicated": "", "owned": SIDE, "blacksmith": LARGE, "runner": SIDE,
-                                       "attempt": "1"})
+                                       "helper": SIDE, "attempt": "1"})
                 out = self.run_main({"DEDICATED_LABEL": DEDICATED, "GITHUB_EVENT_NAME": event}, argv=("dedicated",),
                                     read_runners=lambda t, r: fleet)
                 self.assertEqual(out, {"dedicated": ""})
@@ -244,7 +265,8 @@ def test_a_builder_busy_past_the_wait_falls_back_to_the_owned_pool(self) -> None
         fleet = [runner(*BUILDER, busy=True), runner(SIDE)]
         out = self.run_main({"DEDICATED_LABEL": DEDICATED}, read_runners=lambda t, r: fleet,
                             read_snapshot=lambda t, r: snapshot({}))
-        self.assertEqual(out, {"dedicated": "", "owned": SIDE, "blacksmith": LARGE, "runner": SIDE, "attempt": "1"})
+        self.assertEqual(out, {"dedicated": "", "owned": SIDE, "blacksmith": LARGE, "runner": SIDE, "helper": SIDE,
+                               "attempt": "1"})
         # No idle owned Mac either: Blacksmith.
         out = self.run_main({"DEDICATED_LABEL": DEDICATED}, read_runners=lambda t, r: [runner(*BUILDER, status="offline")],
                             read_snapshot=lambda t, r: snapshot({}))
@@ -375,24 +397,26 @@ def test_the_ghostty_helper_takes_the_owned_or_blacksmith_pick_never_the_dedicat
         self.assertIn("pick-nightly-build-runner", job["needs"])
         source = job["runs-on"].removeprefix("${{").removesuffix("}}")
         pick = "pick-nightly-build-runner"
-        for event, attempt, picked_for, owned, dedicated, blacksmith in itertools.product(
+        for event, attempt, picked_for, helper, dedicated, blacksmith in itertools.product(
                 ("push", "workflow_dispatch"), (1, 2), ("1", "2"), ("", SIDE, DEDICATED), ("", DEDICATED), ("", LARGE)):
-            if (owned == DEDICATED) != (dedicated == DEDICATED):
-                continue
+            if helper == DEDICATED and dedicated != DEDICATED:
+                continue  # the picker names the builder only as `dedicated` (see the Main helper tests)
             context = {"github.repository_owner": "manaflow-ai", "github.repository": "manaflow-ai/cmux",
                        "github.event_name": event, "github.ref": "refs/heads/nightly-next",
                        "github.run_attempt": attempt, "vars.CI_PR_POOL_OWNED": "1",
                        "vars.CI_SIDE_LANE_RUNNER": SIDE, "vars.CI_SEED_TRUSTED_POOL": "glaeda-trusted-std-xcode-26.6",
                        "vars.LINUX_RUNNER": "", "inputs.nightly_next_unsigned_run": "",
                        f"needs.{pick}.outputs.attempt": picked_for,
                        "needs.resolve-nightly-ghostty-cli-helper.outputs.hit": "false",
-                       f"needs.{pick}.outputs.owned": owned, f"needs.{pick}.outputs.dedicated": dedicated,
+                       f"needs.{pick}.outputs.owned": dedicated or helper,
+                       f"needs.{pick}.outputs.helper": helper, f"needs.{pick}.outputs.dedicated": dedicated,
                        f"needs.{pick}.outputs.blacksmith": blacksmith}
-            with self.subTest(event=event, attempt=attempt, picked_for=picked_for, owned=owned, dedicated=dedicated,
+            with self.subTest(event=event, attempt=attempt, picked_for=picked_for, helper=helper, dedicated=dedicated,
                               blacksmith=blacksmith):
                 label = evaluate(parse_expression(source), context)
-                # The same rule as build-nightly-app: only a pick made for this attempt.
-                if picked_for == str(attempt) and owned == SIDE:
+                # The same rule as build-nightly-app: only a pick made for this attempt; the helper
+                # pick, which stands while the app build holds the dedicated builder, never that builder.
+                if picked_for == str(attempt) and helper == SIDE:
                     self.assertEqual(label, SIDE)
                 else:
                     # The picker's moving label, else the default; never a dropped pick.
@@ -417,6 +441,10 @@ def test_the_pickers_never_block_their_job(self) -> None:
                 self.assertGreater(definition["timeout-minutes"], step["timeout-minutes"] + 2)
                 self.assertIn("steps.pick.outputs.owned", definition["outputs"]["owned"])
                 self.assertEqual(definition["outputs"]["blacksmith"], "${{ steps.pick.outputs.blacksmith }}")
+                # The helper's pick stands on the same terms as `owned`.
+                self.assertEqual(definition["outputs"]["helper"],
+                                 definition["outputs"]["owned"].replace("steps.pick.outputs.owned",
+                                                                        "steps.pick.outputs.helper"))
 
     def route_token_if(self, pick: str, **changes) -> bool:
         step = next(s for s in DOC["jobs"][pick]["steps"] if s.get("id") == "route-token")

tests/test_nightly_trusted_routing.py

@@ -132,7 +132,7 @@ def helper(**changes):
                  'vars.CI_SEED_TRUSTED_POOL': 'glaeda-trusted-std-xcode-26.6',
                  # nightly-next reads pick-nightly-build-runner's owned and Blacksmith picks
                  # (tests/test_nightly_build_runner.py covers every combination).
-                 'needs.pick-nightly-build-runner.outputs.owned': 'glaeda-side-std-xcode-26.6',
+                 'needs.pick-nightly-build-runner.outputs.helper': 'glaeda-side-std-xcode-26.6',
                  'needs.pick-nightly-build-runner.outputs.attempt': '1',
                  'needs.pick-nightly-build-runner.outputs.dedicated': '',
                  'needs.pick-nightly-build-runner.outputs.blacksmith': 'blacksmith-6vcpu-macos-26', **changes})
@@ -144,17 +144,21 @@ def helper(**changes):
         self.assertEqual(helper(**{'github.run_attempt': 2, 'needs.pick-nightly-build-runner.outputs.attempt': '2'}),
                          'glaeda-side-std-xcode-26.6')
         # The dedicated builder admits only build-nightly-app: the helper never takes it.
-        self.assertEqual(helper(**{'needs.pick-nightly-build-runner.outputs.owned': 'glaeda-trusted-next-nightly',
+        self.assertEqual(helper(**{'needs.pick-nightly-build-runner.outputs.helper': 'glaeda-trusted-next-nightly',
                                    'needs.pick-nightly-build-runner.outputs.dedicated': 'glaeda-trusted-next-nightly'}),
                          'blacksmith-6vcpu-macos-26')
+        # While the app build holds the dedicated builder, the helper takes its own idle owned pick.
+        self.assertEqual(helper(**{'needs.pick-nightly-build-runner.outputs.owned': 'glaeda-trusted-next-nightly',
+                                   'needs.pick-nightly-build-runner.outputs.dedicated': 'glaeda-trusted-next-nightly'}),
+                         'glaeda-side-std-xcode-26.6')
         self.assertEqual(helper(**{'github.ref': 'refs/heads/main'}), 'glaeda-trusted-std-xcode-26.6')
         self.assertEqual(helper(**{'github.ref': 'refs/heads/main', 'github.event_name': 'schedule'}),
                          'glaeda-trusted-std-xcode-26.6')
         # No owned pick (busy fleet, no routing token): Blacksmith.
         for changes in ({'github.run_attempt': 2},
-                        {'github.event_name': 'workflow_dispatch', 'needs.pick-nightly-build-runner.outputs.owned': ''},
+                        {'github.event_name': 'workflow_dispatch', 'needs.pick-nightly-build-runner.outputs.helper': ''},
                         {'github.ref': 'refs/heads/main', 'vars.CI_PR_POOL_OWNED': '0'},
-                        {'needs.pick-nightly-build-runner.outputs.owned': ''},
+                        {'needs.pick-nightly-build-runner.outputs.helper': ''},
                         {'github.ref': 'refs/heads/main', 'vars.CI_SEED_TRUSTED_POOL': ''},
                         {'github.ref': 'refs/heads/other'}):
             with self.subTest(changes=changes):