ci(nightly-next): decide takes the owned nightly slot first, Blacksmith Linux only on the rescue's re-run

.github/runners.json

@@ -321,6 +321,10 @@
     "nightly-next-launch-smoke.yml#smoke": "apple-gui",
     "nightly-next-train.yml#build": "apple-app-build",
     "nightly.yml#build-nightly-app": "apple-app-build",
+    "nightly.yml#decide": {
+      "class": "portable-checks",
+      "picker_followup": "the run's first job, so no picker can run before it: attempt 1 of a nightly-next push takes the owned nightly slot (vars.CI_NIGHTLY_NEXT_BUILD_RUNNER) and the owned-pool rescue moves it to the Blacksmith Linux set when no slot takes it"
+    },
     "nightly.yml#build-nightly-remote-daemon": "portable-checks",
     "nightly.yml#await-nightly-tree-marker": "portable-checks",
     "nightly.yml#build-nightly-ghostty-cli-helper": "apple-app-build",

.github/workflows/nightly.yml

@@ -190,11 +190,18 @@ jobs:
       attestations: write
       id-token: write
     if: github.event_name != 'workflow_dispatch' || ((inputs.probe_mode || 'off') == 'off' && inputs.promote_nightly_next_sha == '')
-    # This gate only resolves the fixed revision and publication decision.
-    # Keep its default on the hosted runner so a busy paid Linux pool cannot
-    # delay every nightly before the macOS work is admitted; retain the normal
-    # Linux runner fallback for repositories that do not set the override.
-    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
+    # This gate only resolves the fixed revision and publication decision: a checkout,
+    # github-script and two stdlib Python helpers, which run the same on a Mac. It is the
+    # run's first job, so no picker can run before it. Attempt 1 of a nightly-next push
+    # takes the owned nightly slot (vars.CI_NIGHTLY_NEXT_BUILD_RUNNER, glaeda's
+    # nightly-only runner on each mini, admitted ahead of every PR job); the owned-pool
+    # rescue watches that attempt and, when no slot takes decide within
+    # CI_OWNED_POOL_RESCUE_SECONDS and nothing of the run has started, re-runs it, and
+    # attempt 2 takes the Blacksmith Linux set below. Run 41234993713 (3017ef19) queued
+    # decide on blacksmith-32vcpu-ubuntu-2404 with no fleet route at all. Leo,
+    # 2026-10-31: "FOR OUR NIGHTLIES ARE THEY END TO END ON OUR FLEET? WE CAN'T BE
+    # RELYING ON BLACKSMITH THAT CUTS OUT".
+    runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.ref == 'refs/heads/nightly-next' && github.event_name == 'push' && github.run_attempt == 1 && vars.CI_PR_POOL_OWNED == '1' && vars.CI_NIGHTLY_NEXT_BUILD_RUNNER || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
     outputs:
       should_build: ${{ steps.decide.outputs.should_build }}
       head_sha: ${{ steps.decide.outputs.head_sha }}

scripts/ci/owned_pool_rescue.py

@@ -273,6 +273,9 @@
 # The one job of a nightly run the watch judges (the visual gate must never
 # cancel signing or notarization siblings).
 NIGHTLY_APP_JOB = "build-nightly-app"
+# nightly.yml's first job: attempt 1 of a nightly-next push asks for the nightly slot (NIGHTLY_SLOT_LABEL), and
+# the rescue moves it when no slot takes it, so the re-run takes Blacksmith Linux. Every other job waits on it.
+NIGHTLY_DECIDE_JOB = "decide"
 # The trusted owned pool's labels: minis with no pull request runners, whose
 # job-started hook admits only main's push and schedule jobs. Only nightly.yml's
 # app build asks for one through this watch.
@@ -479,7 +482,9 @@ def nightly_watched(job: Mapping[str, Any]) -> bool:
     gate holds nothing.
     """
     return job.get("name") == NIGHTLY_APP_JOB or any(TRUSTED_LABEL.fullmatch(str(label))
-                                                     for label in job.get("labels") or [])
+                                                     for label in job.get("labels") or []) \
+        or job.get("name") == NIGHTLY_DECIDE_JOB and any(NIGHTLY_SLOT_LABEL.fullmatch(str(label))
+                                                         for label in job.get("labels") or [])
 
 
 def queued_pool(job: Mapping[str, Any], *, include_blacksmith_macos: bool = False) -> str | None:

scripts/ci/runner_routing_table.py

@@ -83,6 +83,10 @@
 # nightly.yml's light jobs: build-nightly-app's last step places them (NIGHTLY=1) on a free mini, else
 # Blacksmith. They have no fleet Linux step: a nightly job never takes vars.LINUX_RUNNER as it is.
 NIGHTLY_MINI = re.compile(r"needs\.build-nightly-app\.outputs\.light_mini_runner$")
+# nightly.yml's first job (decide) has no job before it to place it: attempt 1 of a nightly-next push asks for
+# glaeda's nightly slot, which only nightly jobs take, and the owned-pool rescue moves it to Blacksmith when no
+# slot takes it (its picker_followup says so).
+NIGHTLY_SLOT = re.compile(r"vars\.CI_NIGHTLY_NEXT_BUILD_RUNNER$")
 BLACKSMITH_LINUX = re.compile(r"^'blacksmith-\d+vcpu-ubuntu-2404'$")
 
 
@@ -98,9 +102,9 @@ def portable_order(runs_on: str, nightly: bool = False) -> str:
         return "its runs-on is not one expression"
     terms = [term.strip() for term in value_terms(expressions[0])]
     if nightly:
-        mini = next((i for i, term in enumerate(terms) if NIGHTLY_MINI.search(term)), None)
+        mini = next((i for i, term in enumerate(terms) if NIGHTLY_MINI.search(term) or NIGHTLY_SLOT.search(term)), None)
         if mini is None:
-            return "it does not read needs.build-nightly-app.outputs.light_mini_runner"
+            return "it does not read needs.build-nightly-app.outputs.light_mini_runner (or the nightly slot)"
         linux = -1
     else:
         linux = next((i for i, term in enumerate(terms) if PORTABLE_LINUX.search(term)), None)

tests/test_ci_macos_xcode_selection.py

@@ -71,8 +71,8 @@
     ("stable-next.yml", "smoke-27"): "launches the signed cmux NEXT Stable DMG; builds nothing, no Xcode",
     ("stable-next.yml", "smoke-26"): "launches the signed cmux NEXT Stable DMG; builds nothing, no Xcode",
     ("nightly.yml", "decide"):
-        "control job: takes Blacksmith macOS only while Blacksmith Linux overflow is off "
-        "(CI_CLOUD_OVERFLOW_SAVED); git, gh and Python only, no Xcode",
+        "control job: attempt 1 of a nightly-next push takes the owned nightly slot "
+        "(CI_NIGHTLY_NEXT_BUILD_RUNNER); git, github-script and Python only, no Xcode",
     ("nightly.yml", "resolve-nightly-cmux-tui-client"):
         "control job: takes Blacksmith macOS only while Blacksmith Linux overflow is off "
         "(CI_CLOUD_OVERFLOW_SAVED); git, gh and Python only, no Xcode",

tests/test_ci_nightly_off_blacksmith.py

@@ -26,11 +26,33 @@
 OVERFLOW_UNREAD = ("ci-cloud-overflow-probe.yml#probe",)
 LINUX_LABEL = re.compile(r"blacksmith-\d+vcpu-ubuntu-\d{4}|ubuntu-24\.04")
 # The nightly jobs that hold no release secrets and need no Mac.
-LIGHT_NIGHTLY_JOBS = ("decide", "resolve-nightly-cmux-tui-client", "resolve-nightly-ghostty-cli-helper",
+LIGHT_NIGHTLY_JOBS = ("resolve-nightly-cmux-tui-client", "resolve-nightly-ghostty-cli-helper",
                       "pick-nightly-build-runner", "prepare-nightly-variant-primary",
                       "prepare-nightly-variant", "pick-nightly-sign-runner",
                       "find-nightly-next-recovery")
 
+# A runs-on term that can put a nightly job on our own fleet: the owned nightly slot or side lane, or a picker's or
+# placement's owned output (pick-nightly-build-runner, pick-nightly-sign-runner, build-nightly-app's light placement).
+FLEET_ROUTE = re.compile(r"vars\.CI_NIGHTLY_NEXT_BUILD_RUNNER|vars\.CI_SIDE_LANE_RUNNER|vars\.CI_LIGHT_LANE_RUNNER"
+                         r"|needs\.[\w-]+\.outputs\.(?:owned|light_mini_runner|mini_runner|runs_on|helper|dedicated)\b"
+                         r"|matrix\.")
+# nightly.yml jobs whose only route is still Blacksmith Linux, as of 2026-10-31. Debt to move onto the fleet (portable
+# placement, #29130/#29436), never to grow: a new nightly job on Blacksmith Linux alone fails, and an entry that has
+# gained a fleet route (or no longer exists) must leave this list. decide left it first: run 41234993713 queued it on
+# blacksmith-32vcpu-ubuntu-2404 with no fleet route, and every other nightly job waits on it.
+BLACKSMITH_LINUX_ONLY_DEBT = frozenset({
+    "promote-nightly-next", "await-release-surface-ack", "promote-nightly-next-acked",
+    "resolve-nightly-ghostty-cli-helper", "build-nightly-inputs", "pick-nightly-build-runner",
+    "attest-nightly-remote-daemon", "resolve-nightly-cmux-tui-client", "prepare-nightly-variant-primary",
+    "prepare-nightly-variant", "pick-nightly-sign-runner", "signer-build", "signer-app", "signer-app-notary",
+    "signer-dmg", "find-nightly-next-recovery", "route-visual-gate-nightly", "visual-gate-nightly-verdict",
+    "report-visual-gate-nightly", "publish-nightly", "publish-nightly-next-github",
+    "publish-nightly-next-github-names", "keep-published-dsyms", "legacy-identity-target",
+    "publish-nightly-identity-legacy", "nightly-identity-legacy-published", "republish-nightly-deltas",
+    "published-in", "sign-release-notes", "publish-release-notes", "publish-nightly-reel",
+    "report-nightly-failure", "close-nightly-failure-issue",
+})
+
 SINGLE_HOST = re.compile(r"cmux22|glaeda-runner-[a-z0-9][a-z0-9-]*", re.IGNORECASE)
 OLD_NIGHTLY_RUNNER = re.compile(r"CI_NIGHTLY_(?:NEXT_)?TRUSTED_(?:POOL|RUNNER)")
 # The cmux-tui nightly calls the shared package builder, which keeps its own
@@ -216,6 +238,23 @@ def test_light_nightly_jobs_stay_on_linux_while_the_overflow_switch_is_on(self)
                 restore = steps["Restore unsigned nightly products"]["run"]
                 self.assertIn('if [ "$RUNNER_OS" = Linux ]; then', restore)
 
+    def test_no_new_nightly_job_rides_blacksmith_linux_alone(self) -> None:
+        """No pins: a nightly job needs a fleet route ahead of its Blacksmith Linux label.
+
+        The light-job test above only checked that these jobs stay on Linux, so decide's literal
+        blacksmith-32vcpu-ubuntu-2404 pin passed it. This ratchets BLACKSMITH_LINUX_ONLY_DEBT down."""
+        import yaml
+        jobs = yaml.safe_load((WORKFLOWS / "nightly.yml").read_text(encoding="utf-8"))["jobs"]
+        alone = {name for name, job in jobs.items()
+                 if re.search(r"blacksmith-\d+vcpu-ubuntu", str((job or {}).get("runs-on") or ""))
+                 and not FLEET_ROUTE.search(str(job.get("runs-on")))}
+        self.assertEqual(sorted(alone - BLACKSMITH_LINUX_ONLY_DEBT), [],
+                         "these nightly jobs can only take Blacksmith Linux: give them a fleet route first")
+        self.assertEqual(sorted(BLACKSMITH_LINUX_ONLY_DEBT - alone), [],
+                         "these jobs left Blacksmith-only routing (or are gone): drop them from the debt list")
+        self.assertIn("decide", jobs)
+        self.assertNotIn("decide", BLACKSMITH_LINUX_ONLY_DEBT)
+
     def test_no_job_takes_a_mac_only_while_the_overflow_switch_is_on(self) -> None:
         """The overflow switch never sends a job to a macOS label it would not take otherwise.
 

tests/test_ci_owned_pool_rescue.py

@@ -1817,6 +1817,10 @@ def test_only_the_app_build_or_the_trusted_pool_is_judged(self):
         self.assertFalse(rescue.nightly_watched({"name": "visual-gate-nightly", "labels": [SESSION]}))
         self.assertFalse(rescue.nightly_watched({"name": "visual-gate-nightly",
                                                  "labels": ["glaeda-gui-std-xcode-26.6"]}))
+        # decide on the nightly slot is judged, so a busy slot moves it to Blacksmith Linux on the re-run.
+        self.assertTrue(rescue.nightly_watched({"name": "decide", "labels": ["glaeda-nightly-std-xcode-26.6"]}))
+        self.assertFalse(rescue.nightly_watched({"name": "decide", "labels": ["blacksmith-32vcpu-ubuntu-2404"]}))
+        self.assertFalse(rescue.nightly_watched({"name": "other", "labels": ["glaeda-nightly-std-xcode-26.6"]}))
 
 
 SESSION = "glaeda-session-std-xcode-26.6"

tests/test_nightly_trusted_routing.py

@@ -110,6 +110,32 @@ def test_next_build_never_waits_on_one_label_with_no_fallback(self):
         self.assertEqual(select(**{**picked, 'inputs.nightly_next_unsigned_run': '123'}), 'blacksmith-32vcpu-ubuntu-2404')
         self.assertEqual(select(**{**picked, 'github.repository_owner': 'someone'}), 'macos-26')
 
+    def test_decide_takes_the_owned_nightly_slot_then_blacksmith_linux(self):
+        # decide is the run's first job, so no picker runs before it and every other job waits on it. Attempt 1
+        # of a nightly-next push asks for glaeda's nightly slot, which only nightly jobs take; the owned-pool
+        # rescue watches that attempt (owned_pool_rescue.nightly_target()) and moves a decide no slot takes,
+        # and that re-run (attempt 2) takes the Blacksmith Linux set. Run 41234993713 queued decide on
+        # blacksmith-32vcpu-ubuntu-2404 with no fleet route. Leo, 2026-10-31: "FOR OUR NIGHTLIES ARE THEY END
+        # TO END ON OUR FLEET? WE CAN'T BE RELYING ON BLACKSMITH THAT CUTS OUT".
+        source = DOC['jobs']['decide']['runs-on'].removeprefix('${{').removesuffix('}}')
+        slot = 'glaeda-nightly-std-xcode-26.6'
+        base = {'github.repository_owner': 'manaflow-ai', 'github.ref': 'refs/heads/nightly-next',
+                'github.event_name': 'push', 'github.run_attempt': 1, 'vars.CI_PR_POOL_OWNED': '1',
+                'vars.CI_NIGHTLY_NEXT_BUILD_RUNNER': slot, 'vars.LINUX_RUNNER': 'blacksmith-32vcpu-ubuntu-2404'}
+
+        def decide(**changes):
+            return evaluate(parse_expression(source), {**base, **changes})
+
+        self.assertEqual(decide(), slot)
+        for change in ({'github.run_attempt': 2}, {'github.event_name': 'workflow_dispatch'},
+                       {'github.event_name': 'schedule', 'github.ref': 'refs/heads/main'},
+                       {'vars.CI_PR_POOL_OWNED': '0'}, {'vars.CI_NIGHTLY_NEXT_BUILD_RUNNER': ''}):
+            with self.subTest(change=change):
+                self.assertEqual(decide(**change), 'blacksmith-32vcpu-ubuntu-2404')
+        self.assertEqual(decide(**{'github.run_attempt': 2, 'vars.LINUX_RUNNER': 'blacksmith-16vcpu-ubuntu-2404'}),
+                         'blacksmith-16vcpu-ubuntu-2404')
+        self.assertEqual(decide(**{'github.repository_owner': 'someone'}), 'ubuntu-24.04')
+
     def test_only_the_nightly_pickers_read_the_next_build_runner(self):
         readers = sorted(path.name for path in (ROOT / '.github/workflows').glob('*.y*ml')
                          if 'vars.CI_NIGHTLY_NEXT_BUILD_RUNNER' in path.read_text()
@@ -118,7 +144,9 @@ def test_only_the_nightly_pickers_read_the_next_build_runner(self):
         uses = sorted(name for name, job in DOC['jobs'].items() if 'CI_NIGHTLY_NEXT_BUILD_RUNNER' in str(job.get('runs-on', '')))
         # Nightlies never use Blacksmith (Lawrence, 2026-10-30): these jobs' last fallback is the owned nightly
         # slot (then the owned side lane), never a Blacksmith label.
-        self.assertEqual(uses, ['generate-nightly-deltas', 'sign-nightly-identity-legacy', 'verify-nightly-identity-legacy'])
+        # decide's attempt 1 asks for the nightly slot first, then Blacksmith Linux (test_decide_*).
+        self.assertEqual(uses, ['decide', 'generate-nightly-deltas', 'sign-nightly-identity-legacy',
+                                'verify-nightly-identity-legacy'])
         steps = sorted(name for name, job in DOC['jobs'].items() if 'CI_NIGHTLY_NEXT_BUILD_RUNNER' in str(job.get('steps', '')))
         # The verify jobs' owned pick (pick-nightly-sign-runner) offers the same labels as the build's.
         self.assertEqual(steps, ['pick-nightly-build-runner', 'pick-nightly-sign-runner'])