From eba48ff9dd0a766e9b98f36394c75e359d9e1167 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 22:25:49 -0700 Subject: [PATCH 1/4] ci(nightly): no nightly job pinned to the 32 vCPU size; the display visual gate is opt-in Seventeen nightly jobs (both pickers, signer-app/notary/dmg, every publish job) named blacksmith-32vcpu-ubuntu-2404 literally. From about 03:45Z on 2026-10-31 that size waited 8-15 minutes per job while 16 vCPU started, and the nightly chains about ten of these one after another, so every nightly-next run since f8c1ccbed stalled. They now take the same literal-only expression as the other Linux jobs: 16 vCPU when LINUX_RUNNER names it, else 32. Only literal Blacksmith labels result, so no secret-holding job can follow a variable to a reused machine. The visual gate (a display Mac, 37 minutes of f8c1ccbed's critical path) now runs only when NIGHTLY_NEXT_VISUAL_GATE is set to a mode; unset is off. Leo, 2026-10-31: "i dont like doing display tests anymore". Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-d750cf10922c785efa64 --- .github/workflows/nightly.yml | 36 +++++++++++++++++------------------ 1 file changed, 18 insertions(+), 18 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6ef8986e1eda..6fe4486388d5 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -801,7 +801,7 @@ jobs: if: github.event_name == 'workflow_dispatch' && inputs.promote_nightly_next_sha != '' && github.ref == 'refs/heads/main' && github.repository == 'manaflow-ai/cmux' # Holds only the release App credentials (CMUX_RELEASE_APP_*). environment: nightly-next-promote - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 10 permissions: contents: read @@ -931,7 +931,7 @@ jobs: # Holds only the release App credentials (CMUX_RELEASE_APP_*). The approval # was given to release-surface-ack on this run; guard.approvedBy() reads it. environment: nightly-next-promote - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 10 permissions: contents: read @@ -1745,7 +1745,7 @@ jobs: needs: [decide, resolve-nightly-cmux-tui-client] if: ${{ !cancelled() && needs.resolve-nightly-cmux-tui-client.result == 'success' }} continue-on-error: true - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} # A dispatch mints the routing token too, so the key is read in the routing environment, as # the other dispatchable pickers do (iroh-release-gate.yml, test-ios.yml). environment: ci-routing-agent @@ -3939,7 +3939,7 @@ jobs: # A failed pick reads as success to the signing jobs, whose runs-on then falls back to # vars.MACOS_RUNNER_26: a pick never blocks signing. It holds a key: Blacksmith Linux 32. continue-on-error: true - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} # The owned pick mints the routing token (GLAEDA_ROUTE_APP_KEY), so the key is read in the # routing environment, as pick-nightly-build-runner does (tests/test_supplychain_security.py). environment: ci-routing-agent @@ -5270,7 +5270,7 @@ jobs: environment: ${{ needs.decide.outputs.environment }} needs: [decide, build-nightly-app, resolve-nightly-cmux-tui-client, prepare-nightly-variant, signer-build] if: needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true' - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 75 permissions: contents: read @@ -5359,7 +5359,7 @@ jobs: signer-app-notary: environment: ${{ needs.decide.outputs.environment }} needs: [decide, resolve-nightly-cmux-tui-client, signer-build, signer-app] - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 40 permissions: contents: read @@ -5476,7 +5476,7 @@ jobs: signer-dmg: environment: ${{ needs.decide.outputs.environment }} needs: [decide, build-nightly-app, resolve-nightly-cmux-tui-client, signer-build, signer-app, signer-app-notary, signer-dmg-image] - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 40 permissions: contents: read @@ -5787,7 +5787,7 @@ jobs: # with no idle runner; only `enforce` with a failing verdict holds this run's build. route-visual-gate-nightly: needs: [decide, resolve-nightly-cmux-tui-client] - if: github.repository == 'manaflow-ai/cmux' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.should_build == 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true' && needs.decide.outputs.continue_only != 'true' && needs.decide.outputs.no_publish != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true' && vars.NIGHTLY_NEXT_VISUAL_GATE != 'off' + if: github.repository == 'manaflow-ai/cmux' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.should_build == 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true' && needs.decide.outputs.continue_only != 'true' && needs.decide.outputs.no_publish != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true' && vars.NIGHTLY_NEXT_VISUAL_GATE != '' && vars.NIGHTLY_NEXT_VISUAL_GATE != 'off' runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 5 # GLAEDA_ROUTE_APP_KEY lives in the routing environment, like every other route job's. @@ -6100,7 +6100,7 @@ jobs: # failing verdict, or mode `invalid`. A recovered build was judged when it was continued. if: "always() && !cancelled() && needs.decide.result == 'success' && needs.decide.outputs.should_build == 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.decide.outputs.no_publish != 'true' && (((needs.build-sign-notarize-nightly.result == 'success' || needs.signer-dmg.result == 'success') && needs.build-sign-notarize-nightly.outputs.notary_pending != 'true' && (needs.signer-dmg.result == 'success' || needs.verify-signed-nightly.result == 'success') && needs.verify-final-dmg-nightly.result == 'success' && needs.visual-gate-nightly-verdict.outputs.blocks != 'true' && needs.route-visual-gate-nightly.outputs.mode != 'invalid') || (needs.recover-nightly-next-notarization.outputs.accepted == 'true' && needs.verify-recovered-nightly-next.result == 'success'))" # The release App key remains scoped to this protected environment. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 120 env: NIGHTLY_LEGACY_VARIANT: ${{ needs.decide.outputs.legacy_variant }} @@ -6586,7 +6586,7 @@ jobs: environment: ${{ needs.decide.outputs.environment }} permissions: contents: write - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 150 outputs: release_id: ${{ steps.publish_assets.outputs.release_id }} @@ -6708,7 +6708,7 @@ jobs: cancel-in-progress: false permissions: contents: write - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 90 env: NIGHTLY_FEEDS: ${{ needs.decide.outputs.feeds }} @@ -6810,7 +6810,7 @@ jobs: permissions: contents: read actions: read - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 75 env: BUILD: ${{ needs.publish-nightly.outputs.build }} @@ -6863,7 +6863,7 @@ jobs: # A feed read error must not turn a run that published cmux NEXT red; the # macOS job then does not start (it needs this job's success and a build). continue-on-error: true - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 65 permissions: contents: read @@ -7337,7 +7337,7 @@ jobs: # failure) for the whole sign, verify and publish chain. Runs no bundle code # and needs no Apple tool: Blacksmith Linux 32 vCPU (trusted with release # secrets), never vars.LINUX_RUNNER, which may name an owned host. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 90 env: NIGHTLY_BUILD: ${{ needs.legacy-identity-target.outputs.build }} @@ -7720,7 +7720,7 @@ jobs: concurrency: group: release-${{ needs.decide.outputs.track == 'nightly-next' && 'deltas' || 'publish' }}-${{ needs.decide.outputs.release_tag }} cancel-in-progress: false - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} # It holds the publish group, so a hang must not hold it for the default 6 h. nightly-next # makes no gh call here (deltas and feed go to R2) and took at most 1.3 minutes in 56 runs; # other tracks keep the default, as their release uploads may wait out the rate limit. @@ -7944,7 +7944,7 @@ jobs: if: "!cancelled() && needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.no_publish != 'true' && needs.publish-nightly.result == 'success' && needs.publish-nightly.outputs.already_published != 'true' && needs.publish-nightly.outputs.refused != 'true' && needs.publish-nightly.outputs.source != 'recovered'" environment: content-signing # The content-signing key remains scoped to this protected environment. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 70 permissions: contents: read @@ -8079,7 +8079,7 @@ jobs: cancel-in-progress: false if: "!cancelled() && needs.decide.outputs.track == 'nightly-next' && needs.publish-nightly.result == 'success' && needs.publish-nightly.outputs.already_published != 'true' && needs.publish-nightly.outputs.refused != 'true' && needs.sign-release-notes.result == 'success'" environment: ${{ needs.decide.outputs.environment }} - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 70 permissions: contents: read @@ -8128,7 +8128,7 @@ jobs: needs: [decide, build-nightly-app, resolve-nightly-cmux-tui-client, publish-release-notes] if: "!cancelled() && github.repository_owner == 'manaflow-ai' && needs.decide.outputs.track == 'nightly-next' && needs.publish-release-notes.result == 'success'" continue-on-error: true - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 90 # The private-terms secret lives only in this environment, which admits only # the protected branches (main, feat-cmux-next, nightly-next). From 8db21301d11102c41cf9a0028e54ae133592bd89 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 22:37:59 -0700 Subject: [PATCH 2/4] ci(nightly): a newer nightly-next push no longer skips a running build Every nightly-next push run from 04:28Z to 05:23Z on 2026-10-31 found the branch moved by the time it reached resolve-nightly-cmux-tui-client (promotions about every 6 minutes, 12+ minutes of Linux queue before that step) and skipped its build, so no run built. Runs are run-scoped since the shared slot went away; the rulebook has a newer tip build alongside a running one, and publish-nightly only publishes a build newer than the feed. The skip is now opt-in (NIGHTLY_NEXT_SUPERSEDE_SKIP=1). Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-d750cf10922c785efa64 --- .github/workflows/nightly.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 6fe4486388d5..5649789b8859 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -3283,9 +3283,14 @@ jobs: BUILD_SHA: ${{ steps.build.outputs.build_sha }} ARCHS: ${{ needs.decide.outputs.compile_archs }} run: bash scripts/ci/nightly-steps/chief-input-key.sh + # Off unless NIGHTLY_NEXT_SUPERSEDE_SKIP is 1. NEXT runs are run-scoped (no shared slot), and the + # rulebook's Nightly freshness has a newer tip build alongside a running one; publish-nightly + # still publishes only a build newer than the feed. With this on, every run from 04:28Z to + # 05:23Z on 2026-10-31 skipped its build: nightly-next moved (about every 6 minutes) before a + # run reached this step (12+ minutes on a slow Linux queue), so no run built at all. - name: Leave the build slot to a newer nightly-next push id: superseded - if: github.event_name == 'push' && github.ref == 'refs/heads/nightly-next' + if: github.event_name == 'push' && github.ref == 'refs/heads/nightly-next' && vars.NIGHTLY_NEXT_SUPERSEDE_SKIP == '1' env: GH_TOKEN: ${{ github.token }} TIP_SHA: ${{ needs.decide.outputs.head_sha }} From e3e4e3657db2910f4c97d68dc198bfaa9804493e Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 23:02:21 -0700 Subject: [PATCH 3/4] test(nightly): the legacy publisher follows LINUX_RUNNER between two literal Blacksmith labels Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-d750cf10922c785efa64 --- tests/test_nightly_legacy_publish_linux.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/tests/test_nightly_legacy_publish_linux.py b/tests/test_nightly_legacy_publish_linux.py index f49a1df5aa4e..017bbdaa9572 100644 --- a/tests/test_nightly_legacy_publish_linux.py +++ b/tests/test_nightly_legacy_publish_linux.py @@ -62,9 +62,13 @@ class Workflow(unittest.TestCase): def test_publish_runs_on_blacksmith_linux(self) -> None: body = job("publish-nightly-identity-legacy") runs_on = re.search(r"^ runs-on: (.+)$", body, re.MULTILINE).group(1) - self.assertEqual(runs_on, "${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || 'blacksmith-32vcpu-ubuntu-2404' }}") - # It holds the Sparkle and R2 keys: never an owned host, never a Mac. - self.assertNotIn("LINUX_RUNNER", runs_on) + self.assertEqual(runs_on, "${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || " + "vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || " + "'blacksmith-32vcpu-ubuntu-2404' }}") + # It holds the Sparkle and R2 keys: never an owned host, never a Mac. LINUX_RUNNER only + # chooses between two literal Blacksmith labels; its value is never the label. + self.assertNotRegex(runs_on, r"vars\.LINUX_RUNNER \|\|") + self.assertNotIn("&& vars.LINUX_RUNNER", runs_on) def test_publish_uses_no_apple_tool(self) -> None: body = job("publish-nightly-identity-legacy") From c5a0a56c90293bc3d6425f9ec04b666ada62cc8d Mon Sep 17 00:00:00 2001 From: Leo Li Date: Fri, 30 Oct 2026 23:10:15 -0700 Subject: [PATCH 4/4] test(signing): a runs-on selector that only picks between literal ephemeral labels passes rule 7 `vars.X == 'L' && 'L' ||` compares the variable and runs on the literal, so a variable can never route a secret-holding job to a reused machine through it. Co-Authored-By: Claude Opus 5.5 Agent-Session: https://cmux.com/deeplink/agent?id=aid-d750cf10922c785efa64 --- ..._keychain_is_deleted_before_bundle_code.py | 26 ++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/tests/test_ci_signing_keychain_is_deleted_before_bundle_code.py b/tests/test_ci_signing_keychain_is_deleted_before_bundle_code.py index c2669b93fef3..b479d3f51a55 100644 --- a/tests/test_ci_signing_keychain_is_deleted_before_bundle_code.py +++ b/tests/test_ci_signing_keychain_is_deleted_before_bundle_code.py @@ -166,6 +166,16 @@ # `vars.CI_CLOUD_OVERFLOW_SAVED && '' ||`: the overflow failover, # a boolean record choosing between two literal Blacksmith Linux labels. OVERFLOW_FAILOVER = re.compile(r"vars\.CI_CLOUD_OVERFLOW_SAVED\s*&&\s*'blacksmith-\d+vcpu-ubuntu-2404'\s*\|\|\s*") +# `vars.NAME == '