Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 5 additions & 1 deletion .github/nightly-step-coverage.json
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,8 @@
"nightly.yml": {
"decide": {
"Mint the routing token for early nightly controls": {"plumbing": true},
"Place early controls on fleet Linux or free owned minis": {"pr": "cmux-next-guards.yml:ci-guard-group:Run the ci guard group and the routed groups", "note": "guards-local.sh runs test_ci_portable_job_placement.py from the ci group; the reusable workflow excludes that matrix row because this job already runs it"},
"Place credentialed early picker on owned minis": {"pr": "cmux-next-guards.yml:ci-guard-group:Run the ci guard group and the routed groups", "note": "Existing portable placement tests cover unknown API capacity and owned-only eligibility"},
"Admit early controls on reserved fleet Linux": {"pr": "cmux-next-guards.yml:ci-guard-group:Run the ci guard group and the routed groups", "note": "guards-local.sh runs test_ci_portable_job_placement.py and test_nightly_linux_admission.py from the ci group; the reusable workflow excludes that matrix row because this job already runs it"},
"Decide whether a nightly build is needed": {"pr": "ci-guards.yml:workflow-guard-tests:Validate nightly push throttle"}},
"promote-nightly-next": {
"Decide the promotion": {"pr": "ci-guards.yml:workflow-guard-tests:Validate nightly push throttle"},
Expand Down Expand Up @@ -114,13 +115,15 @@
"Retry the daemon attestation": {"nightly_only": "build provenance attestation needs OIDC"},
"Report a missing daemon attestation": {"plumbing": true}},
"build-nightly-inputs": {
"Set up Rust for the third-party notices": {"plumbing": true, "note": "Installs Cargo for exact notice dependency trees on clean fleet Linux runners"},
"Source key of the web bundles": {"plumbing": true},
"Restore the published web bundles": {"plumbing": true, "note": "Restores the same keyed bundle as cmux-next.yml; the installer verifies its source manifest before use"},
"Drop stale submodule checkouts (self-hosted reused workspace)": {"plumbing": true},
"Give this job its own bun": {"plumbing": true},
"Build the web bundles": {"pr": "cmux-next.yml:web-bundles:Build the web bundles for this source key"},
"Generate the third-party notices": {"pr": "cmux-next-guards.yml:rust-notices-tree:Generate the Rust notices from this tree (as the nightly does)"}},
"resolve-nightly-cmux-tui-client": {
"Install gh if this Linux runner lacks it": {"plumbing": true},
"Compute portable OptChat Chief input key": {"plumbing": true, "note": "computes the source key before the Mac build so the Chief cache is portable across nightly runs; no product check"},
"Wait through GitHub API rate limits in this job's gh calls": {"pr": "ci-guards.yml:workflow-guard-tests:Validate nightly publish-path gh calls wait through a rate limit"},
"Validate the saved unsigned nightly source": {"pr": "ci-guards.yml:workflow-guard-tests:Validate unsigned nightly resume provenance and routing"},
Expand Down Expand Up @@ -226,6 +229,7 @@
"Build the unsigned DMG": {"nightly_only": "packages the notarized release-pool app with create-dmg, as notarize-nightly-dmg.sh does"}},
"signer-dmg": {"Install the signer": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the release signer scripts"}, "Sign, notarize and staple the DMG": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the release signer scripts"}, "Write the appcast": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the release signer scripts"}},
"find-nightly-next-recovery": {
"Install gh if this Linux runner lacks it": {"plumbing": true},
"Wait through GitHub API rate limits in this job's gh calls": {"pr": "ci-guards.yml:workflow-guard-tests:Validate nightly publish-path gh calls wait through a rate limit"},

"Pick the Blacksmith macOS label with a moving queue": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the nightly signing jobs take a Blacksmith label with a moving queue"},
Expand Down
4 changes: 3 additions & 1 deletion .github/workflows/ci-guards.yml
Original file line number Diff line number Diff line change
Expand Up @@ -919,7 +919,9 @@ jobs:

- name: Validate portable placement (fleet Linux, then a free mini, then Blacksmith)
if: ${{ !cancelled() && matrix.group == 'ci' }}
run: python3 tests/test_ci_portable_job_placement.py
run: |
python3 tests/test_ci_portable_job_placement.py
python3 tests/test_nightly_linux_admission.py
Comment on lines +922 to +924

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P2: The admission test shares a run step with the placement test, so a placement-test failure exits the shell before the admission test runs. Give it a separate !cancelled() step so both guard failures are reported.

Prompt for AI agents
Check if this issue is valid — if so, understand the root cause and fix it. When an issue isn't valid or won't be fixed in this PR, reply in its thread with the reason and then resolve the thread. At .github/workflows/ci-guards.yml, line 920:

<comment>The admission test shares a `run` step with the placement test, so a placement-test failure exits the shell before the admission test runs. Give it a separate `!cancelled()` step so both guard failures are reported.</comment>

<file context>
@@ -917,7 +917,9 @@ jobs:
       - name: Validate portable placement (fleet Linux, then a free mini, then Blacksmith)
         if: ${{ !cancelled() && matrix.group == 'ci' }}
-        run: python3 tests/test_ci_portable_job_placement.py
+        run: |
+          python3 tests/test_ci_portable_job_placement.py
+          python3 tests/test_nightly_linux_admission.py
</file context>
Suggested change
run: |
python3 tests/test_ci_portable_job_placement.py
python3 tests/test_nightly_linux_admission.py
run: python3 tests/test_ci_portable_job_placement.py
- name: Validate nightly Linux admission
if: ${{ !cancelled() && matrix.group == 'ci' }}
run: python3 tests/test_nightly_linux_admission.py


- name: Validate the rescue's re-runs route alike for both bot identities
if: ${{ !cancelled() && matrix.group == 'ci' }}
Expand Down
68 changes: 53 additions & 15 deletions .github/workflows/nightly.yml
Original file line number Diff line number Diff line change
Expand Up @@ -194,10 +194,14 @@ jobs:
if: github.event_name != 'workflow_dispatch' || ((inputs.probe_mode || 'off') == 'off' && inputs.promote_nightly_next_sha == '')
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.run_attempt == 1 && !startsWith(vars.CI_CONTROL_RUNNER, 'blacksmith-') && vars.CI_CONTROL_RUNNER || github.ref == 'refs/heads/nightly-next' && github.event_name == 'push' && github.run_attempt == 1 && vars.CI_PR_POOL_OWNED == '1' && vars.CI_NIGHTLY_NEXT_BUILD_RUNNER || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
outputs:
early_linux_runner: ${{ steps.place-early.outputs.linux_runner }}
early_linux_jobs: ${{ steps.place-early.outputs.linux_jobs }}
early_mini_runner: ${{ steps.place-early.outputs.mini_runner }}
early_mini_jobs: ${{ steps.place-early.outputs.mini_jobs }}
early_build_nightly_inputs_runner: ${{ steps.place-early.outputs.build_nightly_inputs_runner }}
early_find_nightly_next_recovery_runner: ${{ steps.place-early.outputs.find_nightly_next_recovery_runner }}
early_resolve_nightly_tui_client_runner: ${{ steps.place-early.outputs.resolve_nightly_tui_client_runner }}
early_resolve_nightly_ghostty_cli_helper_runner: ${{ steps.place-early.outputs.resolve_nightly_ghostty_cli_helper_runner }}
early_linux_runner: ${{ steps.place-early-mac.outputs.linux_runner }}
early_linux_jobs: ${{ steps.place-early-mac.outputs.linux_jobs }}
early_mini_runner: ${{ steps.place-early-mac.outputs.mini_runner }}
early_mini_jobs: ${{ steps.place-early-mac.outputs.mini_jobs }}
early_attempt: ${{ github.run_attempt }}
should_build: ${{ steps.decide.outputs.should_build }}
head_sha: ${{ steps.decide.outputs.head_sha }}
Expand Down Expand Up @@ -259,6 +263,8 @@ jobs:
sparse-checkout: |
/scripts/ci/nightly-publication-marker.py
/scripts/ci/portable_job_placement.py
/scripts/ci/nightly_linux_admission.py
/scripts/ci/fleet_admission_transport.py
/scripts/ci/pr_runner_pool.py
/scripts/ci/side_lane_placement.py
/scripts/ci/next_macos_cascade.py
Expand Down Expand Up @@ -792,24 +798,51 @@ jobs:
permission-administration: read
permission-organization-self-hosted-runners: read

# Bounded fleet placement; later jobs consume only their own attempt.
- name: Place early controls on fleet Linux or free owned minis
id: place-early
# The Mac-build picker uses the routing App key. It keeps the existing
# owned-Mac/Blacksmith trust path and never enters the secret-free JIT pool.
- name: Place credentialed early picker on owned minis
id: place-early-mac
if: steps.decide.outputs.track == 'nightly-next' && steps.decide.outputs.should_build == 'true'
continue-on-error: true
timeout-minutes: 5
env:
JOBS: resolve-nightly-cmux-tui-client pick-nightly-build-runner resolve-nightly-ghostty-cli-helper build-nightly-inputs find-nightly-next-recovery
# Keep the compiler artifact executable on the Linux signing jobs.
LINUX_ONLY: signer-build
JOBS: pick-nightly-build-runner
NIGHTLY: "1"
NIGHTLY_WAIT_FOR_FLEET: "1"
FLEET_LINUX: ${{ vars.LINUX_RUNNER }}
FLEET_LINUX: ""
SIDE_LABEL: ${{ vars.CI_PR_POOL_OWNED == '1' && vars.CI_SIDE_LANE_RUNNER || '' }}
ROUTE_TOKEN: ${{ steps.early-route-token.outputs.token }}
SNAPSHOT_TOKEN: ${{ github.token }}
run: python3 scripts/ci/portable_job_placement.py

# The controller reserves unconsumed JIT runners before placement.
# Only a matching 180-second no-start receipt authorizes paid overflow.
- name: Admit early controls on reserved fleet Linux
id: place-early
if: steps.decide.outputs.track == 'nightly-next' && steps.decide.outputs.should_build == 'true'
continue-on-error: true
timeout-minutes: 5
env:
JOBS: resolve-nightly-cmux-tui-client resolve-nightly-ghostty-cli-helper build-nightly-inputs find-nightly-next-recovery
NIGHTLY: "1"
NIGHTLY_WAIT_FOR_FLEET: "1"
CMUX_CI_CONTROLLER: ${{ vars.CMUX_FLEET_CONTROLLER }}
OVERFLOW_LINUX: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
ADMISSION_RECEIPT_PATH: ${{ runner.temp }}/early-nightly-admission.json
SIDE_LABEL: ${{ vars.CI_PR_POOL_OWNED == '1' && vars.CI_SIDE_LANE_RUNNER || '' }}
ROUTE_TOKEN: ${{ steps.early-route-token.outputs.token }}
SNAPSHOT_TOKEN: ${{ github.token }}
run: python3 scripts/ci/nightly_linux_admission.py

- name: Record early nightly admission receipt
if: always() && steps.place-early.outcome != 'skipped'
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: early-nightly-admission-${{ github.run_attempt }}
path: ${{ runner.temp }}/early-nightly-admission.json
if-no-files-found: warn
retention-days: 3

# Moves the protected branch `nightly-next` to a feat-cmux-next commit whose cmux-next.yml
# push run passed its Release compile. Runs only on main, in `release`, with the release App
# (the ruleset's only bypass actor). Guard: scripts/ci/promote-nightly-next.mjs, from this
Expand Down Expand Up @@ -1406,7 +1439,7 @@ jobs:
resolve-nightly-ghostty-cli-helper:
needs: [decide, resolve-nightly-cmux-tui-client]
if: needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.decide.outputs.build_only != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true' && needs.decide.outputs.continue_only != 'true'
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' resolve-nightly-ghostty-cli-helper ') && needs.decide.outputs.early_linux_runner || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_mini_jobs, ' resolve-nightly-ghostty-cli-helper ') && needs.decide.outputs.early_mini_runner || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_resolve_nightly_ghostty_cli_helper_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
timeout-minutes: 15
permissions:
contents: read
Expand Down Expand Up @@ -1672,7 +1705,7 @@ jobs:
name: build nightly inputs
needs: [decide, resolve-nightly-cmux-tui-client]
if: needs.decide.outputs.should_build == 'true' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.continue_only != 'true' && inputs.nightly_next_unsigned_run == '' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true'
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' build-nightly-inputs ') && needs.decide.outputs.early_linux_runner || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_mini_jobs, ' build-nightly-inputs ') && needs.decide.outputs.early_mini_runner || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_build_nightly_inputs_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
permissions:
contents: read
timeout-minutes: 30
Expand Down Expand Up @@ -1731,6 +1764,11 @@ jobs:
if-no-files-found: error
retention-days: 1

- name: Set up Rust for the third-party notices
uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c
with:
toolchain: 1.95.0

- name: Generate the third-party notices
env:
CMUX_NOTICES_ALLOW_CARGO: "1"
Expand Down Expand Up @@ -3162,7 +3200,7 @@ jobs:
resolve-nightly-cmux-tui-client:
needs: decide
if: needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *')
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' resolve-nightly-cmux-tui-client ') && needs.decide.outputs.early_linux_runner || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_mini_jobs, ' resolve-nightly-cmux-tui-client ') && needs.decide.outputs.early_mini_runner || needs.decide.outputs.track == 'nightly-next' && 'cmux-fleet-linux-x64' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_resolve_nightly_tui_client_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
permissions:
contents: read
actions: read
Expand Down Expand Up @@ -5599,7 +5637,7 @@ jobs:
find-nightly-next-recovery:
needs: [decide]
if: needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.no_publish != 'true' && needs.decide.outputs.should_build == 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *')
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' find-nightly-next-recovery ') && needs.decide.outputs.early_linux_runner || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_mini_jobs, ' find-nightly-next-recovery ') && needs.decide.outputs.early_mini_runner || needs.decide.outputs.track == 'nightly-next' && 'cmux-fleet-linux-x64' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_find_nightly_next_recovery_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }}
timeout-minutes: 70
permissions:
actions: read
Expand Down
66 changes: 66 additions & 0 deletions scripts/ci/fleet_admission_transport.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,66 @@
"""Read fleet admission from its configured authenticated transport.

The existing controller uses HTTP over Tailscale. Accept that only for a
numeric address belonging to an online peer of the running local Tailscale
daemon. The encrypted peer connection is the existing authority; no bearer
secret or new runner trust is introduced. HTTPS uses normal certificate and
hostname verification. Neither path follows redirects or environment proxies.
"""
from __future__ import annotations

import ipaddress
import json
import shutil
import subprocess
import urllib.parse
import urllib.request
from pathlib import Path
from typing import Any, Mapping


class NoRedirect(urllib.request.HTTPRedirectHandler):
def redirect_request(self, req, fp, code, msg, headers, newurl):
raise ValueError("fleet admission must not redirect its authority")


def verify_authority(url: str) -> None:
parsed = urllib.parse.urlsplit(url)
if not parsed.hostname or parsed.username or parsed.password or parsed.fragment:
raise ValueError("invalid fleet admission authority")
if parsed.scheme == "https":
return
if parsed.scheme != "http":
raise ValueError("fleet admission needs HTTPS or the existing Tailscale transport")
try:
address = ipaddress.ip_address(parsed.hostname)
except ValueError as error:
raise ValueError("HTTP fleet admission needs a numeric Tailscale peer address") from error
if not (address in ipaddress.ip_network("100.64.0.0/10")
or address in ipaddress.ip_network("fd7a:115c:a1e0::/48")):
raise ValueError("plain HTTP fleet admission is outside Tailscale")
executable = shutil.which("tailscale")
if executable is None:
app = Path("/Applications/Tailscale.app/Contents/MacOS/Tailscale")
executable = str(app) if app.is_file() else None
if executable is None:
raise ValueError("cannot authenticate the controller's Tailscale peer")
result = subprocess.run([executable, "status", "--json"], check=True,
capture_output=True, text=True, timeout=5)
status = json.loads(result.stdout)
peers = status.get("Peer") or {}
if status.get("BackendState") != "Running" or not any(
peer.get("Online") is True and str(address) in (peer.get("TailscaleIPs") or [])
for peer in peers.values()):
raise ValueError("controller is not an online authenticated Tailscale peer")


def exchange(url: str, payload: Mapping[str, Any] | None = None) -> dict[str, Any]:
verify_authority(url)
data = json.dumps(payload).encode() if payload is not None else None
request = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"})
opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect())
with opener.open(request, timeout=35) as response:
value = json.load(response)
if not isinstance(value, dict):
raise ValueError("admission response must be an object")
return value
Loading
Loading