From c5d4373f3e1605e0dbd10eab10f54efc7852cf44 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 02:51:06 +0000 Subject: [PATCH 1/8] test(ci): require nightly controls to consume fleet admission receipts Agent-Session: https://cmux.com/deeplink/agent?id=aid-f484598f7f17f6e93407 --- scripts/ci/nightly_linux_admission.py | 173 ++++++++++++++++++++++++++ tests/test_nightly_linux_admission.py | 159 +++++++++++++++++++++++ 2 files changed, 332 insertions(+) create mode 100644 scripts/ci/nightly_linux_admission.py create mode 100644 tests/test_nightly_linux_admission.py diff --git a/scripts/ci/nightly_linux_admission.py b/scripts/ci/nightly_linux_admission.py new file mode 100644 index 000000000000..040d25b37103 --- /dev/null +++ b/scripts/ci/nightly_linux_admission.py @@ -0,0 +1,173 @@ +#!/usr/bin/env python3 +"""Admit secret-free nightly controls on reserved fleet Linux before paid overflow. + +The controller owns the 180-second lease and dispatches the existing HQ JIT +provisioner. A fleet receipt names an unconsumed runner for each requested job; +only a matching expired lease authorizes Blacksmith. Unknown admission keeps +work on the owned fleet. This client never cancels a job or workflow. +""" +from __future__ import annotations + +import datetime as dt +import json +import os +import re +import sys +import time +import urllib.request +from pathlib import Path +from typing import Any, Mapping + +import portable_job_placement as portable + +FLEET_LINUX = "cmux-fleet-linux-x64" +OVERFLOW = {"blacksmith-16vcpu-ubuntu-2404", "blacksmith-32vcpu-ubuntu-2404"} +JOBS = frozenset({"resolve-nightly-cmux-tui-client", "pick-nightly-build-runner", + "resolve-nightly-ghostty-cli-helper", "build-nightly-inputs", + "find-nightly-next-recovery", "signer-build"}) +WAIT_SECONDS = 180 +CLIENT_DEADLINE = 240 + + +def identity(env: Mapping[str, str]) -> dict[str, Any]: + jobs = sorted(set(env.get("JOBS", "").split())) + if not jobs or not set(jobs) <= JOBS: + raise ValueError("unknown nightly control job") + if env.get("GITHUB_REPOSITORY") != "manaflow-ai/cmux": + raise ValueError("fleet admission is only for manaflow-ai/cmux") + sha = env.get("GITHUB_SHA", "") + if not re.fullmatch(r"[0-9a-f]{40}", sha): + raise ValueError("source SHA must be a full commit") + run_id, attempt = int(env["GITHUB_RUN_ID"]), int(env["GITHUB_RUN_ATTEMPT"]) + if min(run_id, attempt) <= 0: + raise ValueError("run and attempt must be positive") + return {"repository": env["GITHUB_REPOSITORY"], "run_id": run_id, + "run_attempt": attempt, "source_sha": sha, "jobs": jobs} + + +def timestamp(value: Any) -> dt.datetime: + if not isinstance(value, str): + raise ValueError("missing receipt timestamp") + parsed = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + if parsed.tzinfo is None or parsed.utcoffset() != dt.timedelta(0): + raise ValueError("receipt timestamps must be UTC") + return parsed + + +def validate(receipt: Mapping[str, Any], request: Mapping[str, Any]) -> str: + for key in ("repository", "run_id", "run_attempt", "source_sha"): + if type(receipt.get(key)) is not type(request[key]) or receipt[key] != request[key]: + raise ValueError(f"admission receipt mismatches {key}") + if receipt.get("jobs") != request["jobs"]: + raise ValueError("admission receipt mismatches jobs") + lease = receipt.get("lease_id", "") + if not isinstance(lease, str) or not re.fullmatch(r"[0-9a-f]{24}", lease): + raise ValueError("invalid lease identity") + created, expires = timestamp(receipt.get("created_at")), timestamp(receipt.get("expires_at")) + if (expires - created).total_seconds() != WAIT_SECONDS: + raise ValueError("admission lease must give the fleet 180 seconds") + status = receipt.get("status") + if status == "pending": + return status + decided = timestamp(receipt.get("decided_at")) + if status == "fleet": + expected = {job: f"cmux-nightly-{lease}-{job}" for job in request["jobs"]} + if receipt.get("runners") != expected or not created <= decided < expires: + raise ValueError("fleet receipt does not reserve matching runners before expiry") + elif status == "overflow": + elapsed = receipt.get("no_start_seconds") + if (not isinstance(elapsed, (int, float)) or isinstance(elapsed, bool) + or elapsed < WAIT_SECONDS or decided < expires or receipt.get("runners") != {}): + raise ValueError("overflow requires a recorded 180-second fleet no-start") + else: + raise ValueError("unknown admission status") + return str(status) + + +def exchange(url: str, payload: Mapping[str, Any] | None = None) -> dict[str, Any]: + data = json.dumps(payload).encode() if payload is not None else None + request = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}) + with urllib.request.urlopen(request, timeout=35) as response: + value = json.load(response) + if not isinstance(value, dict): + raise ValueError("admission response must be an object") + return value + + +def admit(env: Mapping[str, str], request: Mapping[str, Any]) -> dict[str, Any]: + controller = env.get("CMUX_CI_CONTROLLER", "").rstrip("/") + if not controller: + raise ValueError("CMUX_FLEET_CONTROLLER is not configured") + endpoint = f"{controller}/v1/nightly-linux-admission" + deadline = time.monotonic() + CLIENT_DEADLINE + receipt = exchange(endpoint, request) + lease = receipt.get("lease_id") + # Each GET waits for a controller state change. This is a bounded long-poll, + # not repeated GitHub reads; only the controller decides fleet vs overflow. + while validate(receipt, request) == "pending": + if time.monotonic() >= deadline: + raise TimeoutError("nightly fleet admission did not return a terminal receipt") + receipt = exchange(f"{endpoint}/{lease}?wait=30") + if receipt.get("lease_id") != lease: + raise ValueError("admission changed lease identity") + return receipt + + +def routes(env: Mapping[str, str], request: Mapping[str, Any], receipt: Mapping[str, Any] | None, + runners=None, snapshot=None) -> dict[str, str]: + status = validate(receipt, request) if receipt is not None else "unknown" + if status == "fleet": + return dict(receipt["runners"]) + # A configured label is not an admission receipt. Only owned-mini placement + # is used here; the Linux choice comes from the lease or the declared JIT pool. + placement_env = {**env, "FLEET_LINUX": "", "LINUX_ONLY": "signer-build", + "NIGHTLY": "1", "NIGHTLY_WAIT_FOR_FLEET": "1"} + placed = portable.decide(placement_env, runners, snapshot, dt.datetime.now(dt.timezone.utc)) + fallback = FLEET_LINUX + if status == "overflow": + candidate = env.get("OVERFLOW_LINUX", "") + if candidate not in OVERFLOW: + raise ValueError("overflow runner is outside the declared set") + fallback = candidate + result = {job: fallback for job in request["jobs"]} + for job in placed.mini_jobs: + result[job] = placed.mini_runner + return result + + +def write(env: Mapping[str, str], selected: Mapping[str, str], receipt: Mapping[str, Any] | None, + error: str = "") -> None: + if env.get("GITHUB_OUTPUT"): + with open(env["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: + for job, label in selected.items(): + output.write(f"{job.replace('-', '_')}_runner={label}\n") + record = {"receipt": receipt, "runners": selected, "error": error} + if env.get("ADMISSION_RECEIPT_PATH"): + Path(env["ADMISSION_RECEIPT_PATH"]).write_text(json.dumps(record, sort_keys=True) + "\n") + if env.get("GITHUB_STEP_SUMMARY"): + with open(env["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as summary: + summary.write("### Early nightly fleet admission\n\n```json\n" + json.dumps(record, indent=2) + "\n```\n") + print(json.dumps(record, sort_keys=True)) + + +def main(env: Mapping[str, str] = os.environ) -> int: + request = identity(env) + receipt = None + error = "" + runners = snapshot = None + try: + receipt = admit(env, request) + if receipt["status"] == "overflow": + runners, snapshot = portable.read(env, dt.datetime.now(dt.timezone.utc)) + selected = routes(env, request, receipt, runners, snapshot) + except (Exception, portable.Deadline) as failure: + # Unavailable telemetry is never permission to start paid work. + error = f"fleet admission unavailable: {type(failure).__name__}" + receipt = None + selected = routes(env, request, None) + write(env, selected, receipt, error) + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_nightly_linux_admission.py b/tests/test_nightly_linux_admission.py new file mode 100644 index 000000000000..77cb97ccbda7 --- /dev/null +++ b/tests/test_nightly_linux_admission.py @@ -0,0 +1,159 @@ +#!/usr/bin/env python3 +"""Nightly paid overflow needs a matching, expired fleet admission lease.""" +from __future__ import annotations + +import datetime as dt +import json +import sys +import tempfile +import unittest +from pathlib import Path +from unittest import mock + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts/ci")) +sys.path.insert(0, str(ROOT / "tests")) +import nightly_linux_admission as admission +from test_nightly_trusted_routing import evaluate, parse_expression + +ENV = {"GITHUB_REPOSITORY": "manaflow-ai/cmux", "GITHUB_RUN_ID": "123", + "GITHUB_RUN_ATTEMPT": "1", "GITHUB_SHA": "a" * 40, + "JOBS": "control", "SIDE_LABEL": "glaeda-side-std-xcode-26.6", + "OVERFLOW_LINUX": "blacksmith-16vcpu-ubuntu-2404"} +ENV["JOBS"] = " ".join(sorted(admission.JOBS)) +REQUEST = admission.identity(ENV) +LEASE = "b" * 24 +CREATED = dt.datetime(2026, 11, 1, tzinfo=dt.timezone.utc) + + +def receipt(status="pending"): + value = {**REQUEST, "lease_id": LEASE, "status": status, + "created_at": CREATED.isoformat(), + "expires_at": (CREATED + dt.timedelta(seconds=180)).isoformat()} + if status == "fleet": + value.update(decided_at=(CREATED + dt.timedelta(seconds=15)).isoformat(), + runners={job: f"cmux-nightly-{LEASE}-{job}" for job in REQUEST["jobs"]}) + elif status == "overflow": + value.update(decided_at=value["expires_at"], no_start_seconds=180, runners={}) + return value + + +class Receipts(unittest.TestCase): + def test_reserved_runners_are_bound_to_each_job(self): + value = receipt("fleet") + self.assertEqual(admission.routes(ENV, REQUEST, value), value["runners"]) + self.assertNotIn("blacksmith", json.dumps(value["runners"])) + + def test_overflow_needs_all_180_seconds(self): + for seconds in (0, 179, 180): + with self.subTest(seconds=seconds): + value = receipt("overflow") + value["no_start_seconds"] = seconds + if seconds < 180: + with self.assertRaises(ValueError): + admission.validate(value, REQUEST) + else: + self.assertEqual(admission.validate(value, REQUEST), "overflow") + + def test_invalid_or_stale_receipts_never_authorize_routing(self): + mutations = [ + {"run_id": 124}, {"run_attempt": 2}, {"source_sha": "c" * 40}, + {"repository": "another/repo"}, {"jobs": ["signer-build"]}, + {"lease_id": "../bad"}, {"no_start_seconds": True}, + {"created_at": "2026-11-01T00:00:00"}, + {"expires_at": (CREATED + dt.timedelta(seconds=179)).isoformat()}, + {"decided_at": (CREATED + dt.timedelta(seconds=179)).isoformat()}, + {"status": "unknown"}, {"runners": {"signer-build": admission.FLEET_LINUX}}, + ] + for mutation in mutations: + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + admission.validate({**receipt("overflow"), **mutation}, REQUEST) + + def test_late_or_foreign_fleet_receipt_is_rejected(self): + for mutation in ({"decided_at": receipt()["expires_at"]}, + {"runners": {job: "blacksmith-16vcpu-ubuntu-2404" for job in REQUEST["jobs"]}}, + {"runners": {job: admission.FLEET_LINUX for job in REQUEST["jobs"]}}): + with self.subTest(mutation=mutation), self.assertRaises(ValueError): + admission.validate({**receipt("fleet"), **mutation}, REQUEST) + + def test_missing_admission_keeps_controls_on_existing_owned_routes(self): + selected = admission.routes(ENV, REQUEST, None) + self.assertEqual(selected["signer-build"], admission.FLEET_LINUX) + for job in admission.JOBS - {"signer-build"}: + self.assertEqual(selected[job], ENV["SIDE_LABEL"]) + + def test_unconfigured_mini_and_reruns_still_default_to_fleet_linux(self): + for config in ({**ENV, "SIDE_LABEL": ""}, {**ENV, "GITHUB_RUN_ATTEMPT": "3"}): + self.assertEqual(set(admission.routes(config, admission.identity(config), None).values()), + {admission.FLEET_LINUX}) + + def test_recorded_overflow_preserves_linux_signer_and_only_uses_declared_paid_set(self): + selected = admission.routes(ENV, REQUEST, receipt("overflow")) + self.assertEqual(selected["signer-build"], ENV["OVERFLOW_LINUX"]) + for label in ("ubuntu-24.04", "untrusted", "glaeda-side-std-xcode-26.6"): + with self.subTest(label=label), self.assertRaises(ValueError): + admission.routes({**ENV, "OVERFLOW_LINUX": label}, REQUEST, receipt("overflow")) + + def test_long_poll_consumes_same_lease_and_stops_at_terminal(self): + with mock.patch.object(admission, "exchange", side_effect=[receipt(), receipt("fleet")]) as exchange: + value = admission.admit({**ENV, "CMUX_CI_CONTROLLER": "http://controller"}, REQUEST) + self.assertEqual(value["status"], "fleet") + self.assertEqual(exchange.call_args.args, (f"http://controller/v1/nightly-linux-admission/{LEASE}?wait=30",)) + self.assertEqual(exchange.call_count, 2) + + def test_changed_lease_during_wait_is_rejected(self): + changed = receipt("overflow") + changed["lease_id"] = "c" * 24 + with mock.patch.object(admission, "exchange", side_effect=[receipt(), changed]): + with self.assertRaises(ValueError): + admission.admit({**ENV, "CMUX_CI_CONTROLLER": "http://controller"}, REQUEST) + + def test_long_poll_has_a_client_deadline(self): + with mock.patch.object(admission, "exchange", return_value=receipt()), \ + mock.patch.object(admission.time, "monotonic", side_effect=[0, 241]): + with self.assertRaises(TimeoutError): + admission.admit({**ENV, "CMUX_CI_CONTROLLER": "http://controller"}, REQUEST) + + def test_unavailable_controller_writes_fleet_outputs_and_no_overflow_receipt(self): + with tempfile.TemporaryDirectory() as scratch: + output, record = Path(scratch) / "output", Path(scratch) / "receipt.json" + config = {**ENV, "GITHUB_OUTPUT": str(output), "ADMISSION_RECEIPT_PATH": str(record)} + with mock.patch.object(admission, "admit", side_effect=TimeoutError): + self.assertEqual(admission.main(config), 0) + self.assertNotIn("blacksmith", output.read_text()) + self.assertIn(f"signer_build_runner={admission.FLEET_LINUX}", output.read_text()) + self.assertIsNone(json.loads(record.read_text())["receipt"]) + + +class Workflow(unittest.TestCase): + @classmethod + def setUpClass(cls): + cls.jobs = yaml.safe_load((ROOT / ".github/workflows/nightly.yml").read_text())["jobs"] + + def test_missing_stale_or_quota_failed_picker_cannot_select_blacksmith_first(self): + for name in admission.JOBS: + source = self.jobs[name]["runs-on"].removeprefix("${{").removesuffix("}}") + key = f"needs.decide.outputs.early_{name.replace('-', '_')}_runner" + for attempt in (1, 2, 3): + for overflow in admission.OVERFLOW: + context = {"github.repository_owner": "manaflow-ai", "github.run_attempt": attempt, + "needs.decide.outputs.early_attempt": "0", key: overflow, + "vars.LINUX_RUNNER": overflow} + with self.subTest(job=name, attempt=attempt, overflow=overflow): + self.assertEqual(evaluate(parse_expression(source), context), admission.FLEET_LINUX) + + def test_each_job_consumes_only_its_current_attempt_receipt(self): + for name in admission.JOBS: + source = self.jobs[name]["runs-on"].removeprefix("${{").removesuffix("}}") + key = f"needs.decide.outputs.early_{name.replace('-', '_')}_runner" + for label in (receipt("fleet")["runners"][name], ENV["OVERFLOW_LINUX"]): + context = {"github.repository_owner": "manaflow-ai", "github.run_attempt": 2, + "needs.decide.outputs.early_attempt": "2", key: label} + with self.subTest(job=name, label=label): + self.assertEqual(evaluate(parse_expression(source), context), label) + + +if __name__ == "__main__": + unittest.main() From 9717dcac5efefe504107075186a069607d631c36 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 02:57:25 +0000 Subject: [PATCH 2/8] ci(nightly): consume reserved fleet leases before Linux overflow Agent-Session: https://cmux.com/deeplink/agent?id=aid-f484598f7f17f6e93407 --- .github/nightly-step-coverage.json | 5 +- .github/workflows/ci-guards.yml | 4 +- .github/workflows/nightly.yml | 82 +++++++++++++++++++------ scripts/ci/nightly_linux_admission.py | 32 ++++++++-- scripts/ci/runner_routing_table.py | 11 +++- tests/test-execution.toml | 4 ++ tests/test_ci_nightly_off_blacksmith.py | 8 ++- tests/test_ci_portable_job_placement.py | 13 ++-- tests/test_nightly_linux_admission.py | 48 ++++++++++++++- tests/test_nightly_trusted_routing.py | 11 ++-- 10 files changed, 170 insertions(+), 48 deletions(-) diff --git a/.github/nightly-step-coverage.json b/.github/nightly-step-coverage.json index 58a5c89322e4..3cdf625ef4e7 100644 --- a/.github/nightly-step-coverage.json +++ b/.github/nightly-step-coverage.json @@ -4,7 +4,8 @@ "nightly.yml": { "decide": { "Mint the routing token for early nightly controls": {"plumbing": true}, - "Place early controls on fleet Linux or free owned minis": {"pr": "cmux-next-guards.yml:ci-guard-group:Run the ci guard group and the routed groups", "note": "guards-local.sh runs test_ci_portable_job_placement.py from the ci group; the reusable workflow excludes that matrix row because this job already runs it"}, + "Place credentialed early picker on owned minis": {"pr": "cmux-next-guards.yml:ci-guard-group:Run the ci guard group and the routed groups", "note": "Existing portable placement tests cover unknown API capacity and owned-only eligibility"}, + "Admit early controls on reserved fleet Linux": {"pr": "cmux-next-guards.yml:ci-guard-group:Run the ci guard group and the routed groups", "note": "guards-local.sh runs test_ci_portable_job_placement.py and test_nightly_linux_admission.py from the ci group; the reusable workflow excludes that matrix row because this job already runs it"}, "Decide whether a nightly build is needed": {"pr": "ci-guards.yml:workflow-guard-tests:Validate nightly push throttle"}}, "promote-nightly-next": { "Decide the promotion": {"pr": "ci-guards.yml:workflow-guard-tests:Validate nightly push throttle"}, @@ -208,7 +209,7 @@ "Launch smoke from a fresh home": {"nightly_only": "launches the final notarized DMG from a fresh home in the console session; only a release run has one (gate tested by tests/test_nightly_next_launch_smoke.py)"}, "Apple's checks of a pool-signed DMG and the app in it": {"nightly_only": "Apple's tools on the notarized release-pool DMG and its app; only a release run has one"}, "Bundle size budget": {"pr": "cmux-next.yml:checks:Script tests (pin-cmux-tui, embed-cef, ensure-cef store)"}}, - "signer-build": {"Set up Rust": {"plumbing": true}, + "signer-build": {"Verify the signer is a Linux x64 executable": {"pr": "cmux-next-guards.yml:ci-guard-group:Run the ci guard group and the routed groups", "note": "test_nightly_linux_admission.py executes this check against ELF, Mach-O, wrong architecture and truncated fixtures"}, "Set up Rust": {"plumbing": true}, "Build the signer": {"nightly_only": "builds the pinned release signer; the build script's pins are tested by tests/test_release_signer_scripts.py"}}, "signer-app": {"Install the signer": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the release signer scripts"}, "Prepare the app": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the release signer scripts"}, "Sign the app, the Computer Use helper notarized first": {"nightly_only": "signs with the release environment's Developer ID; the signer's rules are tested by its script tests"}, diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 04cdb75b551b..bb59c047f317 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -917,7 +917,9 @@ jobs: - name: Validate portable placement (fleet Linux, then a free mini, then Blacksmith) if: ${{ !cancelled() && matrix.group == 'ci' }} - run: python3 tests/test_ci_portable_job_placement.py + run: | + python3 tests/test_ci_portable_job_placement.py + python3 tests/test_nightly_linux_admission.py - name: Validate the rescue's re-runs route alike for both bot identities if: ${{ !cancelled() && matrix.group == 'ci' }} diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 8f153ce87999..c34620f71cff 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -194,10 +194,15 @@ jobs: if: github.event_name != 'workflow_dispatch' || ((inputs.probe_mode || 'off') == 'off' && inputs.promote_nightly_next_sha == '') runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.run_attempt == 1 && !startsWith(vars.CI_CONTROL_RUNNER, 'blacksmith-') && vars.CI_CONTROL_RUNNER || github.ref == 'refs/heads/nightly-next' && github.event_name == 'push' && github.run_attempt == 1 && vars.CI_PR_POOL_OWNED == '1' && vars.CI_NIGHTLY_NEXT_BUILD_RUNNER || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} outputs: - early_linux_runner: ${{ steps.place-early.outputs.linux_runner }} - early_linux_jobs: ${{ steps.place-early.outputs.linux_jobs }} - early_mini_runner: ${{ steps.place-early.outputs.mini_runner }} - early_mini_jobs: ${{ steps.place-early.outputs.mini_jobs }} + early_build_nightly_inputs_runner: ${{ steps.place-early.outputs.build_nightly_inputs_runner }} + early_find_nightly_next_recovery_runner: ${{ steps.place-early.outputs.find_nightly_next_recovery_runner }} + early_resolve_nightly_cmux_tui_client_runner: ${{ steps.place-early.outputs.resolve_nightly_cmux_tui_client_runner }} + early_resolve_nightly_ghostty_cli_helper_runner: ${{ steps.place-early.outputs.resolve_nightly_ghostty_cli_helper_runner }} + early_signer_build_runner: ${{ steps.place-early.outputs.signer_build_runner }} + early_linux_runner: ${{ steps.place-early-mac.outputs.linux_runner }} + early_linux_jobs: ${{ steps.place-early-mac.outputs.linux_jobs }} + early_mini_runner: ${{ steps.place-early-mac.outputs.mini_runner }} + early_mini_jobs: ${{ steps.place-early-mac.outputs.mini_jobs }} early_attempt: ${{ github.run_attempt }} should_build: ${{ steps.decide.outputs.should_build }} head_sha: ${{ steps.decide.outputs.head_sha }} @@ -259,6 +264,7 @@ jobs: sparse-checkout: | /scripts/ci/nightly-publication-marker.py /scripts/ci/portable_job_placement.py + /scripts/ci/nightly_linux_admission.py /scripts/ci/pr_runner_pool.py /scripts/ci/side_lane_placement.py /scripts/ci/next_macos_cascade.py @@ -792,23 +798,52 @@ jobs: permission-administration: read permission-organization-self-hosted-runners: read - # Bounded fleet placement; later jobs consume only their own attempt. - - name: Place early controls on fleet Linux or free owned minis + # The Mac-build picker uses the routing App key. It keeps the existing + # owned-Mac/Blacksmith trust path and never enters the secret-free JIT pool. + - name: Place credentialed early picker on owned minis + id: place-early-mac + if: steps.decide.outputs.track == 'nightly-next' && steps.decide.outputs.should_build == 'true' + continue-on-error: true + timeout-minutes: 5 + env: + JOBS: pick-nightly-build-runner + NIGHTLY: "1" + NIGHTLY_WAIT_FOR_FLEET: "1" + FLEET_LINUX: "" + SIDE_LABEL: ${{ vars.CI_PR_POOL_OWNED == '1' && vars.CI_SIDE_LANE_RUNNER || '' }} + ROUTE_TOKEN: ${{ steps.early-route-token.outputs.token }} + SNAPSHOT_TOKEN: ${{ github.token }} + run: python3 scripts/ci/portable_job_placement.py + + # The controller reserves unconsumed JIT runners before placement. + # Only a matching 180-second no-start receipt authorizes paid overflow. + - name: Admit early controls on reserved fleet Linux id: place-early if: steps.decide.outputs.track == 'nightly-next' && steps.decide.outputs.should_build == 'true' continue-on-error: true timeout-minutes: 5 env: - JOBS: resolve-nightly-cmux-tui-client pick-nightly-build-runner resolve-nightly-ghostty-cli-helper build-nightly-inputs find-nightly-next-recovery signer-build + JOBS: resolve-nightly-cmux-tui-client resolve-nightly-ghostty-cli-helper build-nightly-inputs find-nightly-next-recovery signer-build # Keep the compiler artifact executable on the Linux signing jobs. LINUX_ONLY: signer-build NIGHTLY: "1" NIGHTLY_WAIT_FOR_FLEET: "1" - FLEET_LINUX: ${{ vars.LINUX_RUNNER }} + CMUX_CI_CONTROLLER: ${{ vars.CMUX_FLEET_CONTROLLER }} + OVERFLOW_LINUX: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} + ADMISSION_RECEIPT_PATH: ${{ runner.temp }}/early-nightly-admission.json SIDE_LABEL: ${{ vars.CI_PR_POOL_OWNED == '1' && vars.CI_SIDE_LANE_RUNNER || '' }} ROUTE_TOKEN: ${{ steps.early-route-token.outputs.token }} SNAPSHOT_TOKEN: ${{ github.token }} - run: python3 scripts/ci/portable_job_placement.py + run: python3 scripts/ci/nightly_linux_admission.py + + - name: Record early nightly admission receipt + if: always() && steps.place-early.outcome != 'skipped' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: early-nightly-admission-${{ github.run_attempt }} + path: ${{ runner.temp }}/early-nightly-admission.json + if-no-files-found: warn + retention-days: 3 # Moves the protected branch `nightly-next` to a feat-cmux-next commit whose cmux-next.yml # push run passed its Release compile. Runs only on main, in `release`, with the release App @@ -1401,7 +1436,7 @@ jobs: resolve-nightly-ghostty-cli-helper: needs: [decide, resolve-nightly-cmux-tui-client] if: needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.decide.outputs.build_only != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true' && needs.decide.outputs.continue_only != 'true' - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' resolve-nightly-ghostty-cli-helper ') && needs.decide.outputs.early_linux_runner || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_mini_jobs, ' resolve-nightly-ghostty-cli-helper ') && needs.decide.outputs.early_mini_runner || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_resolve_nightly_ghostty_cli_helper_runner || 'cmux-fleet-linux-x64' }} timeout-minutes: 15 permissions: contents: read @@ -1667,7 +1702,7 @@ jobs: name: build nightly inputs needs: [decide, resolve-nightly-cmux-tui-client] if: needs.decide.outputs.should_build == 'true' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.continue_only != 'true' && inputs.nightly_next_unsigned_run == '' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true' - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' build-nightly-inputs ') && needs.decide.outputs.early_linux_runner || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_mini_jobs, ' build-nightly-inputs ') && needs.decide.outputs.early_mini_runner || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_build_nightly_inputs_runner || 'cmux-fleet-linux-x64' }} permissions: contents: read timeout-minutes: 30 @@ -3150,7 +3185,7 @@ jobs: resolve-nightly-cmux-tui-client: needs: decide if: needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' resolve-nightly-cmux-tui-client ') && needs.decide.outputs.early_linux_runner || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_mini_jobs, ' resolve-nightly-cmux-tui-client ') && needs.decide.outputs.early_mini_runner || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_resolve_nightly_cmux_tui_client_runner || 'cmux-fleet-linux-x64' }} permissions: contents: read actions: read @@ -5242,7 +5277,7 @@ jobs: signer-build: needs: [decide] if: needs.decide.outputs.track == 'nightly-next' && (inputs.sign_pool || vars.CI_NIGHTLY_NEXT_SIGN_POOL == '1') && vars.CMUX_NEXT_CEF_ON_DEMAND != '1' && needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.decide.outputs.build_only != 'true' && needs.decide.outputs.continue_only != 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.notary_paused != 'true' && needs.decide.outputs.notary_test != 'true' && needs.decide.outputs.notary_probe != 'true' && needs.decide.outputs.variants == '["arm64"]' && inputs.nightly_next_unsigned_run == '' - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' signer-build ') && needs.decide.outputs.early_linux_runner || github.run_attempt > 1 && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_signer_build_runner || 'cmux-fleet-linux-x64' }} timeout-minutes: 20 permissions: contents: read @@ -5263,17 +5298,26 @@ jobs: key: release-signer-rust-1.95.0-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.publish-tooling/scripts/ci/release-signer/build.sh', '.publish-tooling/scripts/ci/release-signer/apple-codesign-0.29.0.patch') }} - name: Set up Rust if: steps.cache.outputs.cache-hit != 'true' - run: | - set -euo pipefail - export PATH="$HOME/.cargo/bin:$PATH" - rustup toolchain install 1.95.0 --profile minimal - dirname "$(rustup which --toolchain 1.95.0 cargo)" >> "$GITHUB_PATH" + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.95.0 - name: Build the signer id: build run: | set -euo pipefail if [ -x "$RUNNER_TEMP/signer/rcodesign" ]; then echo "sha256=$(sha256sum "$RUNNER_TEMP/signer/rcodesign" | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" else bash .publish-tooling/scripts/ci/release-signer/build.sh "$RUNNER_TEMP/signer" >> "$GITHUB_OUTPUT"; fi + - name: Verify the signer is a Linux x64 executable + run: | + python3 - <<'PYELF' + import os + import pathlib + path = pathlib.Path(os.environ["RUNNER_TEMP"]) / "signer/rcodesign" + with path.open("rb") as binary: + header = binary.read(20) + if len(header) != 20 or header[:6] != b"\x7fELF\x02\x01" or header[18:20] != b"\x3e\x00": + raise SystemExit("nightly signer must be a 64-bit little-endian x86-64 ELF executable") + PYELF - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: nightly-signer-${{ github.run_attempt }} @@ -5565,7 +5609,7 @@ jobs: find-nightly-next-recovery: needs: [decide] if: needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.no_publish != 'true' && needs.decide.outputs.should_build == 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' find-nightly-next-recovery ') && needs.decide.outputs.early_linux_runner || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_mini_jobs, ' find-nightly-next-recovery ') && needs.decide.outputs.early_mini_runner || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_find_nightly_next_recovery_runner || 'cmux-fleet-linux-x64' }} timeout-minutes: 70 permissions: actions: read diff --git a/scripts/ci/nightly_linux_admission.py b/scripts/ci/nightly_linux_admission.py index 040d25b37103..4733ec74f1a0 100644 --- a/scripts/ci/nightly_linux_admission.py +++ b/scripts/ci/nightly_linux_admission.py @@ -10,8 +10,10 @@ import datetime as dt import json +import math import os import re +import signal import sys import time import urllib.request @@ -22,11 +24,11 @@ FLEET_LINUX = "cmux-fleet-linux-x64" OVERFLOW = {"blacksmith-16vcpu-ubuntu-2404", "blacksmith-32vcpu-ubuntu-2404"} -JOBS = frozenset({"resolve-nightly-cmux-tui-client", "pick-nightly-build-runner", +JOBS = frozenset({"resolve-nightly-cmux-tui-client", "resolve-nightly-ghostty-cli-helper", "build-nightly-inputs", "find-nightly-next-recovery", "signer-build"}) WAIT_SECONDS = 180 -CLIENT_DEADLINE = 240 +CLIENT_DEADLINE = 200 def identity(env: Mapping[str, str]) -> dict[str, Any]: @@ -72,12 +74,14 @@ def validate(receipt: Mapping[str, Any], request: Mapping[str, Any]) -> str: decided = timestamp(receipt.get("decided_at")) if status == "fleet": expected = {job: f"cmux-nightly-{lease}-{job}" for job in request["jobs"]} - if receipt.get("runners") != expected or not created <= decided < expires: + if (receipt.get("runners") != expected or not created <= decided < expires + or dt.datetime.now(dt.timezone.utc) >= expires): raise ValueError("fleet receipt does not reserve matching runners before expiry") elif status == "overflow": elapsed = receipt.get("no_start_seconds") if (not isinstance(elapsed, (int, float)) or isinstance(elapsed, bool) - or elapsed < WAIT_SECONDS or decided < expires or receipt.get("runners") != {}): + or not math.isfinite(elapsed) or elapsed < WAIT_SECONDS + or decided < expires or receipt.get("runners") != {}): raise ValueError("overflow requires a recorded 180-second fleet no-start") else: raise ValueError("unknown admission status") @@ -141,7 +145,7 @@ def write(env: Mapping[str, str], selected: Mapping[str, str], receipt: Mapping[ with open(env["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: for job, label in selected.items(): output.write(f"{job.replace('-', '_')}_runner={label}\n") - record = {"receipt": receipt, "runners": selected, "error": error} + record = {"identity": identity(env), "receipt": receipt, "runners": selected, "error": error} if env.get("ADMISSION_RECEIPT_PATH"): Path(env["ADMISSION_RECEIPT_PATH"]).write_text(json.dumps(record, sort_keys=True) + "\n") if env.get("GITHUB_STEP_SUMMARY"): @@ -150,6 +154,22 @@ def write(env: Mapping[str, str], selected: Mapping[str, str], receipt: Mapping[ print(json.dumps(record, sort_keys=True)) +def read_minis(env: Mapping[str, str]): + """Capacity reads cannot extend the placement step beyond its five-minute budget.""" + previous = signal.getsignal(signal.SIGALRM) + + def timeout(_signum, _frame): + raise portable.Deadline("owned-mini capacity read deadline") + + try: + signal.signal(signal.SIGALRM, timeout) + signal.setitimer(signal.ITIMER_REAL, 30) + return portable.read(env, dt.datetime.now(dt.timezone.utc)) + finally: + signal.setitimer(signal.ITIMER_REAL, 0) + signal.signal(signal.SIGALRM, previous) + + def main(env: Mapping[str, str] = os.environ) -> int: request = identity(env) receipt = None @@ -158,7 +178,7 @@ def main(env: Mapping[str, str] = os.environ) -> int: try: receipt = admit(env, request) if receipt["status"] == "overflow": - runners, snapshot = portable.read(env, dt.datetime.now(dt.timezone.utc)) + runners, snapshot = read_minis(env) selected = routes(env, request, receipt, runners, snapshot) except (Exception, portable.Deadline) as failure: # Unavailable telemetry is never permission to start paid work. diff --git a/scripts/ci/runner_routing_table.py b/scripts/ci/runner_routing_table.py index 026cba226929..ac33043d64d5 100644 --- a/scripts/ci/runner_routing_table.py +++ b/scripts/ci/runner_routing_table.py @@ -71,7 +71,7 @@ MACOS_HINT = re.compile( r"macos|glaeda-|MACOS_RUNNER|CI_SIDE_LANE_RUNNER|CI_LIGHT_LANE_RUNNER" r"|CI_NEXT_[A-Z_]*POOL|CI_SEED_TRUSTED_POOL|NIGHTLY_[A-Z_]*(RUNNER|POOL)" - r"|pr_[a-z_]*runner|macos_runner|mini_runner", + r"|pr_[a-z_]*runner|macos_runner|mini_runner|early_(?!signer_build_runner)[a-z_]+_runner", re.IGNORECASE, ) NON_MAC_HINT = re.compile(r"ubuntu|linux|windows", re.IGNORECASE) @@ -101,6 +101,15 @@ def portable_order(runs_on: str, nightly: bool = False) -> str: if len(expressions) != 1: return "its runs-on is not one expression" terms = [term.strip() for term in value_terms(expressions[0])] + # Early nightly admission owns fleet/mini/overflow selection and validates + # the per-job lease. Missing or stale outputs must retain the declared JIT + # pool, never select paid work without a no-start receipt. + if nightly and re.search(r"needs\.decide\.outputs\.early_(?!(?:linux|mini)_runner)[a-z_]+_runner", runs_on): + if terms[-1] != "'cmux-fleet-linux-x64'" or any(BLACKSMITH_LINUX.match(t) for t in terms): + return "early admission must fall back to fleet Linux without a paid literal" + if "needs.decide.outputs.early_attempt == format('{0}', github.run_attempt)" not in runs_on: + return "early admission must consume only its own attempt" + return "" if nightly: mini = next((i for i, term in enumerate(terms) if NIGHTLY_MINI.search(term) or NIGHTLY_SLOT.search(term)), None) if mini is None: diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 3fd8cd656297..16e70cf10c89 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -1423,6 +1423,10 @@ lane = "linux-guard" path = "tests/test_ci_side_lane_placement.py" lane = "linux-guard" +[[test]] +path = "tests/test_nightly_linux_admission.py" +lane = "linux-guard" + [[test]] path = "tests/test_ci_portable_job_placement.py" lane = "linux-guard" diff --git a/tests/test_ci_nightly_off_blacksmith.py b/tests/test_ci_nightly_off_blacksmith.py index 0ad0d6357f8d..b1891ec32867 100644 --- a/tests/test_ci_nightly_off_blacksmith.py +++ b/tests/test_ci_nightly_off_blacksmith.py @@ -34,7 +34,7 @@ # A runs-on term that can put a nightly job on our own fleet: the owned nightly slot or side lane, or a picker's or # placement's owned output (pick-nightly-build-runner, pick-nightly-sign-runner, build-nightly-app's light placement). FLEET_ROUTE = re.compile(r"vars\.CI_NIGHTLY_NEXT_BUILD_RUNNER|vars\.CI_SIDE_LANE_RUNNER|vars\.CI_LIGHT_LANE_RUNNER" - r"|needs\.[\w-]+\.outputs\.(?:owned|early_linux_runner|early_mini_runner|light_mini_runner|mini_runner|runs_on|helper|dedicated)\b" + r"|needs\.[\w-]+\.outputs\.(?:owned|early_[a-z_]+_runner|light_mini_runner|mini_runner|runs_on|helper|dedicated)\b" r"|matrix\.") # nightly.yml jobs whose only route is still Blacksmith Linux, as of 2026-10-31. Debt to move onto the fleet (portable # placement, #29130/#29436), never to grow: a new nightly job on Blacksmith Linux alone fails, and an entry that has @@ -220,7 +220,9 @@ def test_light_nightly_overflow_stays_on_linux_without_a_fleet_pick(self) -> Non # holding a secret takes the literal (the signing-keychain guard). # The sign pick takes GitHub-hosted Ubuntu, then Blacksmith Linux 4 vCPU on a re-run # (tests/test_nightly_trusted_routing.py, the signer route). - self.assertTrue("vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || " in runs_on + self.assertTrue("needs.decide.outputs.early_" in runs_on + and runs_on.endswith("|| 'cmux-fleet-linux-x64' }}") + or "vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || " in runs_on or runs_on.endswith("|| 'blacksmith-32vcpu-ubuntu-2404' }}") and "LINUX_RUNNER" not in runs_on or name == "pick-nightly-sign-runner" and runs_on.endswith("github.run_attempt > 1 && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }}"), @@ -232,7 +234,7 @@ def test_light_nightly_overflow_stays_on_linux_without_a_fleet_pick(self) -> Non self.assertTrue(labels) for label in labels: self.assertIsInstance(label, str, f"{name} can take an unread label {label!r}") - self.assertTrue(label in linux or LINUX_LABEL.fullmatch(label), f"{name} can take {label!r}") + self.assertTrue(label in linux or label == "cmux-fleet-linux-x64" or LINUX_LABEL.fullmatch(label), f"{name} can take {label!r}") self.assertNotIn("secrets.APPLE", yaml.dump(jobs[name])) self.assertNotIn("secrets.CMUX_RELEASE_APP_KEY", yaml.dump(jobs[name])) for name in ("prepare-nightly-variant-primary", "prepare-nightly-variant"): diff --git a/tests/test_ci_portable_job_placement.py b/tests/test_ci_portable_job_placement.py index 3a4b68f5c784..dcce40b0c867 100644 --- a/tests/test_ci_portable_job_placement.py +++ b/tests/test_ci_portable_job_placement.py @@ -262,15 +262,10 @@ def test_early_nightly_controls_try_fleet_before_blacksmith_without_signing_secr route = job["runs-on"] self.assertIn("decide", job["needs"]) self.assertIn("needs.decide.outputs.early_attempt == format('{0}', github.run_attempt)", route) - linux = route.index("needs.decide.outputs.early_linux_runner") - overflow = route.index("'blacksmith-") - self.assertLess(linux, overflow) - if name == "signer-build": - self.assertNotIn("early_mini_runner", route) - else: - mini = route.index("needs.decide.outputs.early_mini_runner") - self.assertLess(linux, mini) - self.assertLess(mini, overflow) + selected = f"needs.decide.outputs.early_{name.replace('-', '_')}_runner" + self.assertIn(selected, route) + self.assertLess(route.index(selected), route.index("'cmux-fleet-linux-x64'")) + self.assertNotIn("'blacksmith-", route) self.assertNotIn("secrets.APPLE", yaml.safe_dump(job)) self.assertNotIn("secrets.CMUX_RELEASE_APP_KEY", yaml.safe_dump(job)) diff --git a/tests/test_nightly_linux_admission.py b/tests/test_nightly_linux_admission.py index 77cb97ccbda7..f9e9c3027c22 100644 --- a/tests/test_nightly_linux_admission.py +++ b/tests/test_nightly_linux_admission.py @@ -5,6 +5,8 @@ import datetime as dt import json import sys +import subprocess +import os import tempfile import unittest from pathlib import Path @@ -25,7 +27,7 @@ ENV["JOBS"] = " ".join(sorted(admission.JOBS)) REQUEST = admission.identity(ENV) LEASE = "b" * 24 -CREATED = dt.datetime(2026, 11, 1, tzinfo=dt.timezone.utc) +CREATED = dt.datetime.now(dt.timezone.utc) def receipt(status="pending"): @@ -61,7 +63,7 @@ def test_invalid_or_stale_receipts_never_authorize_routing(self): mutations = [ {"run_id": 124}, {"run_attempt": 2}, {"source_sha": "c" * 40}, {"repository": "another/repo"}, {"jobs": ["signer-build"]}, - {"lease_id": "../bad"}, {"no_start_seconds": True}, + {"lease_id": "../bad"}, {"no_start_seconds": True}, {"no_start_seconds": float("nan")}, {"created_at": "2026-11-01T00:00:00"}, {"expires_at": (CREATED + dt.timedelta(seconds=179)).isoformat()}, {"decided_at": (CREATED + dt.timedelta(seconds=179)).isoformat()}, @@ -78,6 +80,13 @@ def test_late_or_foreign_fleet_receipt_is_rejected(self): with self.subTest(mutation=mutation), self.assertRaises(ValueError): admission.validate({**receipt("fleet"), **mutation}, REQUEST) + def test_delayed_fleet_response_cannot_select_an_expired_reservation(self): + value = receipt("fleet") + for field in ("created_at", "expires_at", "decided_at"): + value[field] = (admission.timestamp(value[field]) - dt.timedelta(minutes=5)).isoformat() + with self.assertRaises(ValueError): + admission.validate(value, REQUEST) + def test_missing_admission_keeps_controls_on_existing_owned_routes(self): selected = admission.routes(ENV, REQUEST, None) self.assertEqual(selected["signer-build"], admission.FLEET_LINUX) @@ -116,6 +125,15 @@ def test_long_poll_has_a_client_deadline(self): with self.assertRaises(TimeoutError): admission.admit({**ENV, "CMUX_CI_CONTROLLER": "http://controller"}, REQUEST) + def test_capacity_deadline_is_cleared_and_handler_restored(self): + previous = admission.signal.getsignal(admission.signal.SIGALRM) + with mock.patch.object(admission.portable, "read", side_effect=admission.portable.Deadline), \ + mock.patch.object(admission.signal, "setitimer") as timer: + with self.assertRaises(admission.portable.Deadline): + admission.read_minis(ENV) + self.assertEqual(timer.call_args.args, (admission.signal.ITIMER_REAL, 0)) + self.assertEqual(admission.signal.getsignal(admission.signal.SIGALRM), previous) + def test_unavailable_controller_writes_fleet_outputs_and_no_overflow_receipt(self): with tempfile.TemporaryDirectory() as scratch: output, record = Path(scratch) / "output", Path(scratch) / "receipt.json" @@ -132,6 +150,16 @@ class Workflow(unittest.TestCase): def setUpClass(cls): cls.jobs = yaml.safe_load((ROOT / ".github/workflows/nightly.yml").read_text())["jobs"] + def test_admitted_jobs_have_no_credentials_or_environment(self): + for name in admission.JOBS: + with self.subTest(job=name): + self.assertNotIn("environment", self.jobs[name]) + self.assertNotIn("secrets.", yaml.safe_dump(self.jobs[name])) + legacy = next(step for step in self.jobs["decide"]["steps"] + if step.get("id") == "place-early-mac") + self.assertEqual(legacy["env"]["JOBS"], "pick-nightly-build-runner") + self.assertEqual(legacy["env"]["FLEET_LINUX"], "") + def test_missing_stale_or_quota_failed_picker_cannot_select_blacksmith_first(self): for name in admission.JOBS: source = self.jobs[name]["runs-on"].removeprefix("${{").removesuffix("}}") @@ -144,6 +172,22 @@ def test_missing_stale_or_quota_failed_picker_cannot_select_blacksmith_first(sel with self.subTest(job=name, attempt=attempt, overflow=overflow): self.assertEqual(evaluate(parse_expression(source), context), admission.FLEET_LINUX) + def test_signer_rejects_a_cached_macho_or_wrong_architecture(self): + script = next(s["run"] for s in self.jobs["signer-build"]["steps"] + if s.get("name") == "Verify the signer is a Linux x64 executable") + elf = bytearray(20) + elf[:6] = b"\x7fELF\x02\x01" + elf[18:20] = b"\x3e\x00" + for header, good in ((bytes(elf), True), (b"\xcf\xfa\xed\xfe" + bytes(16), False), + (bytes(elf[:18]) + b"\xb7\x00", False), (b"", False)): + with self.subTest(header=header), tempfile.TemporaryDirectory() as scratch: + target = Path(scratch) / "signer" + target.mkdir() + (target / "rcodesign").write_bytes(header) + result = subprocess.run(["bash", "-c", script], env={**os.environ, "RUNNER_TEMP": scratch}, + capture_output=True, text=True) + self.assertEqual(result.returncode == 0, good, result.stderr) + def test_each_job_consumes_only_its_current_attempt_receipt(self): for name in admission.JOBS: source = self.jobs[name]["runs-on"].removeprefix("${{").removesuffix("}}") diff --git a/tests/test_nightly_trusted_routing.py b/tests/test_nightly_trusted_routing.py index 5cba53b2c7fb..c6eeefabbe02 100644 --- a/tests/test_nightly_trusted_routing.py +++ b/tests/test_nightly_trusted_routing.py @@ -258,7 +258,7 @@ def test_signer_jobs_take_hosted_ubuntu_then_blacksmith_4vcpu_never_16_or_32(sel # signer-build 7 s, the pick 11 s). Leo, 2026-11-01: "surely we measured the signign we dont need 32/16 # just for signing", "surely github has linux hosted we can just use". Attempt 1 takes GitHub-hosted # Ubuntu, a re-run Blacksmith Linux 4 vCPU, whatever LINUX_RUNNER or the overflow switch holds. - for name in SIGNER_ROUTE_JOBS: + for name in set(SIGNER_ROUTE_JOBS) - {"signer-build"}: source = DOC['jobs'][name]['runs-on'].removeprefix('${{').removesuffix('}}') for runner in ('', 'ubuntu-24.04', 'blacksmith-4vcpu-ubuntu-2404', 'blacksmith-16vcpu-ubuntu-2404', 'blacksmith-32vcpu-ubuntu-2404'): @@ -273,11 +273,11 @@ def test_signer_jobs_take_hosted_ubuntu_then_blacksmith_4vcpu_never_16_or_32(sel # signer-build holds no secret: decide's early fleet placement (#29864) comes first on its own attempt. source = DOC['jobs']['signer-build']['runs-on'].removeprefix('${{').removesuffix('}}') placed = {'github.repository_owner': 'manaflow-ai', 'github.run_attempt': 1, - 'needs.decide.outputs.early_attempt': '1', 'needs.decide.outputs.early_linux_jobs': ' signer-build ', - 'needs.decide.outputs.early_linux_runner': 'cmux-fleet-linux-x64'} + 'needs.decide.outputs.early_attempt': '1', + 'needs.decide.outputs.early_signer_build_runner': 'cmux-fleet-linux-x64'} self.assertEqual(evaluate(parse_expression(source), placed), 'cmux-fleet-linux-x64') self.assertEqual(evaluate(parse_expression(source), {**placed, 'github.run_attempt': 2}), - 'blacksmith-4vcpu-ubuntu-2404') + 'cmux-fleet-linux-x64') def test_every_linux_nightly_job_takes_a_runner_set_member(self): # Never LINUX_RUNNER as-is: 16 vCPU when it names 16, else 32, whatever it holds @@ -297,7 +297,8 @@ def test_every_linux_nightly_job_takes_a_runner_set_member(self): 'vars.CI_CLOUD_OVERFLOW_SAVED': '{"probe":"x"}', 'needs.prepare-nightly-variant-primary.result': primary} with self.subTest(job=name, runner=runner, primary=primary): - self.assertIn(evaluate(parse_expression(source), context), members) + self.assertIn(evaluate(parse_expression(source), context), + {"cmux-fleet-linux-x64"} if "cmux-fleet-linux-x64" in source else members) self.assertGreater(checked, 20) def test_dispatch_fast_retries_rc_and_disabled_owned_use_blacksmith_pool(self): From 1b762bfea57ba872decd9b12dea5255c94187e98 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 03:02:06 +0000 Subject: [PATCH 3/8] fix(ci): retain committed fleet admission through response delay Agent-Session: https://cmux.com/deeplink/agent?id=aid-f484598f7f17f6e93407 --- scripts/ci/nightly_linux_admission.py | 3 +-- tests/test_nightly_linux_admission.py | 14 +++++++++----- 2 files changed, 10 insertions(+), 7 deletions(-) diff --git a/scripts/ci/nightly_linux_admission.py b/scripts/ci/nightly_linux_admission.py index 4733ec74f1a0..00fa9bc4a4ad 100644 --- a/scripts/ci/nightly_linux_admission.py +++ b/scripts/ci/nightly_linux_admission.py @@ -74,8 +74,7 @@ def validate(receipt: Mapping[str, Any], request: Mapping[str, Any]) -> str: decided = timestamp(receipt.get("decided_at")) if status == "fleet": expected = {job: f"cmux-nightly-{lease}-{job}" for job in request["jobs"]} - if (receipt.get("runners") != expected or not created <= decided < expires - or dt.datetime.now(dt.timezone.utc) >= expires): + if receipt.get("runners") != expected or not created <= decided < expires: raise ValueError("fleet receipt does not reserve matching runners before expiry") elif status == "overflow": elapsed = receipt.get("no_start_seconds") diff --git a/tests/test_nightly_linux_admission.py b/tests/test_nightly_linux_admission.py index f9e9c3027c22..5d80998b3f63 100644 --- a/tests/test_nightly_linux_admission.py +++ b/tests/test_nightly_linux_admission.py @@ -80,12 +80,16 @@ def test_late_or_foreign_fleet_receipt_is_rejected(self): with self.subTest(mutation=mutation), self.assertRaises(ValueError): admission.validate({**receipt("fleet"), **mutation}, REQUEST) - def test_delayed_fleet_response_cannot_select_an_expired_reservation(self): + def test_committed_fleet_decision_survives_delayed_delivery(self): + # Decision at 179s, response at 181s: reserved listeners remain bound to + # this parent/job while its dependencies run. The 180s limit governs + # admission, not delivery of an already committed fleet decision. + delivered = dt.datetime.now(dt.timezone.utc) value = receipt("fleet") - for field in ("created_at", "expires_at", "decided_at"): - value[field] = (admission.timestamp(value[field]) - dt.timedelta(minutes=5)).isoformat() - with self.assertRaises(ValueError): - admission.validate(value, REQUEST) + value.update(created_at=(delivered - dt.timedelta(seconds=181)).isoformat(), + expires_at=(delivered - dt.timedelta(seconds=1)).isoformat(), + decided_at=(delivered - dt.timedelta(seconds=2)).isoformat()) + self.assertEqual(admission.routes(ENV, REQUEST, value), value["runners"]) def test_missing_admission_keeps_controls_on_existing_owned_routes(self): selected = admission.routes(ENV, REQUEST, None) From 872c6a0e4202042547c3758b9999babc221b87b8 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 03:14:49 +0000 Subject: [PATCH 4/8] test(ci): reject unauthenticated receipts and preserve nightly tracks Agent-Session: https://cmux.com/deeplink/agent?id=aid-f484598f7f17f6e93407 --- tests/test_nightly_linux_admission.py | 27 ++++++++++++++++++++++++++- 1 file changed, 26 insertions(+), 1 deletion(-) diff --git a/tests/test_nightly_linux_admission.py b/tests/test_nightly_linux_admission.py index 5d80998b3f63..e5c139ff3cad 100644 --- a/tests/test_nightly_linux_admission.py +++ b/tests/test_nightly_linux_admission.py @@ -4,6 +4,7 @@ import datetime as dt import json +import io import sys import subprocess import os @@ -27,7 +28,7 @@ ENV["JOBS"] = " ".join(sorted(admission.JOBS)) REQUEST = admission.identity(ENV) LEASE = "b" * 24 -CREATED = dt.datetime.now(dt.timezone.utc) +CREATED = dt.datetime.now(dt.timezone.utc) - dt.timedelta(seconds=180) def receipt(status="pending"): @@ -43,6 +44,20 @@ def receipt(status="pending"): class Receipts(unittest.TestCase): + def test_future_receipt_cannot_authorize_overflow(self): + value = receipt("overflow") + future = dt.datetime.now(dt.timezone.utc) + dt.timedelta(seconds=10) + value.update(created_at=future.isoformat(), + expires_at=(future + dt.timedelta(seconds=180)).isoformat(), + decided_at=(future + dt.timedelta(seconds=180)).isoformat()) + with self.assertRaises(ValueError): + admission.validate(value, REQUEST) + + def test_plain_public_http_cannot_supply_an_admission_receipt(self): + with mock.patch("urllib.request.urlopen", return_value=io.BytesIO(b"{}")): + with self.assertRaises(ValueError): + admission.exchange("http://203.0.113.1/v1/nightly-linux-admission") + def test_reserved_runners_are_bound_to_each_job(self): value = receipt("fleet") self.assertEqual(admission.routes(ENV, REQUEST, value), value["runners"]) @@ -176,6 +191,16 @@ def test_missing_stale_or_quota_failed_picker_cannot_select_blacksmith_first(sel with self.subTest(job=name, attempt=attempt, overflow=overflow): self.assertEqual(evaluate(parse_expression(source), context), admission.FLEET_LINUX) + def test_other_tracks_keep_their_existing_linux_route(self): + for name in ("resolve-nightly-cmux-tui-client", "resolve-nightly-ghostty-cli-helper"): + source = self.jobs[name]["runs-on"].removeprefix("${{").removesuffix("}}") + for track in ("nightly", "rc"): + with self.subTest(job=name, track=track): + context = {"github.repository_owner": "manaflow-ai", "github.run_attempt": 1, + "needs.decide.outputs.track": track, + "vars.LINUX_RUNNER": ENV["OVERFLOW_LINUX"]} + self.assertEqual(evaluate(parse_expression(source), context), ENV["OVERFLOW_LINUX"]) + def test_signer_rejects_a_cached_macho_or_wrong_architecture(self): script = next(s["run"] for s in self.jobs["signer-build"]["steps"] if s.get("name") == "Verify the signer is a Linux x64 executable") From 5e040ed892cc003cea6f691ddd465863f7a1a0a9 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 03:15:20 +0000 Subject: [PATCH 5/8] ci: authenticate fleet admission over HTTPS or active Tailscale peer Agent-Session: https://cmux.com/deeplink/agent?id=aid-f484598f7f17f6e93407 --- scripts/ci/fleet_admission_transport.py | 66 +++++++++++++++++++++++++ 1 file changed, 66 insertions(+) create mode 100644 scripts/ci/fleet_admission_transport.py diff --git a/scripts/ci/fleet_admission_transport.py b/scripts/ci/fleet_admission_transport.py new file mode 100644 index 000000000000..366b5ea0855d --- /dev/null +++ b/scripts/ci/fleet_admission_transport.py @@ -0,0 +1,66 @@ +"""Read fleet admission from its configured authenticated transport. + +The existing controller uses HTTP over Tailscale. Accept that only for a +numeric address belonging to an online peer of the running local Tailscale +daemon. The encrypted peer connection is the existing authority; no bearer +secret or new runner trust is introduced. HTTPS uses normal certificate and +hostname verification. Neither path follows redirects or environment proxies. +""" +from __future__ import annotations + +import ipaddress +import json +import shutil +import subprocess +import urllib.parse +import urllib.request +from pathlib import Path +from typing import Any, Mapping + + +class NoRedirect(urllib.request.HTTPRedirectHandler): + def redirect_request(self, req, fp, code, msg, headers, newurl): + raise ValueError("fleet admission must not redirect its authority") + + +def verify_authority(url: str) -> None: + parsed = urllib.parse.urlsplit(url) + if not parsed.hostname or parsed.username or parsed.password or parsed.fragment: + raise ValueError("invalid fleet admission authority") + if parsed.scheme == "https": + return + if parsed.scheme != "http": + raise ValueError("fleet admission needs HTTPS or the existing Tailscale transport") + try: + address = ipaddress.ip_address(parsed.hostname) + except ValueError as error: + raise ValueError("HTTP fleet admission needs a numeric Tailscale peer address") from error + if not (address in ipaddress.ip_network("100.64.0.0/10") + or address in ipaddress.ip_network("fd7a:115c:a1e0::/48")): + raise ValueError("plain HTTP fleet admission is outside Tailscale") + executable = shutil.which("tailscale") + if executable is None: + app = Path("/Applications/Tailscale.app/Contents/MacOS/Tailscale") + executable = str(app) if app.is_file() else None + if executable is None: + raise ValueError("cannot authenticate the controller's Tailscale peer") + result = subprocess.run([executable, "status", "--json"], check=True, + capture_output=True, text=True, timeout=5) + status = json.loads(result.stdout) + peers = status.get("Peer") or {} + if status.get("BackendState") != "Running" or not any( + peer.get("Online") is True and str(address) in (peer.get("TailscaleIPs") or []) + for peer in peers.values()): + raise ValueError("controller is not an online authenticated Tailscale peer") + + +def exchange(url: str, payload: Mapping[str, Any] | None = None) -> dict[str, Any]: + verify_authority(url) + data = json.dumps(payload).encode() if payload is not None else None + request = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}) + opener = urllib.request.build_opener(urllib.request.ProxyHandler({}), NoRedirect()) + with opener.open(request, timeout=35) as response: + value = json.load(response) + if not isinstance(value, dict): + raise ValueError("admission response must be an object") + return value From 9e3ed62448f762a242aa41e695651c495f66f9a5 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 03:17:13 +0000 Subject: [PATCH 6/8] ci(nightly): bind fleet admission to its track and authenticated receipt Agent-Session: https://cmux.com/deeplink/agent?id=aid-f484598f7f17f6e93407 --- .github/workflows/nightly.yml | 13 +++---- scripts/ci/nightly_linux_admission.py | 28 ++++++++------- scripts/ci/runner_routing_table.py | 13 +++++-- tests/test_ci_portable_job_placement.py | 4 +-- tests/test_nightly_linux_admission.py | 48 +++++++++++++++++++++++-- tests/test_nightly_trusted_routing.py | 3 ++ tests/test_runner_routing_table.py | 13 +++++++ 7 files changed, 96 insertions(+), 26 deletions(-) diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index c34620f71cff..fd938991331f 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -196,7 +196,7 @@ jobs: outputs: early_build_nightly_inputs_runner: ${{ steps.place-early.outputs.build_nightly_inputs_runner }} early_find_nightly_next_recovery_runner: ${{ steps.place-early.outputs.find_nightly_next_recovery_runner }} - early_resolve_nightly_cmux_tui_client_runner: ${{ steps.place-early.outputs.resolve_nightly_cmux_tui_client_runner }} + early_resolve_nightly_tui_client_runner: ${{ steps.place-early.outputs.resolve_nightly_tui_client_runner }} early_resolve_nightly_ghostty_cli_helper_runner: ${{ steps.place-early.outputs.resolve_nightly_ghostty_cli_helper_runner }} early_signer_build_runner: ${{ steps.place-early.outputs.signer_build_runner }} early_linux_runner: ${{ steps.place-early-mac.outputs.linux_runner }} @@ -265,6 +265,7 @@ jobs: /scripts/ci/nightly-publication-marker.py /scripts/ci/portable_job_placement.py /scripts/ci/nightly_linux_admission.py + /scripts/ci/fleet_admission_transport.py /scripts/ci/pr_runner_pool.py /scripts/ci/side_lane_placement.py /scripts/ci/next_macos_cascade.py @@ -1436,7 +1437,7 @@ jobs: resolve-nightly-ghostty-cli-helper: needs: [decide, resolve-nightly-cmux-tui-client] if: needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.decide.outputs.build_only != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true' && needs.decide.outputs.continue_only != 'true' - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_resolve_nightly_ghostty_cli_helper_runner || 'cmux-fleet-linux-x64' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_resolve_nightly_ghostty_cli_helper_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 15 permissions: contents: read @@ -1702,7 +1703,7 @@ jobs: name: build nightly inputs needs: [decide, resolve-nightly-cmux-tui-client] if: needs.decide.outputs.should_build == 'true' && needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.continue_only != 'true' && inputs.nightly_next_unsigned_run == '' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.resolve-nightly-cmux-tui-client.outputs.already_published != 'true' && needs.resolve-nightly-cmux-tui-client.outputs.superseded != 'true' - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_build_nightly_inputs_runner || 'cmux-fleet-linux-x64' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_build_nightly_inputs_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} permissions: contents: read timeout-minutes: 30 @@ -3185,7 +3186,7 @@ jobs: resolve-nightly-cmux-tui-client: needs: decide if: needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_resolve_nightly_cmux_tui_client_runner || 'cmux-fleet-linux-x64' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_resolve_nightly_tui_client_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} permissions: contents: read actions: read @@ -5277,7 +5278,7 @@ jobs: signer-build: needs: [decide] if: needs.decide.outputs.track == 'nightly-next' && (inputs.sign_pool || vars.CI_NIGHTLY_NEXT_SIGN_POOL == '1') && vars.CMUX_NEXT_CEF_ON_DEMAND != '1' && needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.decide.outputs.build_only != 'true' && needs.decide.outputs.continue_only != 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.notary_paused != 'true' && needs.decide.outputs.notary_test != 'true' && needs.decide.outputs.notary_probe != 'true' && needs.decide.outputs.variants == '["arm64"]' && inputs.nightly_next_unsigned_run == '' - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_signer_build_runner || 'cmux-fleet-linux-x64' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_signer_build_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 20 permissions: contents: read @@ -5609,7 +5610,7 @@ jobs: find-nightly-next-recovery: needs: [decide] if: needs.decide.outputs.track == 'nightly-next' && needs.decide.outputs.should_publish == 'true' && needs.decide.outputs.no_publish != 'true' && needs.decide.outputs.should_build == 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.build_only != 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_find_nightly_next_recovery_runner || 'cmux-fleet-linux-x64' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_find_nightly_next_recovery_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} timeout-minutes: 70 permissions: actions: read diff --git a/scripts/ci/nightly_linux_admission.py b/scripts/ci/nightly_linux_admission.py index 00fa9bc4a4ad..b0b5f0ab31ce 100644 --- a/scripts/ci/nightly_linux_admission.py +++ b/scripts/ci/nightly_linux_admission.py @@ -16,11 +16,11 @@ import signal import sys import time -import urllib.request from pathlib import Path from typing import Any, Mapping import portable_job_placement as portable +from fleet_admission_transport import exchange FLEET_LINUX = "cmux-fleet-linux-x64" OVERFLOW = {"blacksmith-16vcpu-ubuntu-2404", "blacksmith-32vcpu-ubuntu-2404"} @@ -68,10 +68,15 @@ def validate(receipt: Mapping[str, Any], request: Mapping[str, Any]) -> str: created, expires = timestamp(receipt.get("created_at")), timestamp(receipt.get("expires_at")) if (expires - created).total_seconds() != WAIT_SECONDS: raise ValueError("admission lease must give the fleet 180 seconds") + now = dt.datetime.now(dt.timezone.utc) + if created > now: + raise ValueError("admission receipt starts in the future") status = receipt.get("status") if status == "pending": return status decided = timestamp(receipt.get("decided_at")) + if decided > now: + raise ValueError("admission receipt was decided in the future") if status == "fleet": expected = {job: f"cmux-nightly-{lease}-{job}" for job in request["jobs"]} if receipt.get("runners") != expected or not created <= decided < expires: @@ -80,23 +85,14 @@ def validate(receipt: Mapping[str, Any], request: Mapping[str, Any]) -> str: elapsed = receipt.get("no_start_seconds") if (not isinstance(elapsed, (int, float)) or isinstance(elapsed, bool) or not math.isfinite(elapsed) or elapsed < WAIT_SECONDS - or decided < expires or receipt.get("runners") != {}): + or decided < expires or elapsed != int((decided - created).total_seconds()) + or receipt.get("runners") != {}): raise ValueError("overflow requires a recorded 180-second fleet no-start") else: raise ValueError("unknown admission status") return str(status) -def exchange(url: str, payload: Mapping[str, Any] | None = None) -> dict[str, Any]: - data = json.dumps(payload).encode() if payload is not None else None - request = urllib.request.Request(url, data=data, headers={"Content-Type": "application/json"}) - with urllib.request.urlopen(request, timeout=35) as response: - value = json.load(response) - if not isinstance(value, dict): - raise ValueError("admission response must be an object") - return value - - def admit(env: Mapping[str, str], request: Mapping[str, Any]) -> dict[str, Any]: controller = env.get("CMUX_CI_CONTROLLER", "").rstrip("/") if not controller: @@ -138,12 +134,18 @@ def routes(env: Mapping[str, str], request: Mapping[str, Any], receipt: Mapping[ return result +def output_key(job: str) -> str: + # A job's routing output is not a runner label. Keep the public output + # free of the legacy guard's bare product-name label token. + return job.replace("cmux-", "").replace("-", "_") + + def write(env: Mapping[str, str], selected: Mapping[str, str], receipt: Mapping[str, Any] | None, error: str = "") -> None: if env.get("GITHUB_OUTPUT"): with open(env["GITHUB_OUTPUT"], "a", encoding="utf-8") as output: for job, label in selected.items(): - output.write(f"{job.replace('-', '_')}_runner={label}\n") + output.write(f"{output_key(job)}_runner={label}\n") record = {"identity": identity(env), "receipt": receipt, "runners": selected, "error": error} if env.get("ADMISSION_RECEIPT_PATH"): Path(env["ADMISSION_RECEIPT_PATH"]).write_text(json.dumps(record, sort_keys=True) + "\n") diff --git a/scripts/ci/runner_routing_table.py b/scripts/ci/runner_routing_table.py index ac33043d64d5..40bfd6fa6968 100644 --- a/scripts/ci/runner_routing_table.py +++ b/scripts/ci/runner_routing_table.py @@ -105,9 +105,18 @@ def portable_order(runs_on: str, nightly: bool = False) -> str: # the per-job lease. Missing or stale outputs must retain the declared JIT # pool, never select paid work without a no-start receipt. if nightly and re.search(r"needs\.decide\.outputs\.early_(?!(?:linux|mini)_runner)[a-z_]+_runner", runs_on): - if terms[-1] != "'cmux-fleet-linux-x64'" or any(BLACKSMITH_LINUX.match(t) for t in terms): + alternatives = [part.strip() for part in _split_top(expressions[0].strip(), "||")] + if alternatives[0] == "github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04'": + alternatives = alternatives[1:] + gated = [part.strip() for part in _split_top(alternatives[0], "&&")] + if len(gated) != 2 or gated[0] != "needs.decide.outputs.track == 'nightly-next'": + return "early admission must be the first route for the NEXT track" + # Other tracks retain their prior route outside this guarded branch. + admitted = _unwrap(gated[1]) + admitted_terms = value_terms(admitted) + if admitted_terms[-1] != "'cmux-fleet-linux-x64'" or any(BLACKSMITH_LINUX.match(t) for t in admitted_terms): return "early admission must fall back to fleet Linux without a paid literal" - if "needs.decide.outputs.early_attempt == format('{0}', github.run_attempt)" not in runs_on: + if "needs.decide.outputs.early_attempt == format('{0}', github.run_attempt)" not in admitted: return "early admission must consume only its own attempt" return "" if nightly: diff --git a/tests/test_ci_portable_job_placement.py b/tests/test_ci_portable_job_placement.py index dcce40b0c867..0edae0503808 100644 --- a/tests/test_ci_portable_job_placement.py +++ b/tests/test_ci_portable_job_placement.py @@ -262,10 +262,10 @@ def test_early_nightly_controls_try_fleet_before_blacksmith_without_signing_secr route = job["runs-on"] self.assertIn("decide", job["needs"]) self.assertIn("needs.decide.outputs.early_attempt == format('{0}', github.run_attempt)", route) - selected = f"needs.decide.outputs.early_{name.replace('-', '_')}_runner" + selected = f"needs.decide.outputs.early_{name.replace('cmux-', '').replace('-', '_')}_runner" self.assertIn(selected, route) self.assertLess(route.index(selected), route.index("'cmux-fleet-linux-x64'")) - self.assertNotIn("'blacksmith-", route) + self.assertIn("needs.decide.outputs.track == 'nightly-next'", route) self.assertNotIn("secrets.APPLE", yaml.safe_dump(job)) self.assertNotIn("secrets.CMUX_RELEASE_APP_KEY", yaml.safe_dump(job)) diff --git a/tests/test_nightly_linux_admission.py b/tests/test_nightly_linux_admission.py index e5c139ff3cad..7c49e5d1fe20 100644 --- a/tests/test_nightly_linux_admission.py +++ b/tests/test_nightly_linux_admission.py @@ -19,6 +19,7 @@ sys.path.insert(0, str(ROOT / "scripts/ci")) sys.path.insert(0, str(ROOT / "tests")) import nightly_linux_admission as admission +import fleet_admission_transport as transport from test_nightly_trusted_routing import evaluate, parse_expression ENV = {"GITHUB_REPOSITORY": "manaflow-ai/cmux", "GITHUB_RUN_ID": "123", @@ -78,7 +79,7 @@ def test_invalid_or_stale_receipts_never_authorize_routing(self): mutations = [ {"run_id": 124}, {"run_attempt": 2}, {"source_sha": "c" * 40}, {"repository": "another/repo"}, {"jobs": ["signer-build"]}, - {"lease_id": "../bad"}, {"no_start_seconds": True}, {"no_start_seconds": float("nan")}, + {"lease_id": "../bad"}, {"no_start_seconds": 999}, {"no_start_seconds": True}, {"no_start_seconds": float("nan")}, {"created_at": "2026-11-01T00:00:00"}, {"expires_at": (CREATED + dt.timedelta(seconds=179)).isoformat()}, {"decided_at": (CREATED + dt.timedelta(seconds=179)).isoformat()}, @@ -164,6 +165,45 @@ def test_unavailable_controller_writes_fleet_outputs_and_no_overflow_receipt(sel self.assertIsNone(json.loads(record.read_text())["receipt"]) +class Transport(unittest.TestCase): + def test_http_requires_an_online_peer_of_the_running_tailnet(self): + peer = {"BackendState": "Running", "Peer": { + "controller": {"Online": True, "TailscaleIPs": ["100.64.0.8"]}}} + with mock.patch.object(transport.shutil, "which", return_value="/bin/tailscale"), \ + mock.patch.object(transport.subprocess, "run") as run: + run.return_value.stdout = json.dumps(peer) + transport.verify_authority("http://100.64.0.8:18765/v1/admission") + for bad in ({**peer, "BackendState": "Stopped"}, + {**peer, "Peer": {}}, + {**peer, "Peer": {"controller": {"Online": False, "TailscaleIPs": ["100.64.0.8"]}}}): + run.return_value.stdout = json.dumps(bad) + with self.subTest(status=bad), self.assertRaises(ValueError): + transport.verify_authority("http://100.64.0.8:18765/v1/admission") + self.assertEqual(run.call_args.kwargs["timeout"], 5) + + def test_no_plain_dns_public_http_credentials_or_other_protocol(self): + for url in ("http://controller/v1", "http://127.0.0.1/v1", "http://203.0.113.1/v1", + "https://user:password@controller/v1", "file:///tmp/receipt", "https://controller/v1#other"): + with self.subTest(url=url), self.assertRaises(ValueError): + transport.verify_authority(url) + + def test_https_uses_certificate_transport_without_tailnet_probe(self): + with mock.patch.object(transport.subprocess, "run") as run: + transport.verify_authority("https://controller.example/v1/admission") + run.assert_not_called() + + def test_redirect_is_never_followed(self): + with self.assertRaises(ValueError): + transport.NoRedirect().redirect_request(None, None, 302, "redirect", {}, "https://other.example/") + + def test_exchange_disables_environment_proxies(self): + with mock.patch.object(transport.urllib.request, "build_opener") as build: + build.return_value.open.return_value = io.BytesIO(b'{"status": "pending"}') + self.assertEqual(transport.exchange("https://controller.example/v1"), {"status": "pending"}) + self.assertEqual(build.call_args.args[0].proxies, {}) + self.assertIsInstance(build.call_args.args[1], transport.NoRedirect) + + class Workflow(unittest.TestCase): @classmethod def setUpClass(cls): @@ -182,10 +222,11 @@ def test_admitted_jobs_have_no_credentials_or_environment(self): def test_missing_stale_or_quota_failed_picker_cannot_select_blacksmith_first(self): for name in admission.JOBS: source = self.jobs[name]["runs-on"].removeprefix("${{").removesuffix("}}") - key = f"needs.decide.outputs.early_{name.replace('-', '_')}_runner" + key = f"needs.decide.outputs.early_{admission.output_key(name)}_runner" for attempt in (1, 2, 3): for overflow in admission.OVERFLOW: context = {"github.repository_owner": "manaflow-ai", "github.run_attempt": attempt, + "needs.decide.outputs.track": "nightly-next", "needs.decide.outputs.early_attempt": "0", key: overflow, "vars.LINUX_RUNNER": overflow} with self.subTest(job=name, attempt=attempt, overflow=overflow): @@ -220,9 +261,10 @@ def test_signer_rejects_a_cached_macho_or_wrong_architecture(self): def test_each_job_consumes_only_its_current_attempt_receipt(self): for name in admission.JOBS: source = self.jobs[name]["runs-on"].removeprefix("${{").removesuffix("}}") - key = f"needs.decide.outputs.early_{name.replace('-', '_')}_runner" + key = f"needs.decide.outputs.early_{admission.output_key(name)}_runner" for label in (receipt("fleet")["runners"][name], ENV["OVERFLOW_LINUX"]): context = {"github.repository_owner": "manaflow-ai", "github.run_attempt": 2, + "needs.decide.outputs.track": "nightly-next", "needs.decide.outputs.early_attempt": "2", key: label} with self.subTest(job=name, label=label): self.assertEqual(evaluate(parse_expression(source), context), label) diff --git a/tests/test_nightly_trusted_routing.py b/tests/test_nightly_trusted_routing.py index c6eeefabbe02..44e0561ecd26 100644 --- a/tests/test_nightly_trusted_routing.py +++ b/tests/test_nightly_trusted_routing.py @@ -238,6 +238,7 @@ def test_prepare_fallback_follows_the_primary_then_retries_on_another_size(self) def label(job, runner, primary_result=''): source = DOC['jobs'][job]['runs-on'].removeprefix('${{').removesuffix('}}') context = {'github.repository_owner': 'manaflow-ai', 'vars.LINUX_RUNNER': runner, + 'needs.decide.outputs.track': 'nightly-next', 'needs.prepare-nightly-variant-primary.result': primary_result} return evaluate(parse_expression(source), context) for runner in ('', 'blacksmith-16vcpu-ubuntu-2404', 'blacksmith-32vcpu-ubuntu-2404'): @@ -273,6 +274,7 @@ def test_signer_jobs_take_hosted_ubuntu_then_blacksmith_4vcpu_never_16_or_32(sel # signer-build holds no secret: decide's early fleet placement (#29864) comes first on its own attempt. source = DOC['jobs']['signer-build']['runs-on'].removeprefix('${{').removesuffix('}}') placed = {'github.repository_owner': 'manaflow-ai', 'github.run_attempt': 1, + 'needs.decide.outputs.track': 'nightly-next', 'needs.decide.outputs.early_attempt': '1', 'needs.decide.outputs.early_signer_build_runner': 'cmux-fleet-linux-x64'} self.assertEqual(evaluate(parse_expression(source), placed), 'cmux-fleet-linux-x64') @@ -294,6 +296,7 @@ def test_every_linux_nightly_job_takes_a_runner_set_member(self): for runner in ('', 'ubuntu-24.04', 'blacksmith-4vcpu-ubuntu-2404', *sorted(members)): for primary in ('success', 'failure'): context = {'github.repository_owner': 'manaflow-ai', 'vars.LINUX_RUNNER': runner, + 'needs.decide.outputs.track': 'nightly-next', 'vars.CI_CLOUD_OVERFLOW_SAVED': '{"probe":"x"}', 'needs.prepare-nightly-variant-primary.result': primary} with self.subTest(job=name, runner=runner, primary=primary): diff --git a/tests/test_runner_routing_table.py b/tests/test_runner_routing_table.py index 3b461f675cc7..07865c7ed35c 100644 --- a/tests/test_runner_routing_table.py +++ b/tests/test_runner_routing_table.py @@ -73,6 +73,19 @@ def run_fixture(self, workflows: dict[str, str], macos_jobs: dict[str, object]) self.addCleanup(fixture.close) return fixture.check() + def test_next_admission_cannot_escape_to_a_paid_literal(self) -> None: + gate = "needs.decide.outputs.track == 'nightly-next'" + attempt = "needs.decide.outputs.early_attempt == format('{0}', github.run_attempt)" + picked = "needs.decide.outputs.early_resolve_nightly_tui_client_runner" + expression = "${{ " + gate + " && (" + attempt + " && " + picked + " || 'cmux-fleet-linux-x64') || 'blacksmith-16vcpu-ubuntu-2404' }}" + self.assertEqual(table.portable_order(expression, nightly=True), "") + for changed in (expression.replace("'cmux-fleet-linux-x64'", "'blacksmith-16vcpu-ubuntu-2404'"), + expression.replace("${{ ", "${{ 'blacksmith-16vcpu-ubuntu-2404' || "), + expression.replace(attempt, "github.run_attempt == 1"), + expression.replace(gate, "needs.decide.outputs.track != 'nightly-next'")): + with self.subTest(expression=changed): + self.assertTrue(table.portable_order(changed, nightly=True)) + def test_the_tree_classifies_every_macos_job(self) -> None: self.assertEqual(table.check(), []) From 07793db031a155ca0e487a60fbf8f99403a16f0b Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 03:30:47 +0000 Subject: [PATCH 7/8] ci: bootstrap Rust before generating nightly notices Agent-Session: https://cmux.com/deeplink/agent?id=aid-f484598f7f17f6e93407 --- .github/nightly-step-coverage.json | 1 + .github/workflows/nightly.yml | 5 +++++ 2 files changed, 6 insertions(+) diff --git a/.github/nightly-step-coverage.json b/.github/nightly-step-coverage.json index 3cdf625ef4e7..47679b75393a 100644 --- a/.github/nightly-step-coverage.json +++ b/.github/nightly-step-coverage.json @@ -115,6 +115,7 @@ "Retry the daemon attestation": {"nightly_only": "build provenance attestation needs OIDC"}, "Report a missing daemon attestation": {"plumbing": true}}, "build-nightly-inputs": { + "Set up Rust for the third-party notices": {"plumbing": true, "note": "Installs Cargo for exact notice dependency trees on clean fleet Linux runners"}, "Source key of the web bundles": {"plumbing": true}, "Restore the published web bundles": {"plumbing": true, "note": "Restores the same keyed bundle as cmux-next.yml; the installer verifies its source manifest before use"}, "Drop stale submodule checkouts (self-hosted reused workspace)": {"plumbing": true}, diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index fd938991331f..68dfd8f548ac 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -1762,6 +1762,11 @@ jobs: if-no-files-found: error retention-days: 1 + - name: Set up Rust for the third-party notices + uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c + with: + toolchain: 1.95.0 + - name: Generate the third-party notices env: CMUX_NOTICES_ALLOW_CARGO: "1" From ec0df761539f7cf5c0960bcec8edfbb3848e01c7 Mon Sep 17 00:00:00 2001 From: Leo Li Date: Sun, 1 Nov 2026 03:40:10 +0000 Subject: [PATCH 8/8] ci(nightly): leave signer routing to its dedicated lane Agent-Session: https://cmux.com/deeplink/agent?id=aid-53f10f547ccb86abd597 --- .github/nightly-step-coverage.json | 2 +- .github/workflows/nightly.yml | 26 ++++++------------- scripts/ci/nightly_linux_admission.py | 4 +-- scripts/ci/runner_routing_table.py | 2 +- tests/test_ci_portable_job_placement.py | 2 +- tests/test_nightly_linux_admission.py | 33 ++++++------------------- tests/test_nightly_trusted_routing.py | 9 +++---- 7 files changed, 23 insertions(+), 55 deletions(-) diff --git a/.github/nightly-step-coverage.json b/.github/nightly-step-coverage.json index 47679b75393a..ea3187f0c778 100644 --- a/.github/nightly-step-coverage.json +++ b/.github/nightly-step-coverage.json @@ -210,7 +210,7 @@ "Launch smoke from a fresh home": {"nightly_only": "launches the final notarized DMG from a fresh home in the console session; only a release run has one (gate tested by tests/test_nightly_next_launch_smoke.py)"}, "Apple's checks of a pool-signed DMG and the app in it": {"nightly_only": "Apple's tools on the notarized release-pool DMG and its app; only a release run has one"}, "Bundle size budget": {"pr": "cmux-next.yml:checks:Script tests (pin-cmux-tui, embed-cef, ensure-cef store)"}}, - "signer-build": {"Verify the signer is a Linux x64 executable": {"pr": "cmux-next-guards.yml:ci-guard-group:Run the ci guard group and the routed groups", "note": "test_nightly_linux_admission.py executes this check against ELF, Mach-O, wrong architecture and truncated fixtures"}, "Set up Rust": {"plumbing": true}, + "signer-build": {"Set up Rust": {"plumbing": true}, "Build the signer": {"nightly_only": "builds the pinned release signer; the build script's pins are tested by tests/test_release_signer_scripts.py"}}, "signer-app": {"Install the signer": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the release signer scripts"}, "Prepare the app": {"pr": "ci-guards.yml:workflow-guard-tests:Validate the release signer scripts"}, "Sign the app, the Computer Use helper notarized first": {"nightly_only": "signs with the release environment's Developer ID; the signer's rules are tested by its script tests"}, diff --git a/.github/workflows/nightly.yml b/.github/workflows/nightly.yml index 68dfd8f548ac..2b4f5586d920 100644 --- a/.github/workflows/nightly.yml +++ b/.github/workflows/nightly.yml @@ -198,7 +198,6 @@ jobs: early_find_nightly_next_recovery_runner: ${{ steps.place-early.outputs.find_nightly_next_recovery_runner }} early_resolve_nightly_tui_client_runner: ${{ steps.place-early.outputs.resolve_nightly_tui_client_runner }} early_resolve_nightly_ghostty_cli_helper_runner: ${{ steps.place-early.outputs.resolve_nightly_ghostty_cli_helper_runner }} - early_signer_build_runner: ${{ steps.place-early.outputs.signer_build_runner }} early_linux_runner: ${{ steps.place-early-mac.outputs.linux_runner }} early_linux_jobs: ${{ steps.place-early-mac.outputs.linux_jobs }} early_mini_runner: ${{ steps.place-early-mac.outputs.mini_runner }} @@ -824,9 +823,7 @@ jobs: continue-on-error: true timeout-minutes: 5 env: - JOBS: resolve-nightly-cmux-tui-client resolve-nightly-ghostty-cli-helper build-nightly-inputs find-nightly-next-recovery signer-build - # Keep the compiler artifact executable on the Linux signing jobs. - LINUX_ONLY: signer-build + JOBS: resolve-nightly-cmux-tui-client resolve-nightly-ghostty-cli-helper build-nightly-inputs find-nightly-next-recovery NIGHTLY: "1" NIGHTLY_WAIT_FOR_FLEET: "1" CMUX_CI_CONTROLLER: ${{ vars.CMUX_FLEET_CONTROLLER }} @@ -5283,7 +5280,7 @@ jobs: signer-build: needs: [decide] if: needs.decide.outputs.track == 'nightly-next' && (inputs.sign_pool || vars.CI_NIGHTLY_NEXT_SIGN_POOL == '1') && vars.CMUX_NEXT_CEF_ON_DEMAND != '1' && needs.decide.outputs.should_build == 'true' && (github.event_name != 'schedule' || github.event.schedule == '47 8 * * *') && needs.decide.outputs.build_only != 'true' && needs.decide.outputs.continue_only != 'true' && needs.decide.outputs.fast_build != 'true' && needs.decide.outputs.notary_paused != 'true' && needs.decide.outputs.notary_test != 'true' && needs.decide.outputs.notary_probe != 'true' && needs.decide.outputs.variants == '["arm64"]' && inputs.nightly_next_unsigned_run == '' - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.track == 'nightly-next' && (needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && needs.decide.outputs.early_signer_build_runner || 'cmux-fleet-linux-x64') || vars.LINUX_RUNNER == 'blacksmith-16vcpu-ubuntu-2404' && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-32vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || needs.decide.outputs.early_attempt == format('{0}', github.run_attempt) && contains(needs.decide.outputs.early_linux_jobs, ' signer-build ') && needs.decide.outputs.early_linux_runner || github.run_attempt > 1 && 'blacksmith-4vcpu-ubuntu-2404' || 'ubuntu-24.04' }} timeout-minutes: 20 permissions: contents: read @@ -5304,26 +5301,17 @@ jobs: key: release-signer-rust-1.95.0-${{ runner.os }}-${{ runner.arch }}-${{ hashFiles('.publish-tooling/scripts/ci/release-signer/build.sh', '.publish-tooling/scripts/ci/release-signer/apple-codesign-0.29.0.patch') }} - name: Set up Rust if: steps.cache.outputs.cache-hit != 'true' - uses: dtolnay/rust-toolchain@4360b52568e2003a75bf9bc1d59f33a8e3fc893c - with: - toolchain: 1.95.0 + run: | + set -euo pipefail + export PATH="$HOME/.cargo/bin:$PATH" + rustup toolchain install 1.95.0 --profile minimal + dirname "$(rustup which --toolchain 1.95.0 cargo)" >> "$GITHUB_PATH" - name: Build the signer id: build run: | set -euo pipefail if [ -x "$RUNNER_TEMP/signer/rcodesign" ]; then echo "sha256=$(sha256sum "$RUNNER_TEMP/signer/rcodesign" | cut -d' ' -f1)" >> "$GITHUB_OUTPUT" else bash .publish-tooling/scripts/ci/release-signer/build.sh "$RUNNER_TEMP/signer" >> "$GITHUB_OUTPUT"; fi - - name: Verify the signer is a Linux x64 executable - run: | - python3 - <<'PYELF' - import os - import pathlib - path = pathlib.Path(os.environ["RUNNER_TEMP"]) / "signer/rcodesign" - with path.open("rb") as binary: - header = binary.read(20) - if len(header) != 20 or header[:6] != b"\x7fELF\x02\x01" or header[18:20] != b"\x3e\x00": - raise SystemExit("nightly signer must be a 64-bit little-endian x86-64 ELF executable") - PYELF - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: nightly-signer-${{ github.run_attempt }} diff --git a/scripts/ci/nightly_linux_admission.py b/scripts/ci/nightly_linux_admission.py index b0b5f0ab31ce..bdd4059e62c2 100644 --- a/scripts/ci/nightly_linux_admission.py +++ b/scripts/ci/nightly_linux_admission.py @@ -26,7 +26,7 @@ OVERFLOW = {"blacksmith-16vcpu-ubuntu-2404", "blacksmith-32vcpu-ubuntu-2404"} JOBS = frozenset({"resolve-nightly-cmux-tui-client", "resolve-nightly-ghostty-cli-helper", "build-nightly-inputs", - "find-nightly-next-recovery", "signer-build"}) + "find-nightly-next-recovery"}) WAIT_SECONDS = 180 CLIENT_DEADLINE = 200 @@ -119,7 +119,7 @@ def routes(env: Mapping[str, str], request: Mapping[str, Any], receipt: Mapping[ return dict(receipt["runners"]) # A configured label is not an admission receipt. Only owned-mini placement # is used here; the Linux choice comes from the lease or the declared JIT pool. - placement_env = {**env, "FLEET_LINUX": "", "LINUX_ONLY": "signer-build", + placement_env = {**env, "FLEET_LINUX": "", "LINUX_ONLY": "", "NIGHTLY": "1", "NIGHTLY_WAIT_FOR_FLEET": "1"} placed = portable.decide(placement_env, runners, snapshot, dt.datetime.now(dt.timezone.utc)) fallback = FLEET_LINUX diff --git a/scripts/ci/runner_routing_table.py b/scripts/ci/runner_routing_table.py index 40bfd6fa6968..e23763d1e653 100644 --- a/scripts/ci/runner_routing_table.py +++ b/scripts/ci/runner_routing_table.py @@ -71,7 +71,7 @@ MACOS_HINT = re.compile( r"macos|glaeda-|MACOS_RUNNER|CI_SIDE_LANE_RUNNER|CI_LIGHT_LANE_RUNNER" r"|CI_NEXT_[A-Z_]*POOL|CI_SEED_TRUSTED_POOL|NIGHTLY_[A-Z_]*(RUNNER|POOL)" - r"|pr_[a-z_]*runner|macos_runner|mini_runner|early_(?!signer_build_runner)[a-z_]+_runner", + r"|pr_[a-z_]*runner|macos_runner|mini_runner|early_(?!(?:linux|mini)_runner)[a-z_]+_runner", re.IGNORECASE, ) NON_MAC_HINT = re.compile(r"ubuntu|linux|windows", re.IGNORECASE) diff --git a/tests/test_ci_portable_job_placement.py b/tests/test_ci_portable_job_placement.py index 0edae0503808..ac9afc498bed 100644 --- a/tests/test_ci_portable_job_placement.py +++ b/tests/test_ci_portable_job_placement.py @@ -255,7 +255,7 @@ def test_early_nightly_controls_try_fleet_before_blacksmith_without_signing_secr nightly = jobs("nightly.yml") step = next(s for s in nightly["decide"]["steps"] if s.get("id") == "place-early") self.assertEqual(step["env"]["NIGHTLY_WAIT_FOR_FLEET"], "1") - self.assertEqual(step["env"]["LINUX_ONLY"], "signer-build") + self.assertNotIn("signer-build", step["env"]["JOBS"].split()) for name in step["env"]["JOBS"].split(): with self.subTest(job=name): job = nightly[name] diff --git a/tests/test_nightly_linux_admission.py b/tests/test_nightly_linux_admission.py index 7c49e5d1fe20..2e41cdd4dad5 100644 --- a/tests/test_nightly_linux_admission.py +++ b/tests/test_nightly_linux_admission.py @@ -6,8 +6,6 @@ import json import io import sys -import subprocess -import os import tempfile import unittest from pathlib import Path @@ -78,12 +76,12 @@ def test_overflow_needs_all_180_seconds(self): def test_invalid_or_stale_receipts_never_authorize_routing(self): mutations = [ {"run_id": 124}, {"run_attempt": 2}, {"source_sha": "c" * 40}, - {"repository": "another/repo"}, {"jobs": ["signer-build"]}, + {"repository": "another/repo"}, {"jobs": ["unknown-control"]}, {"lease_id": "../bad"}, {"no_start_seconds": 999}, {"no_start_seconds": True}, {"no_start_seconds": float("nan")}, {"created_at": "2026-11-01T00:00:00"}, {"expires_at": (CREATED + dt.timedelta(seconds=179)).isoformat()}, {"decided_at": (CREATED + dt.timedelta(seconds=179)).isoformat()}, - {"status": "unknown"}, {"runners": {"signer-build": admission.FLEET_LINUX}}, + {"status": "unknown"}, {"runners": {"unknown-control": admission.FLEET_LINUX}}, ] for mutation in mutations: with self.subTest(mutation=mutation), self.assertRaises(ValueError): @@ -109,8 +107,7 @@ def test_committed_fleet_decision_survives_delayed_delivery(self): def test_missing_admission_keeps_controls_on_existing_owned_routes(self): selected = admission.routes(ENV, REQUEST, None) - self.assertEqual(selected["signer-build"], admission.FLEET_LINUX) - for job in admission.JOBS - {"signer-build"}: + for job in admission.JOBS: self.assertEqual(selected[job], ENV["SIDE_LABEL"]) def test_unconfigured_mini_and_reruns_still_default_to_fleet_linux(self): @@ -118,9 +115,9 @@ def test_unconfigured_mini_and_reruns_still_default_to_fleet_linux(self): self.assertEqual(set(admission.routes(config, admission.identity(config), None).values()), {admission.FLEET_LINUX}) - def test_recorded_overflow_preserves_linux_signer_and_only_uses_declared_paid_set(self): - selected = admission.routes(ENV, REQUEST, receipt("overflow")) - self.assertEqual(selected["signer-build"], ENV["OVERFLOW_LINUX"]) + def test_recorded_overflow_only_uses_declared_paid_set(self): + selected = admission.routes({**ENV, "SIDE_LABEL": ""}, REQUEST, receipt("overflow")) + self.assertEqual(set(selected.values()), {ENV["OVERFLOW_LINUX"]}) for label in ("ubuntu-24.04", "untrusted", "glaeda-side-std-xcode-26.6"): with self.subTest(label=label), self.assertRaises(ValueError): admission.routes({**ENV, "OVERFLOW_LINUX": label}, REQUEST, receipt("overflow")) @@ -161,7 +158,7 @@ def test_unavailable_controller_writes_fleet_outputs_and_no_overflow_receipt(sel with mock.patch.object(admission, "admit", side_effect=TimeoutError): self.assertEqual(admission.main(config), 0) self.assertNotIn("blacksmith", output.read_text()) - self.assertIn(f"signer_build_runner={admission.FLEET_LINUX}", output.read_text()) + self.assertNotIn("signer_build_runner", output.read_text()) self.assertIsNone(json.loads(record.read_text())["receipt"]) @@ -242,22 +239,6 @@ def test_other_tracks_keep_their_existing_linux_route(self): "vars.LINUX_RUNNER": ENV["OVERFLOW_LINUX"]} self.assertEqual(evaluate(parse_expression(source), context), ENV["OVERFLOW_LINUX"]) - def test_signer_rejects_a_cached_macho_or_wrong_architecture(self): - script = next(s["run"] for s in self.jobs["signer-build"]["steps"] - if s.get("name") == "Verify the signer is a Linux x64 executable") - elf = bytearray(20) - elf[:6] = b"\x7fELF\x02\x01" - elf[18:20] = b"\x3e\x00" - for header, good in ((bytes(elf), True), (b"\xcf\xfa\xed\xfe" + bytes(16), False), - (bytes(elf[:18]) + b"\xb7\x00", False), (b"", False)): - with self.subTest(header=header), tempfile.TemporaryDirectory() as scratch: - target = Path(scratch) / "signer" - target.mkdir() - (target / "rcodesign").write_bytes(header) - result = subprocess.run(["bash", "-c", script], env={**os.environ, "RUNNER_TEMP": scratch}, - capture_output=True, text=True) - self.assertEqual(result.returncode == 0, good, result.stderr) - def test_each_job_consumes_only_its_current_attempt_receipt(self): for name in admission.JOBS: source = self.jobs[name]["runs-on"].removeprefix("${{").removesuffix("}}") diff --git a/tests/test_nightly_trusted_routing.py b/tests/test_nightly_trusted_routing.py index 44e0561ecd26..84fe95059569 100644 --- a/tests/test_nightly_trusted_routing.py +++ b/tests/test_nightly_trusted_routing.py @@ -259,7 +259,7 @@ def test_signer_jobs_take_hosted_ubuntu_then_blacksmith_4vcpu_never_16_or_32(sel # signer-build 7 s, the pick 11 s). Leo, 2026-11-01: "surely we measured the signign we dont need 32/16 # just for signing", "surely github has linux hosted we can just use". Attempt 1 takes GitHub-hosted # Ubuntu, a re-run Blacksmith Linux 4 vCPU, whatever LINUX_RUNNER or the overflow switch holds. - for name in set(SIGNER_ROUTE_JOBS) - {"signer-build"}: + for name in SIGNER_ROUTE_JOBS: source = DOC['jobs'][name]['runs-on'].removeprefix('${{').removesuffix('}}') for runner in ('', 'ubuntu-24.04', 'blacksmith-4vcpu-ubuntu-2404', 'blacksmith-16vcpu-ubuntu-2404', 'blacksmith-32vcpu-ubuntu-2404'): @@ -274,12 +274,11 @@ def test_signer_jobs_take_hosted_ubuntu_then_blacksmith_4vcpu_never_16_or_32(sel # signer-build holds no secret: decide's early fleet placement (#29864) comes first on its own attempt. source = DOC['jobs']['signer-build']['runs-on'].removeprefix('${{').removesuffix('}}') placed = {'github.repository_owner': 'manaflow-ai', 'github.run_attempt': 1, - 'needs.decide.outputs.track': 'nightly-next', - 'needs.decide.outputs.early_attempt': '1', - 'needs.decide.outputs.early_signer_build_runner': 'cmux-fleet-linux-x64'} + 'needs.decide.outputs.early_attempt': '1', 'needs.decide.outputs.early_linux_jobs': ' signer-build ', + 'needs.decide.outputs.early_linux_runner': 'cmux-fleet-linux-x64'} self.assertEqual(evaluate(parse_expression(source), placed), 'cmux-fleet-linux-x64') self.assertEqual(evaluate(parse_expression(source), {**placed, 'github.run_attempt': 2}), - 'cmux-fleet-linux-x64') + 'blacksmith-4vcpu-ubuntu-2404') def test_every_linux_nightly_job_takes_a_runner_set_member(self): # Never LINUX_RUNNER as-is: 16 vCPU when it names 16, else 32, whatever it holds