Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 27 additions & 1 deletion .github/workflows/cmux-tui-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,11 @@ jobs:
# skipped and the build starts one runner queue earlier.
owns_key: ${{ steps.inspect.outputs.owns_key }}
stale: ${{ steps.stale.outputs.stale }}
# The live GCP Linux label for the daemon tests alone, only when the
# runner census shows an idle Linux member; empty keeps their measured
# Blacksmith overflow. The build legs keep their own routes.
linux_runner: ${{ steps.linux-placement.outputs.linux_runner }}
linux_jobs: ${{ steps.linux-placement.outputs.linux_jobs }}
steps:
- uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2
with:
Expand Down Expand Up @@ -682,6 +687,27 @@ jobs:
SNAPSHOT_TOKEN: ${{ github.token }}
run: python3 scripts/ci/portable_job_placement.py

# The artifact daemon is Linux-only. Emit the fleet Linux label only when
# the runner census shows an idle Linux member beyond its queue; an empty
# output keeps the measured Blacksmith fallback. The trusted runner state
# document answers without routing App reads; the route token is reused
# only when the macOS pick already minted it.
- name: Pick a live Linux runner for the artifact daemon
id: linux-placement
if: steps.daemon-policy.outputs.daemon_tests == 'true'
continue-on-error: true
timeout-minutes: 3
env:
CI_RUNNER_STATE: ${{ vars.CI_RUNNER_STATE }}
JOBS: cmux-next-daemon-tests
LINUX_ONLY: cmux-next-daemon-tests
FLEET_LINUX: cmux-fleet-linux-x64
REQUIRE_LIVE_CAPACITY: "1"
ROUTE_TOKEN: ${{ steps.route-token.outputs.token }}
ROUTE_TOKEN_FALLBACK: ${{ github.token }}
SNAPSHOT_TOKEN: ${{ github.token }}
run: python3 scripts/ci/portable_job_placement.py

# Observation only: an unavailable marker must never hold up a build.
- name: Record tree owner observation
id: tree-source
Expand Down Expand Up @@ -1358,7 +1384,7 @@ jobs:
# its name's "cmux_next_ daemon tests" prefix, so older runs' names with
# a runner suffix still count. Leo, 2026-10-27: "ok do NOT label stuff as
# os=linux...": the name, steps and marker name no runner OS.
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-8vcpu-ubuntu-2404' }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || contains(needs.tree-preflight.outputs.linux_jobs, ' cmux-next-daemon-tests ') && needs.tree-preflight.outputs.linux_runner || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-16vcpu-ubuntu-2404' || 'blacksmith-8vcpu-ubuntu-2404' }}
timeout-minutes: 60
env:
# Cargo's default registry retry is too small for a transient index DNS
Expand Down
6 changes: 3 additions & 3 deletions .github/workflows/cmux-tui-build-package.yml
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ env:
jobs:
plan-build:
name: select binary targets
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || (inputs.linux_runner != '' && inputs.linux_runner || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404') }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || (inputs.linux_runner != '' && !startsWith(inputs.linux_runner, 'blacksmith-') && inputs.linux_runner || !startsWith(vars.LINUX_RUNNER, 'blacksmith-') && vars.LINUX_RUNNER || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-8vcpu-ubuntu-2404' || 'blacksmith-16vcpu-ubuntu-2404') }}
timeout-minutes: 15
outputs:
matrix: ${{ steps.targets.outputs.matrix }}
Expand All @@ -164,7 +164,7 @@ jobs:
LINUX_DARWIN_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || vars.LINUX_RUNNER_DARWIN_CROSS || !startsWith(vars.LINUX_RUNNER, 'blacksmith-') && vars.LINUX_RUNNER || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-8vcpu-ubuntu-2404' || 'blacksmith-16vcpu-ubuntu-2404' }}
LINUX_BUILD_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || (inputs.linux_runner != '' && inputs.linux_runner || vars.LINUX_BUILD_RUNNER || !startsWith(vars.LINUX_RUNNER, 'blacksmith-') && vars.LINUX_RUNNER || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-8vcpu-ubuntu-2404' || 'blacksmith-16vcpu-ubuntu-2404') }}
MACOS_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (inputs.macos_runner != '' && inputs.macos_runner || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }}
LINUX_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || (inputs.linux_runner != '' && inputs.linux_runner || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404') }}
LINUX_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || (inputs.linux_runner != '' && !startsWith(inputs.linux_runner, 'blacksmith-') && inputs.linux_runner || !startsWith(vars.LINUX_RUNNER, 'blacksmith-') && vars.LINUX_RUNNER || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-8vcpu-ubuntu-2404' || 'blacksmith-16vcpu-ubuntu-2404') }}
LINUX_ARM64_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04-arm' || (inputs.linux_arm64_runner != '' && inputs.linux_arm64_runner || vars.LINUX_ARM64_RUNNER || 'blacksmith-4vcpu-ubuntu-2404-arm') }}
VERIFY_LINUX_ARM64: ${{ inputs.verify_linux_arm64 }}
shell: bash
Expand Down Expand Up @@ -1003,7 +1003,7 @@ jobs:
needs.build.result == 'success' &&
(!inputs.include_windows || needs.build-windows.result == 'success')
}}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || (inputs.linux_runner != '' && inputs.linux_runner || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404') }}
runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || (inputs.linux_runner != '' && !startsWith(inputs.linux_runner, 'blacksmith-') && inputs.linux_runner || !startsWith(vars.LINUX_RUNNER, 'blacksmith-') && vars.LINUX_RUNNER || vars.CI_CLOUD_OVERFLOW_SAVED && 'blacksmith-8vcpu-ubuntu-2404' || 'blacksmith-16vcpu-ubuntu-2404') }}
timeout-minutes: 30
permissions:
contents: read
Expand Down
7 changes: 7 additions & 0 deletions scripts/ci/portable_job_placement.py
Original file line number Diff line number Diff line change
Expand Up @@ -154,6 +154,13 @@ def eligible(env: Mapping[str, str]) -> str:
if refusal:
return refusal
label = (env.get("SIDE_LABEL") or "").strip()
jobs = set((env.get("JOBS") or "").split())
linux_only = set((env.get("LINUX_ONLY") or "").split())
# A Linux-only picker (for example the cmux-tui daemon marker) has no
# owned-Mac leg. It must still read the live Linux group even when the
# caller has no side-pool label to offer.
if jobs and jobs.issubset(linux_only):
return ""
if not label.startswith(pool.SIDE_PREFIX) or not pool.persistent(label):
return f"`{label or 'no label'}` is not an owned side label"
if label.startswith("glaeda-aws-"):
Expand Down
12 changes: 11 additions & 1 deletion tests/runs_on_labels.py
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,11 @@
# What scripts/ci/cloud_overflow_switch.py points LINUX_RUNNER at while Blacksmith is down (HOSTED_LINUX).
OVERFLOW_HOSTED_LINUX = "ubuntu-24.04"

# The GCP fleet's Linux label. tree-preflight's linux-placement step (scripts/ci/portable_job_placement.py
# with REQUIRE_LIVE_CAPACITY) emits it only when the runner census shows an idle Linux member, else ''.
GCP_FLEET_LINUX = "cmux-fleet-linux-x64"
FLEET_LINUX_OUTPUTS = ("needs.tree-preflight.outputs.linux_runner",)

TOKEN = re.compile(
r"\s*(?:(?P<str>'(?:[^']|'')*')|(?P<op>&&|\|\||==|!=|<=|>=|[!(),<>])"
r"|(?P<num>-?(?:[0-9]+(?:\.[0-9]*)?|\.[0-9]+)(?:[eE][-+]?[0-9]+)?)|(?P<ident>[A-Za-z_][A-Za-z0-9_.*-]*))"
Expand Down Expand Up @@ -320,10 +325,15 @@ def disallowed_labels(runs_on, variables: Mapping[str, Iterable[str]] | None = N
root: Path = ROOT) -> dict[str, str]:
"""label -> why it may not run this job; empty when every label is Blacksmith or declared fleet Linux.

A FLEET_LINUX_OUTPUTS step output reads as the GCP fleet Linux label it can only hold.

GitHub-hosted labels and the personal workstation are refused first, even if a fleet declares them.
"""
fleet_linux = declared_fleet_linux_labels(root)
fleet_linux = declared_fleet_linux_labels(root) | {GCP_FLEET_LINUX}
personal = personal_machine_names()
if isinstance(runs_on, str):
for output in FLEET_LINUX_OUTPUTS:
runs_on = runs_on.replace(output, f"'{GCP_FLEET_LINUX}'")
labels = resolve_labels(runs_on, variables=variables)
if not labels:
return {"[]": "no label at all: an empty runs-on never schedules"}
Expand Down
53 changes: 52 additions & 1 deletion tests/test_ci_portable_job_placement.py
Original file line number Diff line number Diff line change
Expand Up @@ -50,6 +50,11 @@ def runner(name: str, label: str = SIDE, *, busy: bool = False, status: str = "o
"labels": [{"name": item} for item in ("self-hosted", owned, label)]}


def fleet_runner(name: str = "cmux-fleet-1", *, busy: bool = False, status: str = "online") -> dict:
return {"name": name, "status": status, "busy": busy, "os": "Linux",
"labels": [{"name": item} for item in ("self-hosted", "cmux-fleet-linux-x64")]}


def snapshot(pools: dict | None = None, *, age_minutes: int = 2) -> dict:
generated = (NOW - dt.timedelta(minutes=age_minutes)).strftime("%Y-%m-%dT%H:%M:%SZ")
return {"generated_at": generated, "pools": pools or {}}
Expand Down Expand Up @@ -132,7 +137,6 @@ def test_early_nightly_unknown_capacity_uses_blacksmith_overflow(self):
placed = placement.decide(config, runners, snap, NOW)
self.assertEqual((placed.mini_runner, placed.mini_jobs), ("", ()))
self.assertEqual(placed.linux_jobs, ())
self.assertIn("Blacksmith", placed.why)

def test_offline_runner_without_snapshot_does_not_queue_all_owned_jobs(self):
config = env(JOBS="prepare-one prepare-two prepare-three prepare-four", NIGHTLY="1", NIGHTLY_WAIT_FOR_FLEET="1", REQUIRE_LIVE_CAPACITY="1")
Expand Down Expand Up @@ -228,6 +232,22 @@ def test_fleet_linux_comes_before_the_minis(self):
self.assertEqual(placed.linux_jobs, tuple(JOBS.split()))
self.assertEqual(placed.mini_jobs, ())

def test_live_capacity_requirement_rejects_an_unserved_fleet_label(self):
config = env(JOBS="cmux-next-daemon-tests", LINUX_ONLY="cmux-next-daemon-tests",
FLEET_LINUX="cmux-fleet-linux-x64", REQUIRE_LIVE_CAPACITY="1")
for runners in (None, [], [fleet_runner(status="offline")]):
with self.subTest(runners=runners):
placed = placement.decide(config, runners, snapshot(), NOW)
self.assertEqual(placed.linux_runner, "")
self.assertEqual(placed.linux_jobs, ())

def test_live_capacity_requirement_emits_the_registered_fleet_label(self):
config = env(JOBS="cmux-next-daemon-tests", LINUX_ONLY="cmux-next-daemon-tests",
FLEET_LINUX="cmux-fleet-linux-x64", REQUIRE_LIVE_CAPACITY="1")
placed = placement.decide(config, [fleet_runner()], snapshot(), NOW)
self.assertEqual(placed.linux_runner, "cmux-fleet-linux-x64")
self.assertEqual(placed.linux_jobs, ("cmux-next-daemon-tests",))

def test_a_mac_first_job_takes_a_mini_before_fleet_linux(self):
placed = placement.decide(env(FLEET_LINUX="gcp-linux-32", MAC_FIRST="tooling-tests"),
idle(1), snapshot(), NOW)
Expand Down Expand Up @@ -365,6 +385,21 @@ def test_main_with_no_tokens_writes_empty_routes(self):
self.assertEqual(placement.main(environment), 0)
self.assertIn("mini_runner=\nmini_jobs=\n", output.read_text())

def test_linux_picker_without_route_token_keeps_measured_overflow(self):
with tempfile.TemporaryDirectory() as scratch:
output = Path(scratch) / "out"
summary = Path(scratch) / "summary"
environment = {
**env(JOBS="cmux-next-daemon-tests", LINUX_ONLY="cmux-next-daemon-tests",
FLEET_LINUX="cmux-fleet-linux-x64", REQUIRE_LIVE_CAPACITY="1"),
"GITHUB_OUTPUT": str(output),
"GITHUB_STEP_SUMMARY": str(summary),
"GITHUB_REPOSITORY": "manaflow-ai/cmux",
}
self.assertEqual(placement.main(environment), 0)
text = output.read_text()
self.assertIn("linux_runner=\n", text)


def jobs(name: str) -> dict:
return yaml.safe_load((WORKFLOWS / name).read_text(encoding="utf-8"))["jobs"]
Expand Down Expand Up @@ -461,6 +496,22 @@ def test_the_cmux_tui_macos_targets_take_a_free_mini_before_the_linux_cross_buil
plan = (WORKFLOWS / "cmux-tui-build-package.yml").read_text(encoding="utf-8")
self.assertIn('target["runner"] = darwin_mac_runner', plan)

def test_cmux_tui_daemon_uses_live_gcp_output_before_measured_overflow(self):
workflow = jobs("cmux-tui-artifacts.yml")
preflight = workflow["tree-preflight"]
picker = next(step for step in preflight["steps"] if step.get("id") == "linux-placement")
self.assertEqual(picker["env"]["FLEET_LINUX"], "cmux-fleet-linux-x64")
self.assertEqual(picker["env"]["REQUIRE_LIVE_CAPACITY"], "1")
daemon = workflow["cmux-next-daemon-tests"]["runs-on"]
live = daemon.index("needs.tree-preflight.outputs.linux_runner")
fallback = daemon.index("'blacksmith-8vcpu-ubuntu-2404'")
self.assertLess(live, fallback)
self.assertEqual(workflow["tree-preflight"]["outputs"]["linux_runner"],
"${{ steps.linux-placement.outputs.linux_runner }}")
# The placement counts room for the daemon job only, so no build leg takes its label.
for name in ("build", "build-windows"):
self.assertNotIn("linux_runner", workflow[name].get("with", {}))

def test_the_cmux_tui_mini_pick_spends_routing_reads_only_when_the_macos_targets_may_build(self):
# The pick reads the runners on the routing App (4 requests); a run whose tree is published, that
# is stale, or that defers to a live owner builds nothing, so it must not spend them.
Expand Down
7 changes: 6 additions & 1 deletion tests/test_cmux_tui_publisher_testing.py
Original file line number Diff line number Diff line change
Expand Up @@ -86,7 +86,12 @@ class WiringTests(unittest.TestCase):
def test_both_deferrals_ask_the_script(self):
self.assertIn('cmux_tui_publisher_testing.py --run-id "$source_run" --pool "${DAEMON_POOL:-}"', self.WORKFLOW)
self.assertIn('cmux_tui_publisher_testing.py --run-id "$OWNER_RUN_ID" --pool "${DAEMON_POOL:-}"', self.WORKFLOW)
self.assertEqual(self.WORKFLOW.count(f"DAEMON_POOL: {gate_runs_on(self.WORKFLOW)}"), 2)
# DAEMON_POOL is the gate's Blacksmith route. A publisher job placed on the live fleet Linux label
# reads as off-pool, so a deferring run tests the tree itself instead of waiting on it.
fleet = "contains(needs.tree-preflight.outputs.linux_jobs, ' cmux-next-daemon-tests ') && needs.tree-preflight.outputs.linux_runner || "
gate = gate_runs_on(self.WORKFLOW)
self.assertIn(fleet, gate)
self.assertEqual(self.WORKFLOW.count(f"DAEMON_POOL: {gate.replace(fleet, '')}"), 2)

def test_the_owner_wait_sparse_checkout_has_the_script(self):
owner_wait = self.WORKFLOW[self.WORKFLOW.index("sparse-checkout: |\n scripts/ci/cmux-tui-tree-claim.sh"):]
Expand Down
12 changes: 10 additions & 2 deletions tests/test_tui_publish_workflow_security.py
Original file line number Diff line number Diff line change
Expand Up @@ -2351,9 +2351,17 @@ def test_cmux_next_daemon_tests_take_blacksmith_linux_and_the_mac_shadow_never_t
document = yaml.load(workflow("cmux-tui-artifacts.yml"), Loader=yaml.BaseLoader)
jobs = document["jobs"]
daemon = jobs["cmux-next-daemon-tests"]["runs-on"]
# Every label it can resolve to is Blacksmith Linux; size and overflow wiring are free to change.
# Every label it can resolve to is Blacksmith Linux or the GCP fleet Linux label, which tree-preflight's
# linux-placement step emits only when the runner census shows an idle Linux member; size and overflow
# wiring are free to change.
from runs_on_labels import disallowed_labels
assert disallowed_labels(daemon) == {}, (daemon, disallowed_labels(daemon))
fleet_linux = "cmux-fleet-linux-x64"
picker = next(step for step in jobs["tree-preflight"]["steps"] if step.get("id") == "linux-placement")
assert picker["env"]["FLEET_LINUX"] == fleet_linux, picker["env"]
assert picker["env"]["LINUX_ONLY"] == "cmux-next-daemon-tests", picker["env"]
assert picker["env"]["REQUIRE_LIVE_CAPACITY"] == "1", picker["env"]
placed = daemon.replace("needs.tree-preflight.outputs.linux_runner", f"'{fleet_linux}'")
assert set(disallowed_labels(placed)) <= {fleet_linux}, (daemon, disallowed_labels(placed))
assert "cmux-next-daemon-macos-shadow" not in jobs
shadow_doc = yaml.load(workflow("cmux-next-daemon-macos-shadow.yml"), Loader=yaml.BaseLoader)
shadow = shadow_doc["jobs"]["cmux-next-daemon-macos-shadow"]["runs-on"]
Expand Down
Loading