Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 12 additions & 2 deletions .github/workflows/cmux-tui-artifacts.yml
Original file line number Diff line number Diff line change
Expand Up @@ -1038,7 +1038,7 @@ jobs:
# preflight skips duplicate work after the first complete run.
cancel-in-progress: false
name: cmux_next_ daemon tests (macOS)
# The owned side lane on every attempt, a cmux-tui-pin-* push included.
# The owned side lane on every attempt (a cmux-tui-pin-* push: below).
# Every same-repo branch push runs this job, so naming the PR pool here
# queued them on Blacksmith at its cap while the minis had room, and a
# rerun (cmux-tui-artifacts-retry re-runs a failed run, no owned-pool
Expand All @@ -1056,7 +1056,17 @@ jobs:
# queued an hour behind the Release compile and promotion stalled
# (2026-10-15); a separate pool keeps promotion's two halves from queuing
# behind each other. Clearing it restores the shared promote pool.
runs-on: ${{ inputs.macos_runner || github.repository_owner != 'manaflow-ai' && 'macos-26' || vars.CI_NEXT_DAEMON_POOL != '' && github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && github.run_attempt == 1 && vars.CI_NEXT_DAEMON_POOL || vars.CI_NEXT_PROMOTE_POOL != '' && github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && github.run_attempt == 1 && vars.CI_NEXT_PROMOTE_POOL || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","schedule","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 3 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || 'blacksmith-6vcpu-macos-15' }}
# A cmux-tui-pin-* push (one per merged pull request) takes
# vars.CI_PIN_DAEMON_POOL on every attempt, unless it names the side lane
# or the promotion's daemon pool (CI_NEXT_DAEMON_POOL, else the promote
# pool); else the light minis (vars.CI_LIGHT_LANE_RUNNER while the owned
# pools are on) unless they are the side lane; else Blacksmith macOS 15.
# 2026-10-21: pins flooded the side-std pull request lane, and sharing the
# promotion's daemon pool stalls nightly-next. The light minis may still
# serve both until promotion gets its own label (claude-sr2);
# ci-repo-variables.yml on main warns then, and fails on a pin pool equal
# to either (tests/test_ci_tui_routes_avoid_nightly_label.py).
runs-on: ${{ inputs.macos_runner || github.repository_owner != 'manaflow-ai' && 'macos-26' || vars.CI_NEXT_DAEMON_POOL != '' && github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && github.run_attempt == 1 && vars.CI_NEXT_DAEMON_POOL || vars.CI_NEXT_PROMOTE_POOL != '' && github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && github.run_attempt == 1 && vars.CI_NEXT_PROMOTE_POOL || github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && startsWith(github.ref, 'refs/heads/cmux-tui-pin-') && (vars.CI_PIN_DAEMON_POOL != vars.CI_SIDE_LANE_RUNNER && vars.CI_PIN_DAEMON_POOL != (vars.CI_NEXT_DAEMON_POOL || vars.CI_NEXT_PROMOTE_POOL) && vars.CI_PIN_DAEMON_POOL || vars.CI_PR_POOL_OWNED == '1' && vars.CI_LIGHT_LANE_RUNNER != vars.CI_SIDE_LANE_RUNNER && vars.CI_LIGHT_LANE_RUNNER || 'blacksmith-6vcpu-macos-15') || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","schedule","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 3 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || 'blacksmith-6vcpu-macos-15' }}
timeout-minutes: 60
env:
# Cargo's default registry retry is too small for a transient index DNS
Expand Down
5 changes: 3 additions & 2 deletions scripts/ci/owned_pool_sources.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,8 +53,9 @@
# pr_runner_pool.py's label prefixes; their common prefix is the owned literal.
LABEL_PREFIX_NAMES = ("ROOT_PREFIX", "SIDE_PREFIX", "GUI_PREFIX", "RUNNER_LABEL_PREFIX")
# Pool variables no policy check holds to a label shape yet (docs/ci-runners.md
# on feat-cmux-next: the trusted promote mini and the daemon tests' slot).
EXTRA_OWNED_VARIABLES = frozenset({"CI_NEXT_PROMOTE_POOL", "CI_NEXT_DAEMON_POOL"})
# on feat-cmux-next: the trusted promote mini, the daemon tests' slot and the
# cmux-tui-pin-* pushes' daemon pool).
EXTRA_OWNED_VARIABLES = frozenset({"CI_NEXT_PROMOTE_POOL", "CI_NEXT_DAEMON_POOL", "CI_PIN_DAEMON_POOL"})
# (caller, reusable workflow) calls whose picker never places the call on an
# owned pool, so the caller does not inherit the reusable workflow's owned job.
OFF_POOL_CALLS = {
Expand Down
2 changes: 1 addition & 1 deletion tests/test_ci_nightly_off_blacksmith.py
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ def test_nightlies_skip_the_benchmark_mini(self) -> None:
"""
forbidden = re.compile(
r"cmux11s|glaeda-(side-|root-|gui-|session-|aws-)?std-xcode"
r"|vars\.(CI_SIDE_LANE_RUNNER|CI_NEXT_DAEMON_POOL|CI_NEXT_PROMOTE_POOL|CI_LIGHT_LANE_RUNNER)\b"
r"|vars\.(CI_SIDE_LANE_RUNNER|CI_NEXT_DAEMON_POOL|CI_NEXT_PROMOTE_POOL|CI_PIN_DAEMON_POOL|CI_LIGHT_LANE_RUNNER)\b"
)
allowed = ("vars.CI_SEED_TRUSTED_POOL", "vars.CI_NIGHTLY_NEXT_TRUSTED_POOL")
failures = []
Expand Down
83 changes: 80 additions & 3 deletions tests/test_ci_tui_routes_avoid_nightly_label.py
Original file line number Diff line number Diff line change
Expand Up @@ -34,15 +34,19 @@
SIDE_LABEL = "glaeda-side-std-xcode-26.6"
PROMOTE_LABEL = "glaeda-promote-trusted-std-xcode-26.6"
DAEMON_LABEL = "glaeda-daemon-tests-xcode-26.6"
LIGHT_LABEL = "glaeda-side-light-xcode-26.6"
PIN_LABEL = "glaeda-pin-daemon-xcode-26.6"
PIN_REF = "refs/heads/cmux-tui-pin-0123456789ab"

# The repository's variables as of 2026-10-14 (gh variable list), the ones
# these routes read. MACOS_RUNNER_PR is the nightly label.
LIVE_VARS = {
"CI_PR_POOL_OWNED": "1",
"CI_SIDE_LANE_RUNNER": SIDE_LABEL,
"CI_LIGHT_LANE_RUNNER": "glaeda-side-light-xcode-26.6",
"CI_LIGHT_LANE_RUNNER": LIGHT_LABEL,
"CI_NEXT_PROMOTE_POOL": PROMOTE_LABEL,
"CI_NEXT_DAEMON_POOL": "",
"CI_PIN_DAEMON_POOL": "",
"CI_PAID_MACOS_OVERFLOW": "",
"MACOS_RUNNER_15": "blacksmith-6vcpu-macos-15",
"MACOS_RUNNER_26": NIGHTLY_LABEL,
Expand All @@ -53,7 +57,7 @@
EVENTS = ("push", "pull_request", "pull_request_target", "workflow_dispatch", "schedule")
REFS = (
"refs/heads/feat-cmux-next",
"refs/heads/cmux-tui-pin-0123456789ab",
PIN_REF,
"refs/heads/main",
"refs/heads/feat-some-lane",
)
Expand Down Expand Up @@ -191,11 +195,84 @@ def test_daemon_tests_stay_on_the_owned_minis_on_every_attempt() -> None:
):
assert label == PROMOTE_LABEL, where
continue
# 2026-10-21: a cmux-tui-pin-* push takes its own pool, else the
# light minis, on every attempt; never the side lane.
if context["github.event_name"] == "push" and context["github.ref"] == PIN_REF:
assert label == (LIGHT_LABEL if owned == "1" else "blacksmith-6vcpu-macos-15"), where
continue
if owned != "1" or context["github.event_name"] not in trusted:
assert label == "blacksmith-6vcpu-macos-15", where
continue
# A cmux-tui-pin-* push, and every re-run, stays on the minis.
# Every other re-run stays on the minis.
assert label == SIDE_LABEL, where
# CI_PIN_DAEMON_POOL, when set, takes every attempt of a pin push.
for owned in ("1", ""):
for context in contexts(CI_PR_POOL_OWNED=owned, CI_PIN_DAEMON_POOL=PIN_LABEL):
label = route(runs_on, context)
where = (context["github.repository_owner"], context["github.event_name"],
context["github.ref"], context["github.run_attempt"], owned)
pin_push = (context["github.repository_owner"] == "manaflow-ai"
and context["github.event_name"] == "push" and context["github.ref"] == PIN_REF)
assert (label == PIN_LABEL) == pin_push, where


def test_pin_pushes_while_promotion_shares_the_light_minis() -> None:
# The landing state (claude-sr2, 2026-10-21): CI_PIN_DAEMON_POOL unset and
# CI_NEXT_DAEMON_POOL naming the light minis. Pins take the light minis on
# every attempt; a pin pool naming the promotion's pool or the side lane is
# ignored for the light minis.
runs_on = load("cmux-tui-artifacts.yml")["jobs"]["cmux-next-daemon-tests"]["runs-on"]
for pin in ("", LIGHT_LABEL, SIDE_LABEL):
for context in contexts(CI_NEXT_DAEMON_POOL=LIGHT_LABEL, CI_PIN_DAEMON_POOL=pin):
if (context["github.repository_owner"], context["github.event_name"], context["github.ref"]) == (
"manaflow-ai", "push", PIN_REF):
assert route(runs_on, context) == LIGHT_LABEL, (pin, context["github.run_attempt"])
for context in contexts(CI_NEXT_DAEMON_POOL=PIN_LABEL, CI_PIN_DAEMON_POOL=PIN_LABEL):
if (context["github.repository_owner"], context["github.event_name"], context["github.ref"]) == (
"manaflow-ai", "push", PIN_REF):
assert route(runs_on, context) == LIGHT_LABEL, context["github.run_attempt"]


def _pin_variable_combinations():
"""Every value of the variables the pin route reads, the live ones included."""
for owned, pin, light, daemon, promote, side in itertools.product(
("1", ""),
("", PIN_LABEL, SIDE_LABEL, DAEMON_LABEL, LIGHT_LABEL, PROMOTE_LABEL),
(LIGHT_LABEL, "", SIDE_LABEL, DAEMON_LABEL),
("", DAEMON_LABEL, LIGHT_LABEL, SIDE_LABEL),
(PROMOTE_LABEL, "", LIGHT_LABEL),
(SIDE_LABEL, ""),
):
yield {"CI_PR_POOL_OWNED": owned, "CI_PIN_DAEMON_POOL": pin, "CI_LIGHT_LANE_RUNNER": light,
"CI_NEXT_DAEMON_POOL": daemon, "CI_NEXT_PROMOTE_POOL": promote, "CI_SIDE_LANE_RUNNER": side}


def test_pin_pushes_never_share_the_side_lane_or_the_promotion_daemon_pool() -> None:
# 2026-10-21 (claude-sr2): every merged pull request pushes a
# cmux-tui-pin-* branch, and its daemon tests flooded the pull request side
# lane; on the feat-cmux-next promotion's daemon pool (CI_NEXT_DAEMON_POOL,
# else CI_NEXT_PROMOTE_POOL) they stall nightly-next. Whatever the
# variables say, no job of a pin push lands on the side label, and only
# the light-lane fallback may share the promotion's pool: the light minis
# serve both until promotion gets its own label (ci-repo-variables.yml on
# main warns then).
jobs = load("cmux-tui-artifacts.yml")["jobs"]
routes = {name: job["runs-on"] for name, job in jobs.items() if "runs-on" in job}
assert "cmux-next-daemon-tests" in routes
for overrides in _pin_variable_combinations():
side = overrides["CI_SIDE_LANE_RUNNER"]
promotion = overrides["CI_NEXT_DAEMON_POOL"] or overrides["CI_NEXT_PROMOTE_POOL"]
light_fallback = overrides["CI_LIGHT_LANE_RUNNER"] if overrides["CI_PR_POOL_OWNED"] == "1" else ""
for context in contexts(**overrides):
if (context["github.repository_owner"], context["github.event_name"], context["github.ref"]) != (
"manaflow-ai", "push", PIN_REF):
continue
for name, runs_on in routes.items():
label = route(runs_on, context) if runs_on.strip().startswith("${{") else runs_on
where = (name, context["github.run_attempt"], overrides)
assert not side or label != side, where
assert not promotion or label != promotion or label == light_fallback, where
assert label != NIGHTLY_LABEL, where


def test_daemon_tests_take_their_own_promote_time_pool_when_set() -> None:
Expand Down
15 changes: 14 additions & 1 deletion tests/test_tui_publish_workflow_security.py
Original file line number Diff line number Diff line change
Expand Up @@ -2326,6 +2326,19 @@ def test_cmux_next_daemon_tests_take_the_owned_side_lane_like_the_build_legs() -
"github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && "
"github.run_attempt == 1 && vars.CI_NEXT_PROMOTE_POOL || "
)
# 2026-10-21: a cmux-tui-pin-* push takes CI_PIN_DAEMON_POOL, else the
# light minis, on every attempt, and never the side lane; a pin pool never
# names the promotion's daemon pool (tests/test_ci_tui_routes_avoid_nightly_label.py).
promotion_pool = "(vars.CI_NEXT_DAEMON_POOL || vars.CI_NEXT_PROMOTE_POOL)"
pin = (
"github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && "
"startsWith(github.ref, 'refs/heads/cmux-tui-pin-') && ("
"vars.CI_PIN_DAEMON_POOL != vars.CI_SIDE_LANE_RUNNER && "
f"vars.CI_PIN_DAEMON_POOL != {promotion_pool} && vars.CI_PIN_DAEMON_POOL || "
"vars.CI_PR_POOL_OWNED == '1' && vars.CI_LIGHT_LANE_RUNNER != vars.CI_SIDE_LANE_RUNNER && "
"vars.CI_LIGHT_LANE_RUNNER || "
"'blacksmith-6vcpu-macos-15') || "
)
assert fork_gate in build_route
side_attempt_one = "github.run_attempt == 1 && vars.CI_SIDE_LANE_RUNNER"
assert side_attempt_one in build_route
Expand All @@ -2337,7 +2350,7 @@ def test_cmux_next_daemon_tests_take_the_owned_side_lane_like_the_build_legs() -
"vars.CI_SIDE_LANE_RUNNER",
1,
)
assert daemon == every_attempt.replace(fork_gate, fork_gate + promote, 1), daemon
assert daemon == every_attempt.replace(fork_gate, fork_gate + promote + pin, 1), daemon
assert "MACOS_RUNNER_PR" not in daemon
assert retry_route.split("|| ", 1)[1] in daemon # owned pools off: Blacksmith macOS 15

Expand Down