diff --git a/.github/workflows/cmux-tui-artifacts.yml b/.github/workflows/cmux-tui-artifacts.yml index 7569ca62d484..7bdc2b236f2e 100644 --- a/.github/workflows/cmux-tui-artifacts.yml +++ b/.github/workflows/cmux-tui-artifacts.yml @@ -1038,7 +1038,7 @@ jobs: # preflight skips duplicate work after the first complete run. cancel-in-progress: false name: cmux_next_ daemon tests (macOS) - # The owned side lane on every attempt, a cmux-tui-pin-* push included. + # The owned side lane on every attempt (a cmux-tui-pin-* push: below). # Every same-repo branch push runs this job, so naming the PR pool here # queued them on Blacksmith at its cap while the minis had room, and a # rerun (cmux-tui-artifacts-retry re-runs a failed run, no owned-pool @@ -1056,7 +1056,17 @@ jobs: # queued an hour behind the Release compile and promotion stalled # (2026-10-15); a separate pool keeps promotion's two halves from queuing # behind each other. Clearing it restores the shared promote pool. - runs-on: ${{ inputs.macos_runner || github.repository_owner != 'manaflow-ai' && 'macos-26' || vars.CI_NEXT_DAEMON_POOL != '' && github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && github.run_attempt == 1 && vars.CI_NEXT_DAEMON_POOL || vars.CI_NEXT_PROMOTE_POOL != '' && github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && github.run_attempt == 1 && vars.CI_NEXT_PROMOTE_POOL || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","schedule","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 3 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || 'blacksmith-6vcpu-macos-15' }} + # A cmux-tui-pin-* push (one per merged pull request) takes + # vars.CI_PIN_DAEMON_POOL on every attempt, unless it names the side lane + # or the promotion's daemon pool (CI_NEXT_DAEMON_POOL, else the promote + # pool); else the light minis (vars.CI_LIGHT_LANE_RUNNER while the owned + # pools are on) unless they are the side lane; else Blacksmith macOS 15. + # 2026-10-21: pins flooded the side-std pull request lane, and sharing the + # promotion's daemon pool stalls nightly-next. The light minis may still + # serve both until promotion gets its own label (claude-sr2); + # ci-repo-variables.yml on main warns then, and fails on a pin pool equal + # to either (tests/test_ci_tui_routes_avoid_nightly_label.py). + runs-on: ${{ inputs.macos_runner || github.repository_owner != 'manaflow-ai' && 'macos-26' || vars.CI_NEXT_DAEMON_POOL != '' && github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && github.run_attempt == 1 && vars.CI_NEXT_DAEMON_POOL || vars.CI_NEXT_PROMOTE_POOL != '' && github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && github.run_attempt == 1 && vars.CI_NEXT_PROMOTE_POOL || github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && startsWith(github.ref, 'refs/heads/cmux-tui-pin-') && (vars.CI_PIN_DAEMON_POOL != vars.CI_SIDE_LANE_RUNNER && vars.CI_PIN_DAEMON_POOL != (vars.CI_NEXT_DAEMON_POOL || vars.CI_NEXT_PROMOTE_POOL) && vars.CI_PIN_DAEMON_POOL || vars.CI_PR_POOL_OWNED == '1' && vars.CI_LIGHT_LANE_RUNNER != vars.CI_SIDE_LANE_RUNNER && vars.CI_LIGHT_LANE_RUNNER || 'blacksmith-6vcpu-macos-15') || vars.CI_PR_POOL_OWNED == '1' && contains(fromJSON('["pull_request","push","schedule","workflow_dispatch"]'), github.event_name) && (github.run_attempt <= 3 || github.triggering_actor != 'github-actions[bot]') && vars.CI_SIDE_LANE_RUNNER || 'blacksmith-6vcpu-macos-15' }} timeout-minutes: 60 env: # Cargo's default registry retry is too small for a transient index DNS diff --git a/scripts/ci/owned_pool_sources.py b/scripts/ci/owned_pool_sources.py index 8c35356f6a16..899cf4afc676 100644 --- a/scripts/ci/owned_pool_sources.py +++ b/scripts/ci/owned_pool_sources.py @@ -53,8 +53,9 @@ # pr_runner_pool.py's label prefixes; their common prefix is the owned literal. LABEL_PREFIX_NAMES = ("ROOT_PREFIX", "SIDE_PREFIX", "GUI_PREFIX", "RUNNER_LABEL_PREFIX") # Pool variables no policy check holds to a label shape yet (docs/ci-runners.md -# on feat-cmux-next: the trusted promote mini and the daemon tests' slot). -EXTRA_OWNED_VARIABLES = frozenset({"CI_NEXT_PROMOTE_POOL", "CI_NEXT_DAEMON_POOL"}) +# on feat-cmux-next: the trusted promote mini, the daemon tests' slot and the +# cmux-tui-pin-* pushes' daemon pool). +EXTRA_OWNED_VARIABLES = frozenset({"CI_NEXT_PROMOTE_POOL", "CI_NEXT_DAEMON_POOL", "CI_PIN_DAEMON_POOL"}) # (caller, reusable workflow) calls whose picker never places the call on an # owned pool, so the caller does not inherit the reusable workflow's owned job. OFF_POOL_CALLS = { diff --git a/tests/test_ci_nightly_off_blacksmith.py b/tests/test_ci_nightly_off_blacksmith.py index 54dffe7c5d4a..4519c9bb5c28 100644 --- a/tests/test_ci_nightly_off_blacksmith.py +++ b/tests/test_ci_nightly_off_blacksmith.py @@ -82,7 +82,7 @@ def test_nightlies_skip_the_benchmark_mini(self) -> None: """ forbidden = re.compile( r"cmux11s|glaeda-(side-|root-|gui-|session-|aws-)?std-xcode" - r"|vars\.(CI_SIDE_LANE_RUNNER|CI_NEXT_DAEMON_POOL|CI_NEXT_PROMOTE_POOL|CI_LIGHT_LANE_RUNNER)\b" + r"|vars\.(CI_SIDE_LANE_RUNNER|CI_NEXT_DAEMON_POOL|CI_NEXT_PROMOTE_POOL|CI_PIN_DAEMON_POOL|CI_LIGHT_LANE_RUNNER)\b" ) allowed = ("vars.CI_SEED_TRUSTED_POOL", "vars.CI_NIGHTLY_NEXT_TRUSTED_POOL") failures = [] diff --git a/tests/test_ci_tui_routes_avoid_nightly_label.py b/tests/test_ci_tui_routes_avoid_nightly_label.py index ee70163ffbd6..4ee73cfca0c9 100644 --- a/tests/test_ci_tui_routes_avoid_nightly_label.py +++ b/tests/test_ci_tui_routes_avoid_nightly_label.py @@ -34,15 +34,19 @@ SIDE_LABEL = "glaeda-side-std-xcode-26.6" PROMOTE_LABEL = "glaeda-promote-trusted-std-xcode-26.6" DAEMON_LABEL = "glaeda-daemon-tests-xcode-26.6" +LIGHT_LABEL = "glaeda-side-light-xcode-26.6" +PIN_LABEL = "glaeda-pin-daemon-xcode-26.6" +PIN_REF = "refs/heads/cmux-tui-pin-0123456789ab" # The repository's variables as of 2026-10-14 (gh variable list), the ones # these routes read. MACOS_RUNNER_PR is the nightly label. LIVE_VARS = { "CI_PR_POOL_OWNED": "1", "CI_SIDE_LANE_RUNNER": SIDE_LABEL, - "CI_LIGHT_LANE_RUNNER": "glaeda-side-light-xcode-26.6", + "CI_LIGHT_LANE_RUNNER": LIGHT_LABEL, "CI_NEXT_PROMOTE_POOL": PROMOTE_LABEL, "CI_NEXT_DAEMON_POOL": "", + "CI_PIN_DAEMON_POOL": "", "CI_PAID_MACOS_OVERFLOW": "", "MACOS_RUNNER_15": "blacksmith-6vcpu-macos-15", "MACOS_RUNNER_26": NIGHTLY_LABEL, @@ -53,7 +57,7 @@ EVENTS = ("push", "pull_request", "pull_request_target", "workflow_dispatch", "schedule") REFS = ( "refs/heads/feat-cmux-next", - "refs/heads/cmux-tui-pin-0123456789ab", + PIN_REF, "refs/heads/main", "refs/heads/feat-some-lane", ) @@ -191,11 +195,84 @@ def test_daemon_tests_stay_on_the_owned_minis_on_every_attempt() -> None: ): assert label == PROMOTE_LABEL, where continue + # 2026-10-21: a cmux-tui-pin-* push takes its own pool, else the + # light minis, on every attempt; never the side lane. + if context["github.event_name"] == "push" and context["github.ref"] == PIN_REF: + assert label == (LIGHT_LABEL if owned == "1" else "blacksmith-6vcpu-macos-15"), where + continue if owned != "1" or context["github.event_name"] not in trusted: assert label == "blacksmith-6vcpu-macos-15", where continue - # A cmux-tui-pin-* push, and every re-run, stays on the minis. + # Every other re-run stays on the minis. assert label == SIDE_LABEL, where + # CI_PIN_DAEMON_POOL, when set, takes every attempt of a pin push. + for owned in ("1", ""): + for context in contexts(CI_PR_POOL_OWNED=owned, CI_PIN_DAEMON_POOL=PIN_LABEL): + label = route(runs_on, context) + where = (context["github.repository_owner"], context["github.event_name"], + context["github.ref"], context["github.run_attempt"], owned) + pin_push = (context["github.repository_owner"] == "manaflow-ai" + and context["github.event_name"] == "push" and context["github.ref"] == PIN_REF) + assert (label == PIN_LABEL) == pin_push, where + + +def test_pin_pushes_while_promotion_shares_the_light_minis() -> None: + # The landing state (claude-sr2, 2026-10-21): CI_PIN_DAEMON_POOL unset and + # CI_NEXT_DAEMON_POOL naming the light minis. Pins take the light minis on + # every attempt; a pin pool naming the promotion's pool or the side lane is + # ignored for the light minis. + runs_on = load("cmux-tui-artifacts.yml")["jobs"]["cmux-next-daemon-tests"]["runs-on"] + for pin in ("", LIGHT_LABEL, SIDE_LABEL): + for context in contexts(CI_NEXT_DAEMON_POOL=LIGHT_LABEL, CI_PIN_DAEMON_POOL=pin): + if (context["github.repository_owner"], context["github.event_name"], context["github.ref"]) == ( + "manaflow-ai", "push", PIN_REF): + assert route(runs_on, context) == LIGHT_LABEL, (pin, context["github.run_attempt"]) + for context in contexts(CI_NEXT_DAEMON_POOL=PIN_LABEL, CI_PIN_DAEMON_POOL=PIN_LABEL): + if (context["github.repository_owner"], context["github.event_name"], context["github.ref"]) == ( + "manaflow-ai", "push", PIN_REF): + assert route(runs_on, context) == LIGHT_LABEL, context["github.run_attempt"] + + +def _pin_variable_combinations(): + """Every value of the variables the pin route reads, the live ones included.""" + for owned, pin, light, daemon, promote, side in itertools.product( + ("1", ""), + ("", PIN_LABEL, SIDE_LABEL, DAEMON_LABEL, LIGHT_LABEL, PROMOTE_LABEL), + (LIGHT_LABEL, "", SIDE_LABEL, DAEMON_LABEL), + ("", DAEMON_LABEL, LIGHT_LABEL, SIDE_LABEL), + (PROMOTE_LABEL, "", LIGHT_LABEL), + (SIDE_LABEL, ""), + ): + yield {"CI_PR_POOL_OWNED": owned, "CI_PIN_DAEMON_POOL": pin, "CI_LIGHT_LANE_RUNNER": light, + "CI_NEXT_DAEMON_POOL": daemon, "CI_NEXT_PROMOTE_POOL": promote, "CI_SIDE_LANE_RUNNER": side} + + +def test_pin_pushes_never_share_the_side_lane_or_the_promotion_daemon_pool() -> None: + # 2026-10-21 (claude-sr2): every merged pull request pushes a + # cmux-tui-pin-* branch, and its daemon tests flooded the pull request side + # lane; on the feat-cmux-next promotion's daemon pool (CI_NEXT_DAEMON_POOL, + # else CI_NEXT_PROMOTE_POOL) they stall nightly-next. Whatever the + # variables say, no job of a pin push lands on the side label, and only + # the light-lane fallback may share the promotion's pool: the light minis + # serve both until promotion gets its own label (ci-repo-variables.yml on + # main warns then). + jobs = load("cmux-tui-artifacts.yml")["jobs"] + routes = {name: job["runs-on"] for name, job in jobs.items() if "runs-on" in job} + assert "cmux-next-daemon-tests" in routes + for overrides in _pin_variable_combinations(): + side = overrides["CI_SIDE_LANE_RUNNER"] + promotion = overrides["CI_NEXT_DAEMON_POOL"] or overrides["CI_NEXT_PROMOTE_POOL"] + light_fallback = overrides["CI_LIGHT_LANE_RUNNER"] if overrides["CI_PR_POOL_OWNED"] == "1" else "" + for context in contexts(**overrides): + if (context["github.repository_owner"], context["github.event_name"], context["github.ref"]) != ( + "manaflow-ai", "push", PIN_REF): + continue + for name, runs_on in routes.items(): + label = route(runs_on, context) if runs_on.strip().startswith("${{") else runs_on + where = (name, context["github.run_attempt"], overrides) + assert not side or label != side, where + assert not promotion or label != promotion or label == light_fallback, where + assert label != NIGHTLY_LABEL, where def test_daemon_tests_take_their_own_promote_time_pool_when_set() -> None: diff --git a/tests/test_tui_publish_workflow_security.py b/tests/test_tui_publish_workflow_security.py index 2e17706a965c..c4adda6974a2 100644 --- a/tests/test_tui_publish_workflow_security.py +++ b/tests/test_tui_publish_workflow_security.py @@ -2326,6 +2326,19 @@ def test_cmux_next_daemon_tests_take_the_owned_side_lane_like_the_build_legs() - "github.event_name == 'push' && github.ref == 'refs/heads/feat-cmux-next' && " "github.run_attempt == 1 && vars.CI_NEXT_PROMOTE_POOL || " ) + # 2026-10-21: a cmux-tui-pin-* push takes CI_PIN_DAEMON_POOL, else the + # light minis, on every attempt, and never the side lane; a pin pool never + # names the promotion's daemon pool (tests/test_ci_tui_routes_avoid_nightly_label.py). + promotion_pool = "(vars.CI_NEXT_DAEMON_POOL || vars.CI_NEXT_PROMOTE_POOL)" + pin = ( + "github.repository_owner == 'manaflow-ai' && github.event_name == 'push' && " + "startsWith(github.ref, 'refs/heads/cmux-tui-pin-') && (" + "vars.CI_PIN_DAEMON_POOL != vars.CI_SIDE_LANE_RUNNER && " + f"vars.CI_PIN_DAEMON_POOL != {promotion_pool} && vars.CI_PIN_DAEMON_POOL || " + "vars.CI_PR_POOL_OWNED == '1' && vars.CI_LIGHT_LANE_RUNNER != vars.CI_SIDE_LANE_RUNNER && " + "vars.CI_LIGHT_LANE_RUNNER || " + "'blacksmith-6vcpu-macos-15') || " + ) assert fork_gate in build_route side_attempt_one = "github.run_attempt == 1 && vars.CI_SIDE_LANE_RUNNER" assert side_attempt_one in build_route @@ -2337,7 +2350,7 @@ def test_cmux_next_daemon_tests_take_the_owned_side_lane_like_the_build_legs() - "vars.CI_SIDE_LANE_RUNNER", 1, ) - assert daemon == every_attempt.replace(fork_gate, fork_gate + promote, 1), daemon + assert daemon == every_attempt.replace(fork_gate, fork_gate + promote + pin, 1), daemon assert "MACOS_RUNNER_PR" not in daemon assert retry_route.split("|| ", 1)[1] in daemon # owned pools off: Blacksmith macOS 15