diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index cde1d9301d81..2d0d8eefad82 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -486,9 +486,11 @@ jobs: # `warm`: the SwiftPM cache restore and the seed are skipped, and the # compile rebuilds only what changed since the last job on this Mac. Any # mismatch, miss or failure falls back to the ephemeral flow below. + # Main's full-suite dispatch may run here too (pr_runner_pool.py), and + # the main commit it keeps is what the next pull requests merge onto. # scripts/ci/owned_build_state.py has the rules; nothing is uploaded. id: owned-state - if: steps.reuse-products.outputs.hit != 'true' && github.event_name == 'pull_request' && startsWith(env.CMUX_PRODUCT_RUNNER, 'glaeda-') + if: steps.reuse-products.outputs.hit != 'true' && (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && startsWith(env.CMUX_PRODUCT_RUNNER, 'glaeda-') continue-on-error: true run: | set -euo pipefail @@ -511,7 +513,8 @@ jobs: timeout-minutes: 3 env: SEED_PREFIX: admission-derived-data-v1-${{ runner.os }}-${{ runner.arch }}-${{ steps.owned-state.outputs.fingerprint }}- - MERGED_ONTO: ${{ inputs.source_parent1 || github.event.pull_request.base.sha }} + # Like the seed adoption below: a main dispatch starts at its own commit. + MERGED_ONTO: ${{ github.event_name == 'pull_request' && (inputs.source_parent1 || github.event.pull_request.base.sha) || github.sha }} MAX_DISTANCE: ${{ vars.CI_OWNED_PREFER_SEED }} CMUX_SEED_LOCAL_CACHE: ${{ env.CMUX_OWNED_STATE_ROOT }}/seeds GH_TOKEN: ${{ github.token }} diff --git a/.github/workflows/ci-owned-pool-rescue.yml b/.github/workflows/ci-owned-pool-rescue.yml index 62bfc79c5c62..d0f1329398a0 100644 --- a/.github/workflows/ci-owned-pool-rescue.yml +++ b/.github/workflows/ci-owned-pool-rescue.yml @@ -20,8 +20,9 @@ run-name: owned-pool-rescue-${{ inputs.run_id || github.event.workflow_run.id }} # from 2026-09-24 to 09-25 05:00 UTC, 361 of which were skipped (fork pull # requests) or found no owned pool. The script reads the run from the API and # applies the checks the event filter did (ci.yml, a same-repository pull -# request run, or a dispatch of a DISPATCH_WORKFLOW_PATHS workflow; attempt -# 1), so a dispatch naming any other run does nothing. Only attempt 1 is +# request run or main's full-suite dispatch, which pr_runner_pool.py places +# like one, or a dispatch of a DISPATCH_WORKFLOW_PATHS workflow; attempt 1), +# so a dispatch naming any other run does nothing. Only attempt 1 is # watched, so the re-run this workflow starts is never watched twice. # # Two kinds of source keep the workflow_run trigger: diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a6c85c186a04..afb93460379c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -603,11 +603,12 @@ jobs: id: route-token # The org's manaflow-glaeda-route App, read-only on runners, so the # pool choice below sees which owned Macs are idle now. Same-repository - # pull requests only, on this ephemeral Linux runner: a fork run never - # sees the key, and no Mac ever holds it. Any failure (no key yet, the - # App not installed) leaves the token empty, and the choice falls back - # to the janitor snapshot and CI_OWNED_POOL_SLOTS. - if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && vars.CI_PR_POOL_OWNED == '1' && vars.GLAEDA_ROUTE_APP_ID != '' + # pull requests and main's full-suite dispatch only, on this ephemeral + # Linux runner: a fork run never sees the key, and no Mac ever holds + # it. Any failure (no key yet, the App not installed) leaves the token + # empty, and the choice falls back to the janitor snapshot and + # CI_OWNED_POOL_SLOTS. + if: (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && vars.CI_PR_POOL_OWNED == '1' && vars.GLAEDA_ROUTE_APP_ID != '' continue-on-error: true timeout-minutes: 1 uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 @@ -619,6 +620,11 @@ jobs: - name: Choose the pull request macOS pool id: macos-pool # Fail-safe: any error leaves the outputs empty, which is today's route. + # Main's full-suite dispatch (ci-main-full-suite.yml) is routed too, + # onto the owned pools only, placed like a pull request unless + # CI_OWNED_MAIN_RESERVE holds machines back for pull requests + # (pr_runner_pool.py, "Main's full suite"); ci-macos.yml already reads + # these outputs for a workflow_dispatch on main. continue-on-error: true timeout-minutes: 1 env: @@ -643,7 +649,9 @@ jobs: OWNED_LIGHT_RETRY: ${{ vars.CI_OWNED_LIGHT_RETRY }} # 0 keeps GUI jobs (app-host shards, tests-build-and-lag) off them. POOL_OWNED_GUI: ${{ vars.CI_PR_POOL_OWNED_GUI }} - CMUX_CI_XCODE_APP_PR: ${{ github.event.pull_request.head.repo.full_name == github.repository && vars.CMUX_CI_XCODE_APP_PR || '' }} + # Machines and root runners main's dispatch leaves free (default 0). + OWNED_MAIN_RESERVE: ${{ vars.CI_OWNED_MAIN_RESERVE }} + CMUX_CI_XCODE_APP_PR: ${{ (github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && vars.CMUX_CI_XCODE_APP_PR || '' }} # Handed to the macOS 15 pool's jobs only when the run lands there. CMUX_CI_XCODE_APP_MACOS_15: ${{ vars.CMUX_CI_XCODE_APP_MACOS_15 }} # This run's macOS jobs, from the routing above, so an owned pool is @@ -993,10 +1001,12 @@ jobs: # this workflow with Actions write, and it runs no repository code: # no checkout, one API call. The watcher re-checks the run it is given. # Fail-safe: if the dispatch fails the run is only not watched, and - # ci-status does not wait for this job. + # ci-status does not wait for this job. Main's full-suite dispatch + # (ci-main-full-suite.yml) is placed on the owned pools like a + # same-repository pull request, so it is watched too. name: Watch owned pool jobs needs: changes - if: ${{ needs.changes.outputs.macos_pr_owned_jobs != '' && github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && github.run_attempt == 1 && vars.CI_PR_POOL_OWNED == '1' && (vars.CI_OWNED_POOL_RESCUE || '1') != '0' }} + if: ${{ needs.changes.outputs.macos_pr_owned_jobs != '' && (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && github.run_attempt == 1 && vars.CI_PR_POOL_OWNED == '1' && (vars.CI_OWNED_POOL_RESCUE || '1') != '0' }} runs-on: ubuntu-24.04 # github-hosted-required: one API call; keeps CI's Linux pool free timeout-minutes: 5 # Job level, like reverse-test-impact, so macos-admission-gate never diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 670fd03470d9..365d7f2a8190 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -210,9 +210,29 @@ names no owned pool. | --- | --- | --- | | `CI_PR_POOL_OWNED` | unset (off) | `1` puts owned pools first and turns on the rescue below | | `CI_OWNED_POOL_SLOTS` | unset (no slots) | JSON, owned pool label to machine count, the `conforming_count` from `glaeda-mini-fleet pools --json`: `{"glaeda-std-xcode-26.6": 12, "glaeda-light-xcode-26.6": 2}`. A class (`{"std": 12, "light": 2}`) or a bare count (`12`, the std class) means that class at the lane's Xcode pin | -| `GLAEDA_ROUTE_APP_ID` + secret `GLAEDA_ROUTE_APP_KEY` | unset (snapshot only) | the org's `manaflow-glaeda-route` App. `ci.yml`'s `changes` job mints a token with `administration: read` for same-repository pull requests only, on its ephemeral Linux runner, and the picker lists the repository's runners: the online, idle runners carrying an owned label are that pool's free machines, less what runs of the last `LIVE_WINDOW_MINUTES` took. That replaces `CI_OWNED_POOL_SLOTS` and the snapshot's owned counts and age. Any failure falls back to them | +| `CI_OWNED_MAIN_RESERVE` | `0` | machines, and root runners, main's full-suite dispatch leaves free for pull requests; above 0 it takes an owned pool only whole (below) | +| `GLAEDA_ROUTE_APP_ID` + secret `GLAEDA_ROUTE_APP_KEY` | unset (snapshot only) | the org's `manaflow-glaeda-route` App. `ci.yml`'s `changes` job mints a token with `administration: read` for same-repository pull requests and main's full-suite dispatch only, on its ephemeral Linux runner, and the picker lists the repository's runners: the online, idle runners carrying an owned label are that pool's free machines, less what runs of the last `LIVE_WINDOW_MINUTES` took. That replaces `CI_OWNED_POOL_SLOTS` and the snapshot's owned counts and age. Any failure falls back to them | | `CI_OWNED_LIGHT_RETRY` | unset (off) | `1` lets attempt 2, the full re-run the rescue starts for a job stuck on a full `std` pool, take the `light` pool when the run's whole owned peak is free there and `github-actions[bot]` started the re-run (a person's re-run of attempt 2 stays on Blacksmith). The rescue watches that attempt like attempt 1, and a job stuck or refused there goes to Blacksmith on attempt 3. Only while it is on do the janitor and the picker look up attempt 2's marker. Order: std, light, Blacksmith | +Main's full suite: `ci-main-full-suite.yml` dispatches `ci.yml` on main about +32 times a day, each a full suite. Until this change every one ran compile +admission and the seven app-host shards on Blacksmith (p50 wall about 37 +minutes; admission 887 s on 6vcpu against 663 s on a mini with a 2 s queue). +The dispatch runs main's own code, so the picker places it like a +same-repository pull request, split and the `CI_PR_POOL_QUEUE_ROUNDS` queue +allowance included, on the owned pools only: jobs that do not fit keep +`MACOS_RUNNER_PR` as before. Its run holds 9 root runners +at peak (admission's, then 7 shards, tests-build-and-lag and cli-product-tests); +the Claude wrapper and remote daemon lanes route only for pull requests, so they +are not counted. `CI_OWNED_MAIN_RESERVE` above 0 holds that many machines and +root runners back for pull requests, and then main takes the pool only whole +and only while its peak is free now, with no queue allowance. Main's CI concurrency group +runs one dispatch at a time, so main never holds more than one run's machines. +Its marker, the janitor's `committed` count, the route replay in newer picks, +the owned Mac's kept build state and the rescue all treat it like a +same-repository pull request run. A retry attempt, a dispatch on another +branch, or `CI_PR_POOL_OWNED` off keeps its route. + Each entry of `CI_OWNED_POOL_SLOTS` that is not an owned label or class with a positive whole number of machines counts as none. A full label wins over its class. While owned pools are on, the `changes` job raises a workflow error @@ -261,7 +281,11 @@ If one of its jobs waits for a persistent runner longer than `CI_PR_POOL_QUEUE_ROUNDS` round for a CI run whose picker placed a job in the queue allowance (it uploads a `macos-pool-queued---owned` marker), the watcher confirms the -pull request head has not moved, cancels the run, and re-runs it. A retry +pull request head has not moved, cancels the run, and re-runs it. For main's +full-suite dispatch it checks main's HEAD instead: once main has moved past +the run's commit, the run is cancelled but not re-run, because its completion +makes `ci-main-full-suite.yml` dispatch the newer HEAD; a refused job on +main's run gets its failed jobs re-run whether or not main moved. A retry attempt never takes a persistent pool, so the re-run lands on Blacksmith as a whole, and so does a manual "Re-run all jobs". @@ -307,8 +331,8 @@ minutes on the ephemeral flow instead (run 36048804178). Any mismatch or miss falls back to that flow, and a DerivedData over 40 GB is dropped. Only a successful compile's DerivedData is kept, cloned right after the compile, before the staging and packaging steps rewrite Build/Products. Only pull -request runs keep or read this state, so main's full-suite dispatch never -builds on it. Moves are renames on one volume, glaeda's host lock keeps one job +request runs and main's full-suite dispatch keep or read this state; a main +run leaves the Mac warm for the main commit the next pull requests merge onto. Moves are renames on one volume, glaeda's host lock keeps one job per Mac, and nothing is uploaded: an owned run writes only its own Mac's state. The four steps are non-product recipe steps (`product_input_identity.py`), so no pool's product key changes. Blacksmith and fork runs never take them. @@ -319,7 +343,7 @@ every sweep whatever the queue, which frees minis for current work. The other categories cancel only while more than `CI_JANITOR_QUEUE_THRESHOLD` jobs queue on a pool the run holds, owned pools included. With `CI_PR_POOL_OWNED=1` the janitor also lists the artifacts of each in-flight attempt-1, same-repository -pull request CI run (one request per run, more only past 100 artifacts) to +pull request CI run or main full-suite dispatch (one request per run, more only past 100 artifacts) to read its marker's peak into `committed`. A run's other macOS jobs do not rule it out: `swift-package-tests` always runs on Blacksmith beside a full suite on an owned pool. @@ -338,7 +362,9 @@ The guard keeps the picker the only way onto an owned pool. text, and `runner_label_policy.py` refuses one in any `*RUNNER*` variable, so neither a workflow edit nor `MACOS_RUNNER_PR` can send a job there. `check_owned_pools_route_through_picker` requires the picked label to reach -jobs only as `pr_runner` or on a `pull_request` `runs-on` branch. +jobs only as `pr_runner` or on a `pull_request` `runs-on` branch. `ci-macos.yml` +reads those inputs for a `workflow_dispatch` on `refs/heads/main` too, which +the picker fills only for main's full-suite dispatch. `CI_PR_POOL_ORDER` is the one variable that may name owned labels (the guard's `owned` pattern, which must match `pr_runner_pool.OWNED_LABEL`), and the CI health report checks every other entry in it against the workflow policy. diff --git a/scripts/ci/owned_pool_rescue.py b/scripts/ci/owned_pool_rescue.py index 5e603deb862e..a8974105db3a 100644 --- a/scripts/ci/owned_pool_rescue.py +++ b/scripts/ci/owned_pool_rescue.py @@ -75,6 +75,18 @@ watch lasts E2E_WATCH_LIMIT_SECONDS, since its test job queues only after a sibling wait and a build. +Main's full-suite dispatch of ci.yml (ci-main-full-suite.yml, a +workflow_dispatch on main) is watched exactly like a pull request run: +pr_runner_pool.py may put it on an owned pool like a pull request, and its +`changes` job uploads the +same marker. It has no pull request, so in place of the pull request head it +checks main's HEAD: once main has moved past the run's commit, a stuck run is +cancelled but not re-run, because its completion makes +ci-main-full-suite.yml dispatch the newer HEAD, and a re-run would only queue +the older commit behind it in main's CI concurrency group. A refused job's +failed jobs are re-run whether or not main moved, so a fleet refusal never +leaves main's run red. + Dispatches of test-ios.yml and ios-screenshots.yml are watched exactly like an E2E run (DISPATCH_WORKFLOW_PATHS). Their `runner` job runs ios_runner_pool.py, which may put the iOS jobs on an owned pool with the glaeda-ios-sim capability @@ -103,7 +115,8 @@ It stops watching, doing nothing, when: - owned pools are off (CI_PR_POOL_OWNED is not 1), before any API request; - the run is not attempt 1 of a same-repository pull request run of ci.yml - or a side-lane workflow; + or a side-lane workflow, of main's full-suite dispatch of ci.yml, or of a + dispatch in DISPATCH_WORKFLOW_PATHS; - a side-lane run finished with no job on an owned label, or the fleet accepted all of its owned jobs; - on the attempt 2 it re-ran from failed jobs, no job runs on an owned label; @@ -227,6 +240,8 @@ # The runner's own steps, which run before glaeda's hook decides. SETUP_STEPS = frozenset({"Set up job", "Set up runner"}) MAX_JOB_PAGES = 3 +# Main's full-suite dispatch (ci-main-full-suite.yml) runs ci.yml on this branch. +MAIN_BRANCH = "main" API = "https://api.github.com" @@ -387,6 +402,9 @@ def has_artifact(self, run_id: int, prefix: str, pages: int = 5) -> bool: def pull(self, number: int) -> Mapping[str, Any]: return self.request("GET", f"/pulls/{number}") + def branch_head(self, branch: str) -> str: + return str(((self.request("GET", f"/branches/{branch}") or {}).get("commit") or {}).get("sha") or "") + def cancel(self, run_id: int) -> None: self.request("POST", f"/actions/runs/{run_id}/cancel") @@ -412,6 +430,7 @@ class Target: # so it is watched the attempt-1 way (picker, then marker). full_rerun: bool = False side: bool = False # a side-lane workflow (SIDE_WORKFLOW_PATHS): no picker job + main: bool = False # main's full-suite dispatch of ci.yml: no pull request, main's HEAD instead @property def picker_job(self) -> str: @@ -433,9 +452,12 @@ def target_from_event(event: Mapping[str, Any], repository: str) -> Target | str return (f"started by {path or 'an unknown workflow'}, not {CI_WORKFLOW_PATH}, a side-lane workflow " f"or one of {', '.join(DISPATCH_WORKFLOW_PATHS)}") e2e = path in DISPATCH_WORKFLOW_PATHS + on_main = (path == CI_WORKFLOW_PATH and run.get("event") == "workflow_dispatch" + and run.get("head_branch") == MAIN_BRANCH) expected = "workflow_dispatch" if e2e else "pull_request" - if run.get("event") != expected: - return f"a {run.get('event') or 'unknown'} run of {path}, not a {expected}" + if run.get("event") != expected and not on_main: + what = f"{expected} or a dispatch on {MAIN_BRANCH}" if path == CI_WORKFLOW_PATH else expected + return f"a {run.get('event') or 'unknown'} run of {path}, not a {what}" head = (run.get("head_repository") or {}).get("full_name") or "" if head.casefold() != repository.casefold(): return "a fork head; forks never take a persistent pool" @@ -444,6 +466,8 @@ def target_from_event(event: Mapping[str, Any], repository: str) -> Target | str return f"attempt {attempt}; its first attempt's watch follows it" if e2e: return Target(int(run["id"]), attempt, str(run.get("head_sha") or ""), 0, e2e=True, path=str(path)) + if on_main: + return Target(int(run["id"]), attempt, str(run.get("head_sha") or ""), 0, path=str(path), main=True) pulls = [pr for pr in run.get("pull_requests") or [] if isinstance(pr, Mapping) and pr.get("number")] if len(pulls) != 1: return "the run does not name exactly one pull request" @@ -570,9 +594,15 @@ def next_attempt(target: Target) -> str: def pull_moved(api: GitHub, target: Target, sleep: Callable[[float], None], log: Callable[[str], None]) -> str: - """Why the pull request no longer wants this run, or "" when it still does.""" + """Why the pull request (or main) no longer wants this run, or "" when it still does.""" if target.e2e: return "" # a dispatch has no head to move; a newer one cancels it by concurrency + if target.main: + head = read(lambda: api.branch_head(MAIN_BRANCH), sleep, log) + if head != target.head_sha: + return (f"{MAIN_BRANCH} has moved on, and ci-main-full-suite.yml dispatches its new HEAD " + "once this run completes") + return "" pull = read(lambda: api.pull(target.pr_number), sleep, log) if pull.get("state") != "open": return "the pull request is closed" @@ -592,7 +622,19 @@ def rescue(api: GitHub, target: Target, *, now: Callable[[], dt.datetime], sleep may be re-run: an E2E run stuck in the queue that then finished was likely cancelled by a newer dispatch, which re-running it would cancel. """ - moved = pull_moved(api, target, sleep, log) + # Main's run is re-run after a refusal whether or not main moved: the + # refusal is the fleet's, and a red run would open main's red-CI issue. + keep_main = target.main and failed_only + moved = "" if keep_main else pull_moved(api, target, sleep, log) + if moved and target.main: + # Main's stuck run holds its concurrency group, so nothing newer can + # start until it finishes: cancel it, and its completion dispatches + # the new HEAD. + run = read(lambda: api.run(target.run_id), sleep, log) + if run.get("status") == "completed": + return f"not rescued: {moved}" + api.cancel(target.run_id) + return f"cancelled run {target.run_id}, not re-run: {moved}" if moved: return f"not rescued: {moved}" run = read(lambda: api.run(target.run_id), sleep, log) @@ -634,7 +676,7 @@ def rescue(api: GitHub, target: Target, *, now: Callable[[], dt.datetime], sleep raise Aborted(f"run {target.run_id} did not finish {CANCEL_WAIT_SECONDS}s after cancel; not re-run") # A push during the cancel starts the new head's run; re-running the old # head now would join its concurrency group and cancel it. - moved = pull_moved(api, target, sleep, log) + moved = "" if keep_main else pull_moved(api, target, sleep, log) if moved: return f"cancelled but not re-run: {moved}" if failed_only: @@ -692,7 +734,8 @@ def finish(outcome: str) -> int: if isinstance(target, str): return finish(f"not watched: {target}") subject = ("an E2E dispatch" if target.path == E2E_WORKFLOW_PATH else f"a dispatch of {target.path}") \ - if target.e2e else f"pull request #{target.pr_number}" + if target.e2e else f"main's full-suite dispatch at {target.head_sha[:12]}" if target.main \ + else f"pull request #{target.pr_number}" if target.side: subject += " (side lane)" # Only ci.yml's picker queues on purpose, and says so with a marker (see the docstring). diff --git a/scripts/ci/pr_runner_pool.py b/scripts/ci/pr_runner_pool.py index af71bc9fd396..d6c15791d068 100644 --- a/scripts/ci/pr_runner_pool.py +++ b/scripts/ci/pr_runner_pool.py @@ -180,7 +180,24 @@ 3. The order is std, then light, then Blacksmith. The `persistent` output tells ci.yml to publish the marker the rescue watcher looks for. -Anything uncertain keeps today's route: an event other than pull_request, a +Main's full suite: ci-main-full-suite.yml dispatches ci.yml on main about +32 times a day, each a full suite (compile admission, 7 app-host shards, +tests-build-and-lag, cli-product-tests). That is main's own code, so it may +take an owned pool like a same-repository pull request, and ci-macos.yml +already routes a `workflow_dispatch` on `refs/heads/main` through the same +inputs. It is placed like a pull request, split and queue allowance +(CI_PR_POOL_QUEUE_ROUNDS) included, on the owned pools only; what does not +fit keeps its own route (MACOS_RUNNER_PR), since only an owned pool is a +candidate for it. CI_OWNED_MAIN_RESERVE (0 when unset) holds that many +machines and root runners back for pull requests; with a reserve it takes +an owned pool only whole, and only while its peak is free now (no queue +allowance). Its side lanes (the Claude wrapper and +remote daemon) route only for pull requests, so they are not in its plan. +Main's CI concurrency group holds one run at a time, so main holds at most +one run's machines. ci-owned-pool-rescue.yml watches it like a pull request. + +Anything uncertain keeps today's route: an event other than pull_request or +main's dispatch, a lane (MACOS_RUNNER_PR) naming another pool or unset (the documented way back to the macOS 15 lane), an API error, a missing, stale or malformed snapshot, or an invalid setting. The script then prints an empty runner, and every @@ -250,6 +267,14 @@ # re-run. A human re-run sends them to pr_retry_runner, so the picker must not # take light (or publish its marker) for one. RESCUE_ACTOR = "github-actions[bot]" +MAIN_RESERVE_VARIABLE = "CI_OWNED_MAIN_RESERVE" +# Machines and root runners main's full suite leaves free for pull requests. +# 0: main takes the minis like a pull request. Its run holds 9 root runners +# at peak, so a reserve only lets it in whole when the fleet is nearly idle. +DEFAULT_MAIN_RESERVE = 0 +# The ref of main's full-suite dispatch (ci-main-full-suite.yml). +MAIN_REF = "refs/heads/main" +MAIN_BRANCH = "main" # A pull request run holds several macOS machines at once, each job on its # own. Beside compile admission run the Claude wrapper and remote daemon # lanes; once admission passes, a full suite adds APP_HOST_SHARDS @@ -858,7 +883,7 @@ def pick(load: Mapping[str, Mapping[str, int]], added: Mapping[str, int], usable max_queued: int, jobs: int = MAX_RUN_JOBS, taken: Mapping[str, int] | None = None, split: bool = False, root_free: Mapping[str, int] | None = None, root_jobs: int = 0, - queue_rounds: int = 0) -> tuple[str, bool]: + queue_rounds: int = 0, reserve: int = 0) -> tuple[str, bool]: """The rule itself: first usable pool with headroom, else the shortest queue. An owned pool has headroom while every job of this run (`jobs` of them, @@ -868,8 +893,9 @@ def pick(load: Mapping[str, Mapping[str, int]], added: Mapping[str, int], usable with the most room (the earlier on a tie) has headroom too, if it has any: the jobs that do not fit go to Blacksmith (place()). A pool in `root_free` (it has a root count; the root runners' room, allowance - included) fits the whole run only while it holds `root_jobs`. An owned - pool is never the fallback. + included) fits the whole run only while it holds `root_jobs`. `reserve` + (main's full suite) is kept free on top of both, and turns `split` off. + An owned pool is never the fallback. A Blacksmith pool has headroom while this run's job finds at most `queue_rounds` rounds of queue there once it arrives (queue_allowance()), or @@ -880,9 +906,10 @@ def pick(load: Mapping[str, Mapping[str, int]], added: Mapping[str, int], usable queued = {label: effective_queue(load[label], added[label] + 1) for label in usable} free = {label: owned_room(load[label], added[label], (taken or {}).get(label, 0), queue_rounds) for label in usable if persistent(label)} - fits = [label for label in free if free[label] >= max(1, jobs) - and (root_free or {}).get(label, root_jobs) >= root_jobs] - if split and not fits and free and max(free.values()) >= 1: + reserve = max(0, reserve) + fits = [label for label in free if free[label] >= max(1, jobs) + reserve + and (root_free or {}).get(label, root_jobs + reserve) >= root_jobs + reserve] + if split and not reserve and not fits and free and max(free.values()) >= 1: fits = [max(free, key=lambda label: free[label])] for label in usable: if persistent(label): @@ -916,6 +943,7 @@ def decide( split: bool = False, shards: int = 0, root_jobs: int = 0, + reserve: int = 0, ) -> Choice: """The preference rule over a janitor snapshot. Uncertainty keeps today's route. @@ -936,7 +964,8 @@ def decide( `root_jobs` is this run's peak on root runners, which an owned pool with a root count must have free too. Its root runners are charged one per replayed run (its admission), and a newer run's whole marker peak, since - a marker does not split it. + a marker does not split it. `reserve` (main's full suite) is how many + machines, and root runners, an owned pool must keep free beyond this run. """ if not isinstance(snapshot, Mapping) or not isinstance(snapshot.get("pools"), Mapping): return Choice("", "", "no readable pool snapshot") @@ -980,11 +1009,18 @@ def counted(label: str) -> bool: for _ in range(max(0, routed_since)): earlier, _ = pick(load, added, usable, limits.max_queued, jobs=1, taken=taken, queue_rounds=queue_rounds) added[earlier] += 1 + if reserve: + # Main's full suite with a reserve (CI_OWNED_MAIN_RESERVE) takes an + # owned pool only while its peak and the reserve are free now: no + # queue allowance, which would queue it behind the pull requests the + # reserve keeps room for. The replay above keeps the allowance. + queue_rounds = 0 # The root runners' room: free, plus the same queue allowance over their count. root_free = {label: owned_room(counts, 0, taken.get(label, 0), queue_rounds) - added[label] for label, counts in roots.items() if label in usable} label, headroom = pick(load, added, candidates, limits.max_queued, jobs, taken=taken, split=split, - root_free=root_free, root_jobs=root_jobs, queue_rounds=queue_rounds) + root_free=root_free, root_jobs=root_jobs, queue_rounds=queue_rounds, + reserve=reserve) if persistent(label) and not headroom: return Choice("", "", "every owned pool this run may take is busy, and no other pool is in the order") replayed = sum(added.values()) @@ -1007,8 +1043,9 @@ def room(count: int, capacity: int, allowed: int, what: str) -> str: queue_allowance(roots[label], queue_rounds), "root runners") root = f"; {root_room}, it needs {root_jobs}" machines = room(free, load[label]["capacity"], allowance, "owned machines") + kept = f", and {reserve} kept free for pull requests" if reserve else "" if free >= max(1, jobs) and root_free.get(label, root_jobs) >= root_jobs: - why = f"first pool in order with headroom ({machines}, this run needs {max(1, jobs)}{root}){replay}" + why = f"first pool in order with headroom ({machines}, this run needs {max(1, jobs)}{root}{kept}){replay}" else: why = (f"owned pool with the most room ({machines}, this run needs {max(1, jobs)}{root}): " f"the jobs that fit run there, the rest on the retry runner{replay}") @@ -1102,14 +1139,38 @@ def choose( live_owned: Mapping[str, int] | None = None, shards: int = 0, queue_rounds: str | None = None, + ref: str = "", + main_reserve: str | None = None, ) -> tuple[Choice, Mapping[str, Any] | None]: """The pool for this run and the snapshot it was read from (None when none was read). + Main's full-suite dispatch (`workflow_dispatch` on MAIN_REF) may take an + owned pool only, placed like a pull request unless `main_reserve` holds + machines back (see "Main's full suite" above); anything else keeps its route. + `queue_rounds` is CI_PR_POOL_QUEUE_ROUNDS as settings() reads it; a fork run reads the janitor's copy instead. """ - if event != "pull_request": - return Choice("", "", f"{event or 'unknown'} event; not a pull request"), None + main = event == "workflow_dispatch" and ref == MAIN_REF + if event != "pull_request" and not main: + return Choice("", "", f"{event or 'unknown'} event on {ref or 'an unknown ref'}; " + "not a pull request or main's full-suite dispatch"), None + reserve = 0 + if main: + if (owned or "").strip() != "1": + return Choice("", "", f"main's full-suite dispatch takes only an owned pool, and {OWNED_VARIABLE} " + "is not 1"), None + if run_attempt > 1: + return Choice("", "", f"retry attempt {run_attempt} of main's full-suite dispatch; " + "it keeps its own route"), None + try: + reserve = int(main_reserve) if (main_reserve or "").strip() else DEFAULT_MAIN_RESERVE + except ValueError: + return Choice("", "", f"{MAIN_RESERVE_VARIABLE} is not a number"), None + if reserve < 0: + return Choice("", "", f"{MAIN_RESERVE_VARIABLE} is negative"), None + # Main's own code: the same repository by definition. + head_repo = repo if not head_repo: return Choice("", "", "pull request head repository unknown"), None fork = head_repo != repo @@ -1188,18 +1249,38 @@ def choose( routed_since=routed.unknown, owned_since=routed.owned, ephemeral_since=routed.ephemeral, auto_xcode=fork, owned_slots=owned_capacity, jobs=jobs, split=(split or "").strip() == "1", shards=shards, - root_jobs=root_jobs) + root_jobs=root_jobs, reserve=reserve, + # Main only ever takes an owned pool; the replay still + # spreads newer runs over the whole order. + choose_from=tuple(label for label in limits.order if persistent(label)) if main else None) + if main and not persistent(choice.runner): + # A Blacksmith pick would move main off MACOS_RUNNER_PR; keep its route. + choice = Choice("", "", f"main's full-suite dispatch: no owned pool fits its whole run with " + f"{reserve} machine(s) and root runner(s) left free for pull requests " + f"({choice.reason})") if live and persistent(choice.runner): choice = dataclasses.replace(choice, reason=f"{choice.reason}; owned machines read live from the runners API") if fork and choice.runner: choice = dataclasses.replace(choice, reason=f"fork head; {choice.reason}") if retry and choice.runner: choice = dataclasses.replace(choice, reason=f"retry attempt {run_attempt}; {choice.reason}") + if main and choice.runner: + choice = dataclasses.replace(choice, reason=f"main's full-suite dispatch; {choice.reason}") return choice, snapshot +def main_dispatch(run: Mapping[str, Any]) -> bool: + """A CI run of main's full-suite dispatch (ci-main-full-suite.yml).""" + return run.get("event") == "workflow_dispatch" and run.get("head_branch") == MAIN_BRANCH + + +def routed_run(run: Mapping[str, Any]) -> bool: + """A CI run this picker routes: a pull request, or main's full-suite dispatch.""" + return run.get("event") == "pull_request" or main_dispatch(run) + + def may_hold_owned_pool(run: Mapping[str, Any], *, light_retry: bool = False) -> bool: - """Only attempt 1 of a same-repository pull request run can take an owned pool, + """Only attempt 1 of a same-repository pull request run (or of main's dispatch) can take an owned pool, and attempt 2 too while CI_OWNED_LIGHT_RETRY is 1 (`light_retry`). Attempt 2 then may hold the light tier, or a refused job's retry @@ -1307,7 +1388,10 @@ def runs_since(self, workflow: str, since: str, **filters: str) -> list[Mapping[ def pull_request_routes_since(self, since: str, *, exclude_run_id: int | None, light_retry: bool = False) -> Routed: - """Where the pull request runs since `since` went, so they are not all guessed. + """Where the pull request runs (and main's dispatches) since `since` went, so they are not all guessed. + + One unfiltered page of CI runs, kept to the routed ones (routed_run()), + so main's full-suite dispatch is counted at no extra request. A fork run or a retry attempt never takes an owned pool, so it is off them without a lookup (and a fork's own marker is never trusted). For @@ -1316,8 +1400,8 @@ def pull_request_routes_since(self, since: str, *, exclude_run_id: int | None, was not an owned pool. Any other run (still picking, a lost marker upload, a failed lookup, or past ROUTE_LOOKUPS) is replayed. """ - runs = [run for run in self.runs_since(CI_WORKFLOW, since, event="pull_request") - if run.get("id") != exclude_run_id and run.get("status") != "completed"] + runs = [run for run in self.runs_since(CI_WORKFLOW, since) + if routed_run(run) and run.get("id") != exclude_run_id and run.get("status") != "completed"] owned: dict[str, int] = {} ephemeral = unknown = looked_up = 0 for run in runs: @@ -1433,6 +1517,9 @@ def count_routed(since: str) -> int: since, exclude_run_id=int(run_id) if run_id.isdigit() else None, light_retry=(env.get("OWNED_LIGHT_RETRY") or "").strip() == "1") + event = env.get("EVENT_NAME") or "" + ref = env.get("GITHUB_REF") or "" + on_main = event == "workflow_dispatch" and ref == MAIN_REF # The changes job's routing, when the step runs after it; without it every # run is charged the most machines any run can hold. plan = FULL_RUN if "RUN_MACOS" not in env else run_plan( @@ -1442,6 +1529,10 @@ def count_routed(since: str) -> int: unit_in_admission="false", claude_wrapper=env.get("RUN_CLAUDE_WRAPPER"), cli=env.get("RUN_CLI"), remote_daemon=env.get("RUN_REMOTE_DAEMON"), unit_selectors=env.get("RUN_UNIT_SELECTORS")) + if on_main: + # The side lanes read the pick only on a pull request (ci.yml's + # claude-wrapper, remote-daemon.yml); main's keep their own route. + plan = dataclasses.replace(plan, side=()) # What an owned pool must have free for the whole run: its owned-eligible # jobs at their peak. gui = (env.get("POOL_OWNED_GUI") or "").strip() != "0" @@ -1463,7 +1554,9 @@ def count_routed(since: str) -> int: print(f"::warning title=live owned capacity::could not list runners ({error}); using the snapshot") live_owned = live_runners = None choice, snapshot = choose( - event=env.get("EVENT_NAME") or "", + event=event, + ref=ref, + main_reserve=env.get("OWNED_MAIN_RESERVE"), repo=repo, head_repo=env.get("HEAD_REPO") or "", default_runner=env.get("DEFAULT_RUNNER") or "", @@ -1488,9 +1581,10 @@ def count_routed(since: str) -> int: queue_rounds=env.get("POOL_QUEUE_ROUNDS") or "", ) pr_xcode_app = env.get(PR_XCODE_VARIABLE) - # Only a same-repository pull request reads the slots; ci.yml blanks the pin - # everywhere else, so checking there would flag a class entry on every run. - same_repo_pr = env.get("EVENT_NAME") == "pull_request" and env.get("HEAD_REPO") == repo + # Only a same-repository pull request (and main's dispatch) reads the slots; + # ci.yml blanks the pin everywhere else, so checking there would flag a + # class entry on every run. + same_repo_pr = event == "pull_request" and env.get("HEAD_REPO") == repo or on_main problems = (slot_problems(env.get("OWNED_SLOTS"), pr_xcode_app) if same_repo_pr and (env.get("POOL_OWNED") or "").strip() == "1" else []) for problem in problems: diff --git a/scripts/ci/queue_janitor.py b/scripts/ci/queue_janitor.py index fba64f071bc3..6ab958c499f3 100644 --- a/scripts/ci/queue_janitor.py +++ b/scripts/ci/queue_janitor.py @@ -421,8 +421,9 @@ def may_hold_owned_pool(run: Mapping[str, Any], jobs: Sequence[Mapping[str, Any] light_retry: bool = False) -> bool: """A run whose marker is worth an artifact listing: it may hold an owned pool. - Only attempt 1 of a same-repository pull request run of CI, or of an E2E - or iOS dispatch (the runner job of test-e2e.yml, test-ios.yml and + Only attempt 1 of a same-repository pull request run of CI, of main's + full-suite dispatch of CI (pr_runner_pool.py routes it too), or of an + E2E or iOS dispatch (the runner job of test-e2e.yml, test-ios.yml and ios-screenshots.yml uploads the same marker), can. While CI_OWNED_LIGHT_RETRY is 1 (`light_retry`), attempt 2 can too: the rescue's full re-run picks again and may take the light tier @@ -438,7 +439,8 @@ def may_hold_owned_pool(run: Mapping[str, Any], jobs: Sequence[Mapping[str, Any] return False path = str(run.get("path") or "") if run.get("event") == "workflow_dispatch": - return path.endswith(OWNED_DISPATCH_WORKFLOWS) + return path.endswith(OWNED_DISPATCH_WORKFLOWS) or ( + path.endswith("/ci.yml") and run.get("head_branch") == "main") return run.get("event") == "pull_request" and path.endswith("/ci.yml") diff --git a/tests/test_ci_owned_build_state.py b/tests/test_ci_owned_build_state.py index 1af091efd586..51660085e2c4 100644 --- a/tests/test_ci_owned_build_state.py +++ b/tests/test_ci_owned_build_state.py @@ -440,6 +440,9 @@ def step(self, name): def test_state_steps_run_only_on_an_owned_runner(self): self.assertIn(OWNED, self.by_id["owned-state"]["if"]) self.assertIn("github.event_name == 'pull_request'", self.by_id["owned-state"]["if"]) + # Main's full-suite dispatch may be placed on an owned Mac too (pr_runner_pool.py). + self.assertIn("github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'", + self.by_id["owned-state"]["if"]) # Every other state step follows owned-state. self.assertIn("steps.owned-state.outcome != 'skipped'", self.step("Keep this owned Mac's build state")["if"]) self.assertIn("steps.owned-state.outputs.fingerprint != ''", self.step("Keep this owned Mac's DerivedData")["if"]) diff --git a/tests/test_ci_owned_pool_rescue.py b/tests/test_ci_owned_pool_rescue.py index ca2257b24948..6039b204f626 100644 --- a/tests/test_ci_owned_pool_rescue.py +++ b/tests/test_ci_owned_pool_rescue.py @@ -106,6 +106,10 @@ def pull(self, number): self.calls.append("pull") return {"state": self.state, "head": {"sha": self.head}} + def branch_head(self, branch): + self.calls.append(f"branch:{branch}") + return self.head + def cancel(self, run_id): self.calls.append("cancel") self.cancelled_at, self.cancel_attempt = self.clock.seconds, self.attempt @@ -909,6 +913,105 @@ def jobs(seconds): self.assertIn(f"queued on {MINI}", summary) +def main_event(**overrides): + return event(**{"event": "workflow_dispatch", "head_branch": "main", "pull_requests": [], **overrides}) + + +class MainDispatch(unittest.TestCase): + """Main's full-suite dispatch of ci.yml is watched like a pull request run, against main's HEAD.""" + + def test_only_attempt_1_of_a_same_repository_dispatch_on_main(self): + cases = { + "another branch": main_event(head_branch="topic"), + "fork head": main_event(head_repository={"full_name": "someone/cmux"}), + "attempt 2": main_event(run_attempt=2), + "another workflow": main_event(path=".github/workflows/nightly.yml"), + } + for why, payload in cases.items(): + self.assertIsInstance(rescue.target_from_event(payload, "manaflow-ai/cmux"), str, why) + target = rescue.target_from_event(main_event(), "manaflow-ai/cmux") + self.assertEqual((target.run_id, target.pr_number, target.main, target.e2e, target.picker_job), + (RUN_ID, 0, True, False, "changes")) + self.assertEqual(target.watch_limit, rescue.WATCH_LIMIT_SECONDS) + + def test_an_ephemeral_main_run_stops_after_the_marker_check(self): + clock = Clock() + api = FakeAPI(clock, lambda seconds: [changes()(seconds)]) + code, summary = run_main(api, clock, payload=main_event()) + self.assertEqual(code, 0) + self.assertEqual(api.calls, ["jobs", f"artifact:macos-pool-persistent-{RUN_ID}-1-"]) + self.assertIn("main's full-suite dispatch", summary) + + def test_a_stuck_main_run_is_cancelled_and_rerun_on_blacksmith(self): + clock = Clock() + api = FakeAPI(clock, persistent_run(), marker=True) + code, summary = run_main(api, clock, payload=main_event()) + self.assertEqual(code, 0) + self.assertNotIn("pull", api.calls) + self.assertIn("branch:main", api.calls) + self.assertEqual(api.calls[-1], "rerun") + self.assertIn("cancel", api.calls) + + def test_a_stuck_main_run_is_cancelled_not_rerun_once_main_moves(self): + # The stuck run holds main's concurrency group; cancelling it lets the + # dispatcher start the new HEAD when it completes. + clock = Clock() + api = FakeAPI(clock, persistent_run(), marker=True, head="b" * 40) + _, summary = run_main(api, clock, payload=main_event()) + self.assertIn("cancel", api.calls) + self.assertNotIn("rerun", api.calls) + self.assertIn("main has moved on", summary) + self.assertIn("not re-run", summary) + # Main moving during the cancel: cancelled, and its completion dispatches the new HEAD. + clock = Clock() + api = FakeAPI(clock, persistent_run(), marker=True) + heads = iter([HEAD, "b" * 40]) + api.branch_head = lambda branch: next(heads) + _, summary = run_main(api, clock, payload=main_event()) + self.assertIn("cancel", api.calls) + self.assertNotIn("rerun", api.calls) + self.assertIn("cancelled but not re-run", summary) + + def test_a_dispatch_naming_main_s_run_watches_it(self): + # ci.yml's owned-pool-watch job dispatches the rescue for main's run + # too; the run the API returns is main's dispatch. + clock = Clock() + api = FakeAPI(clock, refusing_run(), marker=True) + live_run = api.run + api.run = lambda run_id: {**main_event()["workflow_run"], **live_run(run_id)} + code, summary = run_main(api, clock, env_extra={"WATCH_RUN_ID": str(RUN_ID)}, + payload={"inputs": {"run_id": str(RUN_ID)}}) + self.assertEqual(code, 0) + self.assertEqual(api.calls[0], "run") + self.assertIn(f"watching run {RUN_ID} of main's full-suite dispatch", summary) + self.assertIn("rerun-failed", api.calls) + + def test_a_main_run_queued_on_purpose_gets_the_round_budget(self): + # The picker places main like a pull request, queue allowance included. + clock = Clock() + api = FakeAPI(clock, persistent_run(compile_started_at=40 + 600), marker=True, queued=True) + code, summary = run_main(api, clock, payload=main_event(), + env_extra={"RESCUE_SECONDS": "30", "QUEUE_ROUNDS": ""}) + self.assertEqual(code, 0) + self.assertIn(f"budget {30 + rescue.QUEUE_ROUND_SECONDS}s", summary) + self.assertNotIn("cancel", api.calls) + + def test_a_refused_main_job_reruns_the_failed_jobs(self): + clock = Clock() + api = FakeAPI(clock, refusing_run(), marker=True) + code, summary = run_main(api, clock, payload=main_event()) + self.assertEqual(code, 0) + self.assertIn("rerun-failed", api.calls) + self.assertNotIn("pull", api.calls) + self.assertIn("refused", summary) + # Even once main moved: a fleet refusal must not leave main's run red. + clock = Clock() + api = FakeAPI(clock, refusing_run(), marker=True, head="b" * 40) + code, summary = run_main(api, clock, payload=main_event()) + self.assertEqual(code, 0) + self.assertIn("rerun-failed", api.calls) + + class Workflow(unittest.TestCase): def setUp(self): self.text = (ROOT / ".github/workflows/ci-owned-pool-rescue.yml").read_text(encoding="utf-8") @@ -988,7 +1091,9 @@ def test_ci_and_e2e_dispatch_it_only_for_an_owned_placement(self): screenshots = yaml.safe_load((ROOT / ".github/workflows/ios-screenshots.yml").read_text(encoding="utf-8")) self.assertNotIn("owned-pool-watch", screenshots["jobs"]) ci = yaml.safe_load((ROOT / ".github/workflows/ci.yml").read_text(encoding="utf-8"))["jobs"] - self.assertIn("github.event.pull_request.head.repo.full_name == github.repository", + self.assertIn("(github.event_name == 'pull_request' && " + "github.event.pull_request.head.repo.full_name == github.repository || " + "github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')", ci["owned-pool-watch"]["if"]) self.assertNotIn("owned-pool-watch", ci["ci-status"]["needs"]) diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py index 9466da0dfa6a..ceee41ebfcb2 100644 --- a/tests/test_ci_pr_runner_pool.py +++ b/tests/test_ci_pr_runner_pool.py @@ -60,7 +60,7 @@ def backlog(small=21, large=0, old=4, large_reserved=0, old_reserved=0, age=5, s def choose(snap, *, event="pull_request", head="manaflow-ai/cmux", default=SMALL, overflow="", order="", max_queued="", pins=PINS, fetch=None, routed=0, attempt=1, owned="", owned_slots="", jobs=pool.MAX_RUN_JOBS, split="", live_owned=None, root_jobs=0, shards=0, light_retry="", - actor="", queue_rounds=None): + actor="", queue_rounds=None, **extra): def count_routed(since): if isinstance(routed, Exception): raise routed @@ -70,7 +70,7 @@ def count_routed(since): overflow=overflow, order=order, max_queued=max_queued, xcode_pins=pins, owned=owned, owned_slots=owned_slots, jobs=jobs, split=split, live_owned=live_owned, root_jobs=root_jobs, shards=shards, light_retry=light_retry, triggering_actor=actor, queue_rounds=queue_rounds, - fetch=fetch or (lambda: snap), count_routed=count_routed, now=NOW, run_attempt=attempt, + fetch=fetch or (lambda: snap), count_routed=count_routed, now=NOW, run_attempt=attempt, **extra, )[0] @@ -804,7 +804,8 @@ def test_runs_off_the_owned_pools_still_queue_on_blacksmith(self): self.assertEqual(choose(snap, routed=pool.Routed(ephemeral=1)).runner, SMALL) def test_route_lookup_reads_markers_then_the_changes_job(self): - same = {"head_repository": {"id": 5}, "repository": {"id": 5}} + same = {"head_repository": {"id": 5}, "repository": {"id": 5}, "event": "pull_request"} + dispatch = {**same, "event": "workflow_dispatch", "head_branch": "main"} skipped = [{"name": pool.MARKER_STEP, "conclusion": "skipped"}] runs = [{"id": 1, "run_attempt": 1, "status": "in_progress", **same}, {"id": 2, "run_attempt": 1, "status": "in_progress", **same}, @@ -813,9 +814,14 @@ def test_route_lookup_reads_markers_then_the_changes_job(self): {"id": 5, "run_attempt": 1, "status": "in_progress", **same}, {"id": 6, "run_attempt": 1, "status": "in_progress", **same}, {"id": 7, "run_attempt": 1, "status": "in_progress", - "head_repository": {"id": 8}, "repository": {"id": 5}}, + "head_repository": {"id": 8}, "repository": {"id": 5}, "event": "pull_request"}, {"id": 8, "run_attempt": 2, "status": "in_progress", **same}, - {"id": 9, "run_attempt": 1, "status": "in_progress", **same}] + {"id": 9, "run_attempt": 1, "status": "in_progress", **same}, + # Main's full-suite dispatch is routed and looked up like a pull request. + {"id": 10, "run_attempt": 1, "status": "in_progress", **dispatch}, + # Neither a merge group nor a dispatch on another branch is routed here. + {"id": 11, "run_attempt": 1, "status": "in_progress", **same, "event": "merge_group"}, + {"id": 12, "run_attempt": 1, "status": "in_progress", **dispatch, "head_branch": "topic"}] responses = { # A marker, with an absurd peak capped at MAX_RUN_JOBS. "/actions/runs/1/artifacts?per_page=100": {"artifacts": [ @@ -837,6 +843,8 @@ def test_route_lookup_reads_markers_then_the_changes_job(self): # Run 6's lookup fails. Run 7 (fork) is never looked up. Run 8 # (attempt 2) is looked up only with CI_OWNED_LIGHT_RETRY on, when # it may hold the light tier; its lookup fails, so it is replayed. + "/actions/runs/10/artifacts?per_page=100": {"artifacts": [ + {"name": f"macos-pool-persistent-10-1-9-{MINI}", "expired": False}]}, } def get(path): @@ -845,12 +853,14 @@ def get(path): return responses[path] client = pool.GitHub("token", "manaflow-ai/cmux") - with unittest.mock.patch.object(client, "runs_since", return_value=runs), \ + with unittest.mock.patch.object(client, "runs_since", return_value=runs) as runs_since, \ unittest.mock.patch.object(client, "get", side_effect=get): routed = client.pull_request_routes_since("2026-09-24T00:00:00Z", exclude_run_id=9) light = client.pull_request_routes_since("2026-09-24T00:00:00Z", exclude_run_id=9, light_retry=True) - self.assertEqual(routed, pool.Routed(unknown=3, owned={MINI: pool.MAX_RUN_JOBS}, ephemeral=3)) - self.assertEqual(light, pool.Routed(unknown=4, owned={MINI: pool.MAX_RUN_JOBS}, ephemeral=2)) + self.assertEqual(routed, pool.Routed(unknown=3, owned={MINI: pool.MAX_RUN_JOBS + 9}, ephemeral=3)) + self.assertEqual(light, pool.Routed(unknown=4, owned={MINI: pool.MAX_RUN_JOBS + 9}, ephemeral=2)) + # One unfiltered page per lookup: main's dispatches cost no extra request. + runs_since.assert_called_with(pool.CI_WORKFLOW, "2026-09-24T00:00:00Z") def test_marker_step_and_routing_job_names_match_ci_yml(self): workflow = (Path(__file__).resolve().parents[1] / ".github/workflows/ci.yml").read_text() @@ -858,7 +868,8 @@ def test_marker_step_and_routing_job_names_match_ci_yml(self): self.assertIn(f"\n {pool.ROUTING_JOB}:\n", workflow) def test_route_lookups_stop_at_the_cap(self): - same = {"head_repository": {"id": 5}, "repository": {"id": 5}, "run_attempt": 1, "status": "queued"} + same = {"head_repository": {"id": 5}, "repository": {"id": 5}, "run_attempt": 1, "status": "queued", + "event": "pull_request"} runs = [{"id": n, **same} for n in range(1, pool.ROUTE_LOOKUPS + 4)] client = pool.GitHub("token", "manaflow-ai/cmux") with unittest.mock.patch.object(client, "runs_since", return_value=runs), \ @@ -1697,6 +1708,138 @@ def sim_fleet(running=0, queued=0, committed=0, **kwargs) -> dict: return snap +class MainFullSuite(unittest.TestCase): + """Main's full-suite dispatch takes the owned pools like a pull request, or whole behind a reserve.""" + + SLOTS = json.dumps({MINI: 36, ROOT_MINI: 14}) + PLAN = dataclasses.replace(pool.FULL_RUN, side=()) + + def snap(self, busy=0, roots_busy=0): + snap = fleet(busy=busy) + snap["pools"][ROOT_MINI] = {"queued": 0, "running": roots_busy} + return snap + + def main_choice(self, snap, **kwargs): + kwargs = {"event": "workflow_dispatch", "head": "", "pins": OWNED_PINS, "owned": "1", + "owned_slots": self.SLOTS, "jobs": pool.owned_peak(self.PLAN), + "root_jobs": pool.root_peak(self.PLAN), **kwargs} + ref = kwargs.pop("ref", pool.MAIN_REF) + reserve = kwargs.pop("reserve", None) + return choose(snap, **kwargs, ref=ref, main_reserve=reserve) + + def test_its_run_holds_nine_machines_and_nine_root_runners(self): + self.assertEqual((pool.owned_peak(self.PLAN), pool.root_peak(self.PLAN)), (9, 9)) + # By default main holds nothing back: it takes the minis like a pull request. + self.assertEqual(pool.DEFAULT_MAIN_RESERVE, 0) + + def test_an_idle_fleet_takes_the_whole_run(self): + choice = self.main_choice(self.snap()) + self.assertEqual((choice.runner, choice.root_runner), (MINI, ROOT_MINI)) + self.assertIn("main's full-suite dispatch", choice.reason) + self.assertEqual(pool.place(self.PLAN, choice.owned_budget, root_budget=choice.root_budget)[0], + ("admission", *(f"shard-{index}" for index in range(1, 8)), "lag", "cli-product")) + + def test_the_reserve_holds_root_runners_and_machines_back_for_pull_requests(self): + # 14 - 2 busy = 12 root runners free: 9 for main leaves 3, under a reserve of 4. + for snap in (self.snap(roots_busy=2), self.snap(busy=24)): + choice = self.main_choice(snap, reserve="4") + self.assertEqual(choice.runner, "", choice.reason) + self.assertIn("left free for pull requests", choice.reason) + self.assertIn("4 kept free for pull requests", self.main_choice(self.snap(), reserve="4").reason) + # Behind a reserve main never splits, however many machines are free. + self.assertEqual(self.main_choice(self.snap(roots_busy=2), split="1", reserve="4").runner, "") + # With none (the default) it splits like a pull request: what fits takes the minis. + self.assertEqual(self.main_choice(self.snap(roots_busy=6), split="1").runner, MINI) + # A smaller reserve lets it in; the variable is read as a count. + self.assertEqual(self.main_choice(self.snap(roots_busy=2), reserve="3").runner, MINI) + self.assertEqual(self.main_choice(self.snap(roots_busy=5), reserve="0").runner, MINI) + self.assertEqual(self.main_choice(self.snap(roots_busy=6), reserve="0").runner, "") + # A pull request with the same load is not held to the reserve. + pull = owned_choice(self.snap(roots_busy=5), owned_slots=self.SLOTS, jobs=9, root_jobs=9) + self.assertEqual(pull.runner, MINI) + + def test_queues_a_round_like_a_pull_request_unless_a_reserve_is_set(self): + # Every mini busy: a round of queue (CI_PR_POOL_QUEUE_ROUNDS=1) lets a + # pull request and main take the fleet anyway, and the picker says the + # placed jobs queue on purpose (the rescue's longer budget). + full = self.snap(busy=36, roots_busy=14) + pull = owned_choice(full, owned_slots=self.SLOTS, jobs=9, root_jobs=9, queue_rounds="1") + self.assertEqual(pull.runner, MINI) + choice = self.main_choice(full, queue_rounds="1") + self.assertEqual((choice.runner, choice.root_runner), (MINI, ROOT_MINI), choice.reason) + self.assertIn("may queue behind them", choice.reason) + owned_jobs = pool.place(self.PLAN, choice.owned_budget, root_budget=choice.root_budget)[0] + self.assertTrue(pool.used_queue(choice, self.PLAN, owned_jobs, len(owned_jobs))) + # Rounds 0 is the old rule for both: the run's peak must be free now. + self.assertEqual(self.main_choice(full, queue_rounds="0").runner, "") + # A reserve keeps main off the queue allowance: it takes the pool only + # while its peak and the reserve are free now. + self.assertEqual(self.main_choice(full, queue_rounds="1", reserve="1").runner, "") + self.assertEqual(self.main_choice(self.snap(roots_busy=5), queue_rounds="1", reserve="1").runner, "") + self.assertEqual(self.main_choice(self.snap(roots_busy=4), queue_rounds="1", reserve="1").runner, MINI) + + def test_never_a_blacksmith_pick(self): + # A pull request would overflow to 12vcpu here; main keeps MACOS_RUNNER_PR. + self.assertEqual(owned_choice(self.snap(busy=36), owned_slots=self.SLOTS, jobs=9, root_jobs=9).runner, LARGE) + self.assertEqual(self.main_choice(self.snap(busy=36)).runner, "") + + def test_anything_else_keeps_its_route(self): + for kwargs in ({"ref": "refs/heads/topic"}, {"ref": ""}, {"owned": ""}, {"attempt": 2}, + {"reserve": "many"}, {"reserve": "-1"}, {"default": ""}, {"overflow": "0"}, + {"event": "merge_group"}, {"event": "push"}): + choice = self.main_choice(self.snap(), **kwargs) + self.assertEqual((choice.runner, choice.root_runner), ("", ""), kwargs) + + def test_main_routes_the_full_suite_without_its_side_lanes(self): + with tempfile.TemporaryDirectory() as tmp: + snapshot = Path(tmp, "snap.json") + fresh = self.snap() + fresh["generated_at"] = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + snapshot.write_text(json.dumps(fresh)) + out = Path(tmp, "out") + env = {"EVENT_NAME": "workflow_dispatch", "GITHUB_REF": "refs/heads/main", + "GITHUB_REPOSITORY": "manaflow-ai/cmux", "HEAD_REPO": "", "DEFAULT_RUNNER": SMALL, + "POOL_OWNED": "1", "POOL_OWNED_SPLIT": "1", "OWNED_SLOTS": '{"std": 36, "root-std": 14}', + "CMUX_CI_XCODE_APP_PR": PR_XCODE, "CMUX_CI_XCODE_APP_MACOS_15": XCODE_15, + "GITHUB_RUN_ATTEMPT": "1", "GITHUB_OUTPUT": str(out), "RUN_MACOS": "true", + "RUN_FULL_SUITE": "true", "RUN_CLI": "true", "RUN_CLAUDE_WRAPPER": "true", + "RUN_REMOTE_DAEMON": "true"} + with unittest.mock.patch("sys.stdout", io.StringIO()): + self.assertEqual(pool.main(["--snapshot", str(snapshot)], env), 0) + values = dict(line.split("=", 1) for line in out.read_text().splitlines()) + self.assertEqual((values["runner"], values["persistent"], values["root_runner"], values["jobs"]), + (MINI, "true", ROOT_MINI, "9")) + self.assertEqual(values["owned_jobs"], " admission " + " ".join(f"shard-{index}" for index in range(1, 8)) + + " lag cli-product ") + self.assertTrue(values["retry_runner"].startswith("blacksmith-")) + # A dispatch on another branch writes the default route. + env.update(GITHUB_REF="refs/heads/topic") + out.write_text("") + with unittest.mock.patch("sys.stdout", io.StringIO()): + pool.main(["--snapshot", str(snapshot)], env) + values = dict(line.split("=", 1) for line in out.read_text().splitlines()) + self.assertEqual((values["runner"], values["owned_jobs"]), ("", "")) + + def test_the_janitor_reads_main_dispatch_markers(self): + repo = {"id": 7} + run = {"event": "workflow_dispatch", "head_branch": "main", "run_attempt": 1, + "path": ".github/workflows/ci.yml", "repository": repo, "head_repository": repo} + self.assertTrue(janitor.may_hold_owned_pool(run, [])) + self.assertTrue(pool.may_hold_owned_pool(run)) + for change in ({"head_branch": "topic"}, {"run_attempt": 2}, {"path": ".github/workflows/nightly.yml"}): + self.assertFalse(janitor.may_hold_owned_pool({**run, **change}, []), change) + + def test_ci_yml_routes_main_dispatch_through_the_picker(self): + steps = yaml.safe_load((WORKFLOWS / "ci.yml").read_text())["jobs"]["changes"]["steps"] + ids = [step.get("id") for step in steps] + mint, picker = steps[ids.index("route-token")], steps[ids.index("macos-pool")] + self.assertNotIn("if", picker) + self.assertIn("github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'", mint["if"]) + self.assertIn("github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main'", + picker["env"]["CMUX_CI_XCODE_APP_PR"]) + self.assertEqual(picker["env"]["OWNED_MAIN_RESERVE"], "${{ vars.CI_OWNED_MAIN_RESERVE }}") + + class IOSRouting(unittest.TestCase): """ios_runner_pool.py: the E2E rule, owned Macs only, with counted simulator capacity.""" diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index cc2496947b0a..eab83ff1de63 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -1358,9 +1358,12 @@ check_no_self_hosted_fleet_runners() { check_owned_pools_route_through_picker() { # An owned pool label never appears in workflow text (the fleet pattern # refuses it), so the only way one reaches runs-on is the value - # pr_runner_pool.py writes for a pull request run: it hands owned labels - # only to same-repository heads on a first attempt, and only once - # CI_PR_POOL_OWNED is 1. This check keeps that the only way. The picker's + # pr_runner_pool.py writes for a pull request run or main's full-suite + # dispatch: it hands owned labels only to same-repository heads (and to a + # workflow_dispatch on refs/heads/main, which runs main's own code) on a + # first attempt, and only once CI_PR_POOL_OWNED is 1. ci-macos.yml reads + # the pick for both through the inputs below. This check keeps that the + # only way. The picker's # runner output feeds exactly the macos_pr_runner output and the rescue # marker; macos_pr_runner reaches a job only as a `pr_runner` input written # exactly one way, or inside a runs-on branch that a pull_request condition diff --git a/tests/test_seed_derived_data.py b/tests/test_seed_derived_data.py index 1e2d6462ba5c..9f5197a1b24f 100644 --- a/tests/test_seed_derived_data.py +++ b/tests/test_seed_derived_data.py @@ -1173,6 +1173,66 @@ def test_a_warm_admission_takes_the_warm_labels_on_attempt_one_only(self): shard = macos["app-host-unit-tests"]["runs-on"].replace("format(' shard-{0} ', matrix.shard)", "' shard-1 '") self.assertEqual(evaluate(shard, context), product_runner) + def test_main_full_suite_dispatch_takes_the_owned_pool_the_picker_names(self): + # pr_runner_pool.py may put main's full-suite dispatch on an owned + # pool; every job of it reads the same inputs a pull request's do. + macos = load("ci-macos.yml")["jobs"] + root, mini, retry = "glaeda-root-std-xcode-26.6", "glaeda-std-xcode-26.6", "blacksmith-6vcpu-macos-26" + owned_jobs = " admission shard-1 shard-2 lag cli-product " + for attempt, actor, runner in (("1", "github-actions[bot]", root), + ("2", "github-actions[bot]", root), + ("2", "someone", retry)): + context = github_context("workflow_dispatch") + context["github"].update(repository="manaflow-ai/cmux", run_attempt=attempt, triggering_actor=actor, + sha="head") + context["inputs"].update(pr_runner=mini, pr_retry_runner=retry, pr_refused_retry_runner=mini, + pr_root_runner=root, pr_owned_jobs=owned_jobs, source_parent1="parent") + with self.subTest(attempt=attempt, actor=actor): + admission = macos["macos-compile-admission"] + self.assertEqual(evaluate(admission["runs-on"], context), runner) + self.assertEqual(evaluate(admission["env"]["CMUX_PRODUCT_RUNNER"], context), runner) + self.assertEqual(evaluate(admission["env"]["CMUX_CI_XCODE_APP"], context), "/Applications/Xcode-pr.app") + self.assertEqual(evaluate(macos["tests-build-and-lag"]["runs-on"], context), runner) + context["needs"] = {"macos-compile-admission": {"outputs": {"runner": runner}}} + shard = macos["app-host-unit-tests"]["runs-on"].replace("format(' shard-{0} ', matrix.shard)", "' shard-1 '") + self.assertEqual(evaluate(shard, context), runner) + self.assertEqual(evaluate(macos["cli-product-tests"]["runs-on"], context), runner) + # An owned Mac's kept build is reused on main too, starting at main's own commit. + context["env"] = {"CMUX_PRODUCT_RUNNER": runner} + context["steps"] = {"reuse-products": {"outputs": {"hit": "false"}}, + "owned-state": {"outputs": {"warm": "true"}}} + _, state = named(steps("ci-macos.yml", "macos-compile-admission"), "Reuse this owned Mac's build state") + self.assertIs(evaluate(state["if"], context), runner.startswith("glaeda-")) + _, prefer = named(steps("ci-macos.yml", "macos-compile-admission"), + "Prefer a near seed over this owned Mac's DerivedData") + self.assertEqual(evaluate(prefer["env"]["MERGED_ONTO"], context), "head") + # A dispatch on another branch never reads the owned state. + topic = github_context("workflow_dispatch", ref="refs/heads/topic") + topic["env"] = {"CMUX_PRODUCT_RUNNER": mini} + topic["steps"] = {"reuse-products": {"outputs": {"hit": "false"}}} + _, state = named(steps("ci-macos.yml", "macos-compile-admission"), "Reuse this owned Mac's build state") + self.assertIs(evaluate(state["if"], topic), False) + + def test_main_full_suite_dispatch_reads_the_route_token_and_the_lane_pin(self): + changes = load("ci.yml")["jobs"]["changes"]["steps"] + mint = next(step for step in changes if step.get("id") == "route-token") + picker = next(step for step in changes if step.get("id") == "macos-pool") + for event_name, ref, head, minted, pin in ( + ("workflow_dispatch", "refs/heads/main", None, True, "/Applications/Xcode-pr.app"), + ("workflow_dispatch", "refs/heads/topic", None, False, ""), + ("merge_group", "refs/heads/gh-readonly-queue/main/x", None, False, ""), + ("pull_request", "refs/pull/1/merge", "manaflow-ai/cmux", True, "/Applications/Xcode-pr.app"), + ("pull_request", "refs/pull/1/merge", "someone/cmux", False, ""), + ): + context = github_context(event_name, ref=ref, CI_PR_POOL_OWNED="1", GLAEDA_ROUTE_APP_ID="42") + context["github"].update(repository="manaflow-ai/cmux", + event={"pull_request": {"head": {"repo": {"full_name": head}}}} if head else {}) + with self.subTest(event_name=event_name, ref=ref, head=head): + self.assertIs(evaluate("${{ " + mint["if"] + " }}", context), minted) + self.assertEqual(evaluate(picker["env"]["CMUX_CI_XCODE_APP_PR"], context), pin) + off = github_context("workflow_dispatch", CI_PR_POOL_OWNED="", GLAEDA_ROUTE_APP_ID="42") + self.assertIs(evaluate("${{ " + mint["if"] + " }}", off), False) + def test_the_expression_evaluator_follows_actions_semantics(self): context = {"vars": {"A": "a", "EMPTY": ""}} # Short-circuited operands are never evaluated, and JSON is indexed.