diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 894086cc9a57..30ec36494a0c 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -2949,12 +2949,12 @@ jobs: ghostty_helper_toolchain_sha256: ${{ steps.ghostty-helper-identity.outputs.toolchain_sha256 }} ghostty_helper_sdk: ${{ steps.ghostty-helper-identity.outputs.sdk }} # Build the release helper with SDK 15, then run package tests with SDK 26. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-15' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-15' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && contains(inputs.pr_owned_jobs, ' swift-package ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15') }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-15' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-15' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && contains(inputs.pr_owned_jobs, ' swift-package ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15') }} timeout-minutes: 60 env: # The owned label's Xcode (the lane pin) exactly when runs-on took it; # the pool picker's Wiring tests keep the two conditions equal. - CMUX_CI_XCODE_APP: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && contains(inputs.pr_owned_jobs, ' swift-package ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) && (inputs.pr_xcode_app || vars.CMUX_CI_XCODE_APP_PR) || vars.CMUX_CI_XCODE_APP_MACOS_15 }} + CMUX_CI_XCODE_APP: ${{ (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && contains(inputs.pr_owned_jobs, ' swift-package ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) && (inputs.pr_xcode_app || vars.CMUX_CI_XCODE_APP_PR) || vars.CMUX_CI_XCODE_APP_MACOS_15 }} CMUX_CI_HELPER_XCODE_APP: ${{ vars.CMUX_CI_HELPER_XCODE_APP_MACOS_15 }} CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" steps: @@ -3930,8 +3930,13 @@ jobs: # compiles into the same artifact shape as nightly and stable releases. # Release builds need enough disk for a universal Release build plus the # restored SwiftPM cache, which the macOS 26 image already carries. The - # variable names that image, not this lane. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }} + # variable names that image, not this lane. A same-repository pull request + # or main's full-suite dispatch takes the owned side label when the picker + # placed ' release-build ' in pr_owned_jobs (pr_runner_pool.RELEASE_BUILD_JOB): + # same Xcode 26.6, and glaeda's hook classes the job isolated. Attempt 2 of a + # refused job tries the owned pool once more; any other retry takes the + # macOS 26 variable. + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && contains(inputs.pr_owned_jobs, ' release-build ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }} timeout-minutes: 60 permissions: actions: read @@ -3941,7 +3946,7 @@ jobs: CMUX_CI_XCODE_APP: ${{ vars.CMUX_CI_XCODE_APP_MACOS_26 }} CMUX_CI_REQUIRED_MACOS_SDK_MAJOR: "26" CMUX_SKIP_ZIG_BUILD: "1" - CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }} + CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && contains(inputs.pr_owned_jobs, ' release-build ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }} CMUX_RELEASE_SOURCE_REVISION: ${{ github.event.pull_request.head.sha || github.sha }} steps: - name: Clear stale git locks (self-hosted reused workspace) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 17a6058fabee..26b044cd740a 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -764,10 +764,12 @@ jobs: # watched, which is how every run behaved before the rescue existed. - name: Mark a run on a persistent macOS pool id: macos-pool-marker - if: ${{ steps.macos-pool.outputs.persistent == 'true' }} + # marker_pool is the owned pick, or the owned pool whose idle side + # runners took a Blacksmith pick's side lanes (side_only_placement). + if: ${{ steps.macos-pool.outputs.marker_pool != '' }} continue-on-error: true env: - POOL: ${{ steps.macos-pool.outputs.runner }} + POOL: ${{ steps.macos-pool.outputs.marker_pool }} run: | set -euo pipefail marker="$RUNNER_TEMP/macos-pool-persistent.json" @@ -780,7 +782,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: # The janitor reads the run's peak and pool from the name. - name: macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.macos-pool.outputs.jobs }}-${{ steps.macos-pool.outputs.runner }} + name: macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}-${{ steps.macos-pool.outputs.jobs }}-${{ steps.macos-pool.outputs.marker_pool }} path: ${{ steps.macos-pool-marker.outputs.path }} if-no-files-found: error retention-days: 1 @@ -1176,7 +1178,7 @@ jobs: name: Claude wrapper regressions needs: [changes, static-preflight] if: ${{ !cancelled() && needs.changes.result == 'success' && needs.static-preflight.result == 'success' && (needs.changes.outputs.claude_wrapper == 'true' || (needs.changes.outputs.macos == 'true' && needs.changes.outputs.full_suite == 'true')) }} - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(needs.changes.outputs.macos_pr_runner, 'blacksmith-') && needs.changes.outputs.macos_pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && github.run_attempt == 2 && contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && (needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_refused_retry_runner) || github.event_name == 'pull_request' && (github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ')) && needs.changes.outputs.macos_pr_retry_runner || github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(needs.changes.outputs.macos_pr_runner, 'blacksmith-') && needs.changes.outputs.macos_pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && github.run_attempt == 2 && contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && (needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_refused_retry_runner) || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ')) && needs.changes.outputs.macos_pr_retry_runner || github.event_name == 'pull_request' && (github.run_attempt <= 2 && contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && (needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_runner) || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }} # Normally 1-2 minutes, but it spawns hundreds of short bash, perl and node # processes, so a mini saturated by a compile beside it stretches it: # 120-527 s at a mean load of 36-100 on 14 cores (glaeda-cmux-jobs.jsonl, diff --git a/.github/workflows/remote-daemon.yml b/.github/workflows/remote-daemon.yml index 093e9bddeb08..5f0f3eb5b2ab 100644 --- a/.github/workflows/remote-daemon.yml +++ b/.github/workflows/remote-daemon.yml @@ -121,7 +121,7 @@ jobs: # Plain `go test` with no Xcode or GUI: any Mac will do. Follow the same # lanes as the other pull-request macOS jobs instead of pinning the # contended macOS 26 pool. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || github.event_name == 'pull_request' && (github.run_attempt == 2 && contains(inputs.pr_owned_jobs, ' remote-daemon ') && (inputs.pr_side_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' remote-daemon ')) && inputs.pr_retry_runner || inputs.pr_side_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && contains(inputs.pr_owned_jobs, ' remote-daemon ') && (inputs.pr_side_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' remote-daemon ')) && inputs.pr_retry_runner || github.run_attempt == 1 && contains(inputs.pr_owned_jobs, ' remote-daemon ') && inputs.pr_side_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15' }} timeout-minutes: 15 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 668befcde8c0..09c12572c6ec 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -653,14 +653,14 @@ overflow and ci-owned-pool-rescue.yml as the way off a busy or refusing mini. | Jobs | Route | Why | | --- | --- | --- | | `ci-macos.yml` compile admission, app-host shards, `tests-build-and-lag`, `cli-product-tests` | owned via `pr_runner_pool.py` (root label), pull requests and main's full-suite dispatch | canonical-root jobs | -| `ci.yml` `claude-wrapper`, `remote-daemon.yml` macOS tests | owned side lane via the picker (the side label) | light | +| `ci.yml` `claude-wrapper`, `remote-daemon.yml` macOS tests | owned side lane via the picker (the side label), pull requests and main's full-suite dispatch | light | | `ci-macos.yml` `swift-package-tests` | owned side lane via the picker (the side label) when the run builds no Release helper; else Blacksmith macOS 15 | the helper needs an SDK 15 Xcode | | the seven side-lane workflows above | `CI_SIDE_LANE_RUNNER` on attempt 1 of a pull request | light; other events stay on Blacksmith | | `test-e2e.yml` (and `dispatch-focused-test.py`) | owned via `e2e_runner_pool.py`; UI runs with `CI_E2E_OWNED_UI=1` | root jobs; Blacksmith when no root runner is free | | `test-ios.yml`, `ios-screenshots.yml` | owned via `ios_runner_pool.py` behind `CI_IOS_OWNED=1` | needs the `glaeda-ios-sim` label (an iOS 26.x simulator runtime) | | `app-host-test-rerun.yml` `rerun` | Blacksmith | restores a product into a fixed canonical root; needs a root route and a glaeda class first | | `cmux-tui.yml` macOS `lint`, `test`, `cdp-browser-smoke` | Blacksmith | could move; glaeda classes unknown ids as compile (root), and these ids are generic | -| `ci-macos.yml` `release-build` | `MACOS_RUNNER_26` | could move; needs a picker key and a glaeda class | +| `ci-macos.yml` `release-build` | owned side lane via the picker (`release-build`, the side label), pull requests and main's full-suite dispatch; else `MACOS_RUNNER_26` | isolated: an unsigned universal Release into its own DerivedData, Xcode 26.6 | | low-volume dispatches: `test-macos-suite`, `tmux-corpus`, `perf-activation`, command palette benchmarks, `iroh-release-gate` version skew | Blacksmith or the caller's runner input | a few runs a week; benchmarks want a quiet machine | | `relay-tls` `system-keychain` | Blacksmith | edits the System keychain trust store | | `plain-paste-worker`, `ci-macos-compat`, `seed-swiftpm-manifests`, release and nightly Ghostty helpers | Blacksmith macOS 15 / 14 | an OS or SDK the minis lack | diff --git a/scripts/ci/pr_runner_pool.py b/scripts/ci/pr_runner_pool.py index dac9e9712249..113336484daf 100644 --- a/scripts/ci/pr_runner_pool.py +++ b/scripts/ci/pr_runner_pool.py @@ -248,9 +248,9 @@ wait up to the queue rounds and the bound (owned_room()). CI_OWNED_MAIN_RESERVE (0 when unset) holds that many machines and root runners back for pull requests; with a reserve it takes an owned pool only whole, and only while its peak is free now (no queue -allowance). Its side lanes (the Claude wrapper and -remote daemon) route only for pull requests, so they are not in its plan. -Main's CI concurrency group holds one run at a time, so main holds at most +allowance). Its side lanes (the Claude wrapper, the remote daemon and the +universal Release build) are in its plan like a pull request's, and read the +pick through the same inputs. Main's CI concurrency group holds one run at a time, so main holds at most one run's machines. ci-owned-pool-rescue.yml watches it like a pull request. Anything uncertain keeps today's route: an event other than pull_request or @@ -359,7 +359,10 @@ # suite with release_build false, which then peaks at all three side lanes # beside admission and its nine follow-on jobs. MAX_RUN_JOBS counts all three; # the replay charge leaves out the package lane, which a compile-only run -# carries only on a package change. +# carries only on a package change. release-build (RELEASE_BUILD_JOB) is the +# package lane's alternative: it runs only on a full suite with release_build, +# exactly when swift-package-tests builds the SDK 15 helper on Blacksmith, so a +# run still has at most three side lanes. APP_HOST_SHARDS = 7 SIDE_LANES = 3 MAX_RUN_JOBS = SIDE_LANES + APP_HOST_SHARDS + 2 @@ -475,6 +478,46 @@ def side_runner(choice: "Choice", owned_slots: Mapping[str, int]) -> str: return side_label(choice.runner) +# Side lanes of a run the picker put on Blacksmith (every owned pool too busy +# for its root jobs) may still take owned side runners: those are a different +# resource from the root runners the pick waited on, and on 2026-09-26 up to 18 +# of 42 std side runners and both light ones sat idle while such runs sent +# swift-package-tests to Blacksmith macOS 15 (about 470 jobs a day). Only +# runners idle now count, with SIDE_ONLY_MARGIN left over for the runs picking +# at the same moment, since these jobs have no queue allowance to spend: the +# light minis first (an M4 beats a 6 vCPU Blacksmith machine), then the std +# minis. ci-owned-pool-rescue.yml watches the run by its marker like any other +# owned placement, and a refused lane retries once on the side label before +# its Blacksmith default. +SIDE_ONLY_MARGIN = 1 + + +def side_only_placement(keys: Sequence[str], runners: Sequence[Mapping[str, Any]], + pools: Sequence[str]) -> tuple[str, tuple[str, ...]]: + """(owned pool, side lanes) that idle side runners can take now, or ("", ()). + + `keys` are the run's side lanes in priority order and `pools` the owned pool + labels for the lane's Xcode. The light pool is tried first; a pool takes + every lane it has room for beyond SIDE_ONLY_MARGIN, and the pool with the + most room wins when none fits them all. + """ + if not keys: + return "", () + best: tuple[str, tuple[str, ...]] = ("", ()) + for pool in sorted(pools, key=lambda label: 0 if "-light-" in label else 1): + side = side_label(pool) + if not side: + continue + idle = sum(1 for runner in runners + if runner.get("status") == "online" and not runner.get("busy") and side in runner_labels(runner)) + room = min(len(keys), idle - SIDE_ONLY_MARGIN) + if room > len(best[1]): + best = (pool, tuple(sorted(keys, key=priority)[:room])) + if room == len(keys): + break + return best + + def pool_label(label: str) -> str: """The owned pool a root or side label's runners belong to; any other label unchanged.""" for prefix in (ROOT_PREFIX, SIDE_PREFIX): @@ -591,13 +634,16 @@ def run_plan(*, macos: str | None, full_suite: str | None, unit_suite: str | Non that does not know) counts one shard, as before. swift-package-tests is a side lane only when package_lane_owned() says it may take the pool; `swift_packages` None (a caller that does not pass it) leaves it out. + release-build is a side lane on a full suite with `release_build` true; + None leaves it out. """ full = flag(macos) and flag(full_suite) side = tuple(key for key, on in (("claude-wrapper", flag(claude_wrapper) or full), ("remote-daemon", flag(remote_daemon)), (SWIFT_PACKAGE_JOB, package_lane_owned( full=full, full_suite=full_suite, swift_packages=swift_packages, - release_build=release_build))) if on) + release_build=release_build)), + (RELEASE_BUILD_JOB, full and flag(release_build))) if on) if not (flag(macos) or flag(cli)): return RunJobs(False, (), side) unit = flag(macos) and flag(unit_suite) and not flag(unit_in_admission) @@ -619,6 +665,11 @@ def run_plan(*, macos: str | None, full_suite: str | None, unit_suite: str | Non # Blacksmith macOS 15 image carries (the minis have Xcode 26.6 alone), so only # a run without that helper build places it on an owned pool. SWIFT_PACKAGE_JOB = "swift-package" +# ci-macos.yml release-build: the unsigned universal Release app nightly signs, +# into its own workspace DerivedData with the lane's Xcode 26.6 (glaeda's hook +# classes it isolated: no GUI, product, canonical root or secrets). It runs +# after admission and swift-package-tests on its own machine. +RELEASE_BUILD_JOB = "release-build" def package_lane_owned(*, full: bool, full_suite: str | None, swift_packages: str | None, @@ -641,12 +692,14 @@ def run_jobs(**routing: str | None) -> int: # Owned placement priority: the heavy compile, then GUI jobs (the longest # Blacksmith queues), then light jobs. GUI jobs need the mini's console # session; CI_PR_POOL_OWNED_GUI=0 keeps them off. -LIGHT_JOBS = ("cli-product", "remote-daemon", "claude-wrapper", SWIFT_PACKAGE_JOB) +# release-build is not light (a 15-minute universal compile), but it follows +# cli-product: it is the side lane that saves the most Blacksmith time. +LIGHT_JOBS = ("cli-product", RELEASE_BUILD_JOB, "remote-daemon", "claude-wrapper", SWIFT_PACKAGE_JOB) # glaeda's canonical-root jobs: admission and every job after it (RunJobs.after: # the shards, tests-build-and-lag, cli-product-tests). The side lanes are not. ROOT_JOBS = "admission, shards, lag, cli-product" # The side lanes (RunJobs.side): light, no canonical root; they take side_runner() on a pool with a root count. -SIDE_LANE_JOBS = ("claude-wrapper", "remote-daemon", SWIFT_PACKAGE_JOB) +SIDE_LANE_JOBS = ("claude-wrapper", "remote-daemon", SWIFT_PACKAGE_JOB, RELEASE_BUILD_JOB) def gui_job(key: str) -> bool: @@ -1960,7 +2013,10 @@ def summary(choice: Choice, snapshot: Mapping[str, Any] | None, *, now: dt.datet f"and a re-run of failed jobs, goes to: `{choice.retry_runner}`") if choice.root_runner: lines.append(f"- Root jobs among them ({ROOT_JOBS}) take `{choice.root_runner}`") - if side: + if side and not persistent(choice.runner) and owned_jobs: + lines.append(f"- Side lanes on idle owned side runners (SIDE_ONLY_MARGIN): {', '.join(owned_jobs)} " + f"take `{side}`") + elif side: lines.append(f"- Side lanes among them ({', '.join(SIDE_LANE_JOBS)}) take `{side}`") if admission_runner: labels = " + ".join(f"`{label}`" for label in json.loads(admission_runner)) @@ -2022,10 +2078,6 @@ def count_routed(since: str) -> int: cli=env.get("RUN_CLI"), remote_daemon=env.get("RUN_REMOTE_DAEMON"), unit_selectors=env.get("RUN_UNIT_SELECTORS"), swift_packages=env.get("RUN_SWIFT_PACKAGES"), release_build=env.get("RUN_RELEASE_BUILD")) - if on_main: - # The side lanes read the pick only on a pull request (ci.yml's - # claude-wrapper, remote-daemon.yml); main's keep their own route. - plan = dataclasses.replace(plan, side=()) # What an owned pool must have free for the whole run: its owned-eligible # jobs at their peak. gui = (env.get("POOL_OWNED_GUI") or "").strip() != "0" @@ -2090,6 +2142,16 @@ def count_routed(since: str) -> int: # run takes retry_runner. The marker's jobs are the owned machines held. owned_jobs, held = (place(plan, choice.owned_budget, gui, choice.root_budget if choice.root_runner else None) if persistent(choice.runner) else ((), plan.peak)) + # A Blacksmith pick's side lanes on idle owned side runners (SIDE_ONLY_MARGIN). + # A macOS 15 pick's Xcode is not the minis', so swift-package-tests, which + # selects the run's Xcode on an owned Mac, stays with it. + side_pool = "" + if (choice.runner and not persistent(choice.runner) and live_runners is not None and attempt in ("", "1") + and same_repo_pr and (env.get("POOL_OWNED") or "").strip() == "1"): + keys = tuple(key for key in plan.side if key != SWIFT_PACKAGE_JOB or not choice.xcode_app) + side_pool, side_keys = side_only_placement(keys, live_runners, owned_pools(pr_xcode_app)) + if side_keys: + owned_jobs, held = side_keys, len(side_keys) # Admission on a root runner whose kept build is of this run's merge base # (see "Warm affinity" above). Attempt 1 only: only it is placed, and # ci-macos.yml reads both outputs on attempt 1 only. @@ -2120,7 +2182,7 @@ def count_routed(since: str) -> int: admission_runner = "" print(f"::warning title=warm routing::{type(error).__name__}: {error}"[:300]) owned_slots = slots(env.get("OWNED_SLOTS"), pr_xcode_app) - side = side_runner(choice, owned_slots) + side = side_label(side_pool) if side_pool else side_runner(choice, owned_slots) text = summary(choice, snapshot, now=now, owned_slots=owned_slots, problems=problems, owned_jobs=owned_jobs, admission_runner=admission_runner, side=side) print(text) @@ -2131,6 +2193,10 @@ def count_routed(since: str) -> int: with open(env["GITHUB_OUTPUT"], "a", encoding="utf-8") as handle: handle.write(f"runner={choice.runner}\nxcode_app={choice.xcode_app}\n" f"persistent={'true' if persistent(choice.runner) else 'false'}\n" + # The owned pool the rescue marker names: the pick, or + # the pool whose side runners took a Blacksmith pick's + # side lanes; "" when no job of the run is owned. + f"marker_pool={choice.runner if persistent(choice.runner) else side_pool}\n" f"retry_runner={choice.retry_runner}\njobs={held}\n" f"shard_runner={choice.shard_runner}\n" # Attempt 2 of an owned job the fleet refused tries it diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py index 7ce70f69fad1..6929fdbbf0bb 100644 --- a/tests/test_ci_pr_runner_pool.py +++ b/tests/test_ci_pr_runner_pool.py @@ -297,7 +297,7 @@ def test_main_writes_outputs_and_summary(self): self.assertEqual(pool.main(["--snapshot", str(snap_path)], env), 0) finally: sys.stdout = old - self.assertEqual(out.read_text(), f"runner={LARGE}\nxcode_app=\npersistent=false\n" + self.assertEqual(out.read_text(), f"runner={LARGE}\nxcode_app=\npersistent=false\nmarker_pool=\n" f"retry_runner=\njobs={pool.MAX_RUN_JOBS}\nshard_runner=\n" f"refused_retry_runner=\nroot_runner=\nside_runner=\n" "admission_runner=\nadmission_warm=\nowned_jobs=\n") @@ -489,7 +489,8 @@ def side_lane(key: str) -> str: return (f"github.run_attempt == 2 && contains(inputs.pr_owned_jobs, {key}) " "&& (inputs.pr_side_runner || inputs.pr_refused_retry_runner) " f"|| (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, {key})) && inputs.pr_retry_runner " - "|| inputs.pr_side_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15'") + f"|| github.run_attempt == 1 && contains(inputs.pr_owned_jobs, {key}) && inputs.pr_side_runner " + "|| inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15'") def warm_lane(index: str = "") -> str: @@ -2134,12 +2135,15 @@ def test_changes_job_chooses_once(self): self.assertEqual(step["env"]["DEFAULT_RUNNER"], "${{ vars.MACOS_RUNNER_PR }}") def test_a_persistent_choice_publishes_the_rescue_marker(self): + # marker_pool is the owned pick, or the pool whose side runners took a + # Blacksmith pick's side lanes: either way the run has owned jobs. steps = self.workflow("ci.yml")["jobs"]["changes"]["steps"] mark = next(step for step in steps if step.get("id") == "macos-pool-marker") - self.assertEqual(mark["if"], "${{ steps.macos-pool.outputs.persistent == 'true' }}") + self.assertEqual(mark["if"], "${{ steps.macos-pool.outputs.marker_pool != '' }}") + self.assertEqual(mark["env"]["POOL"], "${{ steps.macos-pool.outputs.marker_pool }}") upload = next(step for step in steps if step.get("name") == "Upload the persistent pool marker") self.assertEqual(upload["with"]["name"], "macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}" - "-${{ steps.macos-pool.outputs.jobs }}-${{ steps.macos-pool.outputs.runner }}") + "-${{ steps.macos-pool.outputs.jobs }}-${{ steps.macos-pool.outputs.marker_pool }}") def test_the_picker_reads_the_runs_routing(self): changes = self.workflow("ci.yml")["jobs"]["changes"] @@ -2158,8 +2162,11 @@ def lanes(self, name): def test_every_pr_route_in_the_run_reads_the_choice(self): expected = { - # The Claude wrapper, a side lane: the side label first. - "ci.yml": "needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_runner " + # The Claude wrapper, a side lane: the side label first, where the + # picker placed it (side_only_placement() names a side label for a + # Blacksmith pick whose other jobs keep the pick). + "ci.yml": "github.run_attempt <= 2 && contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') " + "&& needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_runner " "|| vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15'", # Compile admission (and its CMUX_PRODUCT_RUNNER mirror) and # tests-build-and-lag each test their own owned_jobs key, and are @@ -2181,12 +2188,18 @@ def test_a_rerun_of_failed_shards_leaves_the_owned_pool(self): "format(' shard-{0} ', matrix.shard))) && inputs.pr_retry_runner " "|| inputs.pr_shard_runner || needs.macos-compile-admission.outputs.runner }}") wrapper = self.workflow("ci.yml")["jobs"]["claude-wrapper"]["runs-on"] - self.assertIn("github.event_name == 'pull_request' && github.run_attempt == 2 && contains(" + routed = "(github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')" + self.assertIn(f"{routed} && github.run_attempt == 2 && contains(" "needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && " "(needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_refused_retry_runner) " - "|| github.event_name == 'pull_request' && " + f"|| {routed} && " "(github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs, " "' claude-wrapper ')) && needs.changes.outputs.macos_pr_retry_runner", wrapper) + # Main's dispatch takes the side label only where the picker placed the wrapper. + self.assertIn("|| github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && " + "contains(needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && " + "(needs.changes.outputs.macos_pr_side_runner || needs.changes.outputs.macos_pr_runner) " + "|| vars.CI_PAID_MACOS_OVERFLOW == '1'", wrapper) def test_callers_pass_the_choice(self): jobs = self.workflow("ci.yml")["jobs"] @@ -2205,10 +2218,11 @@ def test_callers_pass_the_choice(self): self.assertEqual(jobs["remote-daemon"]["with"]["pr_side_runner"], "${{ needs.changes.outputs.macos_pr_side_runner }}") self.assertEqual(jobs["macos"]["with"]["pr_side_runner"], "${{ needs.changes.outputs.macos_pr_side_runner }}") - # In ci-macos.yml only swift-package-tests reads it; its root jobs never do. + # In ci-macos.yml only the side lanes read it (swift-package-tests and + # release-build); its root jobs never do. macos_jobs = self.workflow("ci-macos.yml")["jobs"] readers = sorted(name for name, job in macos_jobs.items() if "pr_side_runner" in yaml.safe_dump(job)) - self.assertEqual(readers, ["swift-package-tests"]) + self.assertEqual(readers, ["release-build", "swift-package-tests"]) self.assertEqual(self.workflow("ci.yml")["jobs"]["changes"]["outputs"]["macos_pr_side_runner"], "${{ steps.macos-pool.outputs.side_runner }}") self.assertEqual(jobs["macos"]["with"]["pr_admission_runner"], @@ -2307,7 +2321,8 @@ def test_package_tests_take_an_owned_mac_only_where_the_picker_placed_them(self) # (MACOS_RUNNER_PR) or the retry pool; only the owned label, on the # attempts that read it, when owned_jobs names ' swift-package '. job = self.workflow("ci-macos.yml")["jobs"]["swift-package-tests"] - owned = ("github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && " + owned = ("(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || " + "github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && " "contains(inputs.pr_owned_jobs, ' swift-package ') && " "(github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && " "(inputs.pr_side_runner || inputs.pr_refused_retry_runner))") @@ -2333,6 +2348,44 @@ def test_package_tests_take_an_owned_mac_only_where_the_picker_placed_them(self) self.assertIn("inputs.full_suite == 'true' && inputs.release_build == 'true'", step.get("if", ""), step.get("name")) + def test_release_build_takes_an_owned_mac_only_where_the_picker_placed_them(self): + # The universal Release build: the side label when owned_jobs names + # ' release-build ' (same repository or main's dispatch), else the + # macOS 26 variable, and its product-contract mirror says the same. + job = self.workflow("ci-macos.yml")["jobs"]["release-build"] + owned = ("(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || " + "github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && " + "contains(inputs.pr_owned_jobs, ' release-build ') && " + "(github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && " + "(inputs.pr_side_runner || inputs.pr_refused_retry_runner))") + expected = ("${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && " + "github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || " + f"{owned} || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }}}}") + self.assertEqual(job["runs-on"], expected) + self.assertEqual(job["env"]["CMUX_PRODUCT_RUNNER"], expected) + self.assertEqual(pool.RELEASE_BUILD_JOB, "release-build") + + def test_release_build_is_a_side_lane_of_a_full_suite_with_release_build(self): + full = dict(macos="true", full_suite="true", unit_suite="false", unit_in_admission="false", + claude_wrapper="true", cli="true", remote_daemon="false") + plan = pool.run_plan(**full, swift_packages="true", release_build="true") + # The helper build keeps swift-package-tests on Blacksmith; release-build takes its place. + self.assertEqual(plan.side, ("claude-wrapper", "release-build")) + self.assertNotIn("release-build", pool.run_plan(**full, swift_packages="true", release_build="false").side) + self.assertNotIn("release-build", pool.run_plan(**full, swift_packages="true").side) + self.assertNotIn("release-build", pool.run_plan(**{**full, "full_suite": "false"}, + release_build="true").side) + # Still at most three side lanes, so the most machines a run holds is unchanged. + self.assertLessEqual(pool.run_plan(**{**full, "remote_daemon": "true"}, swift_packages="true", + release_build="true").peak, pool.MAX_RUN_JOBS) + keys, held = pool.place(plan, plan.peak) + self.assertIn("release-build", keys) + self.assertEqual(held, plan.peak) + # Behind the root jobs and cli-product, ahead of the light side lanes. + order = sorted(("claude-wrapper", "remote-daemon", "release-build", "cli-product", "lag"), key=pool.priority) + self.assertEqual(order, ["lag", "cli-product", "release-build", "remote-daemon", "claude-wrapper"]) + self.assertIn("release-build", pool.SIDE_LANE_JOBS) + def test_the_picker_reads_the_package_lane_routing(self): env = next(step for step in self.workflow("ci.yml")["jobs"]["changes"]["steps"] if step.get("id") == "macos-pool")["env"] @@ -2340,6 +2393,93 @@ def test_the_picker_reads_the_package_lane_routing(self): self.assertEqual(env["RUN_RELEASE_BUILD"], "${{ steps.detect.outputs.release_build }}") +SIDE_STD = "glaeda-side-std-xcode-26.6" +SIDE_LIGHT = "glaeda-side-light-xcode-26.6" + + +def side_runners(std=0, light=0, busy_std=0): + """Owned runners: idle and busy std side runners, idle light side runners, busy root runners.""" + runner = lambda busy, *labels: {"status": "online", "busy": busy, "labels": [{"name": name} for name in labels]} + return ([runner(False, MINI, SIDE_STD) for _ in range(std)] + [runner(True, MINI, SIDE_STD) for _ in range(busy_std)] + + [runner(False, LIGHT, SIDE_LIGHT) for _ in range(light)] + + [runner(True, MINI, ROOT_MINI) for _ in range(4)] + + [runner(True, LIGHT, "glaeda-root-light-xcode-26.6") for _ in range(2)]) + + +class SideOnly(unittest.TestCase): + """A Blacksmith pick's side lanes may take idle owned side runners (side_only_placement).""" + + POOLS = (MINI, LIGHT) + KEYS = ("claude-wrapper", "swift-package") + + def test_light_first_with_a_margin(self): + self.assertEqual(pool.side_only_placement(self.KEYS, side_runners(std=9, light=3), self.POOLS), + (LIGHT, ("claude-wrapper", "swift-package"))) + # Two idle light runners leave room for one lane beyond the margin; std fits both. + self.assertEqual(pool.side_only_placement(self.KEYS, side_runners(std=9, light=2), self.POOLS), + (MINI, ("claude-wrapper", "swift-package"))) + + def test_the_most_room_wins_when_no_pool_fits_every_lane(self): + keys = ("remote-daemon", "claude-wrapper", "release-build") + pool_, placed = pool.side_only_placement(keys, side_runners(std=3, light=2), self.POOLS) + # std has room for two, taken in priority order (release-build first). + self.assertEqual((pool_, placed), (MINI, ("release-build", "remote-daemon"))) + + def test_nothing_without_idle_side_runners(self): + for runners in (side_runners(), side_runners(std=1, light=1), side_runners(busy_std=9)): + self.assertEqual(pool.side_only_placement(self.KEYS, runners, self.POOLS), ("", ())) + self.assertEqual(pool.side_only_placement((), side_runners(std=9), self.POOLS), ("", ())) + # Offline runners and root runners are not side runners. + offline = [{"status": "offline", "busy": False, "labels": [{"name": MINI}, {"name": SIDE_STD}]}] * 5 + self.assertEqual(pool.side_only_placement(self.KEYS, offline, self.POOLS), ("", ())) + + def run_main(self, runners, **env_extra): + fresh = fleet(busy=11) + fresh["generated_at"] = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + with tempfile.TemporaryDirectory() as tmp, \ + unittest.mock.patch.object(pool.GitHub, "snapshot", return_value=fresh), \ + unittest.mock.patch.object(pool.GitHub, "pull_request_routes_since", return_value=pool.Routed()), \ + unittest.mock.patch.object(pool.GitHub, "runners", return_value=runners), \ + unittest.mock.patch("sys.stdout", io.StringIO()): + out = Path(tmp, "out") + env = {"EVENT_NAME": "pull_request", "GITHUB_REPOSITORY": "manaflow-ai/cmux", "GH_TOKEN": "t", + "HEAD_REPO": "manaflow-ai/cmux", "DEFAULT_RUNNER": SMALL, "POOL_OWNED": "1", + "OWNED_SLOTS": json.dumps({MINI: 11, ROOT_MINI: 4, LIGHT: 5, + "glaeda-root-light-xcode-26.6": 2}), "ROUTE_TOKEN": "app-token", + "POOL_QUEUE_ROUNDS": "0", "CMUX_CI_XCODE_APP_PR": PR_XCODE, "CMUX_CI_XCODE_APP_MACOS_15": XCODE_15, + "GITHUB_RUN_ATTEMPT": "1", "GITHUB_OUTPUT": str(out), "RUN_MACOS": "true", + "RUN_FULL_SUITE": "false", "RUN_CLAUDE_WRAPPER": "true", "RUN_SWIFT_PACKAGES": "true", + "RUN_RELEASE_BUILD": "false", **env_extra} + self.assertEqual(pool.main([], env), 0) + return dict(line.split("=", 1) for line in out.read_text().splitlines()) + + def test_a_blacksmith_pick_sends_its_side_lanes_to_idle_side_runners(self): + # Every root runner is busy, so the run takes Blacksmith; three light side runners are idle. + values = self.run_main(side_runners(light=3)) + self.assertTrue(values["runner"].startswith("blacksmith-"), values["runner"]) + self.assertEqual((values["persistent"], values["retry_runner"]), ("false", "")) + self.assertEqual((values["owned_jobs"], values["side_runner"], values["marker_pool"], values["jobs"]), + (" claude-wrapper swift-package ", SIDE_LIGHT, LIGHT, "2")) + self.assertEqual(values["refused_retry_runner"], "") + + def test_no_side_runner_no_change(self): + values = self.run_main(side_runners()) + self.assertTrue(values["runner"].startswith("blacksmith-"), values["runner"]) + self.assertEqual((values["owned_jobs"], values["side_runner"], values["marker_pool"]), ("", "", "")) + + def test_forks_retries_and_macos_15_picks(self): + # A fork head never reads owned runners. + values = self.run_main(side_runners(light=3, std=9), HEAD_REPO="someone/cmux", ROUTE_TOKEN="") + self.assertEqual((values["owned_jobs"], values["marker_pool"]), ("", "")) + values = self.run_main(side_runners(light=3, std=9), GITHUB_RUN_ATTEMPT="2") + self.assertEqual((values["owned_jobs"], values["marker_pool"]), ("", "")) + # A macOS 15 pick selects another Xcode, which swift-package-tests would carry onto the mini. + with unittest.mock.patch.object(pool, "choose", return_value=(pool.Choice(OLD, XCODE_15, "full"), None)): + values = self.run_main(side_runners(light=3, std=9)) + self.assertEqual((values["runner"], values["owned_jobs"], values["side_runner"]), + (OLD, " claude-wrapper ", SIDE_LIGHT)) + + IOS_SIM = "glaeda-ios-sim" SIGNING_WORKFLOWS = ("ios-testflight.yml", "ios-app-store.yml", "ios-appstore-upload.yml") IOS_SLOTS = {MINI: 40, ROOT_MINI: 10, IOS_SIM: 2} @@ -2465,7 +2605,7 @@ def test_anything_else_keeps_its_route(self): choice = self.main_choice(self.snap(), **kwargs) self.assertEqual((choice.runner, choice.root_runner), ("", ""), kwargs) - def test_main_routes_the_full_suite_without_its_side_lanes(self): + def test_main_routes_the_full_suite_with_its_side_lanes(self): with tempfile.TemporaryDirectory() as tmp: snapshot = Path(tmp, "snap.json") fresh = self.snap() @@ -2482,10 +2622,11 @@ def test_main_routes_the_full_suite_without_its_side_lanes(self): with unittest.mock.patch("sys.stdout", io.StringIO()): self.assertEqual(pool.main(["--snapshot", str(snapshot)], env), 0) values = dict(line.split("=", 1) for line in out.read_text().splitlines()) + # The nine root jobs at their peak, plus the Claude wrapper and the remote daemon beside them. self.assertEqual((values["runner"], values["persistent"], values["root_runner"], values["jobs"]), - (MINI, "true", ROOT_MINI, "9")) + (MINI, "true", ROOT_MINI, "11")) self.assertEqual(values["owned_jobs"], " admission " + " ".join(f"shard-{index}" for index in range(1, 8)) - + " lag cli-product ") + + " lag cli-product remote-daemon claude-wrapper ") self.assertTrue(values["retry_runner"].startswith("blacksmith-")) # A dispatch on another branch writes the default route. env.update(GITHUB_REF="refs/heads/topic") diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index cffc063c031b..53ab8a3c749a 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -93,7 +93,7 @@ check_release_build_runner_disk_capacity() { # paid-overflow gate appearing here, which does not belong: MACOS_RUNNER_26 # is the free macOS 26 pool and is read ungated everywhere. See # docs/ci-runners.md for why the gate must not grow to cover it. - if ! awk -v release_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }}" ' + if ! awk -v release_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && contains(inputs.pr_owned_jobs, ' release-build ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) || vars.MACOS_RUNNER_26 || 'blacksmith-6vcpu-macos-26') }}" ' /^ release-build:/ { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } in_job && index($0, release_runner) { saw_release_runner=1 } @@ -298,7 +298,7 @@ check_release_helper_artifact_from_package_lane() { # The one arm besides the dual-Xcode pool: the side label, else the owned # label, only when the picker placed ' swift-package ' in pr_owned_jobs, # which it does only for a run that skips the SDK 15 helper steps (pr_runner_pool.package_lane_owned()). - if ! awk -v dual_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-15' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-15' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository && contains(inputs.pr_owned_jobs, ' swift-package ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15') }}" ' + if ! awk -v dual_runner="runs-on: \${{ github.repository_owner != 'manaflow-ai' && 'macos-15' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-6vcpu-macos-15' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && contains(inputs.pr_owned_jobs, ' swift-package ') && (github.run_attempt == 1 && (inputs.pr_side_runner || inputs.pr_runner) || github.run_attempt == 2 && (inputs.pr_side_runner || inputs.pr_refused_retry_runner)) || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_DUAL_XCODE || 'blacksmith-6vcpu-macos-15') }}" ' /^ swift-package-tests:/ { in_job=1; next } in_job && /^ [^[:space:]#][^:]*:[[:space:]]*(#.*)?$/ { in_job=0 } @@ -1326,6 +1326,9 @@ ROOT_OUTPUT = "needs.changes.outputs.macos_pr_root_runner" ADMISSION_PICKED = "steps.macos-pool.outputs.admission_runner" ADMISSION_OUTPUT = "needs.changes.outputs.macos_pr_admission_runner" SIDE_PICKED = "steps.macos-pool.outputs.side_runner" +# The owned pool the rescue marker names (the pick, or the pool whose side +# runners took the side lanes of a Blacksmith pick): only the marker reads it. +MARKER_PICKED = "steps.macos-pool.outputs.marker_pool" SIDE_OUTPUT = "needs.changes.outputs.macos_pr_side_runner" PASSED = "${{ needs.changes.outputs.macos_pr_runner }}" # Each input the picked pools reach a reusable workflow through, and its value. @@ -1336,7 +1339,7 @@ INPUTS = {"pr_runner": PASSED, "pr_retry_runner": "${{ " + RETRY_OUTPUT + " }}", "pr_admission_runner": "${{ " + ADMISSION_OUTPUT + " }}", "pr_side_runner": "${{ " + SIDE_OUTPUT + " }}"} MARKER = ("macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}" - "-${{ steps.macos-pool.outputs.jobs }}-${{ steps.macos-pool.outputs.runner }}") + "-${{ steps.macos-pool.outputs.jobs }}-${{ steps.macos-pool.outputs.marker_pool }}") # The runs-on branches that may read the picked pool, each behind its # pull_request condition; a fork head keeps only a Blacksmith pick. GUARDED = ( @@ -1346,16 +1349,24 @@ GUARDED = ( "github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR" " || 'blacksmith-6vcpu-macos-15')", # A side lane: the side label of the pool first, when the picker named one. - "github.event_name == 'pull_request' && (needs.changes.outputs.macos_pr_side_runner" + "github.event_name == 'pull_request' && (github.run_attempt <= 2 && contains(" + "needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && needs.changes.outputs.macos_pr_side_runner" " || needs.changes.outputs.macos_pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15')", # Attempt 2 of a refused owned job: the owned pool once more. - "github.event_name == 'pull_request' && github.run_attempt == 2 && contains(needs.changes.outputs.macos_pr_owned_jobs," + "(github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')" + " && github.run_attempt == 2 && contains(needs.changes.outputs.macos_pr_owned_jobs," " ' claude-wrapper ') && (needs.changes.outputs.macos_pr_side_runner" " || needs.changes.outputs.macos_pr_refused_retry_runner)", # A re-run of failed jobs on an owned-pool run, or a job the picker did not # place on the owned pool: the Blacksmith pool the picker named for it. - "github.event_name == 'pull_request' && (github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs," + "(github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main')" + " && (github.run_attempt > 1 || !contains(needs.changes.outputs.macos_pr_owned_jobs," " ' claude-wrapper ')) && needs.changes.outputs.macos_pr_retry_runner", + # The full-suite dispatch on main (code already on main, which + # pr_runner_pool.py places like a pull request): only the job it placed. + "github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main' && contains(" + "needs.changes.outputs.macos_pr_owned_jobs, ' claude-wrapper ') && (needs.changes.outputs.macos_pr_side_runner" + " || needs.changes.outputs.macos_pr_runner)", ) @@ -1383,6 +1394,11 @@ for file in sorted(Path(sys.argv[1]).glob("*.y*ml")): or path[:3] == ("jobs", "changes", "steps") and path[-2:] == ("with", "name") and value == MARKER)) if not allowed: violations.append(f"{where}: reads the picker's runner outside macos_pr_runner and the rescue marker") + if MARKER_PICKED in value and not (file.name == "ci.yml" and path[:3] == ("jobs", "changes", "steps") and ( + path[-2:] == ("env", "POOL") and value == "${{ " + MARKER_PICKED + " }}" + or path[-2:] == ("with", "name") and value == MARKER + or path[-1:] == ("if",) and value == "${{ " + MARKER_PICKED + " != '' }}")): + violations.append(f"{where}: reads the marker pool outside the rescue marker") if RETRY_PICKED in value and not ( file.name == "ci.yml" and path == ("jobs", "changes", "outputs", "macos_pr_retry_runner") and value == "${{ " + RETRY_PICKED + " }}"):