diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 25633f897915..7abeb500dbf9 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -372,6 +372,10 @@ jobs: if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_owned_pool_rescue.py + - name: Validate the owned Mac warm labels + if: ${{ matrix.group == 'ci' }} + run: python3 tests/test_ci_owned_warm_labels.py + - name: Validate CI health report if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_health_report.py diff --git a/.github/workflows/ci-macos.yml b/.github/workflows/ci-macos.yml index 2f30b2f7ff2f..5c0717c52e61 100644 --- a/.github/workflows/ci-macos.yml +++ b/.github/workflows/ci-macos.yml @@ -119,6 +119,14 @@ on: required: false default: "" type: string + # Set only beside pr_root_runner, when an idle root runner kept a build + # of this run's merge base: the JSON array ["", + # "glaeda-warm-"] (pr_runner_pool.py, warm affinity). Compile + # admission's attempt 1 takes it as its runs-on; retries do not. + pr_admission_runner: + required: false + default: "" + type: string # Set only when pr_runner is persistent: the jobs of attempt 1 that take # it, as " ". Every other job takes pr_retry_runner, so a # run can use the owned machines that are free and overflow the rest @@ -168,8 +176,10 @@ jobs: # Pull requests and main's full-suite dispatch (ci-main-full-suite.yml) # compile on the pool and Xcode seed-derived-data.yml builds with, so both # can adopt its DerivedData seed below. Merge groups and dispatches on - # other branches keep the macos-15 lane. - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }} + # other branches keep the macos-15 lane. On an owned pool, attempt 1 may + # also require the warm label of a root runner that kept a build of the + # run's merge base (pr_admission_runner). + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || github.run_attempt == 1 && inputs.pr_admission_runner && fromJSON(inputs.pr_admission_runner) || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }} # A changed-suites run adds its tests after the compile: the same # 30-minute batch ceiling the separate worker had. timeout-minutes: ${{ inputs.unit_in_admission == 'true' && 105 || 75 }} @@ -211,8 +221,10 @@ jobs: # Names the pool this job actually ran on. The app-host product contract # keys on the toolchain and the build path instead, so a product built # here at the canonical root matches on any pool with the same Xcode. - # The macOS runner guard still requires it to track runs-on. - CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }} + # The macOS runner guard still requires it to track runs-on. A warm + # admission (pr_admission_runner) names its first label, the root label: + # the consumers need a root runner, not this one. + CMUX_PRODUCT_RUNNER: ${{ github.repository_owner != 'manaflow-ai' && 'macos-26' || (github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && (startsWith(inputs.pr_runner, 'blacksmith-') && inputs.pr_runner || 'blacksmith-6vcpu-macos-15') || (github.event_name == 'pull_request' || github.event_name == 'workflow_dispatch' && github.ref == 'refs/heads/main') && (github.run_attempt == 2 && github.triggering_actor == 'github-actions[bot]' && contains(inputs.pr_owned_jobs, ' admission ') && (inputs.pr_root_runner || inputs.pr_refused_retry_runner) || (github.run_attempt > 1 || !contains(inputs.pr_owned_jobs, ' admission ')) && inputs.pr_retry_runner || github.run_attempt == 1 && inputs.pr_admission_runner && fromJSON(inputs.pr_admission_runner)[0] || inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15') || vars.CI_PAID_MACOS_OVERFLOW == '1' && vars.MACOS_RUNNER_15 || 'blacksmith-6vcpu-macos-15') }} # What the changed-suites steps at the end of this job read, with the # values `app-host unit tests` gives its changed-suites worker, shard 8. # The compile reads none of them: it runs plain xcodebuild and bakes the @@ -734,6 +746,36 @@ jobs: FINGERPRINT: ${{ steps.owned-state.outputs.fingerprint }} run: python3 scripts/ci/owned_build_state.py keep "$CMUX_OWNED_STATE_ROOT" "$CMUX_COMPILE_ADMISSION_DERIVED_DATA" "$FINGERPRINT" + # The main commits this Mac's kept DerivedData starts from cheaply, for + # ci-owned-warm-labels.yml to label its root runner with, so a later run + # merging onto one of them can ask for this Mac (pr_runner_pool.py, warm + # affinity). Only once owned_build_state.py has `warm-keys`; before + # that, and on any failure, nothing is uploaded and nothing changes. + - name: List the commits this owned Mac starts from warm + id: owned-warm-keys + if: ${{ !cancelled() && steps.owned-state.outputs.fingerprint != '' }} + continue-on-error: true + run: | + set -euo pipefail + usage="$(python3 scripts/ci/owned_build_state.py --help 2>&1 || true)" + case "$usage" in + *"owned_build_state.py warm-keys"*) ;; + *) echo "owned_build_state.py has no warm-keys yet"; exit 0 ;; + esac + python3 scripts/ci/owned_build_state.py warm-keys "$CMUX_OWNED_STATE_ROOT" "$RUNNER_NAME" "$CMUX_PRODUCT_RUNNER" \ + > "$RUNNER_TEMP/warm-keys.json" + echo "path=$RUNNER_TEMP/warm-keys.json" >> "$GITHUB_OUTPUT" + + - name: Upload the owned Mac's warm keys + if: ${{ !cancelled() && steps.owned-warm-keys.outputs.path != '' }} + continue-on-error: true + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: owned-warm-keys-${{ github.run_id }}-${{ github.run_attempt }} + path: ${{ steps.owned-warm-keys.outputs.path }} + retention-days: 1 + if-no-files-found: ignore + - name: Generate Xcode build metrics receipt id: build-metrics if: always() && !cancelled() && steps.hosted-compile.outcome != 'skipped' diff --git a/.github/workflows/ci-owned-warm-labels.yml b/.github/workflows/ci-owned-warm-labels.yml new file mode 100644 index 000000000000..6c2463859eba --- /dev/null +++ b/.github/workflows/ci-owned-warm-labels.yml @@ -0,0 +1,84 @@ +name: CI owned warm labels +run-name: owned-warm-labels-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + +# A compile admission on an owned Mac keeps its DerivedData, and uploads the +# main commits that build starts from cheaply (owned-warm-keys--, +# ci-macos.yml). When the CI run completes, this labels the runner that ran it +# glaeda-warm- for each, and takes those labels off the other runners of +# its pool, so pr_runner_pool.py can send a later admission merging onto one of +# them to that Mac (warm affinity). scripts/ci/owned_warm_labels.py has the +# rules. +# +# Changing runner labels needs the org route App with administration: write, +# so it is triggered by workflow_run and its code comes from main: a pull +# request cannot change it. It trusts only the keys from the artifact; the +# runner comes from the jobs API. Off unless CI_OWNED_WARM_LABELS is 1. It only +# makes API requests, so it runs on a GitHub-hosted runner rather than holding +# a slot in CI's Linux pool. +on: + workflow_run: + workflows: [CI] + types: [completed] + +permissions: {} + +env: + # Renaming the source workflow silently stops every `workflow_run` consumer + # of it; tests/test_ci_workflow_run_sources.py derives the expected name from + # this path. The script confirms the run's own `path` again before acting. + SOURCE_WORKFLOW_PATHS: .github/workflows/ci.yml + +concurrency: + group: owned-warm-labels-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + cancel-in-progress: true + +jobs: + label: + name: Label the owned Mac that kept this run's build + if: ${{ vars.CI_OWNED_WARM_LABELS == '1' && vars.GLAEDA_ROUTE_APP_ID != '' && github.event.workflow_run.head_repository.full_name == github.repository }} + runs-on: ubuntu-24.04 # github-hosted-required: only calls the Actions API; keeps CI's Linux pool free + timeout-minutes: 5 + permissions: + actions: read + contents: read + steps: + # Most runs have no warm keys (no owned Mac, or no warm-keys yet); the + # download then fails and every step after it is skipped. + - name: Download the warm keys + id: keys + continue-on-error: true + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + name: owned-warm-keys-${{ github.event.workflow_run.id }}-${{ github.event.workflow_run.run_attempt }} + run-id: ${{ github.event.workflow_run.id }} + github-token: ${{ github.token }} + path: ${{ runner.temp }}/owned-warm-keys + + - name: Checkout trusted labeler + if: steps.keys.outcome == 'success' + uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 + with: + ref: main + persist-credentials: false + + - name: Mint the runner labeling token + id: route-token + if: steps.keys.outcome == 'success' + continue-on-error: true + timeout-minutes: 1 + uses: actions/create-github-app-token@bcd2ba49218906704ab6c1aa796996da409d3eb1 # v3.2.0 + with: + app-id: ${{ vars.GLAEDA_ROUTE_APP_ID }} + private-key: ${{ secrets.GLAEDA_ROUTE_APP_KEY }} + permission-administration: write + + - name: Label the runner + if: steps.keys.outcome == 'success' && steps.route-token.outputs.token != '' + env: + GH_TOKEN: ${{ github.token }} + ROUTE_TOKEN: ${{ steps.route-token.outputs.token }} + RUN_ID: ${{ github.event.workflow_run.id }} + RUN_ATTEMPT: ${{ github.event.workflow_run.run_attempt }} + RUN_PATH: ${{ github.event.workflow_run.path }} + KEYS_FILE: ${{ runner.temp }}/owned-warm-keys/warm-keys.json + run: python3 scripts/ci/owned_warm_labels.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 122feb8fab88..33ba92b7ffd4 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -111,6 +111,10 @@ jobs: # macos_pr_owned_jobs (admission, shards, lag, cli-product) take it in # place of the pool label, on attempt 1 and after a refusal. macos_pr_root_runner: ${{ steps.macos-pool.outputs.root_runner }} + # Set only beside a root runner, when an idle one carries the warm label + # of this run's merge base: ["", "glaeda-warm-"], + # the labels compile admission's attempt 1 takes instead. + macos_pr_admission_runner: ${{ steps.macos-pool.outputs.admission_runner }} permissions: actions: read contents: read @@ -649,6 +653,10 @@ jobs: RUN_CLAUDE_WRAPPER: ${{ steps.standalone.outputs.claude_wrapper }} RUN_CLI: ${{ steps.detect.outputs.cli }} RUN_REMOTE_DAEMON: ${{ steps.standalone.outputs.remote_daemon }} + # The main commit this run merges onto, whose warm label (a root + # runner that kept a build of it) compile admission may take. + MERGED_ONTO: ${{ steps.source-identity.outputs.parent1 || github.event.pull_request.base.sha }} + WARM_LABELS: ${{ vars.CI_OWNED_WARM_LABELS }} run: python3 scripts/ci/pr_runner_pool.py # A job queued on a persistent pool (owned Macs) waits for it however long @@ -1197,6 +1205,7 @@ jobs: pr_owned_jobs: ${{ needs.changes.outputs.macos_pr_owned_jobs }} pr_refused_retry_runner: ${{ needs.changes.outputs.macos_pr_refused_retry_runner }} pr_root_runner: ${{ needs.changes.outputs.macos_pr_root_runner }} + pr_admission_runner: ${{ needs.changes.outputs.macos_pr_admission_runner }} pr_xcode_app: ${{ needs.changes.outputs.macos_pr_xcode_app }} tests: diff --git a/docs/ci-runners.md b/docs/ci-runners.md index f5f5c2bedb58..e99a84c04050 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -230,6 +230,22 @@ Claude wrapper lanes always do. A root count above its pool's is an error. With 8 std minis and 2 light ones: `{"std": 32, "light": 4, "root-std": 8, "root-light": 2}`. +Warm affinity (`CI_OWNED_WARM_LABELS=1`, off by default): an owned Mac keeps +compile admission's DerivedData, and admission uploads the main commits that +build starts from cheaply (`owned_build_state.py warm-keys`). When the CI run +completes, `ci-owned-warm-labels.yml` (from main, with the route App's +administration: write) labels the runner that ran admission +`glaeda-warm-` for each, at most 4, and removes those labels from the +other runners of its root pool, so one runner per pool carries each commit. +With live runners, the picker sends a run's admission to +`["", "glaeda-warm-"]` when an idle root runner +carries that label (the `admission_runner` output, attempt 1 only); otherwise +admission takes the root label as before. The picker also reads the variable, so +turning it off ignores labels already set. v1 matches the merge base exactly; +it does not rank runners by commit distance. A warm runner taken between the +pick and the queue leaves admission waiting, and the rescue moves it to +Blacksmith like any other stuck owned job. + An owned pool is persistent, which needs one more rule because GitHub never re-routes a queued job: one queued there waits for that pool however long it stays busy. An offline mini still counts as a slot, and the snapshot can be diff --git a/scripts/ci/owned_warm_labels.py b/scripts/ci/owned_warm_labels.py new file mode 100644 index 000000000000..5ef04a9d5898 --- /dev/null +++ b/scripts/ci/owned_warm_labels.py @@ -0,0 +1,218 @@ +#!/usr/bin/env python3 +"""Label an owned Mac's root runner with the main commits it starts from warm. + +An owned Mac keeps compile admission's DerivedData between jobs +(owned_build_state.py `keep`), so a later admission merging onto the main +commit that build sat on recompiles only its own diff. GitHub hands a +root-label job to any free root runner, though, so that admission usually +lands on another Mac and compiles from a seed instead. ci-macos.yml compile +admission uploads `owned-warm-keys--` on an owned Mac: the +output of `owned_build_state.py warm-keys`, + + {"runner": "", "pool": "glaeda-root-std-xcode-26.6", + "keys": ["", ...]} + +with the kept build's merge base first. ci-owned-warm-labels.yml runs this +script from main when that CI run completes. It gives the runner that ran +admission one `glaeda-warm-` label per key (at most MAX_KEYS), removes +its other `glaeda-warm-` labels, and removes its keys from every other runner +of the same root pool, so one runner per pool carries each key. The picker +(pr_runner_pool.py, warm affinity) then sends an admission whose merge base +has a label to that runner when it is idle. + +The runner is the one the jobs API says ran admission, never the name in the +artifact, which the pull request's own code wrote: an artifact naming another +runner, or another pool than the runner's root label, changes nothing. A key +that is not 12 hex digits is dropped. A wrong key only sends an admission to a +Mac whose build is further away, which compiles as it would elsewhere. + +Needs GH_TOKEN (actions: read, for the run's jobs) and ROUTE_TOKEN (the org +route App with administration: write, for the runner labels). +""" +from __future__ import annotations + +import dataclasses +import json +import os +import sys +import urllib.error +import urllib.parse +import urllib.request +from collections.abc import Mapping, Sequence +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +from pr_runner_pool import ROOT_PREFIX, WARM_KEY, WARM_PREFIX, persistent # noqa: E402 + +CI_WORKFLOW_PATH = ".github/workflows/ci.yml" +# ci-macos.yml's compile admission job; the jobs API prefixes the caller's job name. +ADMISSION_JOB = "macOS compile admission" +# Labels one runner carries at most: its kept build's merge base and a few +# commits past it that still build cheaply from it. +MAX_KEYS = 4 +PAGE_SIZE = 100 +API = "https://api.github.com" + + +def keys(document: Any) -> list[str]: + """The artifact's valid keys, deduplicated in order, at most MAX_KEYS.""" + raw = document.get("keys") if isinstance(document, Mapping) else None + found: list[str] = [] + for key in raw if isinstance(raw, list) else []: + key = str(key).strip().lower()[:12] + if WARM_KEY.fullmatch(key) and key not in found: + found.append(key) + return found[:MAX_KEYS] + + +def label_names(runner: Mapping[str, Any]) -> set[str]: + return {str(item.get("name")) for item in runner.get("labels") or [] if isinstance(item, Mapping)} + + +def root_pool(runner: Mapping[str, Any]) -> str: + """The runner's root label (glaeda-root--xcode-), or "".""" + return next((name for name in sorted(label_names(runner)) + if name.startswith(ROOT_PREFIX) and persistent(name)), "") + + +@dataclasses.dataclass(frozen=True) +class Change: + runner_id: int + add: tuple[str, ...] = () + remove: tuple[str, ...] = () + + +def plan(runners: Sequence[Mapping[str, Any]], runner_id: int, wanted: Sequence[str]) -> tuple[list[Change], str]: + """The label changes that make `runner_id` the one runner of its pool warm for `wanted`. + + Other runners of the pool lose those keys first, then the runner drops its + stale warm labels and gains the new ones. Returns the changes and why. + """ + target = next((runner for runner in runners if runner.get("id") == runner_id), None) + if target is None: + return [], f"runner {runner_id} is not registered here" + pool = root_pool(target) + if not pool: + return [], f"runner {target.get('name')} carries no root label" + labels = [WARM_PREFIX + key for key in wanted] + changes = [] + for runner in runners: + if runner is target or pool not in label_names(runner): + continue + taken = tuple(label for label in labels if label in label_names(runner)) + if taken: + changes.append(Change(int(runner["id"]), remove=taken)) + have = label_names(target) + add = tuple(label for label in labels if label not in have) + stale = tuple(sorted(name for name in have if name.startswith(WARM_PREFIX) and name not in labels)) + if add or stale: + changes.append(Change(runner_id, add=add, remove=stale)) + return changes, f"{target.get('name')} in {pool}: {', '.join(labels) or 'no warm keys'}" + + +def admission_job(jobs: Sequence[Mapping[str, Any]]) -> Mapping[str, Any] | None: + """The run's compile admission job that ran on a runner, or None.""" + for job in jobs: + name = str(job.get("name") or "") + if (name == ADMISSION_JOB or name.endswith(" / " + ADMISSION_JOB)) and job.get("runner_id"): + return job + return None + + +class GitHub: + def __init__(self, token: str, repo: str) -> None: + self.repo = repo + self.headers = { + "Accept": "application/vnd.github+json", + "Authorization": f"Bearer {token}", + "X-GitHub-Api-Version": "2022-11-28", + "User-Agent": "cmux-ci-owned-warm-labels", + } + + def request(self, method: str, path: str, body: Any = None) -> Any: + data = None if body is None else json.dumps(body).encode() + headers = {**self.headers, **({"Content-Type": "application/json"} if data else {})} + request = urllib.request.Request(f"{API}/repos/{self.repo}{path}", data=data, headers=headers, method=method) + with urllib.request.urlopen(request, timeout=15) as response: + payload = response.read() + return json.loads(payload) if payload else None + + def jobs(self, run_id: int, attempt: int) -> list[Mapping[str, Any]]: + found: list[Mapping[str, Any]] = [] + for page in range(1, 4): + batch = self.request("GET", f"/actions/runs/{run_id}/attempts/{attempt}/jobs" + f"?per_page={PAGE_SIZE}&page={page}").get("jobs") or [] + found.extend(job for job in batch if isinstance(job, Mapping)) + if len(batch) < PAGE_SIZE: + break + return found + + def runners(self) -> list[Mapping[str, Any]]: + found: list[Mapping[str, Any]] = [] + for page in range(1, 6): + batch = self.request("GET", f"/actions/runners?per_page={PAGE_SIZE}&page={page}").get("runners") or [] + found.extend(runner for runner in batch if isinstance(runner, Mapping)) + if len(batch) < PAGE_SIZE: + break + return found + + def apply(self, change: Change) -> None: + for label in change.remove: + try: + self.request("DELETE", f"/actions/runners/{change.runner_id}/labels/{urllib.parse.quote(label, safe='')}") + except urllib.error.HTTPError as error: + if error.code != 404: # already gone + raise + if change.add: + self.request("POST", f"/actions/runners/{change.runner_id}/labels", {"labels": list(change.add)}) + + +def run(env: Mapping[str, str], actions: GitHub, admin: GitHub) -> str: + """Label the runner that ran this run's admission; returns what it did.""" + if (env.get("RUN_PATH") or "").strip() != CI_WORKFLOW_PATH: + return f"not a run of {CI_WORKFLOW_PATH}" + try: + document = json.loads(Path(env["KEYS_FILE"]).read_text(encoding="utf-8")) + except (KeyError, OSError, ValueError) as error: + return f"no readable warm keys ({error})" + if not isinstance(document, Mapping): + return "the warm keys are not a JSON object" + wanted = keys(document) + job = admission_job(actions.jobs(int(env["RUN_ID"]), int(env.get("RUN_ATTEMPT") or 1))) + if job is None: + return "no compile admission job ran on a runner" + if str(document.get("runner") or "") != str(job.get("runner_name") or ""): + return f"the keys name runner {document.get('runner')!r}, but admission ran on {job.get('runner_name')!r}" + runners = admin.runners() + target = next((runner for runner in runners if runner.get("id") == job["runner_id"]), {}) + if document.get("pool") and document["pool"] != root_pool(target): + return f"the keys name pool {document['pool']!r}, but the runner's root label is {root_pool(target)!r}" + changes, why = plan(runners, int(job["runner_id"]), wanted) + for change in changes: + admin.apply(change) + return f"{why} ({len(changes)} runner(s) changed)" + + +def main(env: Mapping[str, str] | None = None) -> int: + env = os.environ if env is None else env + repo = env.get("GITHUB_REPOSITORY") or "" + token, route_token = env.get("GH_TOKEN") or "", env.get("ROUTE_TOKEN") or "" + if not (repo and token and route_token and (env.get("RUN_ID") or "").isdigit()): + print("GITHUB_REPOSITORY, GH_TOKEN, ROUTE_TOKEN and RUN_ID are required; nothing labeled") + return 0 + try: + result = run(env, GitHub(token, repo), GitHub(route_token, repo)) + except (urllib.error.URLError, OSError, ValueError, KeyError) as error: + # Labels are only a routing hint: a failure leaves the old ones. + print(f"::warning title=owned warm labels::{error}") + return 0 + print(result) + if env.get("GITHUB_STEP_SUMMARY"): + with open(env["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as handle: + handle.write(f"### Owned warm labels\n\n{result}\n") + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/scripts/ci/pr_runner_pool.py b/scripts/ci/pr_runner_pool.py index 841579b82248..ecce8e4b0c87 100644 --- a/scripts/ci/pr_runner_pool.py +++ b/scripts/ci/pr_runner_pool.py @@ -101,6 +101,19 @@ pool label either way. A root job also holds one of the pool's machines, so it counts against both. +Warm affinity: an owned Mac keeps compile admission's DerivedData +(owned_build_state.py), and ci-owned-warm-labels.yml labels its root runner +`glaeda-warm-` for each main commit that build starts from cheaply +(owned_warm_labels.py). When admission is placed on a pool with a root count +and the runners were read live, the picker looks for an idle root runner of +that pool carrying the label of this run's merge base (MERGED_ONTO, the merge +commit's first parent) and, if one does, writes `admission_runner`, the JSON +array `["", "glaeda-warm-"]`, which admission's attempt 1 +takes as its runs-on. Otherwise it is empty and admission takes the root +label. The match is exact: v1 does not rank runners by commit distance. A +warm runner taken between the pick and the queue leaves admission waiting on +the label, and ci-owned-pool-rescue.yml moves it to Blacksmith. + GUI jobs (app-host shards, tests-build-and-lag) take an owned pool unless `vars.CI_PR_POOL_OWNED_GUI == '0'`: the minis' runners are LaunchAgents in the logged-in user's Aqua session, and each mini runs one job at a time. With @@ -183,6 +196,9 @@ # pools: slots() leaves them out, and capability_slots() reads their count # (machines, one simulator job each) from CI_OWNED_POOL_SLOTS. CAPABILITY_LABELS = ("glaeda-ios-sim",) +# `glaeda-warm-`: a root runner whose kept build starts from that main commit. +WARM_PREFIX = "glaeda-warm-" +WARM_KEY = re.compile(r"[0-9a-f]{12}") XCODE_APP = re.compile(r"/Xcode_([0-9]+(?:\.[0-9]+)*)\.app/?") PR_XCODE_VARIABLE = "CMUX_CI_XCODE_APP_PR" OWNED_VARIABLE = "CI_PR_POOL_OWNED" @@ -672,6 +688,26 @@ def live_owned_free(runners: Sequence[Mapping[str, Any]], labels: Sequence[str]) return free +def warm_label(commit: str | None) -> str: + """The warm label for a commit (its first 12 hex digits), or "" for anything else.""" + key = (commit or "").strip().lower()[:12] + return WARM_PREFIX + key if WARM_KEY.fullmatch(key) else "" + + +def warm_admission_runner(runners: Sequence[Mapping[str, Any]], root: str, merged_onto: str | None) -> str: + """Admission's runs-on labels as JSON when an idle `root` runner is warm for `merged_onto`, else "".""" + warm = warm_label(merged_onto) + if not warm or not root.startswith(ROOT_PREFIX): + return "" + for runner in runners: + if runner.get("status") != "online" or runner.get("busy"): + continue + names = {str(item.get("name")) for item in runner.get("labels") or [] if isinstance(item, Mapping)} + if root in names and warm in names: + return json.dumps([root, warm], separators=(",", ":")) + return "" + + def pick(load: Mapping[str, Mapping[str, int]], added: Mapping[str, int], usable: Sequence[str], max_queued: int, jobs: int = MAX_RUN_JOBS, taken: Mapping[str, int] | None = None, split: bool = False, @@ -1159,7 +1195,7 @@ def pull_request_runs_since(self, since: str, *, exclude_run_id: int | None) -> def summary(choice: Choice, snapshot: Mapping[str, Any] | None, *, now: dt.datetime, owned_slots: Mapping[str, int] | None = None, problems: Sequence[str] = (), - owned_jobs: Sequence[str] = ()) -> str: + owned_jobs: Sequence[str] = (), admission_runner: str = "") -> str: runner = choice.runner or "each job's default (MACOS_RUNNER_PR or its fallback)" lines = ["### macOS pool for this run", "", f"- Pool: `{runner}`", f"- Why: {choice.reason}"] if choice.xcode_app: @@ -1169,6 +1205,9 @@ def summary(choice: Choice, snapshot: Mapping[str, Any] | None, *, now: dt.datet f"and a re-run of failed jobs, goes to: `{choice.retry_runner}`") if choice.root_runner: lines.append(f"- Root jobs among them ({ROOT_JOBS}) take `{choice.root_runner}`") + if admission_runner: + labels = " + ".join(f"`{label}`" for label in json.loads(admission_runner)) + lines.append(f"- Compile admission takes {labels}: an idle root runner kept a build of this run's merge base") for problem in problems: lines.append(f"- **Error:** {problem}; that pool gets no machines") if isinstance(snapshot, Mapping) and isinstance(snapshot.get("pools"), Mapping): @@ -1224,16 +1263,18 @@ def count_routed(since: str) -> int: # only) reads which owned runners are idle now. Without it, or on any # error, the slot counts and the snapshot decide as before. live_owned = None + live_runners: list[Mapping[str, Any]] | None = None route_token = (env.get("ROUTE_TOKEN") or "").strip() if route_token and repo and not args.snapshot and (env.get("POOL_OWNED") or "").strip() == "1": try: labels = owned_pools(env.get(PR_XCODE_VARIABLE)) # Each pool's root runners too; choose() keeps those with a root count. labels += tuple(root_label(label) for label in labels) - live_owned = live_owned_free(GitHub(route_token, repo).runners(), labels) if labels else None + live_runners = GitHub(route_token, repo).runners() if labels else None + live_owned = live_owned_free(live_runners, labels) if live_runners is not None else None except Exception as error: # noqa: BLE001 - the snapshot path still decides print(f"::warning title=live owned capacity::could not list runners ({error}); using the snapshot") - live_owned = None + live_owned = live_runners = None choice, snapshot = choose( event=env.get("EVENT_NAME") or "", repo=repo, @@ -1270,8 +1311,16 @@ def count_routed(since: str) -> int: # run takes retry_runner. The marker's jobs are the owned machines held. owned_jobs, held = (place(plan, choice.owned_budget, gui, choice.root_budget if choice.root_runner else None) if persistent(choice.runner) else ((), plan.peak)) + # Admission on a root runner whose kept build is of this run's merge base + # (see "Warm affinity" above); attempt 1 only, since only it is placed. + admission_runner = "" + # CI_OWNED_WARM_LABELS off ignores labels already set, so the switch alone + # turns affinity off without clearing them. + if (env.get("WARM_LABELS") == "1" and choice.root_runner and ADMISSION_JOB in owned_jobs + and live_runners is not None): + admission_runner = warm_admission_runner(live_runners, choice.root_runner, env.get("MERGED_ONTO")) text = summary(choice, snapshot, now=now, owned_slots=slots(env.get("OWNED_SLOTS"), pr_xcode_app), problems=problems, - owned_jobs=owned_jobs) + owned_jobs=owned_jobs, admission_runner=admission_runner) print(text) if env.get("GITHUB_STEP_SUMMARY"): with open(env["GITHUB_STEP_SUMMARY"], "a", encoding="utf-8") as handle: @@ -1288,6 +1337,9 @@ def count_routed(since: str) -> int: # What the root jobs in owned_jobs take instead of # the pool label, on attempt 1 and on that attempt 2. f"root_runner={choice.root_runner}\n" + # JSON labels for admission's attempt 1: the root label + # and the warm label of this run's merge base, or "". + f"admission_runner={admission_runner}\n" # Space-delimited with a space at each end, so each job's # contains(' ') test matches whole keys only. f"owned_jobs={' ' + ' '.join(owned_jobs) + ' ' if owned_jobs else ''}\n") diff --git a/scripts/ci/product_input_identity.py b/scripts/ci/product_input_identity.py index a3a4c2e33372..d048d108284d 100644 --- a/scripts/ci/product_input_identity.py +++ b/scripts/ci/product_input_identity.py @@ -166,6 +166,9 @@ def profile_schemes(name: str | None = None) -> tuple[str, ...]: "Record this owned Mac's build inputs", "Keep this owned Mac's DerivedData", "Keep this owned Mac's build state", + # What the kept DerivedData starts from, for the warm runner labels. + "List the commits this owned Mac starts from warm", + "Upload the owned Mac's warm keys", "Validate Swift warning budget", "Run early CLI binary smoke checks", "Start product publication timer", diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 853de55d450f..153d2dad8d30 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -431,6 +431,10 @@ lane = "linux-guard" path = "tests/test_ci_owned_pool_rescue.py" lane = "linux-guard" +[[test]] +path = "tests/test_ci_owned_warm_labels.py" +lane = "linux-guard" + [[test]] path = "tests/test_ci_pr_runner_pool.py" lane = "linux-guard" diff --git a/tests/test_ci_change_areas.py b/tests/test_ci_change_areas.py index 279c6503ca89..883725609524 100755 --- a/tests/test_ci_change_areas.py +++ b/tests/test_ci_change_areas.py @@ -4518,6 +4518,15 @@ def product_runner_output(key: str) -> str: PRODUCT_RUNNER_OUTPUT = product_runner_output(PRODUCT_RUNNER_KEYS["app-host-unit-tests"]) +# Attempt 1 of compile admission may take the warm labels pr_admission_runner +# carries (pr_runner_pool.py, warm affinity), a JSON array; CMUX_PRODUCT_RUNNER +# restates the first, the root label, which the consumers take. +WARM_ADMISSION = "github.run_attempt == 1 && inputs.pr_admission_runner && fromJSON(inputs.pr_admission_runner)" + + +def admission_route(runs_on: str) -> str: + """Compile admission's runs-on without its warm labels: the route its consumers restate.""" + return runs_on.replace(WARM_ADMISSION + " || ", "") PRODUCT_XCODE_OUTPUT = "${{ needs.macos-compile-admission.outputs.xcode_app }}" @@ -4531,7 +4540,8 @@ def product_consumer_route_violations(workflow: dict) -> list[str]: """ producer = workflow["jobs"]["macos-compile-admission"] violations = [] - if producer["env"].get("CMUX_PRODUCT_RUNNER") != producer["runs-on"]: + if (producer["env"].get("CMUX_PRODUCT_RUNNER") or "").replace(WARM_ADMISSION + "[0]", WARM_ADMISSION) \ + != producer["runs-on"]: violations.append("macos-compile-admission: CMUX_PRODUCT_RUNNER does not restate runs-on") outputs = producer.get("outputs", {}) if outputs.get("runner") != "${{ env.CMUX_PRODUCT_RUNNER }}": @@ -4549,7 +4559,7 @@ def product_consumer_route_violations(workflow: dict) -> list[str]: # Its own owned_jobs key, so it never follows admission's placement. and "' lag '" in runs_on and runs_on.replace("vars.MACOS_RUNNER_DISPLAY", "vars.MACOS_RUNNER_15").replace( - "' lag '", "' admission '") == producer["runs-on"] + "' lag '", "' admission '") == admission_route(producer["runs-on"]) and xcode == producer["env"]["CMUX_CI_XCODE_APP"] ): continue @@ -4586,7 +4596,7 @@ def test_product_consumer_guard_follows_a_new_admission_route() -> None: for key in ("CMUX_PRODUCT_RUNNER", "CMUX_CI_XCODE_APP"): assert pull_request in producer["env"][key], key producer["env"][key] = producer["env"][key].replace(pull_request, new_route, 1) - producer["runs-on"] = producer["env"]["CMUX_PRODUCT_RUNNER"] + producer["runs-on"] = producer["env"]["CMUX_PRODUCT_RUNNER"].replace(WARM_ADMISSION + "[0]", WARM_ADMISSION) violations = product_consumer_route_violations(workflow) assert [line.split(":", 1)[0] for line in violations] == ["tests-build-and-lag"], violations diff --git a/tests/test_ci_owned_build_state.py b/tests/test_ci_owned_build_state.py index 7ebb6166a49c..1af091efd586 100644 --- a/tests/test_ci_owned_build_state.py +++ b/tests/test_ci_owned_build_state.py @@ -401,7 +401,8 @@ def test_each_workflow_call_is_one_the_script_accepts(self): workflow = yaml.safe_load((ROOT / ".github/workflows/ci-macos.yml").read_text()) steps = workflow["jobs"]["macos-compile-admission"]["steps"] calls = [step for step in steps if "owned_build_state.py" in str(step.get("run", ""))] - self.assertEqual(len(calls), 6) + # check, prefer, adopt, record, keep, warm-keys (skipped until the script has it), save. + self.assertEqual(len(calls), 7) with tempfile.TemporaryDirectory() as tmp: base = Path(tmp) (base / "derived").mkdir() @@ -481,7 +482,8 @@ def test_the_product_key_does_not_see_owned_state(self): for name in ("Reuse this owned Mac's build state", "Prefer a near seed over this owned Mac's DerivedData", "Adopt this owned Mac's DerivedData", "Record this owned Mac's build inputs", "Keep this owned Mac's DerivedData", - "Keep this owned Mac's build state"): + "Keep this owned Mac's build state", "List the commits this owned Mac starts from warm", + "Upload the owned Mac's warm keys"): self.assertIn(name, identity.NON_PRODUCT_RECIPE_STEPS) steps = identity.recipe_projection(text)["steps"] for name, block in steps.items(): diff --git a/tests/test_ci_owned_warm_labels.py b/tests/test_ci_owned_warm_labels.py new file mode 100644 index 000000000000..75a8396ed276 --- /dev/null +++ b/tests/test_ci_owned_warm_labels.py @@ -0,0 +1,195 @@ +#!/usr/bin/env python3 +"""Tests for scripts/ci/owned_warm_labels.py (no network: HTTP is mocked).""" + +from __future__ import annotations + +import importlib.util +import io +import json +import sys +import tempfile +import unittest +import unittest.mock +import urllib.error +from pathlib import Path + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts/ci")) + + +def load(name: str, path: Path): + spec = importlib.util.spec_from_file_location(name, path) + module = importlib.util.module_from_spec(spec) + assert spec.loader is not None + sys.modules[name] = module + spec.loader.exec_module(module) + return module + + +labels = load("owned_warm_labels", ROOT / "scripts/ci/owned_warm_labels.py") + +MINI = "glaeda-std-xcode-26.6" +ROOT_STD = "glaeda-root-std-xcode-26.6" +ROOT_LIGHT = "glaeda-root-light-xcode-26.6" +A, B, C = "aaaaaaaaaaaa", "bbbbbbbbbbbb", "cccccccccccc" + + +def warm(key: str) -> str: + return "glaeda-warm-" + key + + +def runner(runner_id, *names, name=None): + return {"id": runner_id, "name": name or f"cmux{runner_id}", "status": "online", "busy": False, + "labels": [{"name": label, "type": "custom"} for label in names]} + + +class Response(io.BytesIO): + def __enter__(self): + return self + + def __exit__(self, *args): + return False + + +class FakeHTTP: + """Answers urllib.request.urlopen for the jobs, runners and label endpoints.""" + + def __init__(self, jobs, runners): + self.jobs, self.runners, self.calls = jobs, runners, [] + + def __call__(self, request, timeout=None): + method, url = request.get_method(), request.full_url + body = json.loads(request.data) if request.data else None + self.calls.append((method, url.removeprefix("https://api.github.com/repos/manaflow-ai/cmux"), body)) + if "/jobs?" in url: + return Response(json.dumps({"jobs": self.jobs}).encode()) + if url.split("?")[0].endswith("/actions/runners"): + return Response(json.dumps({"runners": self.runners}).encode()) + if method == "DELETE" and "glaeda-warm-dddddddddddd" in url: + raise urllib.error.HTTPError(url, 404, "not found", {}, None) + return Response(b"") + + def writes(self): + return [(method, path, body) for method, path, body in self.calls if method != "GET"] + + +class Plan(unittest.TestCase): + def test_keys_are_validated_deduplicated_and_capped(self): + document = {"keys": [A.upper(), A, "nope", "../../x", B + "ffff", C, "dddddddddddd", "eeeeeeeeeeee", 7]} + self.assertEqual(labels.keys(document), [A, B, C, "dddddddddddd"]) + self.assertEqual(len(labels.keys(document)), labels.MAX_KEYS) + self.assertEqual(labels.keys({"keys": "aaaaaaaaaaaa"}), []) + self.assertEqual(labels.keys(["aaaaaaaaaaaa"]), []) + + def test_the_runner_takes_its_keys_and_the_rest_of_its_pool_loses_them(self): + runners = [runner(1, MINI, ROOT_STD, warm(C)), + runner(2, MINI, ROOT_STD, warm(A)), + # Another pool keeps its own copy of a key. + runner(3, "glaeda-light-xcode-26.6", ROOT_LIGHT, warm(A)), + runner(4, MINI, ROOT_STD, warm(B), warm(C))] + changes, why = labels.plan(runners, 1, [A, B]) + self.assertEqual(changes, [labels.Change(2, remove=(warm(A),)), + labels.Change(4, remove=(warm(B),)), + labels.Change(1, add=(warm(A), warm(B)), remove=(warm(C),))]) + self.assertIn(ROOT_STD, why) + + def test_nothing_changes_for_a_labeled_runner_or_one_without_a_root_label(self): + runners = [runner(1, MINI, ROOT_STD, warm(A))] + self.assertEqual(labels.plan(runners, 1, [A])[0], []) + self.assertEqual(labels.plan([runner(1, MINI)], 1, [A])[0], []) + self.assertEqual(labels.plan(runners, 9, [A])[0], []) + # No keys: the runner drops the ones it had. + self.assertEqual(labels.plan(runners, 1, [])[0], [labels.Change(1, remove=(warm(A),))]) + + def test_the_admission_job_is_found_under_its_caller(self): + jobs = [{"name": "macOS / app-host unit tests (1)", "runner_id": 5}, + {"name": "macOS / macOS compile admission", "runner_id": 0}, + {"name": "macOS / macOS compile admission", "runner_id": 7, "runner_name": "cmux7"}] + self.assertEqual(labels.admission_job(jobs)["runner_id"], 7) + self.assertIsNone(labels.admission_job(jobs[:2])) + + +class Run(unittest.TestCase): + def run_script(self, document, *, jobs=None, runners=None, path=".github/workflows/ci.yml"): + jobs = jobs if jobs is not None else [ + {"name": "macOS / macOS compile admission", "runner_id": 1, "runner_name": "cmux1"}] + runners = runners if runners is not None else [ + runner(1, MINI, ROOT_STD, warm("dddddddddddd")), runner(2, MINI, ROOT_STD, warm(A))] + http = FakeHTTP(jobs, runners) + with tempfile.TemporaryDirectory() as tmp, \ + unittest.mock.patch.object(labels.urllib.request, "urlopen", side_effect=http), \ + unittest.mock.patch("sys.stdout", io.StringIO()) as stdout: + keys_file = Path(tmp, "warm-keys.json") + keys_file.write_text(document if isinstance(document, str) else json.dumps(document)) + env = {"GITHUB_REPOSITORY": "manaflow-ai/cmux", "GH_TOKEN": "actions-token", "ROUTE_TOKEN": "app-token", + "RUN_ID": "42", "RUN_ATTEMPT": "2", "RUN_PATH": path, "KEYS_FILE": str(keys_file)} + self.assertEqual(labels.main(env), 0) + return http, stdout.getvalue() + + def test_labels_the_runner_that_ran_admission(self): + http, output = self.run_script({"runner": "cmux1", "pool": ROOT_STD, "keys": [A, B]}) + self.assertIn(("GET", "/actions/runs/42/attempts/2/jobs?per_page=100&page=1", None), http.calls) + self.assertEqual(http.writes(), [ + ("DELETE", f"/actions/runners/2/labels/{warm(A)}", None), + # Already gone (404) is fine. + ("DELETE", "/actions/runners/1/labels/glaeda-warm-dddddddddddd", None), + ("POST", "/actions/runners/1/labels", {"labels": [warm(A), warm(B)]}), + ]) + self.assertIn("2 runner(s) changed", output) + + def test_the_jobs_listing_uses_the_actions_token_and_the_labels_the_apps(self): + seen = [] + http = FakeHTTP([{"name": "macOS / macOS compile admission", "runner_id": 1, "runner_name": "cmux1"}], + [runner(1, MINI, ROOT_STD)]) + + def urlopen(request, timeout=None): + seen.append((request.get_method(), request.full_url.split("?")[0].rsplit("/", 1)[-1], + request.get_header("Authorization"))) + return http(request, timeout) + + with tempfile.TemporaryDirectory() as tmp, \ + unittest.mock.patch.object(labels.urllib.request, "urlopen", side_effect=urlopen), \ + unittest.mock.patch("sys.stdout", io.StringIO()): + keys_file = Path(tmp, "warm-keys.json") + keys_file.write_text(json.dumps({"runner": "cmux1", "keys": [A]})) + labels.main({"GITHUB_REPOSITORY": "manaflow-ai/cmux", "GH_TOKEN": "actions-token", + "ROUTE_TOKEN": "app-token", "RUN_ID": "42", "RUN_PATH": ".github/workflows/ci.yml", + "KEYS_FILE": str(keys_file)}) + self.assertEqual(seen, [("GET", "jobs", "Bearer actions-token"), ("GET", "runners", "Bearer app-token"), + ("POST", "labels", "Bearer app-token")]) + + def test_an_artifact_naming_another_runner_or_pool_changes_nothing(self): + for document in ({"runner": "cmux2", "pool": ROOT_STD, "keys": [A]}, + {"runner": "cmux1", "pool": ROOT_LIGHT, "keys": [A]}, + ["not", "an", "object"], + "{not json"): + http, _ = self.run_script(document) + self.assertEqual(http.writes(), [], document) + + def test_other_runs_change_nothing(self): + http, output = self.run_script({"runner": "cmux1", "keys": [A]}, path=".github/workflows/other.yml") + self.assertEqual(http.calls, []) + http, output = self.run_script({"runner": "cmux1", "keys": [A]}, jobs=[]) + self.assertEqual(http.writes(), []) + self.assertIn("no compile admission job", output) + + def test_an_api_failure_warns_and_never_fails_the_job(self): + def broken(request, timeout=None): + raise urllib.error.HTTPError(request.full_url, 403, "forbidden", {}, None) + + with tempfile.TemporaryDirectory() as tmp, \ + unittest.mock.patch.object(labels.urllib.request, "urlopen", side_effect=broken), \ + unittest.mock.patch("sys.stdout", io.StringIO()) as stdout: + keys_file = Path(tmp, "warm-keys.json") + keys_file.write_text(json.dumps({"runner": "cmux1", "keys": [A]})) + self.assertEqual(labels.main({"GITHUB_REPOSITORY": "manaflow-ai/cmux", "GH_TOKEN": "t", + "ROUTE_TOKEN": "a", "RUN_ID": "42", + "RUN_PATH": ".github/workflows/ci.yml", "KEYS_FILE": str(keys_file)}), 0) + self.assertIn("::warning title=owned warm labels::", stdout.getvalue()) + # Missing tokens: nothing is requested. + with unittest.mock.patch("sys.stdout", io.StringIO()): + self.assertEqual(labels.main({"GITHUB_REPOSITORY": "manaflow-ai/cmux", "RUN_ID": "42"}), 0) + + +if __name__ == "__main__": + unittest.main(verbosity=2) diff --git a/tests/test_ci_pr_runner_pool.py b/tests/test_ci_pr_runner_pool.py index 2b307b7336eb..a7d909efefe0 100644 --- a/tests/test_ci_pr_runner_pool.py +++ b/tests/test_ci_pr_runner_pool.py @@ -287,7 +287,7 @@ def test_main_writes_outputs_and_summary(self): sys.stdout = old self.assertEqual(out.read_text(), f"runner={LARGE}\nxcode_app=\npersistent=false\n" f"retry_runner=\njobs={pool.MAX_RUN_JOBS}\nshard_runner=\n" - f"refused_retry_runner=\nroot_runner=\nowned_jobs=\n") + f"refused_retry_runner=\nroot_runner=\nadmission_runner=\nowned_jobs=\n") text = summary.read_text() self.assertIn(f"Pool: `{LARGE}`", text) self.assertIn(f"{SMALL}: 21 queued, 10 running", text) @@ -456,6 +456,18 @@ def root_lane(key: str) -> str: "|| inputs.pr_root_runner || inputs.pr_runner || vars.MACOS_RUNNER_PR || 'blacksmith-6vcpu-macos-15'") +def warm_lane(index: str = "") -> str: + """root_lane() for compile admission: attempt 1 may take the warm labels first. + + runs-on reads the JSON array itself; CMUX_PRODUCT_RUNNER (index "[0]") + names its first label, the root label. + """ + return root_lane("' admission '").replace( + "&& inputs.pr_retry_runner || inputs.pr_root_runner", + "&& inputs.pr_retry_runner || github.run_attempt == 1 && inputs.pr_admission_runner " + f"&& fromJSON(inputs.pr_admission_runner){index} || inputs.pr_root_runner") + + PR_XCODE = "/Applications/Xcode_26.6.app" MINI = "glaeda-std-xcode-26.6" LIGHT = "glaeda-light-xcode-26.6" @@ -1157,6 +1169,83 @@ def test_e2e_takes_the_root_label(self): owned_slots={MINI: 40, ROOT_MINI: 10}).startswith("glaeda-")) +WARM = "glaeda-warm-0123456789ab" +MERGE_BASE = "0123456789ab" + "c" * 28 + + +def live_runner(runner_id, *labels, busy=False, status="online"): + return {"id": runner_id, "name": f"cmux{runner_id}", "status": status, "busy": busy, + "labels": [{"name": name} for name in labels]} + + +class WarmAffinity(unittest.TestCase): + """Compile admission goes to the idle root runner that kept a build of its merge base.""" + + def test_warm_label_takes_twelve_hex_digits(self): + self.assertEqual(pool.warm_label(MERGE_BASE), WARM) + self.assertEqual(pool.warm_label(MERGE_BASE.upper()), WARM) + for commit in ("", None, "0123456789a", "not-a-commit-sha", "../../etc/passwd"): + self.assertEqual(pool.warm_label(commit), "", commit) + + def test_only_an_idle_root_runner_carrying_the_label_is_picked(self): + expected = json.dumps([ROOT_MINI, WARM], separators=(",", ":")) + idle = live_runner(1, MINI, ROOT_MINI, WARM) + self.assertEqual(pool.warm_admission_runner([idle], ROOT_MINI, MERGE_BASE), expected) + self.assertEqual(json.loads(expected), [ROOT_MINI, WARM]) + for runners in ([live_runner(1, MINI, ROOT_MINI, WARM, busy=True)], + [live_runner(1, MINI, ROOT_MINI, WARM, status="offline")], + # The label on a runner of another root pool, or on no root runner. + [live_runner(1, LIGHT, "glaeda-root-light-xcode-26.6", WARM)], + [live_runner(1, MINI, WARM)], + [live_runner(1, MINI, ROOT_MINI, "glaeda-warm-ffffffffffff")]): + self.assertEqual(pool.warm_admission_runner(runners, ROOT_MINI, MERGE_BASE), "", runners) + # No merge base, or no root label: nothing. + self.assertEqual(pool.warm_admission_runner([idle], ROOT_MINI, ""), "") + self.assertEqual(pool.warm_admission_runner([idle], MINI, MERGE_BASE), "") + # A busy warm runner is skipped for an idle one. + both = [live_runner(1, MINI, ROOT_MINI, WARM, busy=True), live_runner(2, MINI, ROOT_MINI, WARM)] + self.assertEqual(pool.warm_admission_runner(both, ROOT_MINI, MERGE_BASE), expected) + + def outputs(self, runners, *, merged_onto=MERGE_BASE, slots='{"std": 40, "root-std": 10}', token="app-token", + warm_labels="1"): + fresh = fleet(busy=0) + fresh["generated_at"] = dt.datetime.now(dt.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ") + with tempfile.TemporaryDirectory() as tmp, \ + unittest.mock.patch.object(pool.GitHub, "snapshot", return_value=fresh), \ + unittest.mock.patch.object(pool.GitHub, "pull_request_routes_since", return_value=pool.Routed()), \ + unittest.mock.patch.object(pool.GitHub, "runners", return_value=runners), \ + unittest.mock.patch("sys.stdout", io.StringIO()): + out, summary = Path(tmp, "out"), Path(tmp, "summary") + env = {"EVENT_NAME": "pull_request", "GITHUB_REPOSITORY": "manaflow-ai/cmux", "GH_TOKEN": "t", + "HEAD_REPO": "manaflow-ai/cmux", "DEFAULT_RUNNER": SMALL, "POOL_OWNED": "1", + "POOL_OWNED_SPLIT": "1", "OWNED_SLOTS": slots, "ROUTE_TOKEN": token, + "CMUX_CI_XCODE_APP_PR": PR_XCODE, "CMUX_CI_XCODE_APP_MACOS_15": XCODE_15, + "GITHUB_RUN_ATTEMPT": "1", "GITHUB_OUTPUT": str(out), "GITHUB_STEP_SUMMARY": str(summary), + "RUN_MACOS": "true", "MERGED_ONTO": merged_onto, "WARM_LABELS": warm_labels} + pool.main([], env) + values = dict(line.split("=", 1) for line in out.read_text().splitlines()) + values["summary"] = summary.read_text() + return values + + def test_main_names_the_warm_runner_for_admission(self): + runners = [live_runner(1, MINI, ROOT_MINI), live_runner(2, MINI, ROOT_MINI, WARM), live_runner(3, MINI)] + values = self.outputs(runners) + self.assertEqual((values["runner"], values["root_runner"]), (MINI, ROOT_MINI)) + self.assertIn(" admission ", values["owned_jobs"]) + self.assertEqual(json.loads(values["admission_runner"]), [ROOT_MINI, WARM]) + self.assertIn(f"`{WARM}`", values["summary"]) + # Another merge base, a busy warm runner, or no root count: admission keeps the root label. + self.assertEqual(self.outputs(runners, merged_onto="f" * 40)["admission_runner"], "") + busy = [live_runner(1, MINI, ROOT_MINI), live_runner(2, MINI, ROOT_MINI, WARM, busy=True)] + self.assertEqual(self.outputs(busy)["admission_runner"], "") + self.assertEqual(self.outputs(runners, slots='{"std": 40}')["admission_runner"], "") + # Without the route token the runners are never read. + self.assertEqual(self.outputs(runners, token="")["admission_runner"], "") + # CI_OWNED_WARM_LABELS off ignores labels already on the runners. + for off in ("", "0"): + self.assertEqual(self.outputs(runners, warm_labels=off)["admission_runner"], "", off) + + class Wiring(unittest.TestCase): """Every pull-request macOS route in one CI run reads the one chosen pool.""" @@ -1202,7 +1291,7 @@ def test_every_pr_route_in_the_run_reads_the_choice(self): # Compile admission (and its CMUX_PRODUCT_RUNNER mirror) and # tests-build-and-lag each test their own owned_jobs key, and are # root jobs; the side lanes are not. - "ci-macos.yml": {root_lane("' admission '"), root_lane("' lag '")}, + "ci-macos.yml": {warm_lane(), warm_lane("[0]"), root_lane("' lag '")}, "remote-daemon.yml": {retry_lane("' remote-daemon '")}, } for name, lane in expected.items(): @@ -1237,6 +1326,9 @@ def test_callers_pass_the_choice(self): # Only ci-macos.yml runs root jobs; the side lanes keep the pool label. self.assertEqual(jobs["macos"]["with"]["pr_root_runner"], "${{ needs.changes.outputs.macos_pr_root_runner }}") self.assertNotIn("pr_root_runner", jobs["remote-daemon"]["with"]) + self.assertEqual(jobs["macos"]["with"]["pr_admission_runner"], + "${{ needs.changes.outputs.macos_pr_admission_runner }}") + self.assertNotIn("pr_admission_runner", jobs["remote-daemon"]["with"]) for name in ("macos", "remote-daemon", "claude-wrapper"): needs = jobs[name]["needs"] self.assertIn("changes", [needs] if isinstance(needs, str) else needs, name) @@ -1275,6 +1367,52 @@ def test_reusable_inputs_default_to_todays_route(self): self.assertEqual((inputs[key]["required"], inputs[key]["default"], inputs[key]["type"]), (False, "", "string"), (name, key)) + def test_warm_admission_is_attempt_one_only_and_reads_the_merge_base(self): + changes = self.workflow("ci.yml")["jobs"]["changes"] + self.assertEqual(changes["outputs"]["macos_pr_admission_runner"], + "${{ steps.macos-pool.outputs.admission_runner }}") + step = next(step for step in changes["steps"] if step.get("id") == "macos-pool") + self.assertEqual(step["env"]["MERGED_ONTO"], + "${{ steps.source-identity.outputs.parent1 || github.event.pull_request.base.sha }}") + inputs = self.workflow("ci-macos.yml")[True]["workflow_call"]["inputs"]["pr_admission_runner"] + self.assertEqual((inputs["required"], inputs["default"], inputs["type"]), (False, "", "string")) + admission = self.workflow("ci-macos.yml")["jobs"]["macos-compile-admission"] + self.assertIn("github.run_attempt == 1 && inputs.pr_admission_runner && fromJSON(inputs.pr_admission_runner) ||", + admission["runs-on"]) + # Only admission reads it; its consumers follow its root label. + text = (WORKFLOWS / "ci-macos.yml").read_text() + self.assertEqual(text.count("fromJSON(inputs.pr_admission_runner)"), 2) + + def test_admission_uploads_its_warm_keys_only_once_the_subcommand_exists(self): + steps = self.workflow("ci-macos.yml")["jobs"]["macos-compile-admission"]["steps"] + names = [step.get("name") for step in steps] + keep = names.index("Keep this owned Mac's DerivedData") + listed = steps[names.index("List the commits this owned Mac starts from warm")] + upload = steps[names.index("Upload the owned Mac's warm keys")] + self.assertLess(keep, names.index("List the commits this owned Mac starts from warm")) + self.assertIs(listed["continue-on-error"], True) + self.assertIs(upload["continue-on-error"], True) + self.assertIn("steps.owned-state.outputs.fingerprint != ''", listed["if"]) + self.assertIn('*"owned_build_state.py warm-keys"*', listed["run"]) + self.assertEqual(upload["with"]["name"], "owned-warm-keys-${{ github.run_id }}-${{ github.run_attempt }}") + self.assertIn("steps.owned-warm-keys.outputs.path != ''", upload["if"]) + + def test_the_labeler_runs_from_main_off_by_default(self): + labeler = self.workflow("ci-owned-warm-labels.yml") + self.assertEqual(labeler[True]["workflow_run"], {"workflows": ["CI"], "types": ["completed"]}) + self.assertEqual(labeler["permissions"], {}) + job = labeler["jobs"]["label"] + self.assertIn("vars.CI_OWNED_WARM_LABELS == '1'", job["if"]) + self.assertIn("vars.GLAEDA_ROUTE_APP_ID != ''", job["if"]) + self.assertIn("github.event.workflow_run.head_repository.full_name == github.repository", job["if"]) + self.assertEqual(job["runs-on"], "ubuntu-24.04") + steps = {step.get("id") or step.get("name"): step for step in job["steps"]} + self.assertEqual(steps["Checkout trusted labeler"]["with"]["ref"], "main") + self.assertEqual(steps["keys"]["with"]["name"], "owned-warm-keys-${{ github.event.workflow_run.id }}-" + "${{ github.event.workflow_run.run_attempt }}") + self.assertEqual(steps["route-token"]["with"]["permission-administration"], "write") + self.assertEqual(steps["Label the runner"]["run"], "python3 scripts/ci/owned_warm_labels.py") + def test_package_tests_stay_off_the_pr_lane(self): # swift-package-tests builds the SDK 15 helper and must never move. block = yaml.safe_dump(self.workflow("ci-macos.yml")["jobs"]["swift-package-tests"]) diff --git a/tests/test_ci_self_hosted_guard.sh b/tests/test_ci_self_hosted_guard.sh index e140a5967069..761e6f0b394d 100755 --- a/tests/test_ci_self_hosted_guard.sh +++ b/tests/test_ci_self_hosted_guard.sh @@ -1381,12 +1381,15 @@ SHARD_PICKED = "steps.macos-pool.outputs.shard_runner" SHARD_OUTPUT = "needs.changes.outputs.macos_pr_shard_runner" ROOT_PICKED = "steps.macos-pool.outputs.root_runner" ROOT_OUTPUT = "needs.changes.outputs.macos_pr_root_runner" +ADMISSION_PICKED = "steps.macos-pool.outputs.admission_runner" +ADMISSION_OUTPUT = "needs.changes.outputs.macos_pr_admission_runner" PASSED = "${{ needs.changes.outputs.macos_pr_runner }}" # Each input the picked pools reach a reusable workflow through, and its value. INPUTS = {"pr_runner": PASSED, "pr_retry_runner": "${{ " + RETRY_OUTPUT + " }}", "pr_refused_retry_runner": "${{ " + REFUSED_OUTPUT + " }}", "pr_shard_runner": "${{ " + SHARD_OUTPUT + " }}", - "pr_root_runner": "${{ " + ROOT_OUTPUT + " }}"} + "pr_root_runner": "${{ " + ROOT_OUTPUT + " }}", + "pr_admission_runner": "${{ " + ADMISSION_OUTPUT + " }}"} MARKER = ("macos-pool-persistent-${{ github.run_id }}-${{ github.run_attempt }}" "-${{ steps.macos-pool.outputs.jobs }}-${{ steps.macos-pool.outputs.runner }}") # The runs-on branches that may read the picked pool, each behind its @@ -1447,19 +1450,23 @@ for file in sorted(Path(sys.argv[1]).glob("*.y*ml")): file.name == "ci.yml" and path == ("jobs", "changes", "outputs", "macos_pr_root_runner") and value == "${{ " + ROOT_PICKED + " }}"): violations.append(f"{where}: reads the picker's root runner outside macos_pr_root_runner") + if ADMISSION_PICKED in value and not ( + file.name == "ci.yml" and path == ("jobs", "changes", "outputs", "macos_pr_admission_runner") + and value == "${{ " + ADMISSION_PICKED + " }}"): + violations.append(f"{where}: reads the picker's admission runner outside macos_pr_admission_runner") if len(path) >= 3 and path[-2] == "with" and path[-1] in INPUTS: if value != INPUTS[path[-1]] or file.name != "ci.yml": violations.append(f"{where}: {path[-1]} must be exactly {INPUTS[path[-1]]}") continue if OUTPUT not in value and RETRY_OUTPUT not in value and REFUSED_OUTPUT not in value \ - and SHARD_OUTPUT not in value and ROOT_OUTPUT not in value: + and SHARD_OUTPUT not in value and ROOT_OUTPUT not in value and ADMISSION_OUTPUT not in value: continue if path[-1:] == ("runs-on",): rest = value for branch in GUARDED: rest = rest.replace(branch, "") if OUTPUT not in rest and RETRY_OUTPUT not in rest and REFUSED_OUTPUT not in rest \ - and SHARD_OUTPUT not in rest and ROOT_OUTPUT not in rest: + and SHARD_OUTPUT not in rest and ROOT_OUTPUT not in rest and ADMISSION_OUTPUT not in rest: continue violations.append(f"{where}: reads macos_pr_runner outside pr_runner or a pull_request runs-on branch") print("\n".join(violations)) @@ -1770,6 +1777,15 @@ from pathlib import Path import yaml +# Attempt 1 of compile admission may take the warm labels in +# pr_admission_runner, a JSON array; the env restates the first, the root label. +WARM_RUNS_ON = "fromJSON(inputs.pr_admission_runner)" + + +def restated(value): + return value.replace(WARM_RUNS_ON + "[0]", WARM_RUNS_ON) + + def mismatched_identities(document): for job_id, job in document.get("jobs", {}).items(): runs_on = job.get("runs-on") @@ -1777,7 +1793,7 @@ def mismatched_identities(document): scopes.extend((f"step {index}", step) for index, step in enumerate(job.get("steps", []))) for scope, owner in scopes: for key, value in (owner.get("env") or {}).items(): - if isinstance(value, str) and "vars.MACOS_RUNNER" in value and value != runs_on: + if isinstance(value, str) and "vars.MACOS_RUNNER" in value and restated(value) != runs_on: yield f"{job_id}/{scope}: {key}\n env value {value}\n runs-on {runs_on}" diff --git a/tests/test_ci_workflow_run_sources.py b/tests/test_ci_workflow_run_sources.py index 6e6aac4cbf5b..0527394c1478 100644 --- a/tests/test_ci_workflow_run_sources.py +++ b/tests/test_ci_workflow_run_sources.py @@ -71,6 +71,12 @@ ".github/workflows/ios-screenshots.yml", "runner", ), + # The warm labeler labels the runner that ran compile admission. + ( + "scripts/ci/owned_warm_labels.py", + ".github/workflows/ci-macos.yml", + "macos-compile-admission", + ), ) # The fail-fast watcher is started by a display name but must not act on one. diff --git a/tests/test_seed_derived_data.py b/tests/test_seed_derived_data.py index 92aac28bd047..afb0adcb5dc5 100644 --- a/tests/test_seed_derived_data.py +++ b/tests/test_seed_derived_data.py @@ -445,7 +445,7 @@ def named(step_list, name): return matches[0], step_list[matches[0]] -TOKEN = re.compile(r"\s*(\|\||&&|==|!=|>=|<=|>|<|!|\(|\)|,|'(?:[^']|'')*'|[0-9]+(?:\.[0-9]+)?|[A-Za-z_][A-Za-z0-9_.-]*)") +TOKEN = re.compile(r"\s*(\|\||&&|==|!=|>=|<=|>|<|!|\(|\)|\[|\]|,|'(?:[^']|'')*'|[0-9]+(?:\.[0-9]+)?|[A-Za-z_][A-Za-z0-9_.-]*)") def evaluate(expression, context): @@ -456,6 +456,8 @@ def evaluate(expression, context): which compares equal to ''. startsWith() and endsWith() compare case-insensitively. `<`, `>`, `<=` and `>=` compare as numbers, the way Actions coerces: null and '' are 0, and a string that is not a number never compares true. + `&&` and `||` short-circuit, as in Actions, so `x && fromJSON(x)` never + parses an empty x; fromJSON() and `[index]` read JSON arrays and objects. """ text = expression.strip() if text.startswith("${{") and text.endswith("}}"): @@ -468,6 +470,8 @@ def evaluate(expression, context): tokens.append(match.group(1)) at = match.end() position = [0] + # How many enclosing operands are short-circuited: parsed, not evaluated. + skipped = [0] def peek(): return tokens[position[0]] if position[0] < len(tokens) else None @@ -477,7 +481,26 @@ def take(): return tokens[position[0] - 1] def primary(): + value = atom() + while peek() == "[": + take() + index = either() + if take() != "]": + raise ValueError("unbalanced brackets") + if isinstance(value, list) and isinstance(index, float): + value = value[int(index)] if 0 <= int(index) < len(value) else None + else: + value = value.get(str(index)) if isinstance(value, dict) else None + return value + + def atom(): token = take() + if token == "fromJSON" and peek() == "(": + take() + text = either() + if take() != ")": + raise ValueError("unbalanced parentheses") + return None if skipped[0] else json.loads(text) if token == "(": value = either() if take() != ")": @@ -550,11 +573,18 @@ def comparison(): left = {">": a > b, "<": a < b, ">=": a >= b, "<=": a <= b}[operator] return left + def operand(parse, skip): + skipped[0] += skip + try: + return parse() + finally: + skipped[0] -= skip + def both(): left = comparison() while peek() == "&&": take() - right = comparison() + right = operand(comparison, not left) left = right if left else left return left @@ -562,7 +592,7 @@ def either(): left = both() while peek() == "||": take() - right = both() + right = operand(both, bool(left)) left = left if left else right return left @@ -1057,8 +1087,43 @@ def test_root_jobs_take_the_root_label_when_the_picker_names_one(self): self.assertEqual(evaluate(shard, context), runner) self.assertEqual(evaluate(macos["cli-product-tests"]["runs-on"], context), runner) + def test_a_warm_admission_takes_the_warm_labels_on_attempt_one_only(self): + # pr_admission_runner names a root runner that kept a build of the + # run's merge base. Admission's attempt 1 asks for both labels; its + # consumers, and every retry, keep the root label. + macos = load("ci-macos.yml")["jobs"] + root, mini, retry = "glaeda-root-std-xcode-26.6", "glaeda-std-xcode-26.6", "blacksmith-12vcpu-macos-26" + warm = json.dumps([root, "glaeda-warm-0123456789ab"]) + owned_jobs = " admission shard-1 lag cli-product " + for attempt, actor, admission_runner, runner in ( + ("1", "someone", warm, [root, "glaeda-warm-0123456789ab"]), + ("1", "someone", "", root), + ("2", "github-actions[bot]", warm, root), + ("2", "someone", warm, retry), + ): + context = github_context("pull_request", ref="refs/pull/1/merge") + context["github"].update(repository="manaflow-ai/cmux", run_attempt=attempt, triggering_actor=actor, + event={"pull_request": {"head": {"repo": {"full_name": "manaflow-ai/cmux"}}}}) + context["inputs"].update(pr_runner=mini, pr_retry_runner=retry, pr_refused_retry_runner=mini, + pr_root_runner=root, pr_admission_runner=admission_runner, + pr_owned_jobs=owned_jobs) + with self.subTest(attempt=attempt, actor=actor, admission_runner=admission_runner): + admission = macos["macos-compile-admission"] + self.assertEqual(evaluate(admission["runs-on"], context), runner) + product_runner = evaluate(admission["env"]["CMUX_PRODUCT_RUNNER"], context) + self.assertEqual(product_runner, runner[0] if isinstance(runner, list) else runner) + self.assertEqual(evaluate(macos["tests-build-and-lag"]["runs-on"], context), product_runner) + context["needs"] = {"macos-compile-admission": {"outputs": {"runner": product_runner}}} + shard = macos["app-host-unit-tests"]["runs-on"].replace("format(' shard-{0} ', matrix.shard)", "' shard-1 '") + self.assertEqual(evaluate(shard, context), product_runner) + def test_the_expression_evaluator_follows_actions_semantics(self): context = {"vars": {"A": "a", "EMPTY": ""}} + # Short-circuited operands are never evaluated, and JSON is indexed. + self.assertEqual(evaluate("${{ vars.EMPTY && fromJSON(vars.EMPTY) || 'y' }}", context), "y") + self.assertEqual(evaluate("${{ vars.A || fromJSON(vars.EMPTY) }}", context), "a") + self.assertEqual(evaluate("${{ fromJSON('[\"x\",\"y\"]')[1] }}", context), "y") + self.assertEqual(evaluate("${{ fromJSON('[\"x\"]') }}", context), ["x"]) self.assertEqual(evaluate("${{ vars.A && 'x' || 'y' }}", context), "x") self.assertEqual(evaluate("${{ vars.EMPTY && 'x' || 'y' }}", context), "y") self.assertEqual(evaluate("${{ vars.MISSING || vars.A }}", context), "a")