diff --git a/.github/runners.json b/.github/runners.json index 01b4f494f601..8ec79e0702e5 100644 --- a/.github/runners.json +++ b/.github/runners.json @@ -52,5 +52,72 @@ "manaflow-ai": "blacksmith" }, - "default_fleet": "hosted" + "default_fleet": "hosted", + + "pools": { + "$comment": [ + "The one routing source (scripts/ci/runner_pools.py, docs/ci-runners.md \"Runner pools\").", + "Each pool is a class of work and its members in order. The RUNNER_POOLS repository variable", + "holds each pool's live member; a job reads fromJSON(vars.RUNNER_POOLS). and names no", + "label itself (tests/test_ci_runner_set.py). runner_pools.py moves a pool to its next member", + "when the queue janitor or the overflow probe sees the current one start nothing.", + "Members are ephemeral only: owned Macs come first through a picker's output, never a pool.", + "Only a pool marked hosted may name a GitHub-hosted label, and it says why." + ], + "linux": { + "why": "Every Linux job: the runner set's Blacksmith sizes (Leo, 2026-10-28: \"just pin the set of the shit that isn't hilariously slow\")", + "members": [ + "blacksmith-32vcpu-ubuntu-2404", + "blacksmith-16vcpu-ubuntu-2404" + ] + }, + "linux_arm64": { + "why": "Native ARM64 Linux package checks", + "members": [ + "blacksmith-4vcpu-ubuntu-2404-arm" + ] + }, + "windows": { + "why": "Windows builds and tests", + "members": [ + "blacksmith-4vcpu-windows-2025" + ] + }, + "macos_15": { + "why": "The macOS 15 image (its default SDK), after any owned-Mac picker", + "members": [ + "blacksmith-6vcpu-macos-15" + ] + }, + "macos_26": { + "why": "The macOS 26 image, after any owned-Mac picker", + "members": [ + "blacksmith-6vcpu-macos-26", + "blacksmith-12vcpu-macos-26" + ] + }, + "macos_26_large": { + "why": "macOS 26 work that needs the 12 vCPU, 48 GB SKU", + "members": [ + "blacksmith-12vcpu-macos-26", + "blacksmith-6vcpu-macos-26" + ] + }, + "provenance": { + "why": "npm and PyPI provenance publishing and artifact attestations", + "hosted": true, + "hosted_why": "npm provenance, PyPI trusted publishing and artifact attestations accept only GitHub's OIDC from a GitHub-hosted runner", + "members": [ + "ubuntu-24.04" + ] + }, + "macos_intel": { + "why": "The Intel macOS leg of the dispatch-only compatibility suite", + "hosted": true, + "hosted_why": "Blacksmith has no Intel macOS image; GitHub's macos-15-intel is the only x86_64 Mac", + "members": [ + "macos-15-intel" + ] + } + } } diff --git a/.github/workflows/ci-cloud-overflow-probe.yml b/.github/workflows/ci-cloud-overflow-probe.yml index 9fd97c09ac4a..47166b490d44 100644 --- a/.github/workflows/ci-cloud-overflow-probe.yml +++ b/.github/workflows/ci-cloud-overflow-probe.yml @@ -137,6 +137,21 @@ jobs: MACOS_RUNNER_TESTS=${{ vars.MACOS_RUNNER_TESTS }} run: python3 scripts/ci/cloud_overflow_switch.py + # The one routing source (docs/ci-runners.md "Runner pools"): point each pool in + # RUNNER_POOLS at its first member that is not dead, from the probe's per-label health + # and the queue janitor's snapshot. Runs whatever the switch decided, with its token. + - name: Point the runner pools at live members + if: always() && steps.switch.outcome != 'skipped' + env: + GH_TOKEN: ${{ github.token }} + GH_REPO: ${{ github.repository }} + SWITCH_TOKEN: ${{ steps.switch-token.outputs.token }} + DRY_RUN: ${{ github.event_name == 'workflow_dispatch' && (inputs.dry_run || inputs.probe_label != '') && 'true' || 'false' }} + RUNNER_POOLS: ${{ vars.RUNNER_POOLS }} + # Read live through the API first; this is the job-start value, the fallback. + CI_CLOUD_LABEL_HEALTH: ${{ vars.CI_CLOUD_LABEL_HEALTH }} + run: python3 scripts/ci/runner_pools.py + # Last, in its own step so the summary above is already saved. If the # switch did not observe every probe start (one may still be queued), # cancel this run even when the switch failed before it could write its diff --git a/.github/workflows/ci-guards.yml b/.github/workflows/ci-guards.yml index 3661d892a301..3c05caedba71 100644 --- a/.github/workflows/ci-guards.yml +++ b/.github/workflows/ci-guards.yml @@ -150,6 +150,10 @@ jobs: if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_cloud_overflow_switch.py + - name: Validate the runner pools and their reconciler + if: ${{ matrix.group == 'ci' }} + run: python3 tests/test_ci_runner_pools.py + - name: Validate fast checks fail over with the overflow switch if: ${{ matrix.group == 'ci' }} run: python3 tests/test_ci_fast_checks_failover.py diff --git a/.github/workflows/ci-repo-variables.yml b/.github/workflows/ci-repo-variables.yml index f65db577508e..a82194265574 100644 --- a/.github/workflows/ci-repo-variables.yml +++ b/.github/workflows/ci-repo-variables.yml @@ -28,6 +28,8 @@ on: - scripts/ci/runner_label_policy.py - scripts/ci/runs_on_values.py - scripts/ci/signing_runner_variables.py + - scripts/ci/runner_pools.py + - .github/runners.json workflow_dispatch: permissions: @@ -88,6 +90,14 @@ jobs: WINDOWS_RUNNER=${{ vars.WINDOWS_RUNNER }} run: python3 scripts/ci/check_repo_variables.py + # RUNNER_POOLS, the one routing source (scripts/ci/runner_pools.py): every pool set, each to + # one of its declared members. A missing pool fails every job that reads it. + - name: Check the runner pools + if: ${{ !cancelled() }} + env: + RUNNER_POOLS: ${{ vars.RUNNER_POOLS }} + run: python3 scripts/ci/runner_pools.py --check + # No runs-on waits on one machine through a variable's value # (scripts/ci/runs_on_values.py). The workflow text passes # `vars.X || ` however X is set, and a set X never falls through, @@ -135,6 +145,7 @@ jobs: MACOS_RUNNER_PR=${{ vars.MACOS_RUNNER_PR }} MACOS_RUNNER_TESTS=${{ vars.MACOS_RUNNER_TESTS }} NIGHTLY_NEXT_TRAIN_RUNNER=${{ vars.NIGHTLY_NEXT_TRAIN_RUNNER }} + RUNNER_POOLS=${{ vars.RUNNER_POOLS }} WINDOWS_RUNNER=${{ vars.WINDOWS_RUNNER }} run: | set -euo pipefail diff --git a/.github/workflows/cmux-tui-spec.yml b/.github/workflows/cmux-tui-spec.yml index ea3d6420b904..33109bfb9067 100644 --- a/.github/workflows/cmux-tui-spec.yml +++ b/.github/workflows/cmux-tui-spec.yml @@ -22,7 +22,7 @@ permissions: jobs: inventory: - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-32vcpu-ubuntu-2404' || fromJSON(vars.RUNNER_POOLS).linux }} timeout-minutes: 5 steps: - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd # v6.0.2 diff --git a/.github/workflows/required-checks-drift.yml b/.github/workflows/required-checks-drift.yml index 2b8491614bf9..2bc2b28cdae4 100644 --- a/.github/workflows/required-checks-drift.yml +++ b/.github/workflows/required-checks-drift.yml @@ -48,7 +48,7 @@ concurrency: jobs: reconcile: name: Reconcile required checks with GitHub - runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-4vcpu-ubuntu-2404' || vars.LINUX_RUNNER || 'blacksmith-4vcpu-ubuntu-2404' }} + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != github.repository && 'blacksmith-32vcpu-ubuntu-2404' || fromJSON(vars.RUNNER_POOLS).linux }} timeout-minutes: 10 steps: - name: Check out the reconciliation diff --git a/docs/ci-runners.md b/docs/ci-runners.md index 0ff4a1b2c93c..04382528ea80 100644 --- a/docs/ci-runners.md +++ b/docs/ci-runners.md @@ -970,6 +970,73 @@ caller. Hosted/isolated fallback remains available when the shared host refuses local admission or is draining, pressured, or unavailable. +## Runner pools + +Leo, 2026-10-28: "I hate pins. Like, just pin the set of the shit that isn't +hilariously slow. Like most of what GitHub has is hilariously slow. We can't +be using them." + +One routing source. `.github/runners.json` declares each pool once under +`pools`: a class of work and its members, in order. The `RUNNER_POOLS` +repository variable holds each pool's live member as one JSON object, and a job +reads it instead of naming a label: + +```yaml +runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || fromJSON(vars.RUNNER_POOLS).linux }} +``` + +A picker's output still comes first where a run needs its own choice (an owned +Mac, warm affinity); the pool is what follows it, never a literal label. +`fromJSON()` of an unset `RUNNER_POOLS` fails the workflow at once, which is +the point: a missing route is a red run, not a job queued forever on a label +nobody serves. + +| Pool | Members (first is preferred) | +| --- | --- | +| `linux` | Blacksmith 32 vCPU, Blacksmith 16 vCPU | +| `linux_arm64` | Blacksmith 4 vCPU ARM64 | +| `windows` | Blacksmith 4 vCPU Windows | +| `macos_15` | Blacksmith macOS 15 | +| `macos_26` | Blacksmith macOS 26 (6, then 12 vCPU) | +| `macos_26_large` | Blacksmith macOS 26 (12, then 6 vCPU) | +| `provenance` | GitHub-hosted `ubuntu-24.04`: npm and PyPI provenance and artifact attestations accept only GitHub's OIDC | +| `macos_intel` | GitHub-hosted `macos-15-intel`: Blacksmith has no Intel Mac (the dispatch-only compatibility leg) | + +Every member outside the two hosted pools is an ephemeral Blacksmith label, so +a job reading a pool never lands on a reused machine, whoever started the run. +Owned Macs come first through a picker job's output (`simple_pool_picker.py`, +`side_lane_placement.py`, the nightly pickers) in the routing order: owned +Macs, then Blacksmith macOS. The pool is what follows the picker. Jobs that +hold signing secrets keep their literal ephemeral label for now and move +through the stall-aware signing picker, not a pool. + +`scripts/ci/runner_pools.py` is the one writer, and it only chooses. The +overflow probe's `watch` job runs it every 10 minutes, after the switch, with +the token the switch already uses (the `manaflow-glaeda-route` App; no new +credential). It reads per-label start evidence: the probe's per-label health +(`CI_CLOUD_LABEL_HEALTH`, every Linux member probed), and the queue janitor's +snapshot (a Blacksmith label whose oldest queued job waited 10 minutes, the +same stall rule as `simple_pool_picker.blacksmith_stalled()`, and the +dead-label entries once #27114 lands). It marks those labels dead, points each +pool at its first unmarked member, and writes `RUNNER_POOLS` only on a change. +A mark expires after 30 minutes unless seen again; a label seen starting a job +is cleared at once. With every member marked, the pool keeps its first. A +malformed `RUNNER_POOLS` is rewritten from these declarations, never crashed +on. Jobs already queued on a dead member keep that label (GitHub binds it when +it creates the job); moving them is the queue janitor's dead-label re-dispatch +(with #27114), whose re-run reads the pool's new member. +With no evidence at all (no probe health yet, no snapshot) it marks nothing +dead: it fails closed and never retires a member on a guess. A pool not marked +`hosted` never resolves to a GitHub-hosted label, whatever is dead. +`ci-repo-variables.yml` runs `runner_pools.py --check` every 15 minutes, so a +hand-edited `RUNNER_POOLS` that drops a pool or names a non-member goes red. +`tests/test_ci_runner_pools.py` checks the declared pools and the reconciler +against a recorded janitor snapshot. + +The first jobs on the pool are `cmux-tui-spec.yml` and +`required-checks-drift.yml`; the rest of the workflows follow by group, and +`tests/test_ci_runner_set.py` lands with the last group. + ## Blacksmith outage: the overflow switch This is the only Blacksmith-outage lever. The 2026-09-29 manual variable diff --git a/scripts/ci/runner_pools.py b/scripts/ci/runner_pools.py new file mode 100644 index 000000000000..884e4577d740 --- /dev/null +++ b/scripts/ci/runner_pools.py @@ -0,0 +1,383 @@ +#!/usr/bin/env python3 +"""Keep the RUNNER_POOLS repository variable pointed at a live member of each runner pool. + +Leo, 2026-10-28: "Bro, I thought we already had logic to automatically route shit. How can we +still have pins like this? I hate pins. Like, just pin the set of the shit that isn't hilariously +slow." and "Where the fuck are the generalized solution to all these fucking problems that these +incessant fucking pins?" + +One routing source. `.github/runners.json` declares each pool once (`pools`: a class of work and +its ordered members). The repository variable RUNNER_POOLS holds, for each pool, the member jobs +run on now, as one JSON object. A job writes + + runs-on: ${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || fromJSON(vars.RUNNER_POOLS).linux }} + +and never a label of its own (tests/test_ci_runner_set.py). A picker's output still comes first +where a run needs its own choice (owned Macs, warm affinity); the pool is what follows it. + +This script is the one writer, and it only chooses: each pass (ci-cloud-overflow-probe.yml's +watch job, every 10 minutes, with the token the overflow switch already uses) it + +- reads per-label start evidence (label_evidence()): + - the overflow probe's per-label health, CI_CLOUD_LABEL_HEALTH (`labels`: label -> "started" + or "stalled"; the switch probes every linux pool member), read live through the API so it is this run's and not the job's start value; + used while its `checked` is at most HEALTH_MAX_AGE old; + - the queue janitor's snapshot (macos-pool-load), only while its `generated_at` is at most + SNAPSHOT_MAX_AGE old: a Blacksmith label in `pools` whose oldest + queued job has waited BLACKSMITH_STALL_MINUTES (simple_pool_picker.blacksmith_stalled(): on + a first-in, first-out queue nothing started there since), and the dead-label entries + (`dead_labels`, cmux#27114: `label` joined with "," and `live`, members that started jobs). + The snapshot's `starved` and `unserved` describe owned (glaeda-*) labels, which are never + pool members, so they are not read; +- marks dead labels in RUNNER_POOLS' `_dead` (label -> when last seen dead). A mark expires after + DEAD_TTL unless seen again; evidence that the label started a job clears it at once; +- points each pool at its first member with no mark. With every member marked, the pool keeps + its first member; +- writes RUNNER_POOLS only when something changed. A value that does not match the schema + (parse_value()) is replaced from the declared defaults, never crashed on. + +Moving jobs already queued on a dead member is not done here: GitHub binds a job's label when it +creates the job, and the queue janitor's dead-label re-dispatch (cmux#27114) re-runs those jobs, +which then read the pool's new member. + +RUNNER_POOLS never names a reused machine: every member is an ephemeral Blacksmith label, so a +job reading it can never land on an owned Mac, whoever started it (a fork's pull request +included). Owned Macs come first through a picker's output (the routing order: owned Macs, then +Blacksmith macOS), never through a pool, and signing jobs move through the signing picker. A pool +that must run GitHub-hosted (npm and PyPI provenance accept only GitHub's OIDC) says +`"hosted": true` with its reason; no other pool may name a GitHub-hosted label. + +--check validates a RUNNER_POOLS value: every pool set, each to one of its own members. +""" + +from __future__ import annotations + +import argparse +import datetime as dt +import json +import os +import re +import sys +from collections.abc import Iterable, Mapping, Sequence +from pathlib import Path +from typing import Any + +sys.path.insert(0, str(Path(__file__).resolve().parent)) +import pr_runner_pool # noqa: E402 (its GitHub client reads the janitor snapshot) +import simple_pool_picker # noqa: E402 (the one Blacksmith stall rule) + +ROOT = Path(__file__).resolve().parents[2] +RUNNERS = ROOT / ".github" / "runners.json" +VARIABLE = "RUNNER_POOLS" +# Written by the overflow switch every probe run (cloud_overflow_switch.HEALTH_VARIABLE, label_health()). +HEALTH_VARIABLE = "CI_CLOUD_LABEL_HEALTH" +DEAD_KEY = "_dead" +UPDATED_KEY = "_updated" +# A dead mark lasts this long unless seen again. The probe runs every 10 minutes and the janitor +# sweeps about as often, so a label that is still dead is marked again before its mark expires, +# and one that recovered is used again within half an hour (or at once when it starts a job). +DEAD_TTL = dt.timedelta(minutes=30) +# Probe health older than this is from a run that did not finish; it decides nothing. +HEALTH_MAX_AGE = dt.timedelta(minutes=25) +# A janitor snapshot older than this (or undated, or from the future) decides nothing: the age the +# pool picker reads it at (pr_runner_pool.MAX_SNAPSHOT_MINUTES). +SNAPSHOT_MAX_AGE = dt.timedelta(minutes=pr_runner_pool.MAX_SNAPSHOT_MINUTES) +UTC = dt.timezone.utc +EPHEMERAL = re.compile(r"^blacksmith-\d+vcpu-[a-z0-9.-]+$") +HOSTED = re.compile(r"^(?:ubuntu|macos|windows)(?:-[\w.-]+)?$") +KEY = re.compile(r"^[a-z][a-z0-9_]*$") + + +def iso(moment: dt.datetime) -> str: + return moment.astimezone(UTC).strftime("%Y-%m-%dT%H:%M:%SZ") + + +def parse_time(value: object) -> dt.datetime | None: + """An ISO 8601 time with a zone; None for anything else (a naive time included).""" + if not isinstance(value, str) or not value: + return None + try: + moment = dt.datetime.fromisoformat(value.replace("Z", "+00:00")) + except ValueError: + return None + return moment if moment.tzinfo is not None else None + + +def load_pools(path: Path = RUNNERS) -> dict[str, dict[str, Any]]: + pools = json.loads(path.read_text(encoding="utf-8"))["pools"] + return {name: spec for name, spec in pools.items() if not name.startswith("$")} + + +def members(spec: Mapping[str, Any]) -> list[str]: + return list(dict.fromkeys(spec["members"])) + + +def spec_problems(pools: Mapping[str, Mapping[str, Any]]) -> list[str]: + """What is wrong with the declared pools (tests/test_ci_runner_pools.py).""" + found = [] + for name, spec in pools.items(): + where = f"pools.{name}" + if not KEY.match(name): + found.append(f"{where}: a pool name is lower case with underscores, so fromJSON(...).{name} reads it") + listed = spec.get("members") + if not isinstance(listed, list) or not listed or not all(isinstance(m, str) and m for m in listed): + found.append(f"{where}: members is a non-empty list of labels") + continue + if not str(spec.get("why", "")).strip(): + found.append(f"{where}: says why (what work it runs)") + for member in listed: + if HOSTED.match(member): + if not spec.get("hosted"): + found.append(f"{where}: {member} is GitHub-hosted, which is not in the runner set") + elif not EPHEMERAL.match(member): + found.append(f"{where}: {member} is not an ephemeral Blacksmith label; owned Macs come only " + "through a picker, never a pool") + if spec.get("hosted") and not str(spec.get("hosted_why", "")).strip(): + found.append(f"{where}: a hosted pool names why it cannot run on the set (hosted_why)") + return found + + +def parse_value(raw: str | None, pools: Mapping[str, Mapping[str, Any]]) -> tuple[dict[str, Any], list[str]]: + """The usable part of a RUNNER_POOLS value, and why any of it was dropped. + + A pool naming a label that is not its member, a `_dead` that is not an object of label -> + zoned time, or a value that is not a JSON object is dropped, and reconcile() fills it again + from the declared members: a malformed variable is repaired, never crashed on. + """ + if not (raw or "").strip(): + return {}, [] + try: + value = json.loads(str(raw)) + except ValueError as error: + return {}, [f"{VARIABLE} is not JSON ({error}); rewriting it from .github/runners.json"] + if not isinstance(value, dict): + return {}, [f"{VARIABLE} is not a JSON object; rewriting it from .github/runners.json"] + clean: dict[str, Any] = {} + dropped: list[str] = [] + for name, spec in pools.items(): + if name not in value: + continue + if isinstance(value[name], str) and value[name] in members(spec): + clean[name] = value[name] + else: + dropped.append(f"{VARIABLE}.{name}={value[name]!r} is not a member of pools.{name}; reset") + marks = value.get(DEAD_KEY, {}) + if isinstance(marks, dict): + good = {label: since for label, since in marks.items() + if isinstance(label, str) and parse_time(since) is not None} + if len(good) != len(marks): + dropped.append(f"{VARIABLE}.{DEAD_KEY} holds entries that are not label -> zoned time; dropped") + clean[DEAD_KEY] = good + else: + dropped.append(f"{VARIABLE}.{DEAD_KEY} is not an object; reset") + if isinstance(value.get(UPDATED_KEY), str) and parse_time(value[UPDATED_KEY]) is not None: + clean[UPDATED_KEY] = value[UPDATED_KEY] + return clean, dropped + + +def value_problems(value: Mapping[str, Any], pools: Mapping[str, Mapping[str, Any]]) -> list[str]: + """Why a RUNNER_POOLS value would misroute a job: a missing pool, or a label that is not its member.""" + found = [] + for name, spec in sorted(pools.items()): + label = value.get(name) + if not isinstance(label, str) or not label: + found.append(f"{VARIABLE} has no {name}: every job reading fromJSON(vars.{VARIABLE}).{name} fails") + elif label not in members(spec): + found.append(f"{VARIABLE}.{name}={label!r} is not a member of pools.{name} " + f"({', '.join(spec['members'])})") + return found + + +def _labels(joined: object) -> set[str]: + """A label set as the janitor writes it: joined with "," (dead_labels) or "+" (fleet keys).""" + if not isinstance(joined, str): + return set() + return {part.strip() for part in re.split(r"[,+]", joined) if part.strip()} + + +def label_evidence(health: Mapping[str, Any] | None, snapshot: Mapping[str, Any] | None, *, + now: dt.datetime) -> tuple[set[str], set[str], list[str]]: + """(dead, started, notes): the labels seen starting nothing, the labels seen starting a job.""" + dead: set[str] = set() + started: set[str] = set() + notes: list[str] = [] + if isinstance(health, Mapping): + checked = parse_time(health.get("checked")) + labels = health.get("labels") + if checked is None or not isinstance(labels, Mapping): + notes.append(f"{HEALTH_VARIABLE} is unreadable; no probe evidence") + elif not (dt.timedelta(minutes=-1) <= now - checked <= HEALTH_MAX_AGE): + notes.append(f"{HEALTH_VARIABLE} is from {iso(checked)}, outside the last {HEALTH_MAX_AGE}; no probe evidence") + else: + for label, state in labels.items(): + if state == "stalled": + dead.add(str(label)) + elif state == "started": + started.add(str(label)) + generated = parse_time(snapshot.get("generated_at")) if isinstance(snapshot, Mapping) else None + if isinstance(snapshot, Mapping) and generated is None: + notes.append("the janitor snapshot has no readable generated_at; no snapshot evidence") + elif generated is not None and not (dt.timedelta(minutes=-1) <= now - generated <= SNAPSHOT_MAX_AGE): + notes.append(f"the janitor snapshot is from {iso(generated)}, outside the last {SNAPSHOT_MAX_AGE}; " + "no snapshot evidence") + elif generated is not None: + age = max(0.0, (now - generated).total_seconds() / 60) + for label, load in (snapshot.get("pools") or {}).items(): + if not (isinstance(label, str) and EPHEMERAL.match(label) and isinstance(load, Mapping)): + continue + try: + queued = int(load.get("queued") or 0) + oldest = float(load.get("oldest_queued_minutes") or 0) + except (TypeError, ValueError): + continue + if simple_pool_picker.blacksmith_stalled(queued, oldest + age): + dead.add(label) + for entry in snapshot.get("dead_labels") or (): + if isinstance(entry, Mapping): + dead |= _labels(entry.get("label")) + live = entry.get("live") + if isinstance(live, list): + started |= {str(label) for label in live if isinstance(label, str)} + # A label both started and stalled in one pass (two probes, or a probe and a stale queue) started. + return dead - started, started, notes + + +def reconcile(current: Mapping[str, Any], pools: Mapping[str, Mapping[str, Any]], *, + dead_now: Iterable[str], served_now: Iterable[str], now: dt.datetime) -> tuple[dict[str, Any], list[str]]: + """The next RUNNER_POOLS value and a line per change. `current` is parse_value()'s output.""" + lines: list[str] = [] + marks: dict[str, str] = {} + for label, since in (current.get(DEAD_KEY) or {}).items(): + seen = parse_time(since) + if seen is not None and now - seen < DEAD_TTL: + marks[label] = since + else: + lines.append(f"{label}: dead mark expired; it may take jobs again") + for label in sorted(set(served_now)): + if marks.pop(label, None): + lines.append(f"{label}: started a job; it takes jobs again") + for label in sorted(set(dead_now)): + if label not in marks: + lines.append(f"{label}: dead (no job started there while one waited)") + marks[label] = iso(now) + known = {label for spec in pools.values() for label in members(spec)} + marks = {label: since for label, since in marks.items() if label in known} + value: dict[str, Any] = {} + for name, spec in sorted(pools.items()): + candidates = members(spec) + chosen = next((label for label in candidates if label not in marks), candidates[0]) + if current.get(name) != chosen: + lines.append(f"pools.{name}: {current.get(name) or 'unset'} -> {chosen}") + value[name] = chosen + value[DEAD_KEY] = dict(sorted(marks.items())) + comparable = {key: current.get(key) for key in value} + if comparable != value or UPDATED_KEY not in current: + value[UPDATED_KEY] = iso(now) + else: + value[UPDATED_KEY] = current[UPDATED_KEY] + return value, lines + + +def render(value: Mapping[str, Any]) -> str: + return json.dumps(value, sort_keys=True, separators=(",", ":")) + + +def _health(switch: Any, env: Mapping[str, str]) -> Mapping[str, Any] | None: + """The probe's per-label health: live through the API when a client can read it, else the job's env.""" + raw = None + if switch is not None and hasattr(switch, "get_variable"): + try: + raw = switch.get_variable(HEALTH_VARIABLE) + except Exception as error: # noqa: BLE001 - missing health only means no probe evidence + print(f"::warning title=Runner pools::could not read {HEALTH_VARIABLE} ({error}); using the job's value") + if raw is None: + raw = env.get(HEALTH_VARIABLE) + try: + value = json.loads(raw) if (raw or "").strip() else None + except ValueError: + value = None + return value if isinstance(value, Mapping) else None + + +class Variables: + """Read and write repository variables with the switch's token (cloud_overflow_switch.GitHub).""" + + def __init__(self, token: str, repo: str) -> None: + import cloud_overflow_switch # noqa: PLC0415 - its client already writes variables + self.client = cloud_overflow_switch.GitHub(token, repo) + + def get_variable(self, name: str) -> str | None: + try: + return str(self.client.request("GET", f"/actions/variables/{name}").get("value")) + except RuntimeError as error: + if "(404)" in str(error): + return None + raise + + def set_variable(self, name: str, value: str) -> None: + self.client.set_variable(name, value) + + +def main(argv: Sequence[str] | None = None, env: Mapping[str, str] | None = None, *, + api: Any = None, switch: Any = None, now: dt.datetime | None = None) -> int: + env = os.environ if env is None else env + parser = argparse.ArgumentParser(description=__doc__.splitlines()[0]) + parser.add_argument("--check", action="store_true", help="validate the live RUNNER_POOLS and exit") + parser.add_argument("--dry-run", action="store_true", + default=(env.get("DRY_RUN", "").strip().lower() in {"1", "true", "yes"})) + args = parser.parse_args(argv) + now = now or dt.datetime.now(UTC) + pools = load_pools() + problems = spec_problems(pools) + if problems: + for problem in problems: + print(f"::error title=Runner pools::{problem}") + return 1 + current, dropped = parse_value(env.get(VARIABLE), pools) + if args.check: + found = dropped + value_problems(current, pools) + for problem in found: + print(f"::error title=Runner pools::{problem}") + return 1 if found else 0 + for note in dropped: + print(f"::warning title=Runner pools::{note}") + + repo = env.get("GH_REPO") or env.get("GITHUB_REPOSITORY") or "" + if switch is None and (env.get("SWITCH_TOKEN") or "").strip() and repo: + switch = Variables(str(env["SWITCH_TOKEN"]).strip(), repo) + if api is None and env.get("GH_TOKEN") and repo: + api = pr_runner_pool.GitHub(str(env["GH_TOKEN"]), repo) + snapshot = None + if api is not None: + try: + snapshot = api.snapshot(now=now) + except Exception as error: # noqa: BLE001 - a missing snapshot only means no janitor evidence + print(f"::warning title=Runner pools::no janitor snapshot ({error}); judging on the probe alone") + dead, started, notes = label_evidence(_health(switch, env), snapshot, now=now) + for note in notes: + print(note) + value, lines = reconcile(current, pools, dead_now=dead, served_now=started, now=now) + for line in lines: + print(line) + summary = env.get("GITHUB_STEP_SUMMARY") + if summary: + with open(summary, "a", encoding="utf-8") as handle: + handle.write("### Runner pools\n\n" + ("\n".join(f"- {line}" for line in lines) or "- no change") + + "\n\n```json\n" + json.dumps(value, indent=2, sort_keys=True) + "\n```\n") + if render(value) == (env.get(VARIABLE) or "").strip(): + print(f"{VARIABLE} unchanged") + return 0 + if args.dry_run: + print(f"{VARIABLE} would be {render(value)} (dry run)") + return 0 + if switch is None: + print(f"::error title=Runner pools::no switch token (GLAEDA_ROUTE_APP_ID); {VARIABLE} stays " + f"{env.get(VARIABLE) or 'unset'}; it should be {render(value)}") + return 1 + switch.set_variable(VARIABLE, render(value)) + print(f"{VARIABLE} = {render(value)}") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tests/fixtures/runner-pools/macos-pool-load-2026-10-28T2253Z.json b/tests/fixtures/runner-pools/macos-pool-load-2026-10-28T2253Z.json new file mode 100644 index 000000000000..d3263d187123 --- /dev/null +++ b/tests/fixtures/runner-pools/macos-pool-load-2026-10-28T2253Z.json @@ -0,0 +1,59 @@ +{ + "version": 1, + "generated_at": "2026-10-28T22:53:37Z", + "pools": { + "blacksmith-6vcpu-macos-15": { + "queued": 0, + "running": 1, + "reserved_queued": 0, + "oldest_queued_minutes": 0 + }, + "blacksmith-6vcpu-macos-26": { + "queued": 0, + "running": 1, + "reserved_queued": 0, + "oldest_queued_minutes": 0 + }, + "glaeda-gui-std-xcode-26.6": { + "queued": 0, + "running": 1, + "reserved_queued": 0, + "oldest_queued_minutes": 0, + "committed": 1 + }, + "glaeda-ios-sim": { + "queued": 0, + "running": 1, + "reserved_queued": 0, + "oldest_queued_minutes": 0, + "committed": 1 + }, + "glaeda-side-std-xcode-26.6": { + "queued": 1, + "running": 2, + "reserved_queued": 0, + "oldest_queued_minutes": 2, + "committed": 3 + }, + "glaeda-std-xcode-26.6": { + "queued": 1, + "running": 4, + "reserved_queued": 0, + "oldest_queued_minutes": 2, + "committed": 5 + } + }, + "settings": { + "lane": "blacksmith-6vcpu-macos-26", + "overflow": "", + "order": "", + "max_queued": "", + "queue_rounds": "2" + }, + "wait_p90_seconds": { + "glaeda-session-std-xcode-26.6": 437, + "glaeda-side-std-xcode-26.6": 1975 + }, + "starved": [], + "unserved": [] +} diff --git a/tests/test-execution.toml b/tests/test-execution.toml index 7cca28fdfa1b..28b4cd54b0a4 100644 --- a/tests/test-execution.toml +++ b/tests/test-execution.toml @@ -1773,6 +1773,10 @@ lane = "linux-guard" path = "tests/test_ci_cloud_overflow_switch.py" lane = "linux-guard" +[[test]] +path = "tests/test_ci_runner_pools.py" +lane = "linux-guard" + [[test]] path = "tests/test_ci_fast_checks_failover.py" lane = "linux-guard" diff --git a/tests/test_ci_repo_variable_defaults.py b/tests/test_ci_repo_variable_defaults.py index 0527f0e6ba1a..00eb50503819 100644 --- a/tests/test_ci_repo_variable_defaults.py +++ b/tests/test_ci_repo_variable_defaults.py @@ -141,6 +141,24 @@ def default_chain_reaches_literal(text: str, end: int) -> bool: rest = rest[chained.end():] +# The one routing source (scripts/ci/runner_pools.py): RUNNER_POOLS has no literal fallback on purpose. +# Leo, 2026-10-28: "I hate pins." An unset RUNNER_POOLS makes fromJSON() fail the run at once (a red run, +# not a job queued on a label nobody serves). A fork pull request, which gets no repository variables, +# never reaches the read: the fork pull request branch in front of it names its own label. +FORK_PULL_REQUEST_BRANCH = re.compile( + r"github\.event_name == 'pull_request' && github\.event\.pull_request\.head\.repo\.full_name" + r" != github\.repository(?: && !contains\(fromJSON\([\w.]+\), github\.event\.pull_request\.head\.repo\.full_name\))?" + r" && '[^']+' \|\| " +) + + +def pool_read_after_fork_branch(expression: str, read: re.Match) -> bool: + """`fromJSON(vars.RUNNER_POOLS).` with the fork pull request branch before it.""" + if read.group(1) != "RUNNER_POOLS" or not expression[: read.start()].endswith("fromJSON("): + return False + return FORK_PULL_REQUEST_BRANCH.search(expression[: read.start()]) is not None + + def check_runs_on(path: Path, errors: list[str]) -> None: for number, line in enumerate(path.read_text(encoding="utf-8").splitlines(), start=1): matched = RUNS_ON.match(line) @@ -152,6 +170,8 @@ def check_runs_on(path: Path, errors: list[str]) -> None: continue if any(default_chain_reaches_literal(expression, read.end()) for read in reads): continue + if all(pool_read_after_fork_branch(expression, read) for read in reads): + continue # A last top-level `|| 'label'` is the fallback of the whole expression # (build-nightly-app: trusted tuple || 'blacksmith-12vcpu-macos-26'). if re.search(r"\|\|\s*'[^']+'\s*\}\}\s*(#.*)?$", line): @@ -270,6 +290,17 @@ def self_test_gate_matcher(errors: list[str]) -> None: ) +def self_test_pool_reads(errors: list[str]) -> None: + fork = ("github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name != " + "github.repository && 'blacksmith-32vcpu-ubuntu-2404' || ") + good = "${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || " + fork + "fromJSON(vars.RUNNER_POOLS).linux }}" + bad = "${{ github.repository_owner != 'manaflow-ai' && 'ubuntu-24.04' || fromJSON(vars.RUNNER_POOLS).linux }}" + for text, expected in ((good, True), (bad, False)): + reads = list(VARS_REFERENCE.finditer(text)) + if all(pool_read_after_fork_branch(text, read) for read in reads) != expected: + errors.append(f"self-test: pool read {'accepted' if not expected else 'refused'} wrongly: {text}") + + def main() -> int: errors: list[str] = [] files = workflow_files() @@ -277,6 +308,7 @@ def main() -> int: print(f"no workflows found under {WORKFLOWS}", file=sys.stderr) return 1 self_test_gate_matcher(errors) + self_test_pool_reads(errors) for path in files: check_runs_on(path, errors) check_cheap_defaults(path, errors) diff --git a/tests/test_ci_runner_pools.py b/tests/test_ci_runner_pools.py new file mode 100644 index 000000000000..eb1e4dfb7301 --- /dev/null +++ b/tests/test_ci_runner_pools.py @@ -0,0 +1,300 @@ +#!/usr/bin/env python3 +"""scripts/ci/runner_pools.py keeps RUNNER_POOLS on a live member of every declared pool.""" + +from __future__ import annotations + +import copy +import datetime as dt +import json +import sys +import tempfile +import unittest +from pathlib import Path + +import yaml + +ROOT = Path(__file__).resolve().parents[1] +sys.path.insert(0, str(ROOT / "scripts" / "ci")) +import runner_pools # noqa: E402 + +# The queue janitor's macos-pool-load snapshot as main's janitor wrote it at 22:53Z on 2026-10-28 +# (runner names and warm state dropped). Its starved and unserved are empty: they describe owned +# glaeda-* labels, never pool members. +RECORDED = json.loads((ROOT / "tests/fixtures/runner-pools/macos-pool-load-2026-10-28T2253Z.json").read_text()) +SNAPSHOT_AT = dt.datetime(2026, 10, 28, 22, 53, 37, tzinfo=dt.timezone.utc) +NOW = SNAPSHOT_AT + dt.timedelta(minutes=2) +POOLS = { + "linux": {"why": "x", "members": ["blacksmith-32vcpu-ubuntu-2404", "blacksmith-16vcpu-ubuntu-2404"]}, + "macos_26": {"why": "x", "members": ["blacksmith-6vcpu-macos-26", "blacksmith-12vcpu-macos-26"]}, + "macos_26_large": {"why": "x", "members": ["blacksmith-12vcpu-macos-26", "blacksmith-6vcpu-macos-26"]}, +} + + +def health(labels, checked=NOW): + # Exactly what the overflow switch writes to CI_CLOUD_LABEL_HEALTH (cloud_overflow_switch.label_health()). + import cloud_overflow_switch as switch + stalled = [label for label, state in labels.items() if state == "stalled"] + return switch.label_health(list(labels), stalled, checked=runner_pools.iso(checked), + run="https://github.com/x/y/actions/runs/1", minutes=5) + + +class Declared(unittest.TestCase): + def test_the_declared_pools_are_valid(self) -> None: + self.assertEqual(runner_pools.spec_problems(runner_pools.load_pools()), []) + + def test_the_linux_pool_is_the_runner_set(self) -> None: + # Leo, 2026-10-28: "just pin the set of the shit that isn't hilariously slow". + self.assertEqual(runner_pools.load_pools()["linux"]["members"], + ["blacksmith-32vcpu-ubuntu-2404", "blacksmith-16vcpu-ubuntu-2404"]) + + def test_no_pool_names_a_reused_machine(self) -> None: + # A job reading a pool, a fork's pull request included, never lands on an owned Mac: + # those come only through a picker. + for name, spec in runner_pools.load_pools().items(): + for label in runner_pools.members(spec): + with self.subTest(pool=name, label=label): + self.assertRegex(label, runner_pools.HOSTED if spec.get("hosted") else runner_pools.EPHEMERAL) + + def test_spec_problems_refuse_owned_hosted_and_unknown_labels(self) -> None: + bad = { + "a": {"why": "x", "members": ["glaeda-std-xcode-26.6"]}, + "b": {"why": "x", "members": ["ubuntu-24.04"]}, + "c": {"why": "x", "hosted": True, "members": ["ubuntu-24.04"]}, + "d": {"why": "x", "members": ["some-host"]}, + "E": {"why": "x", "members": ["blacksmith-32vcpu-ubuntu-2404"]}, + "f": {"members": ["blacksmith-32vcpu-ubuntu-2404"]}, + } + found = "\n".join(runner_pools.spec_problems(bad)) + for name in ("pools.a", "pools.b", "pools.c", "pools.d", "pools.E", "pools.f"): + self.assertIn(name, found) + + def test_the_reconciler_runs_in_the_overflow_watch_with_the_switch_token(self) -> None: + # No new credential: the token the overflow switch already mints writes RUNNER_POOLS. + workflow = yaml.safe_load((ROOT / ".github/workflows/ci-cloud-overflow-probe.yml").read_text()) + steps = workflow["jobs"]["watch"]["steps"] + [step] = [s for s in steps if "runner_pools.py" in str(s.get("run", ""))] + self.assertEqual(step["env"]["SWITCH_TOKEN"], "${{ steps.switch-token.outputs.token }}") + self.assertEqual(step["env"]["RUNNER_POOLS"], "${{ vars.RUNNER_POOLS }}") + self.assertIn("always()", str(step.get("if"))) + names = [s.get("name") for s in steps] + self.assertLess(names.index(step["name"]), names.index("End this run's stuck probe")) + + +class Evidence(unittest.TestCase): + def test_it_reads_the_variable_the_switch_writes(self) -> None: + import cloud_overflow_switch as switch + self.assertEqual(runner_pools.HEALTH_VARIABLE, switch.HEALTH_VARIABLE) + # The switch probes every member of the linux pool, so each one has health of its own. + self.assertEqual(set(switch.PROBE_LABELS), set(runner_pools.load_pools()["linux"]["members"])) + + def test_the_recorded_snapshot_marks_nothing(self) -> None: + dead, started, _ = runner_pools.label_evidence(None, RECORDED, now=NOW) + self.assertEqual((dead, started), (set(), set())) + + def test_a_stalled_blacksmith_mac_in_the_snapshot_is_dead(self) -> None: + # simple_pool_picker.blacksmith_stalled(): a queued job waited 10 minutes, so nothing started. + snapshot = copy.deepcopy(RECORDED) + snapshot["pools"]["blacksmith-6vcpu-macos-26"].update(queued=3, oldest_queued_minutes=9) + dead, _, _ = runner_pools.label_evidence(None, snapshot, now=NOW) + self.assertEqual(dead, {"blacksmith-6vcpu-macos-26"}) + # Owned labels in the snapshot are never pool members and are not read. + snapshot["pools"]["glaeda-std-xcode-26.6"].update(queued=9, oldest_queued_minutes=60) + snapshot["starved"] = [{"label": "glaeda-gui-std-xcode-26.6+glaeda-std-xcode-26.6", "queued": 1}] + dead, _, _ = runner_pools.label_evidence(None, snapshot, now=NOW) + self.assertEqual(dead, {"blacksmith-6vcpu-macos-26"}) + + def test_dead_label_entries_split_on_comma_and_plus(self) -> None: + # cmux#27114's DeadLabel.as_json(): label sets joined with ",", live members listed. + snapshot = dict(RECORDED, dead_labels=[{ + "label": "blacksmith-16vcpu-ubuntu-2404", "family": "linux", "queued": 4, + "oldest_queued_minutes": 6, "live": ["blacksmith-32vcpu-ubuntu-2404"]}]) + dead, started, _ = runner_pools.label_evidence(None, snapshot, now=NOW) + self.assertEqual(dead, {"blacksmith-16vcpu-ubuntu-2404"}) + self.assertEqual(started, {"blacksmith-32vcpu-ubuntu-2404"}) + self.assertEqual(runner_pools._labels("a,b+c"), {"a", "b", "c"}) + + def test_probe_health_per_label(self) -> None: + dead, started, _ = runner_pools.label_evidence(health({ + "blacksmith-32vcpu-ubuntu-2404": "stalled", "blacksmith-16vcpu-ubuntu-2404": "started"}), None, now=NOW) + self.assertEqual(dead, {"blacksmith-32vcpu-ubuntu-2404"}) + self.assertEqual(started, {"blacksmith-16vcpu-ubuntu-2404"}) + + def test_no_evidence_retires_nothing(self) -> None: + # Fail closed: with CI_CLOUD_LABEL_HEALTH absent (cmux#27121 not deployed) and no snapshot, + # nothing is marked dead, and the pools stay on their first members. + dead, started, _ = runner_pools.label_evidence(None, None, now=NOW) + self.assertEqual((dead, started), (set(), set())) + switch = Main.Switch() + self.assertEqual(runner_pools.main([], {"RUNNER_POOLS": ""}, api=Main.Api(None), switch=switch, now=NOW), 0) + value = json.loads(switch.writes[0][1]) + self.assertEqual(value["_dead"], {}) + for name, spec in runner_pools.load_pools().items(): + self.assertEqual(value[name], spec["members"][0]) + + def test_an_undated_stale_or_future_snapshot_decides_nothing(self) -> None: + stalled = copy.deepcopy(RECORDED) + stalled["pools"]["blacksmith-6vcpu-macos-26"].update(queued=3, oldest_queued_minutes=30) + stalled["dead_labels"] = [{"label": "blacksmith-16vcpu-ubuntu-2404", "live": []}] + dead, _, _ = runner_pools.label_evidence(None, stalled, now=NOW) + self.assertEqual(dead, {"blacksmith-6vcpu-macos-26", "blacksmith-16vcpu-ubuntu-2404"}) + for generated in (None, "yesterday", "2026-10-28T22:53:37", + runner_pools.iso(NOW - runner_pools.SNAPSHOT_MAX_AGE - dt.timedelta(minutes=1)), + runner_pools.iso(NOW + dt.timedelta(hours=1))): + with self.subTest(generated=generated): + snapshot = dict(stalled, generated_at=generated) + dead, started, notes = runner_pools.label_evidence(None, snapshot, now=NOW) + self.assertEqual((dead, started), (set(), set())) + self.assertTrue(notes) + + def test_old_or_unreadable_health_decides_nothing(self) -> None: + old = health({"blacksmith-32vcpu-ubuntu-2404": "stalled"}, checked=NOW - dt.timedelta(hours=1)) + future = health({"blacksmith-32vcpu-ubuntu-2404": "stalled"}, checked=NOW + dt.timedelta(hours=1)) + for value in (old, future, {"labels": []}, {"checked": "2026-10-28T22:00:00", "labels": {}}): + dead, started, notes = runner_pools.label_evidence(value, None, now=NOW) + self.assertEqual((dead, started), (set(), set())) + self.assertTrue(notes) + + +class Reconcile(unittest.TestCase): + def run_once(self, current, dead=(), served=(), now=NOW): + return runner_pools.reconcile(current, POOLS, dead_now=dead, served_now=served, now=now) + + def test_first_pass_points_every_pool_at_its_first_member(self) -> None: + value, _ = self.run_once({}) + self.assertEqual(value["linux"], "blacksmith-32vcpu-ubuntu-2404") + self.assertEqual(value["macos_26"], "blacksmith-6vcpu-macos-26") + self.assertEqual(value["macos_26_large"], "blacksmith-12vcpu-macos-26") + self.assertEqual(value["_dead"], {}) + + def test_a_dead_member_flips_its_pools_to_the_next(self) -> None: + value, lines = self.run_once({}, dead={"blacksmith-6vcpu-macos-26"}) + self.assertEqual(value["macos_26"], "blacksmith-12vcpu-macos-26") + self.assertEqual(value["macos_26_large"], "blacksmith-12vcpu-macos-26") + self.assertIn("pools.macos_26: unset -> blacksmith-12vcpu-macos-26", lines) + + def test_a_mark_holds_until_it_expires_or_the_label_starts_a_job(self) -> None: + first, _ = self.run_once({}, dead={"blacksmith-32vcpu-ubuntu-2404"}) + self.assertEqual(first["linux"], "blacksmith-16vcpu-ubuntu-2404") + held, _ = self.run_once(first, now=NOW + dt.timedelta(minutes=20)) + self.assertEqual(held["linux"], "blacksmith-16vcpu-ubuntu-2404") + expired, _ = self.run_once(first, now=NOW + runner_pools.DEAD_TTL) + self.assertEqual(expired["linux"], "blacksmith-32vcpu-ubuntu-2404") + probed, _ = self.run_once(first, served={"blacksmith-32vcpu-ubuntu-2404"}, now=NOW + dt.timedelta(minutes=5)) + self.assertEqual(probed["linux"], "blacksmith-32vcpu-ubuntu-2404") + + def test_every_member_dead_keeps_the_first(self) -> None: + value, _ = self.run_once({}, dead={"blacksmith-32vcpu-ubuntu-2404", "blacksmith-16vcpu-ubuntu-2404"}) + self.assertEqual(value["linux"], "blacksmith-32vcpu-ubuntu-2404") + + def test_no_change_keeps_the_value_byte_for_byte(self) -> None: + value, _ = self.run_once({}) + again, lines = self.run_once(value, now=NOW + dt.timedelta(minutes=10)) + self.assertEqual(runner_pools.render(again), runner_pools.render(value)) + self.assertEqual(lines, []) + + +class NeverHosted(unittest.TestCase): + def test_a_non_hosted_pool_never_resolves_to_a_hosted_label(self) -> None: + # Hosted runners are not in the declared set: whatever is dead, a pool not marked hosted + # picks one of its own Blacksmith members, never ubuntu-* or macos-*. + pools = runner_pools.load_pools() + every = {label for spec in pools.values() for label in runner_pools.members(spec)} + value, _ = runner_pools.reconcile({}, pools, dead_now=every, served_now=(), now=NOW) + for name, spec in pools.items(): + if not spec.get("hosted"): + self.assertRegex(value[name], runner_pools.EPHEMERAL) + current, dropped = runner_pools.parse_value(json.dumps({"linux": "ubuntu-24.04"}), pools) + self.assertNotIn("linux", current) + self.assertTrue(dropped) + + def test_the_overflow_switch_never_writes_a_hosted_label(self) -> None: + # The switch that moves the old variables (LINUX_RUNNER, CI_TRUSTED_RUNNER) never points one + # at GitHub-hosted Ubuntu or macOS, with or without an override asking it to. + import cloud_overflow_switch as switch + for overrides in (None, json.dumps({"LINUX_RUNNER": "ubuntu-24.04"}), + json.dumps({"MACOS_RUNNER_15": "macos-15"})): + values, _ = switch.failover_values("/Applications/Xcode_26.6.app", overrides) + for name, value in values.items(): + with self.subTest(overrides=overrides, variable=name): + self.assertNotRegex(str(value), r"^(?:ubuntu|macos|windows)-") + + +class Schema(unittest.TestCase): + def test_malformed_values_are_repaired_not_crashed_on(self) -> None: + good, _ = runner_pools.reconcile({}, POOLS, dead_now=(), served_now=(), now=NOW) + for raw in ('{"_dead":[]}', '{"_dead":{"blacksmith-32vcpu-ubuntu-2404":"2026-10-28T22:00:00"}}', + '{"_dead":{"blacksmith-32vcpu-ubuntu-2404":7}}', '{"linux":"ubuntu-24.04"}', '{"linux":3}', + '[]', '"x"', 'not json'): + with self.subTest(raw=raw): + current, dropped = runner_pools.parse_value(raw, POOLS) + self.assertTrue(dropped) + value, _ = runner_pools.reconcile(current, POOLS, dead_now=(), served_now=(), now=NOW) + self.assertEqual({k: value[k] for k in POOLS}, {k: good[k] for k in POOLS}) + self.assertEqual(value["_dead"], {}) + + def test_a_valid_value_round_trips(self) -> None: + value, _ = runner_pools.reconcile({}, POOLS, dead_now={"blacksmith-6vcpu-macos-26"}, served_now=(), now=NOW) + current, dropped = runner_pools.parse_value(runner_pools.render(value), POOLS) + self.assertEqual(dropped, []) + self.assertEqual(runner_pools.render(current), runner_pools.render(value)) + + def test_value_problems(self) -> None: + value, _ = runner_pools.reconcile({}, POOLS, dead_now=(), served_now=(), now=NOW) + self.assertEqual(runner_pools.value_problems(value, POOLS), []) + self.assertTrue(runner_pools.value_problems({**value, "linux": "ubuntu-24.04"}, POOLS)) + missing = dict(value) + del missing["macos_26"] + self.assertTrue(runner_pools.value_problems(missing, POOLS)) + + +class Main(unittest.TestCase): + class Api: + def __init__(self, snapshot): + self._snapshot = snapshot + + def snapshot(self, *, now): + return self._snapshot + + class Switch: + def __init__(self, variables=None): + self.writes = [] + self.variables = dict(variables or {}) + + def get_variable(self, name): + return self.variables.get(name) + + def set_variable(self, name, value): + self.writes.append((name, value)) + self.variables[name] = value + + def test_a_stalled_size_flips_linux_once(self) -> None: + switch = self.Switch({runner_pools.HEALTH_VARIABLE: json.dumps(health({ + "blacksmith-32vcpu-ubuntu-2404": "stalled", "blacksmith-16vcpu-ubuntu-2404": "started"}))}) + with tempfile.TemporaryDirectory() as tmp: + env = {"RUNNER_POOLS": "", "GITHUB_STEP_SUMMARY": f"{tmp}/summary"} + self.assertEqual(runner_pools.main([], env, api=self.Api(RECORDED), switch=switch, now=NOW), 0) + [(name, raw)] = switch.writes + self.assertEqual(name, "RUNNER_POOLS") + self.assertEqual(json.loads(raw)["linux"], "blacksmith-16vcpu-ubuntu-2404") + + def test_nothing_changed_writes_nothing(self) -> None: + switch = self.Switch() + env = {"RUNNER_POOLS": ""} + runner_pools.main([], env, api=self.Api(RECORDED), switch=switch, now=NOW) + env["RUNNER_POOLS"] = switch.writes[0][1] + runner_pools.main([], env, api=self.Api(RECORDED), switch=switch, now=NOW + dt.timedelta(minutes=10)) + self.assertEqual(len(switch.writes), 1) + + def test_a_malformed_variable_is_rewritten(self) -> None: + switch = self.Switch() + self.assertEqual(runner_pools.main([], {"RUNNER_POOLS": '{"_dead":[]}'}, api=self.Api(None), + switch=switch, now=NOW), 0) + self.assertEqual(runner_pools.value_problems(json.loads(switch.writes[0][1]), runner_pools.load_pools()), []) + + def test_check_fails_on_an_unset_or_malformed_variable(self) -> None: + self.assertEqual(runner_pools.main(["--check"], {"RUNNER_POOLS": ""}), 1) + self.assertEqual(runner_pools.main(["--check"], {"RUNNER_POOLS": '{"_dead":[]}'}), 1) + + +if __name__ == "__main__": + unittest.main() diff --git a/tests/test_runner_label_policy.py b/tests/test_runner_label_policy.py index d74ae85b96bf..814d6182fd76 100644 --- a/tests/test_runner_label_policy.py +++ b/tests/test_runner_label_policy.py @@ -192,7 +192,9 @@ def test_a_non_string_value_is_ignored_rather_than_crashing(self) -> None: HEALTH_REPORT_WORKFLOW = ROOT / ".github" / "workflows" / "ci-health-report.yml" -NON_LABEL_RUNNER_VARIABLES = {"CI_SEED_KEEP_LOCAL_RUNNERS"} +# RUNNER_POOLS is a JSON map of pool -> label, read as fromJSON(vars.RUNNER_POOLS).; its labels are +# checked by scripts/ci/runner_pools.py --check (ci-repo-variables.yml), not by the label policy. +NON_LABEL_RUNNER_VARIABLES = {"CI_SEED_KEEP_LOCAL_RUNNERS", "RUNNER_POOLS"} def reported_runner_variables() -> set[str]: