From 8365505986a2e279421d889636d42fdcc9894cdb Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Wed, 12 Aug 2026 21:32:25 +0000 Subject: [PATCH 1/4] darwin-ci: bake a macOS 26 guest image too and run one agent per image, so tart hosts serve the latest tier scripts/darwin-ci baked every tart host a single macos-sequoia guest and tagged all of its agents release-tier=previous, so the whole tart fleet served the darwin 14 lane and the darwin 26 lane (release-tier=latest) was left to two bare boxes. Every build schedules one job of each tier, so latest backed up and PR builds' darwin 26 shards expired unrun. - config.tart lists one guest per tier ci.mjs targets (26 tahoe, 15 sequoia); images are named bun-ci- - bake takes a release, and refuses a base image whose sw_vers major does not match the release the agents will advertise - install-agent installs one launchd job + config per image, tagged release=,release-tier=, after retiring whatever agent jobs the host had; the nightly cleanup boots out all of them - the command hook boots the image for the agent's release tag (BUILDKITE_AGENT_META_DATA_RELEASE) - spawn defaults to 1 per image, so a host still runs two guests - main.ts: --release N pins a host to one image, --base overrides that image's base; bases are resolved before the host is touched - test/internal/source-lints/darwin-ci-release-tiers.test.ts checks the configured guests cover every tier ci.mjs schedules darwin aarch64 jobs on, and that agent tags, image names and the hook's lookup agree; source-lints.yml runs on darwin-ci and ci.mjs changes --- .buildkite/ci.mjs | 7 +- .github/workflows/source-lints.yml | 4 + scripts/darwin-ci/README.md | 33 +++-- scripts/darwin-ci/guest/bake.sh | 2 +- scripts/darwin-ci/hooks/command.ts | 10 +- scripts/darwin-ci/lib/agent.ts | 133 +++++++++++------- scripts/darwin-ci/lib/bake.ts | 16 ++- scripts/darwin-ci/lib/config.ts | 33 ++++- scripts/darwin-ci/main.ts | 66 +++++++-- .../darwin-ci-release-tiers.test.ts | 85 +++++++++++ 10 files changed, 300 insertions(+), 89 deletions(-) create mode 100644 test/internal/source-lints/darwin-ci-release-tiers.test.ts diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index 439f91ee22b0..8b1964b0a323 100755 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -399,8 +399,11 @@ function getTestAgent(platform, options) { const { os, arch, profile, tier } = platform; if (os === "darwin") { - // `release-tier` is emitted by scripts/agent.mjs based on the box's macOS - // major version. arm64 splits into `latest` (current macOS) + `previous` + // `release-tier` is emitted from the macOS major version by scripts/agent.mjs + // (bare boxes) and scripts/darwin-ci (tart hosts, which run one agent per + // guest image so every host serves both tiers; test/internal/source-lints/ + // darwin-ci-release-tiers.test.ts keeps its images in step with the tiers + // targeted here). arm64 splits into `latest` (current macOS) + `previous` // (anything older). x64 is NOT tier-targeted — single entry, any Intel // box — because the tier split bottlenecked the smaller pool and Intel // can't run latest anyway. diff --git a/.github/workflows/source-lints.yml b/.github/workflows/source-lints.yml index be44e6e90a55..232d7a0ee717 100644 --- a/.github/workflows/source-lints.yml +++ b/.github/workflows/source-lints.yml @@ -15,6 +15,8 @@ on: - "src/**/*.rs" - "src/jsc/bindings/**" - "scripts/build/**" + - "scripts/darwin-ci/**" + - ".buildkite/ci.mjs" - "scripts/glob-sources.ts" - "test/harness.ts" - "test/tsconfig.json" @@ -27,6 +29,8 @@ on: - "src/**/*.rs" - "src/jsc/bindings/**" - "scripts/build/**" + - "scripts/darwin-ci/**" + - ".buildkite/ci.mjs" - "scripts/glob-sources.ts" - "test/harness.ts" - "test/tsconfig.json" diff --git a/scripts/darwin-ci/README.md b/scripts/darwin-ci/README.md index c1d7b7ad0252..e701b5d33d15 100644 --- a/scripts/darwin-ci/README.md +++ b/scripts/darwin-ci/README.md @@ -12,8 +12,18 @@ Two modes: on the host itself. Agents tag themselves `os=darwin arch=... release= release-tier=...` -and `.buildkite/ci.mjs` selects on those. In tart mode `release` is the -guest's macOS version, and a guest cannot be newer than its host. +and `.buildkite/ci.mjs` selects on those. Every build schedules one darwin +aarch64 test job per tier (`release-tier=latest` and `release-tier=previous`), +so the fleet needs agents on both; a tier with too few agents backs up until +PR jobs expire unrun. In tart mode `release` is the guest's macOS version, and +a guest cannot be newer than its host. + +A tart host therefore bakes one image per guest release in `lib/config.ts` +(`bun-ci-26` for `latest`, `bun-ci-15` for `previous`) and runs one agent per +image, so each host serves both lanes; the command hook boots the image for +the agent's `release` tag. `--release N` restricts a host to one image (a host +still on an older macOS, or without disk for two images); `--spawn N` sets the +agents, i.e. concurrent guests, per image. ## Layout @@ -29,15 +39,20 @@ guest/job.sh runs inside the guest for every job `provision tart` disables remote management, makes sshd key-only, joins the tailnet, installs `buildkite-agent` and Tart, creates an unprivileged auto-login user (Virtualization.framework needs a console -session), bakes the guest image from a public base image plus -`scripts/bootstrap.sh`, and starts the agent as that user with `hooks/` as -its hooks path. It asks for one reboot the first time and is re-run after it. +session), bakes the guest images from their public base images plus +`scripts/bootstrap.sh`, and starts the agents as that user with `hooks/` as +their hooks path. It asks for one reboot the first time and is re-run after it. `provision bare` does the same host setup, then runs `scripts/bootstrap.sh` on the host and installs the `scripts/agent.mjs` service. `bake` is safe on a live host: it builds a staging image and swaps it in only -after the toolchain verifies. Re-run it when toolchain pins move. +after the guest's macOS major matches the image's release and the toolchain +verifies. Re-run it when toolchain pins move, or with `--release N` for one +image. `install-agent` retires whatever agents the host had (including the +single `com.buildkite.buildkite-agent` job from before images were +per-release) before installing one per image; a host from that era also still +has the old `bun-ci-base` image to `tart delete`. ## Bringing up a host @@ -52,10 +67,12 @@ ssh @ 'darwin-ci/host.sh tart --tags ' ``` Approve the Tailscale login it prints, reboot when it asks, run the same -command again, and check the agent appears under `queue=test-darwin`. +command again, and check that `queue=test-darwin` gains one agent per image +(`-tart-26-1` and `-tart-15-1`), one tagged with each +`release-tier`. ## Removing a host -Unload the agent (`launchctl bootout` the `com.buildkite.buildkite-agent` +Unload the agents (`launchctl bootout` each `com.buildkite.buildkite-agent.*` job), drop the host from the token allowlist and the tailnet, and release it. Nothing on a host needs preserving. diff --git a/scripts/darwin-ci/guest/bake.sh b/scripts/darwin-ci/guest/bake.sh index f1dd79d4e3f1..6aae6572e9de 100755 --- a/scripts/darwin-ci/guest/bake.sh +++ b/scripts/darwin-ci/guest/bake.sh @@ -1,5 +1,5 @@ #!/bin/bash -# Runs inside a fresh guest to turn the base image into bun-ci-base. Args: +# Runs inside a fresh guest to turn a base image into its bun-ci- image. Args: set -euo pipefail exec { if (cleaned) return; @@ -48,7 +52,7 @@ try { process.exit(status); async function runInGuest(): Promise { - console.log(`--- :tart: ${vm} from ${config.tart.image}`); + console.log(`--- :tart: ${vm} from ${image}`); const ip = (await boot()) ?? (await retryBoot()) ?? fail(`guest never came up; see /tmp/${vm}.log`); console.log(` guest ${ip}`); const g = guest(ip); @@ -72,7 +76,7 @@ async function runInGuest(): Promise { } async function boot(): Promise { - await tart.cloneLocked(config.tart.image, vm); + await tart.cloneLocked(image, vm); tart.start(vm, `/tmp/${vm}.log`); return tart.waitForSsh(vm); } diff --git a/scripts/darwin-ci/lib/agent.ts b/scripts/darwin-ci/lib/agent.ts index a166f789382f..d2e3a2b722b9 100644 --- a/scripts/darwin-ci/lib/agent.ts +++ b/scripts/darwin-ci/lib/agent.ts @@ -1,14 +1,36 @@ import { $ } from "bun"; -import { join } from "node:path"; +import { readdirSync } from "node:fs"; +import { basename, join } from "node:path"; import { ciUserHome, ciUserId } from "./ci-user"; -import { config, releaseTier } from "./config"; +import { config, guestImage, releaseTier } from "./config"; import { plist } from "./launchd"; import { consoleUser, fail, output, poll, sleep, succeeds, sudoRead, sudoWrite } from "./shell"; const path = "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"; -const agentLabel = "com.buildkite.buildkite-agent"; +const launchAgents = "/Library/LaunchAgents"; +// One launchd job per guest release, `.`. Hosts set up when there +// was one image per host have a single job named just ``; the prefix match +// in installedAgentLabels() retires it too. +const agentLabelPrefix = "com.buildkite.buildkite-agent"; const cleanupLabel = "com.buildkite.cleanup"; +export type TartAgentOptions = { releases: readonly number[]; spawn: number }; +type AgentConfig = { release: number; spawn: number; token: string; home: string }; + +/** buildkite-agent.cfg for the agents that boot the macOS `release` guest image. */ +export function tartAgentConfig({ release, spawn, token, home }: AgentConfig): string { + return [ + `token="${token}"`, + `name="%hostname-tart-${release}-%spawn"`, + `tags="queue=${config.queue},os=darwin,arch=aarch64,distro=macOS,release=${release},release-tier=${releaseTier(release)},tart=true"`, + `hooks-path="${join(config.installDir, "hooks")}"`, + `build-path="${join(home, "tart-builds")}"`, + `spawn=${spawn}`, + `cancel-grace-period=30`, + "", + ].join("\n"); +} + async function agentToken(): Promise { return ( process.env.BUILDKITE_AGENT_TOKEN || @@ -17,63 +39,38 @@ async function agentToken(): Promise { ); } -type TartAgentOptions = { release: number; spawn: number }; - -export async function installTartAgent({ release, spawn }: TartAgentOptions): Promise { +export async function installTartAgent({ releases, spawn }: TartAgentOptions): Promise { const user = config.ciUser; + if (!releases.length) fail("no guest releases to serve"); if ((await consoleUser()) !== user) fail(`${user} is not logged in at the console; run setup-user, reboot, then retry`); - const baked = await succeeds($`sudo -u ${user} -H ${config.tart.bin} get ${config.tart.image}`); - if (!baked) fail(`no ${config.tart.image} image for ${user}; run bake as ${user} first`); + for (const release of releases) { + const image = guestImage(release); + const baked = await succeeds($`sudo -u ${user} -H ${config.tart.bin} get ${image}`); + if (!baked) fail(`no ${image} image for ${user}; run bake --release ${release} as ${user} first`); + } const home = await ciUserHome(); const uid = await ciUserId(); - const configPath = join(home, ".buildkite-agent", "buildkite-agent.cfg"); + const token = await agentToken(); + const configDir = join(home, ".buildkite-agent"); const builds = join(home, "tart-builds"); const logs = join(home, "Library", "Logs", "buildkite-agent"); - const hooks = join(config.installDir, "hooks"); const owner = `${user}:staff`; - await $`sudo install -d -o ${user} -g staff ${builds} ${logs} ${join(home, ".buildkite-agent")}`; + await $`sudo install -d -o ${user} -g staff ${builds} ${logs} ${configDir}`; - await sudoWrite( - configPath, - [ - `token="${await agentToken()}"`, - `name="%hostname-tart-${release}-%spawn"`, - `tags="queue=${config.queue},os=darwin,arch=aarch64,distro=macOS,release=${release},release-tier=${releaseTier(release)},tart=true"`, - `hooks-path="${hooks}"`, - `build-path="${builds}"`, - `spawn=${spawn}`, - `cancel-grace-period=30`, - "", - ].join("\n"), - "600", - owner, - ); - - // a LaunchAgent in the auto-login user's GUI session, because Virtualization.framework will not start guests outside one - await sudoWrite( - `/Library/LaunchAgents/${agentLabel}.plist`, - plist({ - Label: agentLabel, - UserName: user, - LimitLoadToSessionType: "Aqua", - ProcessType: "Interactive", - EnvironmentVariables: { PATH: path }, - ProgramArguments: [config.buildkiteAgent.bin, "start", "--config", configPath], - WorkingDirectory: home, - RunAtLoad: true, - KeepAlive: true, - ThrottleInterval: 10, - StandardOutPath: join(logs, "buildkite-agent.log"), - StandardErrorPath: join(logs, "buildkite-agent.log"), - }), - ); + await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow(); + await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow(); + for (const label of installedAgentLabels()) { + await unload(`gui/${uid}/${label}`); + await $`sudo rm -f ${join(launchAgents, `${label}.plist`)}`; + } + await $`sudo find ${configDir} -name ${"buildkite-agent*.cfg"} -delete`; const nightly = [ `PATH=${path}`, - `launchctl bootout gui/${uid}/${agentLabel}`, + `for plist in ${launchAgents}/${agentLabelPrefix}*.plist; do launchctl bootout gui/${uid}/$(basename $plist .plist); done`, "sleep 40", `for vm in $(sudo -u ${user} tart list --source local --quiet | grep '^bk-'); do sudo -u ${user} tart delete $vm; done`, `rm -rf ${builds}/* /tmp/bk-*.log`, @@ -88,15 +85,47 @@ export async function installTartAgent({ release, spawn }: TartAgentOptions): Pr StartCalendarInterval: { Hour: 5, Minute: 0 }, }), ); - - await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow(); - await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow(); - await unload(`gui/${uid}/${agentLabel}`); - await $`sudo launchctl bootstrap gui/${uid} /Library/LaunchAgents/${agentLabel}.plist`; await $`sudo launchctl bootstrap system /Library/LaunchDaemons/${cleanupLabel}.plist`; + for (const release of releases) { + const label = `${agentLabelPrefix}.${release}`; + const configPath = join(configDir, `buildkite-agent-${release}.cfg`); + const log = join(logs, `buildkite-agent-${release}.log`); + + await sudoWrite(configPath, tartAgentConfig({ release, spawn, token, home }), "600", owner); + + // a LaunchAgent in the auto-login user's GUI session, because Virtualization.framework will not start guests outside one + await sudoWrite( + join(launchAgents, `${label}.plist`), + plist({ + Label: label, + UserName: user, + LimitLoadToSessionType: "Aqua", + ProcessType: "Interactive", + EnvironmentVariables: { PATH: path }, + ProgramArguments: [config.buildkiteAgent.bin, "start", "--config", configPath], + WorkingDirectory: home, + RunAtLoad: true, + KeepAlive: true, + ThrottleInterval: 10, + StandardOutPath: log, + StandardErrorPath: log, + }), + ); + await $`sudo launchctl bootstrap gui/${uid} ${join(launchAgents, `${label}.plist`)}`; + } + await sleep(4000); - console.log(await output($`sudo tail -4 ${join(logs, "buildkite-agent.log")}`)); + for (const release of releases) { + console.log(`--- macOS ${release} (${releaseTier(release)})`); + console.log(await output($`sudo tail -4 ${join(logs, `buildkite-agent-${release}.log`)}`)); + } +} + +function installedAgentLabels(): string[] { + return readdirSync(launchAgents) + .filter(name => name.startsWith(agentLabelPrefix) && name.endsWith(".plist")) + .map(name => basename(name, ".plist")); } export async function installBareAgent(): Promise { diff --git a/scripts/darwin-ci/lib/bake.ts b/scripts/darwin-ci/lib/bake.ts index 16588d5713dd..865ac9221edd 100644 --- a/scripts/darwin-ci/lib/bake.ts +++ b/scripts/darwin-ci/lib/bake.ts @@ -1,13 +1,14 @@ import { join } from "node:path"; -import { config, toolchain } from "./config"; +import { config, guestImage, toolchain } from "./config"; import { ensureHostKey, guest } from "./guest"; import { fail, log } from "./shell"; import { tart } from "./tart"; -type BakeOptions = { base: string; ref: string }; +type BakeOptions = { release: number; base: string; ref: string }; -export async function bake({ base, ref }: BakeOptions): Promise { - const { image, cpu, memoryMb } = config.tart; +export async function bake({ release, base, ref }: BakeOptions): Promise { + const { cpu, memoryMb } = config.tart; + const image = guestImage(release); const staging = `${image}-new`; const publicKey = await ensureHostKey(); @@ -31,6 +32,13 @@ export async function bake({ base, ref }: BakeOptions): Promise { ); const g = guest(ip); + // the agents for this image advertise release= (lib/agent.ts), and the latest lane asserts it + const major = (await g.capture("sw_vers -productVersion")).trim().split(".")[0]; + if (major !== String(release)) { + await tart.destroy(staging); + fail(`${base} is macOS ${major}, not ${release}; pass the matching --base or --release`); + } + await g.push(join(import.meta.dir, "..", "guest", "bake.sh"), "/tmp/bake.sh"); log(`bootstrap toolchain in guest (${config.bun.repo}@${ref})`); diff --git a/scripts/darwin-ci/lib/config.ts b/scripts/darwin-ci/lib/config.ts index 7883ddd47067..c8865574bed5 100644 --- a/scripts/darwin-ci/lib/config.ts +++ b/scripts/darwin-ci/lib/config.ts @@ -7,13 +7,20 @@ export const config = { bun: { repo: "https://github.com/oven-sh/bun.git", ref: "main" }, tart: { bin: "/opt/homebrew/bin/tart", - baseRemote: "ghcr.io/cirruslabs/macos-sequoia-xcode:latest", - image: "bun-ci-base", + // One guest image per `release-tier` that .buildkite/ci.mjs schedules a darwin + // aarch64 lane on (`latest` and `previous`; every build runs one job of each). + // A host bakes every image and runs `spawn` agents per image, so each host + // serves both lanes. `--release N` bakes and serves just one, for a host + // whose macOS is older than the newest guest (a guest cannot be newer than + // its host) or that lacks the disk for two images. + guests: [ + { release: 26, base: "ghcr.io/cirruslabs/macos-tahoe-xcode:latest" }, + { release: 15, base: "ghcr.io/cirruslabs/macos-sequoia-xcode:latest" }, + ], guestUser: "admin", - guestRelease: 15, cpu: 8, memoryMb: 24576, - spawn: 2, + spawn: 1, }, } as const; @@ -24,3 +31,21 @@ export function releaseTier(release: number): "latest" | "previous" | "oldest" { if (release >= 14) return "previous"; return "oldest"; } + +export function guestImage(release: number): string { + return `bun-ci-${release}`; +} + +export function guestBase(release: number): string | undefined { + return config.tart.guests.find(guest => guest.release === release)?.base; +} + +/** + * The guest release the agent running a hook serves. Agents are installed one + * per release and tagged `release=N` (lib/agent.ts); buildkite-agent exposes + * each tag to hooks as BUILDKITE_AGENT_META_DATA_. + */ +export function agentGuestRelease(env: Record): number | undefined { + const release = env.BUILDKITE_AGENT_META_DATA_RELEASE; + return release !== undefined && /^\d+$/.test(release) ? Number(release) : undefined; +} diff --git a/scripts/darwin-ci/main.ts b/scripts/darwin-ci/main.ts index ea36306d9915..3f5867270c69 100644 --- a/scripts/darwin-ci/main.ts +++ b/scripts/darwin-ci/main.ts @@ -4,7 +4,7 @@ import { parseArgs } from "node:util"; import { installBareAgent, installTartAgent } from "./lib/agent"; import { bake } from "./lib/bake"; import { ciUserExists, enableAutoLogin, ensureCiUser } from "./lib/ci-user"; -import { config } from "./lib/config"; +import { config, guestBase, guestImage, releaseTier } from "./lib/config"; import { bootstrapToolchain, brewInstall, @@ -18,27 +18,39 @@ import { } from "./lib/host"; import { consoleUser, fail, step } from "./lib/shell"; +type GuestImages = [release: number, base: string][]; + +const configuredReleases = config.tart.guests.map(({ release }) => release); + const usage = `usage: - main.ts provision [--tags ] converge a freshly imaged host - main.ts setup-user create the auto-login ${config.ciUser} user - main.ts bake [--base ] [--ref ] build ${config.tart.image} (run as ${config.ciUser}) - main.ts install-agent [--release N] [--spawn N] write agent config and launchd jobs + main.ts provision [--tags ] [--release N] [--spawn N] + converge a freshly imaged host + main.ts setup-user create the auto-login ${config.ciUser} user + main.ts bake [--release N [--base ]] [--ref ] + build the guest images (run as ${config.ciUser}) + main.ts install-agent [--release N] [--spawn N] write agent configs and launchd jobs + +A tart host bakes one image per configured guest release (${configuredReleases.join(", ")}) and runs +--spawn agents (default ${config.tart.spawn}) for each, so it serves every darwin lane. --release N limits +the host to one release; --base overrides that release's base image. provision, setup-user and install-agent need passwordless sudo.`; const { positionals, values } = parseArgs({ allowPositionals: true, options: { - base: { type: "string", default: config.tart.baseRemote }, + base: { type: "string" }, ref: { type: "string", default: config.bun.ref }, - release: { type: "string", default: String(config.tart.guestRelease) }, + release: { type: "string" }, spawn: { type: "string", default: String(config.tart.spawn) }, tags: { type: "string" }, }, }); const [subcommand, ...args] = positionals; -const agentOptions = { release: Number(values.release), spawn: Number(values.spawn) }; +const releases = values.release === undefined ? configuredReleases : [positiveInteger("release")]; +if (values.base !== undefined && values.release === undefined) fail(`--base needs --release\n\n${usage}`); +const agentOptions = { releases, spawn: positiveInteger("spawn") }; switch (subcommand) { case "provision": @@ -48,7 +60,10 @@ switch (subcommand) { await setupUser(); break; case "bake": - await bake({ base: values.base!, ref: values.ref! }); + for (const [release, base] of guestImages()) { + step(`bake ${guestImage(release)} (${releaseTier(release)}) from ${base}`); + await bake({ release, base, ref: values.ref! }); + } break; case "install-agent": await installTartAgent(agentOptions); @@ -57,6 +72,20 @@ switch (subcommand) { fail(usage); } +function positiveInteger(name: "release" | "spawn"): number { + const value = values[name]!; + if (!/^[1-9]\d*$/.test(value)) fail(`--${name} must be a positive integer, got ${JSON.stringify(value)}`); + return Number(value); +} + +/** [release, base image] for every image this host will bake; fails before anything is touched if one has no base. */ +function guestImages(): GuestImages { + return releases.map(release => [ + release, + values.base ?? guestBase(release) ?? fail(`no base image is configured for macOS ${release}; pass --base`), + ]); +} + function parseMode(mode: string | undefined): "tart" | "bare" { if (mode === "tart" || mode === "bare") return mode; return fail(usage); @@ -68,6 +97,12 @@ async function setupUser(): Promise { } async function provision(name: string, mode: "tart" | "bare"): Promise { + let images: GuestImages = []; + if (mode === "tart") { + if (process.arch !== "arm64") fail("tart mode needs Apple Silicon; use bare on Intel"); + images = guestImages(); + } + step("remote management off, sshd key-only"); await disableRemoteManagement(); await hardenSshd(); @@ -84,15 +119,14 @@ async function provision(name: string, mode: "tart" | "bare"): Promise { step(`install scripts to ${config.installDir}`); await installSelf(); - if (mode === "tart") await provisionTart(); + if (mode === "tart") await provisionTart(images); else await provisionBare(); step("done"); console.log(`tailscale: ${await tailnetSummary()}`); } -async function provisionTart(): Promise { - if (process.arch !== "arm64") fail("tart mode needs Apple Silicon; use bare on Intel"); +async function provisionTart(images: GuestImages): Promise { const user = config.ciUser; const main = `${config.installDir}/main.ts`; @@ -110,10 +144,12 @@ async function provisionTart(): Promise { return; } - step(`bake ${config.tart.image} as ${user}`); - await $`sudo -u ${user} -H /usr/local/bin/bun ${main} bake --base ${values.base!} --ref ${values.ref!}`; + for (const [release, base] of images) { + step(`bake ${guestImage(release)} (${releaseTier(release)}) from ${base} as ${user}`); + await $`sudo -u ${user} -H /usr/local/bin/bun ${main} bake --release ${release} --base ${base} --ref ${values.ref!}`; + } - step("agent"); + step(`agents for macOS ${releases.join(", ")}`); await installTartAgent(agentOptions); } diff --git a/test/internal/source-lints/darwin-ci-release-tiers.test.ts b/test/internal/source-lints/darwin-ci-release-tiers.test.ts new file mode 100644 index 000000000000..3d895e321dbe --- /dev/null +++ b/test/internal/source-lints/darwin-ci-release-tiers.test.ts @@ -0,0 +1,85 @@ +/** + * .buildkite/ci.mjs schedules the darwin aarch64 test lanes by `release-tier` + * (one job per tier on every build), and the tart hosts provisioned by + * scripts/darwin-ci are the fleet behind those tags: each host bakes one guest + * image per release in its config and runs an agent per image, tagged with that + * image's release and tier, and the command hook boots the image named by the + * agent's release tag. This keeps the three in step. When the config only + * carried a macOS 15 image, every tart agent was `previous`, the `latest` lane + * had two bare boxes for the whole PR stream, and its shards expired unrun. + * + * Configure-time only: ci.mjs is read as text (importing it would generate a + * pipeline) and nothing here talks to tart or launchd. + */ +import { describe, expect, test } from "bun:test"; +import { readFileSync } from "node:fs"; +import { join } from "node:path"; + +import { tartAgentConfig } from "../../../scripts/darwin-ci/lib/agent.ts"; +import { agentGuestRelease, config, guestImage, releaseTier } from "../../../scripts/darwin-ci/lib/config.ts"; + +const repoRoot = join(import.meta.dir, "..", "..", ".."); +const guests = config.tart.guests; + +/** + * Every `release-tier` that ci.mjs puts a darwin aarch64 job on: the + * testPlatforms entries and the traceOrderTargets host. Commented-out lanes + * (darwin x64 today) do not count. + */ +function tiersScheduledByCi(): string[] { + const tiers = new Set(); + for (const line of readFileSync(join(repoRoot, ".buildkite", "ci.mjs"), "utf8").split("\n")) { + if (line.trimStart().startsWith("//")) continue; + if (!/os:\s*"darwin"/.test(line) || !/arch:\s*"aarch64"/.test(line)) continue; + for (const [, tier] of line.matchAll(/\btier:\s*"([a-z]+)"/g)) tiers.add(tier); + } + return [...tiers].sort(); +} + +/** The `release=` tag from a baked agent config, as buildkite-agent hands it to hooks. */ +function releaseTagOf(agentConfig: string): string { + const tags = agentConfig.match(/^tags="([^"]*)"$/m)?.[1] ?? ""; + return ( + tags + .split(",") + .find(tag => tag.startsWith("release=")) + ?.slice("release=".length) ?? "" + ); +} + +describe("scripts/darwin-ci serves every darwin release-tier that .buildkite/ci.mjs schedules", () => { + test("a guest image is configured for each tier ci.mjs targets", () => { + const scheduled = tiersScheduledByCi(); + // The latest lane is the one that was missing; it doubles as a guard that the + // ci.mjs scan above still finds the darwin entries at all. + expect(scheduled).toContain("latest"); + + const served = new Set(guests.map(({ release }) => releaseTier(release))); + expect(scheduled.filter(tier => !served.has(tier))).toEqual([]); + }); + + test("each configured guest bakes to its own image", () => { + const images = guests.map(({ release }) => guestImage(release)); + expect(new Set(images).size).toBe(images.length); + }); + + test("each image's agents are tagged with its release and tier, and the hook boots that image", () => { + for (const { release } of guests) { + const agentConfig = tartAgentConfig({ release, spawn: 1, token: "token", home: "/Users/ci" }); + expect(agentConfig).toContain(`name="%hostname-tart-${release}-%spawn"`); + expect(agentConfig).toContain(`queue=${config.queue},`); + expect(agentConfig).toContain(`release=${release},release-tier=${releaseTier(release)},`); + + // hooks/command.ts boots guestImage(agentGuestRelease(env)); bake.ts produced guestImage(release) + const hookEnv = { BUILDKITE_AGENT_META_DATA_RELEASE: releaseTagOf(agentConfig) }; + expect(agentGuestRelease(hookEnv)).toBe(release); + } + }); + + test("the hook refuses an agent whose release tag is missing or not a macOS major", () => { + expect(agentGuestRelease({})).toBeUndefined(); + expect(agentGuestRelease({ BUILDKITE_AGENT_META_DATA_RELEASE: "" })).toBeUndefined(); + expect(agentGuestRelease({ BUILDKITE_AGENT_META_DATA_RELEASE: "latest" })).toBeUndefined(); + expect(agentGuestRelease({ BUILDKITE_AGENT_META_DATA_RELEASE: "26" })).toBe(26); + }); +}); From bb6e26493184290e8c48ba3cb14705f6da8ea2a2 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Wed, 12 Aug 2026 23:41:45 +0000 Subject: [PATCH 2/4] darwin-ci: retire a host's agents before replacing what they run on; install-agent refreshes the hooks On an already provisioned host, provision replaced the hooks (and now the image names they resolve) while the old agents were still taking jobs, and a standalone install-agent left the previously installed hooks in place next to agent configs written by a newer layout. Both paths now retire the host's agent jobs first, and install-agent rsyncs hooks/ and lib/ into installDir before writing the configs that point at them. Also covers guestBase() in the source-lint test and documents the upgrade path for hosts that still have the single bun-ci-base image. --- scripts/darwin-ci/README.md | 13 +++++++--- scripts/darwin-ci/lib/agent.ts | 26 +++++++++++++++---- scripts/darwin-ci/main.ts | 17 ++++++++---- .../darwin-ci-release-tiers.test.ts | 15 +++++++++-- 4 files changed, 55 insertions(+), 16 deletions(-) diff --git a/scripts/darwin-ci/README.md b/scripts/darwin-ci/README.md index e701b5d33d15..2fe7ac990798 100644 --- a/scripts/darwin-ci/README.md +++ b/scripts/darwin-ci/README.md @@ -49,10 +49,15 @@ their hooks path. It asks for one reboot the first time and is re-run after it. `bake` is safe on a live host: it builds a staging image and swaps it in only after the guest's macOS major matches the image's release and the toolchain verifies. Re-run it when toolchain pins move, or with `--release N` for one -image. `install-agent` retires whatever agents the host had (including the -single `com.buildkite.buildkite-agent` job from before images were -per-release) before installing one per image; a host from that era also still -has the old `bun-ci-base` image to `tart delete`. +image. `install-agent` re-installs `hooks/` and `lib/` alongside the agent +configs that point at them, retiring whatever agent jobs the host had first. + +To move a host that was provisioned with a single `bun-ci-base` image onto +this layout, re-run `provision` (the same `host.sh` command as below): it +retires the host's `com.buildkite.buildkite-agent` job before replacing the +hooks, bakes both images, and installs the new agents, so the host simply +takes no jobs while it bakes. `tart delete bun-ci-base` as the CI user +afterwards; nothing references it any more. ## Bringing up a host diff --git a/scripts/darwin-ci/lib/agent.ts b/scripts/darwin-ci/lib/agent.ts index d2e3a2b722b9..d0b64ab237f5 100644 --- a/scripts/darwin-ci/lib/agent.ts +++ b/scripts/darwin-ci/lib/agent.ts @@ -3,6 +3,7 @@ import { readdirSync } from "node:fs"; import { basename, join } from "node:path"; import { ciUserHome, ciUserId } from "./ci-user"; import { config, guestImage, releaseTier } from "./config"; +import { installSelf } from "./host"; import { plist } from "./launchd"; import { consoleUser, fail, output, poll, sleep, succeeds, sudoRead, sudoWrite } from "./shell"; @@ -10,7 +11,7 @@ const path = "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"; const launchAgents = "/Library/LaunchAgents"; // One launchd job per guest release, `.`. Hosts set up when there // was one image per host have a single job named just ``; the prefix match -// in installedAgentLabels() retires it too. +// in retireTartAgents() covers it too. const agentLabelPrefix = "com.buildkite.buildkite-agent"; const cleanupLabel = "com.buildkite.cleanup"; @@ -62,11 +63,10 @@ export async function installTartAgent({ releases, spawn }: TartAgentOptions): P await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow(); await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow(); - for (const label of installedAgentLabels()) { - await unload(`gui/${uid}/${label}`); - await $`sudo rm -f ${join(launchAgents, `${label}.plist`)}`; - } + await retireTartAgents(); await $`sudo find ${configDir} -name ${"buildkite-agent*.cfg"} -delete`; + // the configs below point at installDir's hooks, which look images up the same way these agents are tagged + await installSelf(); const nightly = [ `PATH=${path}`, @@ -122,6 +122,22 @@ export async function installTartAgent({ releases, spawn }: TartAgentOptions): P } } +/** + * Stop and remove every agent job on this host, whatever layout installed it, so + * no job is picked up while the images and hooks underneath them are replaced. + * Returns the labels retired; nothing to do on a fresh host. + */ +export async function retireTartAgents(): Promise { + const labels = installedAgentLabels(); + if (!labels.length) return labels; + const uid = await ciUserId(); + for (const label of labels) { + await unload(`gui/${uid}/${label}`); + await $`sudo rm -f ${join(launchAgents, `${label}.plist`)}`; + } + return labels; +} + function installedAgentLabels(): string[] { return readdirSync(launchAgents) .filter(name => name.startsWith(agentLabelPrefix) && name.endsWith(".plist")) diff --git a/scripts/darwin-ci/main.ts b/scripts/darwin-ci/main.ts index 3f5867270c69..872d0839f321 100644 --- a/scripts/darwin-ci/main.ts +++ b/scripts/darwin-ci/main.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun import { $ } from "bun"; import { parseArgs } from "node:util"; -import { installBareAgent, installTartAgent } from "./lib/agent"; +import { installBareAgent, installTartAgent, retireTartAgents } from "./lib/agent"; import { bake } from "./lib/bake"; import { ciUserExists, enableAutoLogin, ensureCiUser } from "./lib/ci-user"; import { config, guestBase, guestImage, releaseTier } from "./lib/config"; @@ -24,15 +24,16 @@ const configuredReleases = config.tart.guests.map(({ release }) => release); const usage = `usage: main.ts provision [--tags ] [--release N] [--spawn N] - converge a freshly imaged host + converge a freshly imaged (or previously provisioned) host main.ts setup-user create the auto-login ${config.ciUser} user main.ts bake [--release N [--base ]] [--ref ] build the guest images (run as ${config.ciUser}) - main.ts install-agent [--release N] [--spawn N] write agent configs and launchd jobs + main.ts install-agent [--release N] [--spawn N] install the hooks, then one agent launchd job per image A tart host bakes one image per configured guest release (${configuredReleases.join(", ")}) and runs ---spawn agents (default ${config.tart.spawn}) for each, so it serves every darwin lane. --release N limits -the host to one release; --base overrides that release's base image. +one agent, i.e. one concurrent guest, per image, so it serves every darwin lane. +--spawn N runs N agents per image (default ${config.tart.spawn}), so the host runs images x N guests. +--release N limits the host to one release; --base overrides that release's base image. provision, setup-user and install-agent need passwordless sudo.`; @@ -113,6 +114,12 @@ async function provision(name: string, mode: "tart" | "bare"): Promise { step("tailscale"); await joinTailnet(name, values.tags); + if (mode === "tart") { + // everything below replaces what running agents use (binary, hooks, images); install-agent brings them back + const retired = await retireTartAgents(); + if (retired.length) console.log(`retired ${retired.join(", ")} until the agents are reinstalled`); + } + step(`buildkite-agent ${config.buildkiteAgent.version}`); await installBuildkiteAgent(); diff --git a/test/internal/source-lints/darwin-ci-release-tiers.test.ts b/test/internal/source-lints/darwin-ci-release-tiers.test.ts index 3d895e321dbe..4b4d08587683 100644 --- a/test/internal/source-lints/darwin-ci-release-tiers.test.ts +++ b/test/internal/source-lints/darwin-ci-release-tiers.test.ts @@ -16,7 +16,13 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { tartAgentConfig } from "../../../scripts/darwin-ci/lib/agent.ts"; -import { agentGuestRelease, config, guestImage, releaseTier } from "../../../scripts/darwin-ci/lib/config.ts"; +import { + agentGuestRelease, + config, + guestBase, + guestImage, + releaseTier, +} from "../../../scripts/darwin-ci/lib/config.ts"; const repoRoot = join(import.meta.dir, "..", "..", ".."); const guests = config.tart.guests; @@ -58,9 +64,14 @@ describe("scripts/darwin-ci serves every darwin release-tier that .buildkite/ci. expect(scheduled.filter(tier => !served.has(tier))).toEqual([]); }); - test("each configured guest bakes to its own image", () => { + test("each configured guest bakes from its own base to its own image", () => { const images = guests.map(({ release }) => guestImage(release)); expect(new Set(images).size).toBe(images.length); + + // main.ts resolves bases through guestBase(); a release nobody configured needs an explicit --base + expect(guests.map(({ release }) => guestBase(release))).toEqual(guests.map(({ base }) => base)); + const unconfigured = Math.max(...guests.map(({ release }) => release)) + 1; + expect(guestBase(unconfigured)).toBeUndefined(); }); test("each image's agents are tagged with its release and tier, and the hook boots that image", () => { From edda2991896513a3ac70e8b1c7545080bd3bb44b Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 13 Aug 2026 02:49:08 +0000 Subject: [PATCH 3/4] darwin-ci: bake from a staged copy on re-provision; cap images x spawn at the two guests macOS allows provision retired a host's agents before it had checked anything, so a re-provision that stopped early (console user not yet ci, host too old for the tahoe guest, bake failure, or the nightly reboot firing during the hour-long bake) left the host serving nothing. It now bakes from a staged copy of these scripts while the existing agents keep using the installed hooks and images, and install-agent swaps everything over at the end; retiring also stops the nightly reboot daemon until the agents are back. bake refuses a release newer than the host's macOS before pulling anything. Virtualization.framework runs at most two macOS guests per host, which the old spawn=2 default filled exactly; with spawn now counted per image, images x spawn is checked against that limit up front, and the lint test checks the defaults fit it. --- scripts/darwin-ci/README.md | 32 ++++++++++------- scripts/darwin-ci/lib/agent.ts | 31 ++++++++++++---- scripts/darwin-ci/lib/bake.ts | 8 ++++- scripts/darwin-ci/lib/config.ts | 9 +++-- scripts/darwin-ci/lib/host.ts | 7 ++-- scripts/darwin-ci/main.ts | 35 +++++++++++-------- .../darwin-ci-release-tiers.test.ts | 8 ++++- 7 files changed, 87 insertions(+), 43 deletions(-) diff --git a/scripts/darwin-ci/README.md b/scripts/darwin-ci/README.md index 2fe7ac990798..84d2169d38f8 100644 --- a/scripts/darwin-ci/README.md +++ b/scripts/darwin-ci/README.md @@ -21,9 +21,10 @@ a guest cannot be newer than its host. A tart host therefore bakes one image per guest release in `lib/config.ts` (`bun-ci-26` for `latest`, `bun-ci-15` for `previous`) and runs one agent per image, so each host serves both lanes; the command hook boots the image for -the agent's `release` tag. `--release N` restricts a host to one image (a host -still on an older macOS, or without disk for two images); `--spawn N` sets the -agents, i.e. concurrent guests, per image. +the agent's `release` tag. macOS allows two guests per host, which the default +(two images, `--spawn 1` each) fills exactly. `--release N` restricts a host to +one image (a host still on an older macOS, or without disk for two images); +pair it with `--spawn 2` so that host still runs two guests. ## Layout @@ -42,22 +43,27 @@ unprivileged auto-login user (Virtualization.framework needs a console session), bakes the guest images from their public base images plus `scripts/bootstrap.sh`, and starts the agents as that user with `hooks/` as their hooks path. It asks for one reboot the first time and is re-run after it. +On a host that is already serving it is also the update path: it bakes from a +staged copy of these scripts while the existing agents carry on with the +installed hooks and images, then `install-agent` swaps agents, hooks and images +over together, so a failed bake leaves the host serving as before. `provision bare` does the same host setup, then runs `scripts/bootstrap.sh` on the host and installs the `scripts/agent.mjs` service. -`bake` is safe on a live host: it builds a staging image and swaps it in only -after the guest's macOS major matches the image's release and the toolchain -verifies. Re-run it when toolchain pins move, or with `--release N` for one -image. `install-agent` re-installs `hooks/` and `lib/` alongside the agent -configs that point at them, retiring whatever agent jobs the host had first. +`bake` is safe on a live host: it refuses a release newer than the host's +macOS up front, builds a staging image, and swaps it in only after the guest's +macOS major matches the image's release and the toolchain verifies. Re-run it +when toolchain pins move, or with `--release N` for one image. `install-agent` +retires whatever agent jobs the host had (any layout, plus the nightly reboot +daemon), re-installs `hooks/` and `lib/`, and installs one agent job per image +together with the configs that point at them. To move a host that was provisioned with a single `bun-ci-base` image onto -this layout, re-run `provision` (the same `host.sh` command as below): it -retires the host's `com.buildkite.buildkite-agent` job before replacing the -hooks, bakes both images, and installs the new agents, so the host simply -takes no jobs while it bakes. `tart delete bun-ci-base` as the CI user -afterwards; nothing references it any more. +this layout, re-run `provision` (the same `host.sh` command as below); it +needs disk for the second image while `bun-ci-base` is still present. Once it +has finished, `tart delete bun-ci-base` as the CI user; nothing references it +any more. ## Bringing up a host diff --git a/scripts/darwin-ci/lib/agent.ts b/scripts/darwin-ci/lib/agent.ts index d0b64ab237f5..3939a55f3e9b 100644 --- a/scripts/darwin-ci/lib/agent.ts +++ b/scripts/darwin-ci/lib/agent.ts @@ -18,6 +18,22 @@ const cleanupLabel = "com.buildkite.cleanup"; export type TartAgentOptions = { releases: readonly number[]; spawn: number }; type AgentConfig = { release: number; spawn: number; token: string; home: string }; +/** Guests a host runs at once with these agents installed: every agent boots one. */ +export function concurrentGuests({ releases, spawn }: TartAgentOptions): number { + return releases.length * spawn; +} + +export function checkTartAgentOptions(options: TartAgentOptions): void { + const { releases, spawn } = options; + if (!releases.length) fail("no guest releases to serve"); + const guests = concurrentGuests(options); + if (guests > config.tart.maxGuests) { + fail( + `${releases.length} image(s) x --spawn ${spawn} = ${guests} guests, but macOS runs at most ${config.tart.maxGuests} at once per host`, + ); + } +} + /** buildkite-agent.cfg for the agents that boot the macOS `release` guest image. */ export function tartAgentConfig({ release, spawn, token, home }: AgentConfig): string { return [ @@ -40,9 +56,10 @@ async function agentToken(): Promise { ); } -export async function installTartAgent({ releases, spawn }: TartAgentOptions): Promise { +export async function installTartAgent(options: TartAgentOptions): Promise { + const { releases, spawn } = options; const user = config.ciUser; - if (!releases.length) fail("no guest releases to serve"); + checkTartAgentOptions(options); if ((await consoleUser()) !== user) fail(`${user} is not logged in at the console; run setup-user, reboot, then retry`); for (const release of releases) { @@ -62,7 +79,6 @@ export async function installTartAgent({ releases, spawn }: TartAgentOptions): P await $`sudo install -d -o ${user} -g staff ${builds} ${logs} ${configDir}`; await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow(); - await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow(); await retireTartAgents(); await $`sudo find ${configDir} -name ${"buildkite-agent*.cfg"} -delete`; // the configs below point at installDir's hooks, which look images up the same way these agents are tagged @@ -123,11 +139,12 @@ export async function installTartAgent({ releases, spawn }: TartAgentOptions): P } /** - * Stop and remove every agent job on this host, whatever layout installed it, so - * no job is picked up while the images and hooks underneath them are replaced. - * Returns the labels retired; nothing to do on a fresh host. + * Stop and remove every agent job on this host, whatever layout installed it, and + * the nightly reboot that would otherwise fire while the agents are being replaced. + * installTartAgent() puts both back. Returns the agent labels retired. */ -export async function retireTartAgents(): Promise { +async function retireTartAgents(): Promise { + await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow(); const labels = installedAgentLabels(); if (!labels.length) return labels; const uid = await ciUserId(); diff --git a/scripts/darwin-ci/lib/bake.ts b/scripts/darwin-ci/lib/bake.ts index 865ac9221edd..ea249810d990 100644 --- a/scripts/darwin-ci/lib/bake.ts +++ b/scripts/darwin-ci/lib/bake.ts @@ -1,7 +1,8 @@ +import { $ } from "bun"; import { join } from "node:path"; import { config, guestImage, toolchain } from "./config"; import { ensureHostKey, guest } from "./guest"; -import { fail, log } from "./shell"; +import { fail, log, output } from "./shell"; import { tart } from "./tart"; type BakeOptions = { release: number; base: string; ref: string }; @@ -12,6 +13,11 @@ export async function bake({ release, base, ref }: BakeOptions): Promise { const staging = `${image}-new`; const publicKey = await ensureHostKey(); + const hostMajor = Number((await output($`sw_vers -productVersion`)).split(".")[0]); + if (hostMajor < release) { + fail(`this host runs macOS ${hostMajor} and cannot boot a macOS ${release} guest; pass --release <= ${hostMajor}`); + } + log(`pull ${base}`); await tart.pull(base); diff --git a/scripts/darwin-ci/lib/config.ts b/scripts/darwin-ci/lib/config.ts index c8865574bed5..2e1fd2a213b1 100644 --- a/scripts/darwin-ci/lib/config.ts +++ b/scripts/darwin-ci/lib/config.ts @@ -10,9 +10,10 @@ export const config = { // One guest image per `release-tier` that .buildkite/ci.mjs schedules a darwin // aarch64 lane on (`latest` and `previous`; every build runs one job of each). // A host bakes every image and runs `spawn` agents per image, so each host - // serves both lanes. `--release N` bakes and serves just one, for a host - // whose macOS is older than the newest guest (a guest cannot be newer than - // its host) or that lacks the disk for two images. + // serves both lanes. `--release N` bakes and serves just one (with `--spawn 2` + // to keep the host full), for a host whose macOS is older than the newest + // guest (a guest cannot be newer than its host) or that lacks the disk for + // two images. guests: [ { release: 26, base: "ghcr.io/cirruslabs/macos-tahoe-xcode:latest" }, { release: 15, base: "ghcr.io/cirruslabs/macos-sequoia-xcode:latest" }, @@ -21,6 +22,8 @@ export const config = { cpu: 8, memoryMb: 24576, spawn: 1, + // Virtualization.framework runs at most two macOS guests per host; images x spawn must fit. + maxGuests: 2, }, } as const; diff --git a/scripts/darwin-ci/lib/host.ts b/scripts/darwin-ci/lib/host.ts index 96b5688e8c01..1baae235e3ae 100644 --- a/scripts/darwin-ci/lib/host.ts +++ b/scripts/darwin-ci/lib/host.ts @@ -78,10 +78,11 @@ export async function installBuildkiteAgent(): Promise { await $`rm -rf ${tmp}`; } -export async function installSelf(): Promise { +/** Copy the running tree of these scripts to `dest` (world-readable, so the CI user can run them). */ +export async function installSelf(dest: string = config.installDir): Promise { const source = join(import.meta.dir, ".."); - await $`sudo mkdir -p ${config.installDir}`; - await $`sudo rsync -a --delete --chmod=Fa+r,Da+rx ${source}/ ${config.installDir}/`; + await $`sudo mkdir -p ${dest}`; + await $`sudo rsync -a --delete --chmod=Fa+r,Da+rx ${source}/ ${dest}/`; } export async function bootstrapToolchain(): Promise { diff --git a/scripts/darwin-ci/main.ts b/scripts/darwin-ci/main.ts index 872d0839f321..846109435ffe 100644 --- a/scripts/darwin-ci/main.ts +++ b/scripts/darwin-ci/main.ts @@ -1,7 +1,7 @@ #!/usr/bin/env bun import { $ } from "bun"; import { parseArgs } from "node:util"; -import { installBareAgent, installTartAgent, retireTartAgents } from "./lib/agent"; +import { checkTartAgentOptions, installBareAgent, installTartAgent } from "./lib/agent"; import { bake } from "./lib/bake"; import { ciUserExists, enableAutoLogin, ensureCiUser } from "./lib/ci-user"; import { config, guestBase, guestImage, releaseTier } from "./lib/config"; @@ -32,8 +32,9 @@ const usage = `usage: A tart host bakes one image per configured guest release (${configuredReleases.join(", ")}) and runs one agent, i.e. one concurrent guest, per image, so it serves every darwin lane. ---spawn N runs N agents per image (default ${config.tart.spawn}), so the host runs images x N guests. ---release N limits the host to one release; --base overrides that release's base image. +--spawn N runs N agents per image (default ${config.tart.spawn}); images x N may not exceed the +${config.tart.maxGuests} guests macOS allows per host. --release N limits the host to one release +(add --spawn ${config.tart.maxGuests} to keep it full); --base overrides that release's base image. provision, setup-user and install-agent need passwordless sudo.`; @@ -101,6 +102,7 @@ async function provision(name: string, mode: "tart" | "bare"): Promise { let images: GuestImages = []; if (mode === "tart") { if (process.arch !== "arm64") fail("tart mode needs Apple Silicon; use bare on Intel"); + checkTartAgentOptions(agentOptions); images = guestImages(); } @@ -114,18 +116,9 @@ async function provision(name: string, mode: "tart" | "bare"): Promise { step("tailscale"); await joinTailnet(name, values.tags); - if (mode === "tart") { - // everything below replaces what running agents use (binary, hooks, images); install-agent brings them back - const retired = await retireTartAgents(); - if (retired.length) console.log(`retired ${retired.join(", ")} until the agents are reinstalled`); - } - step(`buildkite-agent ${config.buildkiteAgent.version}`); await installBuildkiteAgent(); - step(`install scripts to ${config.installDir}`); - await installSelf(); - if (mode === "tart") await provisionTart(images); else await provisionBare(); @@ -133,9 +126,15 @@ async function provision(name: string, mode: "tart" | "bare"): Promise { console.log(`tailscale: ${await tailnetSummary()}`); } +/** + * Safe to re-run on a host that is already serving: the images are baked by a + * staged copy of these scripts while the existing agents keep using installDir's + * hooks and images, and install-agent swaps both over at the end. A failed bake + * leaves the host as it was. + */ async function provisionTart(images: GuestImages): Promise { const user = config.ciUser; - const main = `${config.installDir}/main.ts`; + const staging = `${config.installDir}.next`; step("tart"); await brewInstall("cirruslabs/cli/tart"); @@ -151,16 +150,22 @@ async function provisionTart(images: GuestImages): Promise { return; } + step(`stage scripts in ${staging}`); + await installSelf(staging); for (const [release, base] of images) { step(`bake ${guestImage(release)} (${releaseTier(release)}) from ${base} as ${user}`); - await $`sudo -u ${user} -H /usr/local/bin/bun ${main} bake --release ${release} --base ${base} --ref ${values.ref!}`; + await $`sudo -u ${user} -H /usr/local/bin/bun ${staging}/main.ts bake --release ${release} --base ${base} --ref ${values.ref!}`; } - step(`agents for macOS ${releases.join(", ")}`); + step(`scripts to ${config.installDir}, agents for macOS ${releases.join(", ")}`); await installTartAgent(agentOptions); + await $`sudo rm -rf ${staging}`; } async function provisionBare(): Promise { + step(`install scripts to ${config.installDir}`); + await installSelf(); + step("toolchain (scripts/bootstrap.sh)"); await bootstrapToolchain(); diff --git a/test/internal/source-lints/darwin-ci-release-tiers.test.ts b/test/internal/source-lints/darwin-ci-release-tiers.test.ts index 4b4d08587683..76a6ec396e62 100644 --- a/test/internal/source-lints/darwin-ci-release-tiers.test.ts +++ b/test/internal/source-lints/darwin-ci-release-tiers.test.ts @@ -15,7 +15,7 @@ import { describe, expect, test } from "bun:test"; import { readFileSync } from "node:fs"; import { join } from "node:path"; -import { tartAgentConfig } from "../../../scripts/darwin-ci/lib/agent.ts"; +import { concurrentGuests, tartAgentConfig } from "../../../scripts/darwin-ci/lib/agent.ts"; import { agentGuestRelease, config, @@ -74,6 +74,12 @@ describe("scripts/darwin-ci serves every darwin release-tier that .buildkite/ci. expect(guestBase(unconfigured)).toBeUndefined(); }); + test("the default agents per image fit the guests a host can run at once", () => { + // install-agent refuses anything larger, so a default that did not fit would leave every host unprovisionable + const releases = guests.map(({ release }) => release); + expect(concurrentGuests({ releases, spawn: config.tart.spawn })).toBeLessThanOrEqual(config.tart.maxGuests); + }); + test("each image's agents are tagged with its release and tier, and the hook boots that image", () => { for (const { release } of guests) { const agentConfig = tartAgentConfig({ release, spawn: 1, token: "token", home: "/Users/ci" }); From abd93545c4979b8c6e182357320a1b3ed0ac9d87 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 13 Aug 2026 03:55:14 +0000 Subject: [PATCH 4/4] darwin-ci: lint the latest guest against the release ci.mjs pins; retire a bare daemon's plist too; doc fixes The source-lint test now reads each darwin aarch64 lane's release as well as its tier, so bumping the latest lane to the next macOS in ci.mjs fails until config.tart.guests follows, and it checks the served tiers equal the scheduled ones in both directions (an image for an unscheduled tier would take one of the host's two guest slots). install-agent removes the scripts/agent.mjs LaunchDaemon plist it was already booting out, so a converted bare host stays converted across the nightly reboot. README, usage and comments now say what the code does: images are swapped in per bake and only hooks and agents flip at the end, a live bake needs a free guest slot, the bun-ci-base migration needs a macOS 26 host (or --release 15 --spawn 2) and disk for the old and new images at once, and --release/--spawn is the per-host knob for rebalancing the tiers. --- .buildkite/ci.mjs | 10 ++-- scripts/darwin-ci/README.md | 51 +++++++++-------- scripts/darwin-ci/lib/agent.ts | 21 ++++--- scripts/darwin-ci/lib/config.ts | 6 +- scripts/darwin-ci/main.ts | 7 ++- .../darwin-ci-release-tiers.test.ts | 56 ++++++++++++++----- 6 files changed, 94 insertions(+), 57 deletions(-) diff --git a/.buildkite/ci.mjs b/.buildkite/ci.mjs index 8b1964b0a323..57f1116107bb 100755 --- a/.buildkite/ci.mjs +++ b/.buildkite/ci.mjs @@ -400,11 +400,11 @@ function getTestAgent(platform, options) { if (os === "darwin") { // `release-tier` is emitted from the macOS major version by scripts/agent.mjs - // (bare boxes) and scripts/darwin-ci (tart hosts, which run one agent per - // guest image so every host serves both tiers; test/internal/source-lints/ - // darwin-ci-release-tiers.test.ts keeps its images in step with the tiers - // targeted here). arm64 splits into `latest` (current macOS) + `previous` - // (anything older). x64 is NOT tier-targeted — single entry, any Intel + // (bare boxes) and scripts/darwin-ci (tart hosts, which by default run one + // agent per guest image and so serve both tiers; test/internal/source-lints/ + // darwin-ci-release-tiers.test.ts keeps its images in step with the tiers and + // latest release targeted here). arm64 splits into `latest` (current macOS) + // + `previous` (anything older). x64 is NOT tier-targeted — single entry, any Intel // box — because the tier split bottlenecked the smaller pool and Intel // can't run latest anyway. return { diff --git a/scripts/darwin-ci/README.md b/scripts/darwin-ci/README.md index 84d2169d38f8..1bc297d3b920 100644 --- a/scripts/darwin-ci/README.md +++ b/scripts/darwin-ci/README.md @@ -12,19 +12,21 @@ Two modes: on the host itself. Agents tag themselves `os=darwin arch=... release= release-tier=...` -and `.buildkite/ci.mjs` selects on those. Every build schedules one darwin -aarch64 test job per tier (`release-tier=latest` and `release-tier=previous`), -so the fleet needs agents on both; a tier with too few agents backs up until -PR jobs expire unrun. In tart mode `release` is the guest's macOS version, and -a guest cannot be newer than its host. - -A tart host therefore bakes one image per guest release in `lib/config.ts` -(`bun-ci-26` for `latest`, `bun-ci-15` for `previous`) and runs one agent per -image, so each host serves both lanes; the command hook boots the image for -the agent's `release` tag. macOS allows two guests per host, which the default -(two images, `--spawn 1` each) fills exactly. `--release N` restricts a host to -one image (a host still on an older macOS, or without disk for two images); -pair it with `--spawn 2` so that host still runs two guests. +and `.buildkite/ci.mjs` selects on those. Every build schedules the same darwin +aarch64 test step (two shards) once per tier, `release-tier=latest` and +`release-tier=previous`, so the fleet needs agents on both; a tier with too few +agents backs up until PR jobs expire unrun. In tart mode `release` is the +guest's macOS version, and a guest cannot be newer than its host. + +By default a tart host therefore bakes one image per guest release in +`lib/config.ts` (`bun-ci-26` for `latest`, `bun-ci-15` for `previous`) and +runs one agent per image, so each host serves both lanes; the command hook +boots the image for the agent's `release` tag. macOS allows two guests per +host, which the default (two images, `--spawn 1` each) fills exactly. +`--release N --spawn 2` puts a whole host on one image instead: required on a +host whose own macOS is older than the `latest` guest, useful when a host lacks +the disk for two images, and the knob for rebalancing the tiers if one of them +turns out to be the bottleneck. ## Layout @@ -45,8 +47,11 @@ session), bakes the guest images from their public base images plus their hooks path. It asks for one reboot the first time and is re-run after it. On a host that is already serving it is also the update path: it bakes from a staged copy of these scripts while the existing agents carry on with the -installed hooks and images, then `install-agent` swaps agents, hooks and images -over together, so a failed bake leaves the host serving as before. +installed hooks, and only then runs `install-agent`, so a failed bake leaves +the agents and hooks untouched (each image is swapped in as its own bake +verifies, exactly as a standalone `bake` does). Baking needs one of the host's +two guest slots, so on a busy host run it while the agents are idle, or expect +the boot of the staging guest to fail and re-run. `provision bare` does the same host setup, then runs `scripts/bootstrap.sh` on the host and installs the `scripts/agent.mjs` service. @@ -55,15 +60,17 @@ over together, so a failed bake leaves the host serving as before. macOS up front, builds a staging image, and swaps it in only after the guest's macOS major matches the image's release and the toolchain verifies. Re-run it when toolchain pins move, or with `--release N` for one image. `install-agent` -retires whatever agent jobs the host had (any layout, plus the nightly reboot -daemon), re-installs `hooks/` and `lib/`, and installs one agent job per image -together with the configs that point at them. +retires whatever agent jobs the host had (either tart layout or a bare +`scripts/agent.mjs` daemon, plus the nightly reboot daemon), re-installs these +scripts, and installs one agent job per image together with the configs that +point at them. To move a host that was provisioned with a single `bun-ci-base` image onto -this layout, re-run `provision` (the same `host.sh` command as below); it -needs disk for the second image while `bun-ci-base` is still present. Once it -has finished, `tart delete bun-ci-base` as the CI user; nothing references it -any more. +this layout, re-run `provision` (the same `host.sh` command as below), adding +`--release 15 --spawn 2` if the host itself is not on macOS 26 yet. While it +runs the host holds `bun-ci-base`, the new base image(s) and the new image(s) +at once, so check the disk first. Once it has finished, `tart delete +bun-ci-base` as the CI user; nothing references it any more. ## Bringing up a host diff --git a/scripts/darwin-ci/lib/agent.ts b/scripts/darwin-ci/lib/agent.ts index 3939a55f3e9b..dd6157d9a9d7 100644 --- a/scripts/darwin-ci/lib/agent.ts +++ b/scripts/darwin-ci/lib/agent.ts @@ -11,9 +11,11 @@ const path = "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"; const launchAgents = "/Library/LaunchAgents"; // One launchd job per guest release, `.`. Hosts set up when there // was one image per host have a single job named just ``; the prefix match -// in retireTartAgents() covers it too. +// in retireAgents() covers it too. const agentLabelPrefix = "com.buildkite.buildkite-agent"; const cleanupLabel = "com.buildkite.cleanup"; +// what scripts/agent.mjs installs on a bare host (label `buildkite-agent`) +const bareAgentPlist = "/Library/LaunchDaemons/buildkite-agent.plist"; export type TartAgentOptions = { releases: readonly number[]; spawn: number }; type AgentConfig = { release: number; spawn: number; token: string; home: string }; @@ -78,8 +80,7 @@ export async function installTartAgent(options: TartAgentOptions): Promise await $`sudo install -d -o ${user} -g staff ${builds} ${logs} ${configDir}`; - await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow(); - await retireTartAgents(); + await retireAgents(); await $`sudo find ${configDir} -name ${"buildkite-agent*.cfg"} -delete`; // the configs below point at installDir's hooks, which look images up the same way these agents are tagged await installSelf(); @@ -139,20 +140,22 @@ export async function installTartAgent(options: TartAgentOptions): Promise } /** - * Stop and remove every agent job on this host, whatever layout installed it, and - * the nightly reboot that would otherwise fire while the agents are being replaced. - * installTartAgent() puts both back. Returns the agent labels retired. + * Stop and remove every agent job on this host, whichever layout installed it + * (per-image or single-image tart jobs, or scripts/agent.mjs's bare daemon), and + * the nightly reboot that would otherwise fire while they are being replaced. + * installTartAgent() puts the tart jobs and the reboot back. */ -async function retireTartAgents(): Promise { +async function retireAgents(): Promise { await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow(); + await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow(); + await $`sudo rm -f ${bareAgentPlist}`; const labels = installedAgentLabels(); - if (!labels.length) return labels; + if (!labels.length) return; const uid = await ciUserId(); for (const label of labels) { await unload(`gui/${uid}/${label}`); await $`sudo rm -f ${join(launchAgents, `${label}.plist`)}`; } - return labels; } function installedAgentLabels(): string[] { diff --git a/scripts/darwin-ci/lib/config.ts b/scripts/darwin-ci/lib/config.ts index 2e1fd2a213b1..ea31a4aa8a6a 100644 --- a/scripts/darwin-ci/lib/config.ts +++ b/scripts/darwin-ci/lib/config.ts @@ -8,9 +8,9 @@ export const config = { tart: { bin: "/opt/homebrew/bin/tart", // One guest image per `release-tier` that .buildkite/ci.mjs schedules a darwin - // aarch64 lane on (`latest` and `previous`; every build runs one job of each). - // A host bakes every image and runs `spawn` agents per image, so each host - // serves both lanes. `--release N` bakes and serves just one (with `--spawn 2` + // aarch64 lane on (`latest` and `previous`; every build runs the same test step + // once per tier). A host bakes every image and runs `spawn` agents per image, + // so by default each host serves both lanes. `--release N` bakes and serves just one (with `--spawn 2` // to keep the host full), for a host whose macOS is older than the newest // guest (a guest cannot be newer than its host) or that lacks the disk for // two images. diff --git a/scripts/darwin-ci/main.ts b/scripts/darwin-ci/main.ts index 846109435ffe..f74edf90d690 100644 --- a/scripts/darwin-ci/main.ts +++ b/scripts/darwin-ci/main.ts @@ -23,7 +23,7 @@ type GuestImages = [release: number, base: string][]; const configuredReleases = config.tart.guests.map(({ release }) => release); const usage = `usage: - main.ts provision [--tags ] [--release N] [--spawn N] + main.ts provision [--tags ] [tart: --release N --spawn N --base --ref ] converge a freshly imaged (or previously provisioned) host main.ts setup-user create the auto-login ${config.ciUser} user main.ts bake [--release N [--base ]] [--ref ] @@ -129,8 +129,9 @@ async function provision(name: string, mode: "tart" | "bare"): Promise { /** * Safe to re-run on a host that is already serving: the images are baked by a * staged copy of these scripts while the existing agents keep using installDir's - * hooks and images, and install-agent swaps both over at the end. A failed bake - * leaves the host as it was. + * hooks (each bake swaps its own image in once it verifies), and install-agent + * replaces hooks and agents together at the end. A failed bake leaves the + * agents and hooks as they were. */ async function provisionTart(images: GuestImages): Promise { const user = config.ciUser; diff --git a/test/internal/source-lints/darwin-ci-release-tiers.test.ts b/test/internal/source-lints/darwin-ci-release-tiers.test.ts index 76a6ec396e62..ab45a88e0aa6 100644 --- a/test/internal/source-lints/darwin-ci-release-tiers.test.ts +++ b/test/internal/source-lints/darwin-ci-release-tiers.test.ts @@ -4,9 +4,10 @@ * scripts/darwin-ci are the fleet behind those tags: each host bakes one guest * image per release in its config and runs an agent per image, tagged with that * image's release and tier, and the command hook boots the image named by the - * agent's release tag. This keeps the three in step. When the config only - * carried a macOS 15 image, every tart agent was `previous`, the `latest` lane - * had two bare boxes for the whole PR stream, and its shards expired unrun. + * agent's release tag. This keeps the three in step, including when a new macOS + * moves the `latest` lane's release in ci.mjs. When the config only carried a + * macOS 15 image, every tart agent was `previous`, the `latest` lane had two + * bare boxes for the whole PR stream, and its shards expired unrun. * * Configure-time only: ci.mjs is read as text (importing it would generate a * pipeline) and nothing here talks to tart or launchd. @@ -27,19 +28,27 @@ import { const repoRoot = join(import.meta.dir, "..", "..", ".."); const guests = config.tart.guests; +type Lane = { tier: string; release: number }; + /** - * Every `release-tier` that ci.mjs puts a darwin aarch64 job on: the - * testPlatforms entries and the traceOrderTargets host. Commented-out lanes - * (darwin x64 today) do not count. + * Every darwin aarch64 lane ci.mjs schedules by `release-tier`: the + * testPlatforms entries and the traceOrderTargets host, each written as one + * `{ os: "darwin", arch: "aarch64", release: "N", tier: "..." }` line. + * Commented-out lanes (darwin x64 today) do not count. */ -function tiersScheduledByCi(): string[] { - const tiers = new Set(); +function darwinLanesInCi(): Lane[] { + const lanes: Lane[] = []; for (const line of readFileSync(join(repoRoot, ".buildkite", "ci.mjs"), "utf8").split("\n")) { if (line.trimStart().startsWith("//")) continue; if (!/os:\s*"darwin"/.test(line) || !/arch:\s*"aarch64"/.test(line)) continue; - for (const [, tier] of line.matchAll(/\btier:\s*"([a-z]+)"/g)) tiers.add(tier); + const tier = line.match(/\btier:\s*"([a-z]+)"/)?.[1]; + if (tier) lanes.push({ tier, release: Number(line.match(/\brelease:\s*"(\d+)"/)?.[1]) }); } - return [...tiers].sort(); + return lanes; +} + +function unique(values: T[]): T[] { + return [...new Set(values)].sort((a, b) => (a < b ? -1 : a > b ? 1 : 0)); } /** The `release=` tag from a baked agent config, as buildkite-agent hands it to hooks. */ @@ -54,14 +63,31 @@ function releaseTagOf(agentConfig: string): string { } describe("scripts/darwin-ci serves every darwin release-tier that .buildkite/ci.mjs schedules", () => { - test("a guest image is configured for each tier ci.mjs targets", () => { - const scheduled = tiersScheduledByCi(); + test("the configured guest images serve exactly the tiers ci.mjs targets", () => { + const scheduled = unique(darwinLanesInCi().map(({ tier }) => tier)); // The latest lane is the one that was missing; it doubles as a guard that the // ci.mjs scan above still finds the darwin entries at all. expect(scheduled).toContain("latest"); - const served = new Set(guests.map(({ release }) => releaseTier(release))); - expect(scheduled.filter(tier => !served.has(tier))).toEqual([]); + // Exact equality in both directions: a tier with no image starves its lane, + // and an image for a tier with no lane takes one of the host's two guest + // slots (config.tart.maxGuests) away from a lane that needs it. + expect(unique(guests.map(({ release }) => releaseTier(release)))).toEqual(scheduled); + }); + + test("the latest guest is the macOS release ci.mjs pins on the latest lane", () => { + // getTestBunStep() passes the lane's `release` as EXPECTED_PLATFORM_RELEASE on + // darwin aarch64 latest, and runner.node.mjs fails the job on any other macOS + // major; bake.ts guarantees bun-ci- really is macOS N. Previous lanes float. + const pinned = unique( + darwinLanesInCi() + .filter(({ tier }) => tier === "latest") + .map(({ release }) => release), + ); + const latestGuests = unique( + guests.map(({ release }) => release).filter(release => releaseTier(release) === "latest"), + ); + expect(latestGuests).toEqual(pinned); }); test("each configured guest bakes from its own base to its own image", () => { @@ -93,7 +119,7 @@ describe("scripts/darwin-ci serves every darwin release-tier that .buildkite/ci. } }); - test("the hook refuses an agent whose release tag is missing or not a macOS major", () => { + test("the hook refuses an agent whose release tag is missing or not numeric", () => { expect(agentGuestRelease({})).toBeUndefined(); expect(agentGuestRelease({ BUILDKITE_AGENT_META_DATA_RELEASE: "" })).toBeUndefined(); expect(agentGuestRelease({ BUILDKITE_AGENT_META_DATA_RELEASE: "latest" })).toBeUndefined();