darwin-ci: bake a macOS 26 guest image too and run one agent per image, so tart hosts serve the latest tier

.buildkite/ci.mjs

@@ -399,9 +399,12 @@ function getTestAgent(platform, options) {
   const { os, arch, profile, tier } = platform;
 
   if (os === "darwin") {
-    // `release-tier` is emitted by scripts/agent.mjs based on the box's macOS
-    // major version. arm64 splits into `latest` (current macOS) + `previous`
-    // (anything older). x64 is NOT tier-targeted — single entry, any Intel
+    // `release-tier` is emitted from the macOS major version by scripts/agent.mjs
+    // (bare boxes) and scripts/darwin-ci (tart hosts, which by default run one
+    // agent per guest image and so serve both tiers; test/internal/source-lints/
+    // darwin-ci-release-tiers.test.ts keeps its images in step with the tiers and
+    // latest release targeted here). arm64 splits into `latest` (current macOS)
+    // + `previous` (anything older). x64 is NOT tier-targeted — single entry, any Intel
     // box — because the tier split bottlenecked the smaller pool and Intel
     // can't run latest anyway.
     return {

.github/workflows/source-lints.yml

@@ -15,6 +15,8 @@ on:
       - "src/**/*.rs"
       - "src/jsc/bindings/**"
       - "scripts/build/**"
+      - "scripts/darwin-ci/**"
+      - ".buildkite/ci.mjs"
       - "scripts/glob-sources.ts"
       - "test/harness.ts"
       - "test/tsconfig.json"
@@ -27,6 +29,8 @@ on:
       - "src/**/*.rs"
       - "src/jsc/bindings/**"
       - "scripts/build/**"
+      - "scripts/darwin-ci/**"
+      - ".buildkite/ci.mjs"
       - "scripts/glob-sources.ts"
       - "test/harness.ts"
       - "test/tsconfig.json"

scripts/darwin-ci/README.md

@@ -12,9 +12,22 @@ Two modes:
   on the host itself.
 
 Agents tag themselves `os=darwin arch=... release=<macOS major> release-tier=...`
-and `.buildkite/ci.mjs` selects on those. In tart mode `release` is the
+and `.buildkite/ci.mjs` selects on those. Every build schedules the same darwin
+aarch64 test step (two shards) once per tier, `release-tier=latest` and
+`release-tier=previous`, so the fleet needs agents on both; a tier with too few
+agents backs up until PR jobs expire unrun. In tart mode `release` is the
 guest's macOS version, and a guest cannot be newer than its host.
 
+By default a tart host therefore bakes one image per guest release in
+`lib/config.ts` (`bun-ci-26` for `latest`, `bun-ci-15` for `previous`) and
+runs one agent per image, so each host serves both lanes; the command hook
+boots the image for the agent's `release` tag. macOS allows two guests per
+host, which the default (two images, `--spawn 1` each) fills exactly.
+`--release N --spawn 2` puts a whole host on one image instead: required on a
+host whose own macOS is older than the `latest` guest, useful when a host lacks
+the disk for two images, and the knob for rebalancing the tiers if one of them
+turns out to be the bottleneck.
+
 ## Layout
 
 ```
@@ -29,15 +42,35 @@ guest/job.sh       runs inside the guest for every job
 `provision <hostname> tart` disables remote management, makes sshd key-only,
 joins the tailnet, installs `buildkite-agent` and Tart, creates an
 unprivileged auto-login user (Virtualization.framework needs a console
-session), bakes the guest image from a public base image plus
-`scripts/bootstrap.sh`, and starts the agent as that user with `hooks/` as
-its hooks path. It asks for one reboot the first time and is re-run after it.
+session), bakes the guest images from their public base images plus
+`scripts/bootstrap.sh`, and starts the agents as that user with `hooks/` as
+their hooks path. It asks for one reboot the first time and is re-run after it.
+On a host that is already serving it is also the update path: it bakes from a
+staged copy of these scripts while the existing agents carry on with the
+installed hooks, and only then runs `install-agent`, so a failed bake leaves
+the agents and hooks untouched (each image is swapped in as its own bake
+verifies, exactly as a standalone `bake` does). Baking needs one of the host's
+two guest slots, so on a busy host run it while the agents are idle, or expect
+the boot of the staging guest to fail and re-run.
 
 `provision <hostname> bare` does the same host setup, then runs
 `scripts/bootstrap.sh` on the host and installs the `scripts/agent.mjs` service.
 
-`bake` is safe on a live host: it builds a staging image and swaps it in only
-after the toolchain verifies. Re-run it when toolchain pins move.
+`bake` is safe on a live host: it refuses a release newer than the host's
+macOS up front, builds a staging image, and swaps it in only after the guest's
+macOS major matches the image's release and the toolchain verifies. Re-run it
+when toolchain pins move, or with `--release N` for one image. `install-agent`
+retires whatever agent jobs the host had (either tart layout or a bare
+`scripts/agent.mjs` daemon, plus the nightly reboot daemon), re-installs these
+scripts, and installs one agent job per image together with the configs that
+point at them.
+
+To move a host that was provisioned with a single `bun-ci-base` image onto
+this layout, re-run `provision` (the same `host.sh` command as below), adding
+`--release 15 --spawn 2` if the host itself is not on macOS 26 yet. While it
+runs the host holds `bun-ci-base`, the new base image(s) and the new image(s)
+at once, so check the disk first. Once it has finished, `tart delete
+bun-ci-base` as the CI user; nothing references it any more.
 
 ## Bringing up a host
 
@@ -52,10 +85,12 @@ ssh <admin>@<host> 'darwin-ci/host.sh <hostname> tart --tags <tailscale tags>'
 ```
 
 Approve the Tailscale login it prints, reboot when it asks, run the same
-command again, and check the agent appears under `queue=test-darwin`.
+command again, and check that `queue=test-darwin` gains one agent per image
+(`<hostname>-tart-26-1` and `<hostname>-tart-15-1`), one tagged with each
+`release-tier`.
 
 ## Removing a host
 
-Unload the agent (`launchctl bootout` the `com.buildkite.buildkite-agent`
+Unload the agents (`launchctl bootout` each `com.buildkite.buildkite-agent.*`
 job), drop the host from the token allowlist and the tailnet, and release it.
 Nothing on a host needs preserving.

scripts/darwin-ci/guest/bake.sh

@@ -1,5 +1,5 @@
 #!/bin/bash
-# Runs inside a fresh guest to turn the base image into bun-ci-base. Args: <bun repo> <ref> <buildkite-agent version> <tools to verify>
+# Runs inside a fresh guest to turn a base image into its bun-ci-<release> image. Args: <bun repo> <ref> <buildkite-agent version> <tools to verify>
 set -euo pipefail
 exec </dev/null
 repo=$1 ref=$2 agent_version=$3 toolchain=$4

scripts/darwin-ci/hooks/command.ts

@@ -1,7 +1,7 @@
 import { $ } from "bun";
 import { unlinkSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
-import { config } from "../lib/config";
+import { agentGuestRelease, guestImage } from "../lib/config";
 import { guest } from "../lib/guest";
 import { fail } from "../lib/shell";
 import { tart } from "../lib/tart";
@@ -29,6 +29,10 @@ if (!command.includes("runner.node.mjs")) {
   process.exit((await $`bash -c ${command}`.nothrow()).exitCode);
 }
 
+const image = guestImage(
+  agentGuestRelease(env) ?? fail("this agent has no release tag; re-run main.ts install-agent on the host"),
+);
+
 let cleaned = false;
 async function cleanup(): Promise<void> {
   if (cleaned) return;
@@ -48,7 +52,7 @@ try {
 process.exit(status);
 
 async function runInGuest(): Promise<number> {
-  console.log(`--- :tart: ${vm} from ${config.tart.image}`);
+  console.log(`--- :tart: ${vm} from ${image}`);
   const ip = (await boot()) ?? (await retryBoot()) ?? fail(`guest never came up; see /tmp/${vm}.log`);
   console.log(`    guest ${ip}`);
   const g = guest(ip);
@@ -72,7 +76,7 @@ async function runInGuest(): Promise<number> {
 }
 
 async function boot(): Promise<string | undefined> {
-  await tart.cloneLocked(config.tart.image, vm);
+  await tart.cloneLocked(image, vm);
   tart.start(vm, `/tmp/${vm}.log`);
   return tart.waitForSsh(vm);
 }

scripts/darwin-ci/lib/agent.ts

@@ -1,13 +1,54 @@
 import { $ } from "bun";
-import { join } from "node:path";
+import { readdirSync } from "node:fs";
+import { basename, join } from "node:path";
 import { ciUserHome, ciUserId } from "./ci-user";
-import { config, releaseTier } from "./config";
+import { config, guestImage, releaseTier } from "./config";
+import { installSelf } from "./host";
 import { plist } from "./launchd";
 import { consoleUser, fail, output, poll, sleep, succeeds, sudoRead, sudoWrite } from "./shell";
 
 const path = "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin";
-const agentLabel = "com.buildkite.buildkite-agent";
+const launchAgents = "/Library/LaunchAgents";
+// One launchd job per guest release, `<prefix>.<release>`. Hosts set up when there
+// was one image per host have a single job named just `<prefix>`; the prefix match
+// in retireAgents() covers it too.
+const agentLabelPrefix = "com.buildkite.buildkite-agent";
 const cleanupLabel = "com.buildkite.cleanup";
+// what scripts/agent.mjs installs on a bare host (label `buildkite-agent`)
+const bareAgentPlist = "/Library/LaunchDaemons/buildkite-agent.plist";
+
+export type TartAgentOptions = { releases: readonly number[]; spawn: number };
+type AgentConfig = { release: number; spawn: number; token: string; home: string };
+
+/** Guests a host runs at once with these agents installed: every agent boots one. */
+export function concurrentGuests({ releases, spawn }: TartAgentOptions): number {
+  return releases.length * spawn;
+}
+
+export function checkTartAgentOptions(options: TartAgentOptions): void {
+  const { releases, spawn } = options;
+  if (!releases.length) fail("no guest releases to serve");
+  const guests = concurrentGuests(options);
+  if (guests > config.tart.maxGuests) {
+    fail(
+      `${releases.length} image(s) x --spawn ${spawn} = ${guests} guests, but macOS runs at most ${config.tart.maxGuests} at once per host`,
+    );
+  }
+}
+
+/** buildkite-agent.cfg for the agents that boot the macOS `release` guest image. */
+export function tartAgentConfig({ release, spawn, token, home }: AgentConfig): string {
+  return [
+    `token="${token}"`,
+    `name="%hostname-tart-${release}-%spawn"`,
+    `tags="queue=${config.queue},os=darwin,arch=aarch64,distro=macOS,release=${release},release-tier=${releaseTier(release)},tart=true"`,
+    `hooks-path="${join(config.installDir, "hooks")}"`,
+    `build-path="${join(home, "tart-builds")}"`,
+    `spawn=${spawn}`,
+    `cancel-grace-period=30`,
+    "",
+  ].join("\n");
+}
 
 async function agentToken(): Promise<string> {
   return (
@@ -17,63 +58,36 @@ async function agentToken(): Promise<string> {
   );
 }
 
-type TartAgentOptions = { release: number; spawn: number };
-
-export async function installTartAgent({ release, spawn }: TartAgentOptions): Promise<void> {
+export async function installTartAgent(options: TartAgentOptions): Promise<void> {
+  const { releases, spawn } = options;
   const user = config.ciUser;
+  checkTartAgentOptions(options);
   if ((await consoleUser()) !== user)
     fail(`${user} is not logged in at the console; run setup-user, reboot, then retry`);
-  const baked = await succeeds($`sudo -u ${user} -H ${config.tart.bin} get ${config.tart.image}`);
-  if (!baked) fail(`no ${config.tart.image} image for ${user}; run bake as ${user} first`);
+  for (const release of releases) {
+    const image = guestImage(release);
+    const baked = await succeeds($`sudo -u ${user} -H ${config.tart.bin} get ${image}`);
+    if (!baked) fail(`no ${image} image for ${user}; run bake --release ${release} as ${user} first`);
+  }
 
   const home = await ciUserHome();
   const uid = await ciUserId();
-  const configPath = join(home, ".buildkite-agent", "buildkite-agent.cfg");
+  const token = await agentToken();
+  const configDir = join(home, ".buildkite-agent");
   const builds = join(home, "tart-builds");
   const logs = join(home, "Library", "Logs", "buildkite-agent");
-  const hooks = join(config.installDir, "hooks");
   const owner = `${user}:staff`;
 
-  await $`sudo install -d -o ${user} -g staff ${builds} ${logs} ${join(home, ".buildkite-agent")}`;
+  await $`sudo install -d -o ${user} -g staff ${builds} ${logs} ${configDir}`;
 
-  await sudoWrite(
-    configPath,
-    [
-      `token="${await agentToken()}"`,
-      `name="%hostname-tart-${release}-%spawn"`,
-      `tags="queue=${config.queue},os=darwin,arch=aarch64,distro=macOS,release=${release},release-tier=${releaseTier(release)},tart=true"`,
-      `hooks-path="${hooks}"`,
-      `build-path="${builds}"`,
-      `spawn=${spawn}`,
-      `cancel-grace-period=30`,
-      "",
-    ].join("\n"),
-    "600",
-    owner,
-  );
-
-  // a LaunchAgent in the auto-login user's GUI session, because Virtualization.framework will not start guests outside one
-  await sudoWrite(
-    `/Library/LaunchAgents/${agentLabel}.plist`,
-    plist({
-      Label: agentLabel,
-      UserName: user,
-      LimitLoadToSessionType: "Aqua",
-      ProcessType: "Interactive",
-      EnvironmentVariables: { PATH: path },
-      ProgramArguments: [config.buildkiteAgent.bin, "start", "--config", configPath],
-      WorkingDirectory: home,
-      RunAtLoad: true,
-      KeepAlive: true,
-      ThrottleInterval: 10,
-      StandardOutPath: join(logs, "buildkite-agent.log"),
-      StandardErrorPath: join(logs, "buildkite-agent.log"),
-    }),
-  );
+  await retireAgents();
+  await $`sudo find ${configDir} -name ${"buildkite-agent*.cfg"} -delete`;
+  // the configs below point at installDir's hooks, which look images up the same way these agents are tagged
+  await installSelf();
 
   const nightly = [
     `PATH=${path}`,
-    `launchctl bootout gui/${uid}/${agentLabel}`,
+    `for plist in ${launchAgents}/${agentLabelPrefix}*.plist; do launchctl bootout gui/${uid}/$(basename $plist .plist); done`,
     "sleep 40",
     `for vm in $(sudo -u ${user} tart list --source local --quiet | grep '^bk-'); do sudo -u ${user} tart delete $vm; done`,
     `rm -rf ${builds}/* /tmp/bk-*.log`,
@@ -88,15 +102,66 @@ export async function installTartAgent({ release, spawn }: TartAgentOptions): Pr
       StartCalendarInterval: { Hour: 5, Minute: 0 },
     }),
   );
-
-  await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow();
-  await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow();
-  await unload(`gui/${uid}/${agentLabel}`);
-  await $`sudo launchctl bootstrap gui/${uid} /Library/LaunchAgents/${agentLabel}.plist`;
   await $`sudo launchctl bootstrap system /Library/LaunchDaemons/${cleanupLabel}.plist`;
 
+  for (const release of releases) {
+    const label = `${agentLabelPrefix}.${release}`;
+    const configPath = join(configDir, `buildkite-agent-${release}.cfg`);
+    const log = join(logs, `buildkite-agent-${release}.log`);
+
+    await sudoWrite(configPath, tartAgentConfig({ release, spawn, token, home }), "600", owner);
+
+    // a LaunchAgent in the auto-login user's GUI session, because Virtualization.framework will not start guests outside one
+    await sudoWrite(
+      join(launchAgents, `${label}.plist`),
+      plist({
+        Label: label,
+        UserName: user,
+        LimitLoadToSessionType: "Aqua",
+        ProcessType: "Interactive",
+        EnvironmentVariables: { PATH: path },
+        ProgramArguments: [config.buildkiteAgent.bin, "start", "--config", configPath],
+        WorkingDirectory: home,
+        RunAtLoad: true,
+        KeepAlive: true,
+        ThrottleInterval: 10,
+        StandardOutPath: log,
+        StandardErrorPath: log,
+      }),
+    );
+    await $`sudo launchctl bootstrap gui/${uid} ${join(launchAgents, `${label}.plist`)}`;
+  }
+
   await sleep(4000);
-  console.log(await output($`sudo tail -4 ${join(logs, "buildkite-agent.log")}`));
+  for (const release of releases) {
+    console.log(`--- macOS ${release} (${releaseTier(release)})`);
+    console.log(await output($`sudo tail -4 ${join(logs, `buildkite-agent-${release}.log`)}`));
+  }
+}
+
+/**
+ * Stop and remove every agent job on this host, whichever layout installed it
+ * (per-image or single-image tart jobs, or scripts/agent.mjs's bare daemon), and
+ * the nightly reboot that would otherwise fire while they are being replaced.
+ * installTartAgent() puts the tart jobs and the reboot back.
+ */
+async function retireAgents(): Promise<void> {
+  await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow();
+  await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow();
+  await $`sudo rm -f ${bareAgentPlist}`;
+  const labels = installedAgentLabels();
+  if (!labels.length) return;
+  const uid = await ciUserId();
+  for (const label of labels) {
+    await unload(`gui/${uid}/${label}`);
+    await $`sudo rm -f ${join(launchAgents, `${label}.plist`)}`;
+  }
+}
+
+function installedAgentLabels(): string[] {
+  return readdirSync(launchAgents)
+    .filter(name => name.startsWith(agentLabelPrefix) && name.endsWith(".plist"))
+    .map(name => basename(name, ".plist"));
 }
 
 export async function installBareAgent(): Promise<void> {