From 88bc38469cb3a3a6cbb14937bb80101533dc44f9 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Tue, 11 Aug 2026 18:13:08 -0700 Subject: [PATCH] darwin-ci: address review of the tart host scripts Follow-ups from the review of #37633, which added scripts/darwin-ci. The scripts now use Bun.spawn with argv arrays instead of Bun.$: a host runs them on whatever bun host.sh installed when it was provisioned, and Bun.$'s quoting differs between releases ($.escape does not produce bash-safe strings, so job.env broke on any commit message containing a quote; literal `*` words were glob-expanded, so collectReports never ran). - shell.ts: `run`/`output` throw on failure, `probe` is for commands whose failure is an answer; `shellQuote` for strings handed to a remote shell - tart.ts: the clone lock is a symlink holding the owner's pid, broken only when that pid is dead; stop/destroy tolerate exactly "not running" / "does not exist" and propagate everything else - bake.ts: validates the ref up front, quotes every argument passed into the guest, and swaps the image in under the clone lock keeping the previous image until the rename succeeds - release-tier.mjs: one copy of the release-tier thresholds shared with scripts/agent.mjs, so bare and tart hosts cannot disagree - job.sh: run BUILDKITE_COMMAND through `bash -e -o pipefail` (sharding already comes from the forwarded env), stop hiding warm-up install output - command hook: host steps also run under `bash -e -o pipefail`; job.env is 0600 and always removed; a cleanup failure no longer masks the test status - pre-exit: an orphan that fails to delete is logged and the rest still reaped - ci-user: createhomedir/dseditgroup only when needed and no longer nothrow; ciUserId rejects uid 0 - host.sh: brew installer and bun pinned by commit/version and sha256 - host.ts: profiles touched by explicit path, agent download cleaned up in finally; agent.ts nightly reap uses `sudo -H` like its siblings - main.ts: --release/--spawn must be positive integers No-Verification-Needed: CI provisioning scripts, no bun runtime surface --- scripts/agent.mjs | 26 +++----- scripts/darwin-ci/README.md | 8 ++- scripts/darwin-ci/guest/job.sh | 15 +++-- scripts/darwin-ci/hooks/command.ts | 29 +++++---- scripts/darwin-ci/hooks/pre-exit.ts | 23 +++++-- scripts/darwin-ci/host.sh | 26 ++++++-- scripts/darwin-ci/lib/agent.ts | 64 +++++++++++++------- scripts/darwin-ci/lib/bake.ts | 30 ++++++++-- scripts/darwin-ci/lib/ci-user.ts | 66 ++++++++++++++++---- scripts/darwin-ci/lib/config.ts | 6 -- scripts/darwin-ci/lib/guest.ts | 37 ++++++------ scripts/darwin-ci/lib/host.ts | 68 ++++++++++++--------- scripts/darwin-ci/lib/release-tier.mjs | 10 ++++ scripts/darwin-ci/lib/shell.ts | 80 ++++++++++++++++++------- scripts/darwin-ci/lib/tart.ts | 83 ++++++++++++++++++-------- scripts/darwin-ci/main.ts | 42 ++++++++++--- 16 files changed, 415 insertions(+), 198 deletions(-) create mode 100644 scripts/darwin-ci/lib/release-tier.mjs diff --git a/scripts/agent.mjs b/scripts/agent.mjs index ca4e7bda731c..b9e23e4c207f 100755 --- a/scripts/agent.mjs +++ b/scripts/agent.mjs @@ -3,9 +3,10 @@ // An agent that starts buildkite-agent and runs others services. import { copyFileSync, existsSync, readFileSync, realpathSync } from "node:fs"; -import { join } from "node:path"; +import { dirname, join } from "node:path"; import { fileURLToPath } from "node:url"; import { parseArgs } from "node:util"; +import { darwinReleaseTier } from "./darwin-ci/lib/release-tier.mjs"; import { getAbi, getAbiVersion, @@ -36,22 +37,8 @@ import { const BUILDKITE_TOKEN_SECRET = "buildkite/agent-token"; const AZURE_KEYVAULT = "bun-ci"; const AZURE_TOKEN_SECRET = "buildkite-agent-token"; - -// macOS major-version thresholds for the `release-tier` agent tag. -// >= LATEST -> "latest" (current macOS; arm64-only in practice) -// >= PREVIOUS -> "previous" (recent-but-not-current; 14/15 today) -// else -> "oldest" (min-supported; 13 today) -// Bump LATEST when a new macOS ships and the first runner on it is online. -// Bump PREVIOUS when the floor of "recent" moves. -const LATEST_DARWIN_RELEASE = 26; -const PREVIOUS_DARWIN_RELEASE = 14; - -function darwinReleaseTier(distroVersion) { - const major = parseInt(distroVersion?.split(".")[0] || "0"); - if (major >= LATEST_DARWIN_RELEASE) return "latest"; - if (major >= PREVIOUS_DARWIN_RELEASE) return "previous"; - return "oldest"; -} +// must match the import above; install() copies it next to agent.mjs at the same relative path +const RELEASE_TIER_MODULE = "darwin-ci/lib/release-tier.mjs"; /** * @param {"install" | "start"} action @@ -149,7 +136,8 @@ async function doBuildkiteAgent(action, cliOptions = {}) { // Copy this script and its imports into homePath so the launchd plist // doesn't depend on the checkout that ran `install` sticking around. const srcDir = fileURLToPath(new URL(".", import.meta.url)); - for (const f of ["agent.mjs", "utils.mjs"]) { + for (const f of ["agent.mjs", "utils.mjs", RELEASE_TIER_MODULE]) { + mkdir(dirname(join(homePath, f))); copyFileSync(join(srcDir, f), join(homePath, f)); } // Stable node path (the Homebrew/usr-local symlink, not a Cellar version @@ -370,7 +358,7 @@ async function doBuildkiteAgent(action, cliOptions = {}) { // ci.mjs targets darwin test jobs by `release-tier` so each PR runs on // distinct OS-age pools without needing per-box config. arm64 uses // latest+previous; x64 uses previous+oldest (Intel can't run latest). - "release-tier": isMacOS ? darwinReleaseTier(distroVersion) : undefined, + "release-tier": isMacOS ? darwinReleaseTier(parseInt(distroVersion?.split(".")[0] || "0")) : undefined, "ephemeral": ephemeral || false, "cloud": cloud, }; diff --git a/scripts/darwin-ci/README.md b/scripts/darwin-ci/README.md index c1d7b7ad0252..5dcd2a1165d0 100644 --- a/scripts/darwin-ci/README.md +++ b/scripts/darwin-ci/README.md @@ -21,6 +21,7 @@ guest's macOS version, and a guest cannot be newer than its host. host.sh first contact: installs brew and bun, then runs `main.ts provision` main.ts provision | setup-user | bake | install-agent lib/ host hardening, tailscale, the unprivileged CI user, tart, bake, agent config +lib/release-tier.mjs the `release-tier` thresholds, also imported by scripts/agent.mjs for bare hosts hooks/ agent hooks for tart hosts (command, pre-exit, environment) guest/bake.sh runs inside the guest once, at bake time guest/job.sh runs inside the guest for every job @@ -37,7 +38,12 @@ its hooks path. It asks for one reboot the first time and is re-run after it. `scripts/bootstrap.sh` on the host and installs the `scripts/agent.mjs` service. `bake` is safe on a live host: it builds a staging image and swaps it in only -after the toolchain verifies. Re-run it when toolchain pins move. +after the toolchain verifies, under the same lock the command hook clones +with. Re-run it when toolchain pins move. + +Everything here runs on whatever bun `host.sh` pinned when the host was first +provisioned, so it sticks to `Bun.spawn` with argv arrays and stays off +`Bun.$`. ## Bringing up a host diff --git a/scripts/darwin-ci/guest/job.sh b/scripts/darwin-ci/guest/job.sh index 3545ac487f10..1d40fa43bce8 100755 --- a/scripts/darwin-ci/guest/job.sh +++ b/scripts/darwin-ci/guest/job.sh @@ -6,20 +6,19 @@ export PATH=/usr/local/bin:/opt/homebrew/bin:/opt/rust/bin:$PATH # does not source; without them the proxies in /opt/rust/bin look for a toolchain in ~/.rustup and find none. export RUSTUP_HOME=/opt/rust CARGO_HOME=/opt/rust export BUILDKITE_BUILD_CHECKOUT_PATH=$HOME/work -set -a; source ~/job.env; set +a -cd ~/work +set -a; source ~/job.env || exit 1; set +a +cd ~/work || exit 1 echo "[guest] macOS $(sw_vers -productVersion) shard=${BUILDKITE_PARALLEL_JOB:-0}/${BUILDKITE_PARALLEL_JOB_COUNT:-1}" -bun install >/dev/null 2>&1 || true -(cd test && bun install >/dev/null 2>&1) || true - -shard=() -[ -n "${BUILDKITE_PARALLEL_JOB:-}" ] && shard=(--shard="$BUILDKITE_PARALLEL_JOB" --max-shards="$BUILDKITE_PARALLEL_JOB_COUNT") +# warms node_modules with the guest's bun; the runner repeats both installs with the bun under test and reports those +for dir in . test; do + (cd "$dir" && bun install) >"$HOME/install.out" 2>&1 || { echo "[guest] bun install in $dir failed:"; cat "$HOME/install.out"; } +done # the runner can leak servers that keep node alive after the run, so wait on it directly and stream its log log=$HOME/runner.out : >"$log" -$BUILDKITE_COMMAND "${shard[@]}" >"$log" 2>&1 & +bash -e -o pipefail -c "$BUILDKITE_COMMAND" >"$log" 2>&1 & runner=$! tail -f "$log" & tailer=$! diff --git a/scripts/darwin-ci/hooks/command.ts b/scripts/darwin-ci/hooks/command.ts index 2a064168c2ee..50b669da7f7e 100644 --- a/scripts/darwin-ci/hooks/command.ts +++ b/scripts/darwin-ci/hooks/command.ts @@ -1,9 +1,8 @@ -import { $ } from "bun"; import { unlinkSync, writeFileSync } from "node:fs"; import { join } from "node:path"; import { config } from "../lib/config"; import { guest } from "../lib/guest"; -import { fail } from "../lib/shell"; +import { fail, runInherit, shellQuote } from "../lib/shell"; import { tart } from "../lib/tart"; const env = process.env; @@ -26,14 +25,16 @@ const hostOnlyEnv = new Set([ if (!command.includes("runner.node.mjs")) { console.log("--- running on host (not a test step)"); - process.exit((await $`bash -c ${command}`.nothrow()).exitCode); + // same flags buildkite-agent's own command runner uses, so a failing middle line still fails the step + process.exit(await runInherit(["bash", "-e", "-o", "pipefail", "-c", command], { cwd: checkout })); } let cleaned = false; async function cleanup(): Promise { if (cleaned) return; cleaned = true; - await tart.destroy(vm); + // pre-exit destroys it again and reaps whatever this misses, so a failure here must not override the test status + await tart.destroy(vm).catch((error: unknown) => console.error(`failed to destroy ${vm}: ${error}`)); } for (const signal of ["SIGINT", "SIGTERM", "SIGHUP"] as const) { process.on(signal, () => cleanup().finally(() => process.exit(130))); @@ -61,10 +62,11 @@ async function runInGuest(): Promise { console.log(`+++ :test_tube: ${command}`); const socket = "/tmp/bk-job-api.sock"; const forward = ["-o", "StreamLocalBindUnlink=yes", "-R", `${socket}:${env.BUILDKITE_AGENT_JOB_API_SOCKET}`]; - const status = await g.run( - `BUILDKITE_AGENT_JOB_API_SOCKET=${socket} BUILDKITE_AGENT_JOB_API_TOKEN=${env.BUILDKITE_AGENT_JOB_API_TOKEN ?? ""} /bin/bash ~/job.sh`, - forward, - ); + const jobEnv = [ + `BUILDKITE_AGENT_JOB_API_SOCKET=${socket}`, + `BUILDKITE_AGENT_JOB_API_TOKEN=${shellQuote(env.BUILDKITE_AGENT_JOB_API_TOKEN ?? "")}`, + ]; + const status = await g.run(`${jobEnv.join(" ")} /bin/bash ~/job.sh`, forward); console.log("--- :outbox_tray: collect reports"); await g.collectReports("work", checkout); @@ -87,9 +89,12 @@ async function retryBoot(): Promise { async function pushEnv(g: ReturnType): Promise { const lines = Object.entries(env) .filter(([key]) => /^(BUILDKITE|BUN_|EXPECTED_PLATFORM_)|^(CI|ASAN_OPTIONS)$/.test(key) && !hostOnlyEnv.has(key)) - .map(([key, value]) => `${key}=${$.escape(String(value ?? ""))}`); + .map(([key, value]) => `${key}=${shellQuote(value ?? "")}`); const file = `/tmp/${vm}.env`; - writeFileSync(file, lines.join("\n") + "\n"); - await g.push(file, "job.env"); - unlinkSync(file); + writeFileSync(file, lines.join("\n") + "\n", { mode: 0o600 }); + try { + await g.push(file, "job.env"); + } finally { + unlinkSync(file); + } } diff --git a/scripts/darwin-ci/hooks/pre-exit.ts b/scripts/darwin-ci/hooks/pre-exit.ts index 9025d8b55ea8..c06d706f12bf 100644 --- a/scripts/darwin-ci/hooks/pre-exit.ts +++ b/scripts/darwin-ci/hooks/pre-exit.ts @@ -1,11 +1,21 @@ -import { $ } from "bun"; import { readdirSync, statSync } from "node:fs"; import { homedir } from "node:os"; import { join } from "node:path"; import { succeeds } from "../lib/shell"; import { tart } from "../lib/tart"; -await tart.destroy(`bk-${process.env.BUILDKITE_JOB_ID ?? "none"}`); +let failed = false; +async function reap(name: string, action: () => Promise): Promise { + try { + await action(); + } catch (error) { + failed = true; + console.error(`pre-exit: could not remove ${name}: ${error}`); + } +} + +const own = `bk-${process.env.BUILDKITE_JOB_ID ?? "none"}`; +await reap(own, () => tart.destroy(own)); // a bk-* guest with no `tart run` behind it belongs to a job that died without its cleanup running const vms = join(homedir(), ".tart", "vms"); @@ -13,7 +23,10 @@ const tenMinutesAgo = Date.now() - 10 * 60_000; for (const name of readdirSync(vms).filter((name: string) => name.startsWith("bk-"))) { const stat = statSync(join(vms, name), { throwIfNoEntry: false }); if (!stat || stat.mtimeMs > tenMinutesAgo) continue; - if (await succeeds($`pgrep -f ${`tart run ${name}`}`)) continue; - await tart.remove(name); - console.log(`pre-exit: reaped orphan guest ${name}`); + if (await succeeds(["pgrep", "-f", `tart run ${name}`])) continue; + await reap(name, async () => { + await tart.remove(name); + console.log(`pre-exit: reaped orphan guest ${name}`); + }); } +process.exit(failed ? 1 : 0); diff --git a/scripts/darwin-ci/host.sh b/scripts/darwin-ci/host.sh index 56a00208eca7..78225f946fd2 100755 --- a/scripts/darwin-ci/host.sh +++ b/scripts/darwin-ci/host.sh @@ -4,20 +4,36 @@ set -euo pipefail here=$(cd "$(dirname "$0")" && pwd) sudo -n true 2>/dev/null || { echo "needs passwordless sudo for $(whoami)"; exit 1; } -if [ "$(uname -m)" = arm64 ]; then prefix=/opt/homebrew; target=aarch64; else prefix=/usr/local; target=x64; fi +# https://github.com/Homebrew/install/commits/master +brew_installer_commit=a34ae4ee9151cbce4c3b33bca7043a972b7ae9a5 +brew_installer_sha256=12479a24be3f5307eecac7cde670fad7118640f031229e964f544b1367b52a41 +# https://github.com/oven-sh/bun/releases/download/bun-v$bun_version/SHASUMS256.txt +bun_version=1.3.14 +bun_sha256_aarch64=d8b96221828ad6f97ac7ac0ab7e95872341af763001e8803e8267652c2652620 +bun_sha256_x64=4183df3374623e5bab315c547cfa0974533cd457d86b73b639f7a87974cd6633 + +if [ "$(uname -m)" = arm64 ]; then prefix=/opt/homebrew; target=aarch64; bun_sha256=$bun_sha256_aarch64; else prefix=/usr/local; target=x64; bun_sha256=$bun_sha256_x64; fi + +tmp=$(mktemp -d) +trap 'rm -rf "$tmp"' EXIT + +fetch() { # + curl -fsSL -o "$2" "$1" + echo "$3 $2" | shasum -a 256 -c - >/dev/null || { echo "checksum mismatch for $1"; exit 1; } +} if [ ! -x "$prefix/bin/brew" ]; then - NONINTERACTIVE=1 /bin/bash -c "$(curl -fsSL https://raw.githubusercontent.com/Homebrew/install/HEAD/install.sh)" + fetch "https://raw.githubusercontent.com/Homebrew/install/$brew_installer_commit/install.sh" "$tmp/brew-install.sh" "$brew_installer_sha256" + NONINTERACTIVE=1 /bin/bash "$tmp/brew-install.sh" fi eval "$("$prefix/bin/brew" shellenv)" if [ ! -x /usr/local/bin/bun ]; then - tmp=$(mktemp -d) - curl -fsSL -o "$tmp/bun.zip" "https://github.com/oven-sh/bun/releases/latest/download/bun-darwin-$target.zip" + fetch "https://github.com/oven-sh/bun/releases/download/bun-v$bun_version/bun-darwin-$target.zip" "$tmp/bun.zip" "$bun_sha256" unzip -q "$tmp/bun.zip" -d "$tmp" sudo mkdir -p /usr/local/bin sudo install -m 755 "$tmp/bun-darwin-$target/bun" /usr/local/bin/bun - rm -rf "$tmp" fi +rm -rf "$tmp"; trap - EXIT exec /usr/local/bin/bun "$here/main.ts" provision "$@" diff --git a/scripts/darwin-ci/lib/agent.ts b/scripts/darwin-ci/lib/agent.ts index a166f789382f..c91215ee1a3b 100644 --- a/scripts/darwin-ci/lib/agent.ts +++ b/scripts/darwin-ci/lib/agent.ts @@ -1,9 +1,22 @@ -import { $ } from "bun"; import { join } from "node:path"; import { ciUserHome, ciUserId } from "./ci-user"; -import { config, releaseTier } from "./config"; +import { config } from "./config"; +import { bootstrapCheckout } from "./host"; import { plist } from "./launchd"; -import { consoleUser, fail, output, poll, sleep, succeeds, sudoRead, sudoWrite } from "./shell"; +import { darwinReleaseTier } from "./release-tier.mjs"; +import { + consoleUser, + fail, + poll, + probe, + run, + runInheritOrThrow, + sleep, + spawn, + succeeds, + sudoRead, + sudoWrite, +} from "./shell"; const path = "/usr/local/bin:/opt/homebrew/bin:/usr/bin:/bin:/usr/sbin:/sbin"; const agentLabel = "com.buildkite.buildkite-agent"; @@ -17,13 +30,14 @@ async function agentToken(): Promise { ); } -type TartAgentOptions = { release: number; spawn: number }; +export type TartAgentOptions = { release: number; spawn: number }; -export async function installTartAgent({ release, spawn }: TartAgentOptions): Promise { +export async function installTartAgent({ release, spawn: workers }: TartAgentOptions): Promise { const user = config.ciUser; if ((await consoleUser()) !== user) fail(`${user} is not logged in at the console; run setup-user, reboot, then retry`); - const baked = await succeeds($`sudo -u ${user} -H ${config.tart.bin} get ${config.tart.image}`); + const asCiUser = ["sudo", "-u", user, "-H"]; + const baked = await succeeds([...asCiUser, config.tart.bin, "get", config.tart.image]); if (!baked) fail(`no ${config.tart.image} image for ${user}; run bake as ${user} first`); const home = await ciUserHome(); @@ -34,17 +48,17 @@ export async function installTartAgent({ release, spawn }: TartAgentOptions): Pr const hooks = join(config.installDir, "hooks"); const owner = `${user}:staff`; - await $`sudo install -d -o ${user} -g staff ${builds} ${logs} ${join(home, ".buildkite-agent")}`; + await run(["sudo", "install", "-d", "-o", user, "-g", "staff", builds, logs, join(home, ".buildkite-agent")]); await sudoWrite( configPath, [ `token="${await agentToken()}"`, `name="%hostname-tart-${release}-%spawn"`, - `tags="queue=${config.queue},os=darwin,arch=aarch64,distro=macOS,release=${release},release-tier=${releaseTier(release)},tart=true"`, + `tags="queue=${config.queue},os=darwin,arch=aarch64,distro=macOS,release=${release},release-tier=${darwinReleaseTier(release)},tart=true"`, `hooks-path="${hooks}"`, `build-path="${builds}"`, - `spawn=${spawn}`, + `spawn=${workers}`, `cancel-grace-period=30`, "", ].join("\n"), @@ -71,11 +85,12 @@ export async function installTartAgent({ release, spawn }: TartAgentOptions): Pr }), ); + const tartAsCiUser = `${asCiUser.join(" ")} ${config.tart.bin}`; const nightly = [ `PATH=${path}`, `launchctl bootout gui/${uid}/${agentLabel}`, "sleep 40", - `for vm in $(sudo -u ${user} tart list --source local --quiet | grep '^bk-'); do sudo -u ${user} tart delete $vm; done`, + `for vm in $(${tartAsCiUser} list --source local --quiet | grep '^bk-'); do ${tartAsCiUser} delete $vm; done`, `rm -rf ${builds}/* /tmp/bk-*.log`, "shutdown -r now", ].join("; "); @@ -89,28 +104,33 @@ export async function installTartAgent({ release, spawn }: TartAgentOptions): Pr }), ); - await $`sudo launchctl bootout system/buildkite-agent`.quiet().nothrow(); - await $`sudo launchctl bootout system/${cleanupLabel}`.quiet().nothrow(); + await spawn(["sudo", "launchctl", "bootout", "system/buildkite-agent"]); + await spawn(["sudo", "launchctl", "bootout", `system/${cleanupLabel}`]); await unload(`gui/${uid}/${agentLabel}`); - await $`sudo launchctl bootstrap gui/${uid} /Library/LaunchAgents/${agentLabel}.plist`; - await $`sudo launchctl bootstrap system /Library/LaunchDaemons/${cleanupLabel}.plist`; + await run(["sudo", "launchctl", "bootstrap", `gui/${uid}`, `/Library/LaunchAgents/${agentLabel}.plist`]); + await run(["sudo", "launchctl", "bootstrap", "system", `/Library/LaunchDaemons/${cleanupLabel}.plist`]); await sleep(4000); - console.log(await output($`sudo tail -4 ${join(logs, "buildkite-agent.log")}`)); + console.log( + (await probe(["sudo", "tail", "-4", join(logs, "buildkite-agent.log")])) ?? "(agent log not written yet)", + ); } export async function installBareAgent(): Promise { - const checkout = join(process.env.HOME!, "bun-bootstrap"); const token = await agentToken(); - await $`sudo env BUILDKITE_AGENT_TOKEN=${token} PATH=${path} node scripts/agent.mjs install`.cwd(checkout); - await sleep(4000); - console.log( - await output($`tail -4 ${join(process.env.HOME!, "Library", "Logs", "buildkite-agent", "buildkite-agent.log")}`), + await runInheritOrThrow( + ["sudo", "env", `PATH=${path}`, `BUILDKITE_AGENT_TOKEN=${token}`, "node", "scripts/agent.mjs", "install"], + { + cwd: bootstrapCheckout, + }, ); + await sleep(4000); + const log = join(process.env.HOME!, "Library", "Logs", "buildkite-agent", "buildkite-agent.log"); + console.log((await probe(["tail", "-4", log])) ?? "(agent log not written yet)"); } // bootout returns before a busy agent has finished its cancel grace period, and bootstrap fails with EIO until it has async function unload(target: string): Promise { - await $`sudo launchctl bootout ${target}`.quiet().nothrow(); - await poll(30, 2000, async () => ((await succeeds($`sudo launchctl print ${target}`)) ? undefined : true)); + await spawn(["sudo", "launchctl", "bootout", target]); + await poll(30, 2000, async () => ((await succeeds(["sudo", "launchctl", "print", target])) ? undefined : true)); } diff --git a/scripts/darwin-ci/lib/bake.ts b/scripts/darwin-ci/lib/bake.ts index 16588d5713dd..08c19b4a4113 100644 --- a/scripts/darwin-ci/lib/bake.ts +++ b/scripts/darwin-ci/lib/bake.ts @@ -1,7 +1,7 @@ import { join } from "node:path"; import { config, toolchain } from "./config"; import { ensureHostKey, guest } from "./guest"; -import { fail, log } from "./shell"; +import { fail, log, shellQuote, succeeds } from "./shell"; import { tart } from "./tart"; type BakeOptions = { base: string; ref: string }; @@ -9,6 +9,7 @@ type BakeOptions = { base: string; ref: string }; export async function bake({ base, ref }: BakeOptions): Promise { const { image, cpu, memoryMb } = config.tart; const staging = `${image}-new`; + if (!(await succeeds(["git", "check-ref-format", "--allow-onelevel", ref]))) fail(`invalid ref: ${ref}`); const publicKey = await ensureHostKey(); log(`pull ${base}`); @@ -27,24 +28,41 @@ export async function bake({ base, ref }: BakeOptions): Promise { (await tart.waitForAgent(staging)) ?? fail("tart guest agent not responding"); await tart.exec( staging, - `mkdir -p ~/.ssh && chmod 700 ~/.ssh && echo '${publicKey}' > ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys`, + `mkdir -p ~/.ssh && chmod 700 ~/.ssh && echo ${shellQuote(publicKey)} > ~/.ssh/authorized_keys && chmod 600 ~/.ssh/authorized_keys`, ); const g = guest(ip); await g.push(join(import.meta.dir, "..", "guest", "bake.sh"), "/tmp/bake.sh"); log(`bootstrap toolchain in guest (${config.bun.repo}@${ref})`); + const bakeArgs = [config.bun.repo, ref, config.buildkiteAgent.version, toolchain.join(" ")].map(shellQuote); const status = await g.run( - `/bin/bash -l /tmp/bake.sh ${config.bun.repo} ${ref} ${config.buildkiteAgent.version} '${toolchain.join(" ")}' 2>&1 | tee /tmp/bake.log; grep -qx BAKE_OK /tmp/bake.log`, + `/bin/bash -l /tmp/bake.sh ${bakeArgs.join(" ")} 2>&1 | tee /tmp/bake.log; grep -qx BAKE_OK /tmp/bake.log`, ); if (status !== 0) fail(`bake failed in guest; ${image} untouched, ${staging} left running at ${ip} for inspection`); - await g.run("sync; sudo shutdown -h now").catch(() => {}); + await g.run("sync; sudo shutdown -h now"); log("waiting for guest to power off"); await Promise.race([vm.exited, Bun.sleep(180_000)]); await tart.stop(staging); - await tart.remove(image); - await tart.rename(staging, image); + await promote(staging, image); log(`${image} ready`); } + +// swap under the image lock so no job clones mid-rename; the previous image is kept until the new one is in place +async function promote(staging: string, image: string): Promise { + const previous = `${image}-previous`; + await tart.withImageLock(async () => { + await tart.destroy(previous); + const hadPrevious = await tart.exists(image); + if (hadPrevious) await tart.rename(image, previous); + try { + await tart.rename(staging, image); + } catch (error) { + if (hadPrevious) await tart.rename(previous, image); + throw error; + } + }); + await tart.destroy(previous); +} diff --git a/scripts/darwin-ci/lib/ci-user.ts b/scripts/darwin-ci/lib/ci-user.ts index 2ab9df71fcb6..db893d157465 100644 --- a/scripts/darwin-ci/lib/ci-user.ts +++ b/scripts/darwin-ci/lib/ci-user.ts @@ -1,31 +1,49 @@ -import { $ } from "bun"; +import { existsSync } from "node:fs"; import { config } from "./config"; -import { fail, output, succeeds, sudoRead, sudoWrite } from "./shell"; +import { fail, output, run, succeeds, sudoRead, sudoWrite } from "./shell"; const user = config.ciUser; const passwordFile = "/var/root/ci-user-password"; const kcpasswordKey = [0x7d, 0x89, 0x52, 0x23, 0xd2, 0xbc, 0xdd, 0xea, 0xa3, 0xb9, 0x1f]; -export const ciUserExists = () => succeeds($`id ${user}`); +export const ciUserExists = () => succeeds(["id", user]); export async function ciUserHome(): Promise { - const line = await output($`dscl . -read /Users/${user} NFSHomeDirectory`); + const line = await output(["dscl", ".", "-read", `/Users/${user}`, "NFSHomeDirectory"]); return line.split(/\s+/)[1] ?? fail(`no home directory for ${user}`); } -export const ciUserId = async () => Number(await output($`id -u ${user}`)); +export async function ciUserId(): Promise { + const uid = Number(await output(["id", "-u", user])); + return uid > 0 ? uid : fail(`unexpected uid ${uid} for ${user}`); +} export async function ensureCiUser(): Promise { if (await ciUserExists()) return; - await $`sudo /bin/sh -c ${`umask 077; openssl rand -hex 24 > ${passwordFile}`}`; + await run(["sudo", "/bin/sh", "-c", `umask 077; openssl rand -hex 24 > ${passwordFile}`]); const password = (await sudoRead(passwordFile)) ?? fail(`could not read ${passwordFile}`); - await $`sudo sysadminctl -addUser ${user} -fullName CI -shell /bin/zsh -home /Users/${user} -password ${password}`.quiet(); - await $`sudo createhomedir -c -u ${user}`.quiet().nothrow(); - await $`sudo dseditgroup -o edit -d ${user} -t user admin`.quiet().nothrow(); + await run([ + "sudo", + "sysadminctl", + "-addUser", + user, + "-fullName", + "CI", + "-shell", + "/bin/zsh", + "-home", + `/Users/${user}`, + "-password", + password, + ]); + if (!existsSync(await ciUserHome())) await run(["sudo", "createhomedir", "-c", "-u", user]); + const membership = await output(["dsmemberutil", "checkmembership", "-U", user, "-G", "admin"]); + if (!membership.includes("is not a member")) + await run(["sudo", "dseditgroup", "-o", "edit", "-d", user, "-t", "user", "admin"]); } // loginwindow reads the auto-login password from /etc/kcpassword, XOR'd with a fixed key and padded to 12 bytes -function kcpassword(password: string): Uint8Array { +function kcpassword(password: string): Uint8Array { const bytes = [...Buffer.from(password), 0]; while (bytes.length % 12) bytes.push(0); return Uint8Array.from(bytes, (byte, i) => byte ^ kcpasswordKey[i % kcpasswordKey.length]); @@ -36,7 +54,29 @@ export async function enableAutoLogin(): Promise { (await sudoRead(passwordFile)) ?? fail(`${passwordFile} missing; cannot configure auto-login for an existing ${user}`); await sudoWrite("/etc/kcpassword", kcpassword(password), "600"); - await $`sudo defaults write /Library/Preferences/com.apple.loginwindow autoLoginUser ${user}`; - await $`sudo defaults write /Library/Preferences/com.apple.loginwindow DisableScreenLockImmediate -bool true`; - await $`sudo pmset -a sleep 0 displaysleep 0 disksleep 0 womp 1 autorestart 1`.quiet(); + await run(["sudo", "defaults", "write", "/Library/Preferences/com.apple.loginwindow", "autoLoginUser", user]); + await run([ + "sudo", + "defaults", + "write", + "/Library/Preferences/com.apple.loginwindow", + "DisableScreenLockImmediate", + "-bool", + "true", + ]); + await run([ + "sudo", + "pmset", + "-a", + "sleep", + "0", + "displaysleep", + "0", + "disksleep", + "0", + "womp", + "1", + "autorestart", + "1", + ]); } diff --git a/scripts/darwin-ci/lib/config.ts b/scripts/darwin-ci/lib/config.ts index 7883ddd47067..42049257a2b7 100644 --- a/scripts/darwin-ci/lib/config.ts +++ b/scripts/darwin-ci/lib/config.ts @@ -18,9 +18,3 @@ export const config = { } as const; export const toolchain = ["bun", "node", "cmake", "ninja", "ccache", "cargo", "go", "clang-21"]; - -export function releaseTier(release: number): "latest" | "previous" | "oldest" { - if (release >= 26) return "latest"; - if (release >= 14) return "previous"; - return "oldest"; -} diff --git a/scripts/darwin-ci/lib/guest.ts b/scripts/darwin-ci/lib/guest.ts index 44fc6bf7ef27..734745a64b9c 100644 --- a/scripts/darwin-ci/lib/guest.ts +++ b/scripts/darwin-ci/lib/guest.ts @@ -1,7 +1,7 @@ -import { $ } from "bun"; import { homedir } from "node:os"; import { join } from "node:path"; import { config } from "./config"; +import { run, runInherit } from "./shell"; export const hostKey = join(homedir(), ".ssh", "id_ed25519"); @@ -24,37 +24,38 @@ const sshOptions = [ export async function ensureHostKey(): Promise { if (!(await Bun.file(hostKey).exists())) { - await $`ssh-keygen -q -t ed25519 -N "" -C ${`${process.env.USER}@darwin-ci`} -f ${hostKey}`; + await run(["ssh-keygen", "-q", "-t", "ed25519", "-N", "", "-C", `${process.env.USER}@darwin-ci`, "-f", hostKey]); } return (await Bun.file(`${hostKey}.pub`).text()).trim(); } export function guest(ip: string) { const target = `${config.tart.guestUser}@${ip}`; - const rsh = `ssh ${sshOptions.join(" ")}`; + const rsh = ["-e", `ssh ${sshOptions.join(" ")}`]; return { ip, - run(command: string, forward: string[] = []): Promise { - const proc = Bun.spawn(["ssh", ...sshOptions, ...forward, target, command], { - stdin: "ignore", - stdout: "inherit", - stderr: "inherit", - }); - return proc.exited; - }, + // the remote sshd hands `command` to the guest user's shell, so callers quote its arguments with shellQuote + run: (command: string, forward: string[] = []) => runInherit(["ssh", ...sshOptions, ...forward, target, command]), - capture: (command: string) => $`ssh ${sshOptions} ${target} ${command} < /dev/null`.text(), - - push: (local: string, remote: string) => $`scp ${sshOptions} -q ${local} ${target}:${remote}`.quiet(), + push: (local: string, remote: string) => run(["scp", ...sshOptions, "-q", local, `${target}:${remote}`]), syncTo: (localDir: string, remoteDir: string) => - $`rsync -a --delete -e ${rsh} ${localDir}/ ${target}:${remoteDir}/`.quiet(), + run(["rsync", "-a", "--delete", ...rsh, `${localDir}/`, `${target}:${remoteDir}/`]), collectReports: (remoteDir: string, localDir: string) => - $`rsync -a -e ${rsh} --include=*/ --include=*.xml --include=*.junit --exclude=* --prune-empty-dirs ${target}:${remoteDir}/ ${localDir}/` - .quiet() - .nothrow(), + run([ + "rsync", + "-a", + ...rsh, + "--include=*/", + "--include=*.xml", + "--include=*.junit", + "--exclude=*", + "--prune-empty-dirs", + `${target}:${remoteDir}/`, + `${localDir}/`, + ]), }; } diff --git a/scripts/darwin-ci/lib/host.ts b/scripts/darwin-ci/lib/host.ts index 96b5688e8c01..b09ab22e89aa 100644 --- a/scripts/darwin-ci/lib/host.ts +++ b/scripts/darwin-ci/lib/host.ts @@ -1,9 +1,8 @@ -import { $ } from "bun"; -import { mkdtempSync } from "node:fs"; -import { tmpdir } from "node:os"; +import { mkdtempSync, rmSync } from "node:fs"; +import { homedir, tmpdir } from "node:os"; import { join } from "node:path"; import { config, toolchain } from "./config"; -import { fail, output, poll, sleep, succeeds, sudoWrite } from "./shell"; +import { fail, poll, probe, run, runInherit, runInheritOrThrow, sleep, spawn, succeeds, sudoWrite } from "./shell"; export const brewPrefix = process.arch === "arm64" ? "/opt/homebrew" : "/usr/local"; const brew = `${brewPrefix}/bin/brew`; @@ -11,9 +10,9 @@ const tailscale = `${brewPrefix}/bin/tailscale`; export async function disableRemoteManagement(): Promise { const kickstart = "/System/Library/CoreServices/RemoteManagement/ARDAgent.app/Contents/Resources/kickstart"; - await $`sudo ${kickstart} -deactivate -stop`.quiet().nothrow(); - await $`sudo launchctl disable system/com.apple.screensharing`.quiet().nothrow(); - await $`sudo launchctl bootout system/com.apple.screensharing`.quiet().nothrow(); + await spawn(["sudo", kickstart, "-deactivate", "-stop"]); + await spawn(["sudo", "launchctl", "disable", "system/com.apple.screensharing"]); + await spawn(["sudo", "launchctl", "bootout", "system/com.apple.screensharing"]); } export async function hardenSshd(): Promise { @@ -21,28 +20,32 @@ export async function hardenSshd(): Promise { "/etc/ssh/sshd_config.d/000-hardening.conf", "PasswordAuthentication no\nKbdInteractiveAuthentication no\nPermitRootLogin no\n", ); - await $`sudo launchctl kickstart -k system/com.openssh.sshd`.quiet(); + await run(["sudo", "launchctl", "kickstart", "-k", "system/com.openssh.sshd"]); } export async function setHostname(name: string): Promise { for (const key of ["ComputerName", "LocalHostName", "HostName"]) { - await $`sudo scutil --set ${key} ${name}`; + await run(["sudo", "scutil", "--set", key, name]); } - // bootstrap.sh only appends PATH entries to profiles that already exist - await $`touch ~/.profile ~/.zshrc ~/.bash_profile`; +} + +// bootstrap.sh only appends PATH entries to profiles that already exist +export async function ensureShellProfiles(): Promise { + const home = homedir(); + await run(["touch", join(home, ".profile"), join(home, ".zshrc"), join(home, ".bash_profile")]); } export async function brewInstall(formula: string): Promise { const name = formula.split("/").pop()!; - if (await succeeds($`${brew} list ${name}`)) return; - await $`${brew} install ${formula}`; + if (await succeeds([brew, "list", name])) return; + await runInheritOrThrow([brew, "install", formula]); } export async function joinTailnet(hostname: string, tags: string | undefined): Promise { await brewInstall("tailscale"); - await $`sudo ${brewPrefix}/bin/tailscaled install-system-daemon`.quiet().nothrow(); + await spawn(["sudo", `${brewPrefix}/bin/tailscaled`, "install-system-daemon"]); await sleep(3000); - if (await succeeds($`sudo ${tailscale} status`)) return; + if (await succeeds(["sudo", tailscale, "status"])) return; const tagArgs = tags ? [`--advertise-tags=${tags}`] : []; Bun.spawn(["sudo", tailscale, "up", "--ssh", ...tagArgs, `--hostname=${hostname}`], { @@ -52,7 +55,7 @@ export async function joinTailnet(hostname: string, tags: string | undefined): P }).unref(); const url = await poll(15, 2000, async () => { - const status = JSON.parse((await output($`sudo ${tailscale} status --json`)) || "{}"); + const status = JSON.parse((await probe(["sudo", tailscale, "status", "--json"])) ?? "{}"); return typeof status.AuthURL === "string" && status.AuthURL ? (status.AuthURL as string) : undefined; }); console.log( @@ -63,39 +66,46 @@ export async function joinTailnet(hostname: string, tags: string | undefined): P } export async function tailnetSummary(): Promise { - const line = await output($`sudo ${tailscale} status --self --peers=false`); - return line.split("\n")[0] || "not connected"; + const status = await probe(["sudo", tailscale, "status", "--self", "--peers=false"]); + return status?.split("\n")[0] || "not connected"; } export async function installBuildkiteAgent(): Promise { const { version, bin } = config.buildkiteAgent; - if ((await output($`${bin} --version`)).includes(version)) return; + if ((await probe([bin, "--version"]))?.includes(version)) return; const arch = process.arch === "arm64" ? "arm64" : "amd64"; const url = `https://github.com/buildkite/agent/releases/download/v${version}/buildkite-agent-darwin-${arch}-${version}.tar.gz`; const tmp = mkdtempSync(join(tmpdir(), "buildkite-agent-")); - await $`curl -fsSL ${url} | tar -xz -C ${tmp}`; - await $`sudo mkdir -p /usr/local/bin && sudo install -m 755 ${join(tmp, "buildkite-agent")} ${bin}`; - await $`rm -rf ${tmp}`; + try { + await run(["curl", "-fsSL", "-o", join(tmp, "agent.tgz"), url]); + await run(["tar", "-xzf", join(tmp, "agent.tgz"), "-C", tmp]); + await run(["sudo", "mkdir", "-p", "/usr/local/bin"]); + await run(["sudo", "install", "-m", "755", join(tmp, "buildkite-agent"), bin]); + } finally { + rmSync(tmp, { recursive: true, force: true }); + } } export async function installSelf(): Promise { const source = join(import.meta.dir, ".."); - await $`sudo mkdir -p ${config.installDir}`; - await $`sudo rsync -a --delete --chmod=Fa+r,Da+rx ${source}/ ${config.installDir}/`; + await run(["sudo", "mkdir", "-p", config.installDir]); + await run(["sudo", "rsync", "-a", "--delete", "--chmod=Fa+r,Da+rx", `${source}/`, `${config.installDir}/`]); } +export const bootstrapCheckout = join(homedir(), "bun-bootstrap"); + export async function bootstrapToolchain(): Promise { - const checkout = join(process.env.HOME!, "bun-bootstrap"); - await $`rm -rf ${checkout}`; - await $`git clone -q --depth=1 --branch ${config.bun.ref} ${config.bun.repo} ${checkout}`; - await $`./scripts/bootstrap.sh`.cwd(checkout).nothrow(); + rmSync(bootstrapCheckout, { recursive: true, force: true }); + await run(["git", "clone", "-q", "--depth=1", "--branch", config.bun.ref, config.bun.repo, bootstrapCheckout]); + const status = await runInherit(["./scripts/bootstrap.sh"], { cwd: bootstrapCheckout }); + if (status !== 0) console.log(`bootstrap.sh exited ${status}; verifying toolchain`); await verifyToolchain(); } export async function verifyToolchain(): Promise { const missing: string[] = []; for (const tool of toolchain) { - if (!(await succeeds($`bash -lc ${`command -v ${tool}`}`))) missing.push(tool); + if (!(await succeeds(["bash", "-lc", `command -v ${tool}`]))) missing.push(tool); } if (missing.length) fail(`toolchain incomplete after bootstrap: ${missing.join(", ")}`); } diff --git a/scripts/darwin-ci/lib/release-tier.mjs b/scripts/darwin-ci/lib/release-tier.mjs new file mode 100644 index 000000000000..a499f81b3d6b --- /dev/null +++ b/scripts/darwin-ci/lib/release-tier.mjs @@ -0,0 +1,10 @@ +// `release-tier` agent tag thresholds, shared by scripts/agent.mjs (bare hosts) and darwin-ci (tart hosts); bump LATEST once the first runner on a new macOS is online +export const LATEST_DARWIN_RELEASE = 26; +export const PREVIOUS_DARWIN_RELEASE = 14; + +/** @param {number} major */ +export function darwinReleaseTier(major) { + if (major >= LATEST_DARWIN_RELEASE) return "latest"; + if (major >= PREVIOUS_DARWIN_RELEASE) return "previous"; + return "oldest"; +} diff --git a/scripts/darwin-ci/lib/shell.ts b/scripts/darwin-ci/lib/shell.ts index af03e49dd525..1494a691ee40 100644 --- a/scripts/darwin-ci/lib/shell.ts +++ b/scripts/darwin-ci/lib/shell.ts @@ -1,5 +1,3 @@ -import { $ } from "bun"; - export function log(message: string): void { console.log(`${new Date().toTimeString().slice(0, 8)} ${message}`); } @@ -15,47 +13,89 @@ export function fail(message: string): never { export const sleep = (ms: number) => Bun.sleep(ms); -export async function succeeds(proc: $.ShellPromise): Promise { - return (await proc.quiet().nothrow()).exitCode === 0; +export const shellQuote = (arg: string) => `'${arg.replace(/'/g, `'\\''`)}'`; + +export type Result = { exitCode: number; stdout: string; stderr: string }; +export type SpawnOptions = { + cwd?: string; + stdin?: string | Uint8Array; + env?: Record; +}; + +export async function spawn(argv: string[], { cwd, stdin, env }: SpawnOptions = {}): Promise { + const proc = Bun.spawn(argv, { + cwd, + env, + stdin: stdin === undefined ? "ignore" : new Blob([stdin]), + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([ + new Response(proc.stdout).text(), + new Response(proc.stderr).text(), + proc.exited, + ]); + return { exitCode, stdout, stderr }; +} + +// only the leading words: argv may carry a password further along +const describe = (argv: string[]) => argv.slice(0, 3).join(" "); + +export async function run(argv: string[], options?: SpawnOptions): Promise { + const { exitCode, stdout, stderr } = await spawn(argv, options); + if (exitCode !== 0) throw new Error(`${describe(argv)} exited ${exitCode}\n${stderr}`.trimEnd()); + return stdout; +} + +// stdio passed through, for long-running commands whose progress the operator wants to see +export function runInherit(argv: string[], { cwd, env }: SpawnOptions = {}): Promise { + return Bun.spawn(argv, { cwd, env, stdin: "ignore", stdout: "inherit", stderr: "inherit" }).exited; +} + +export async function runInheritOrThrow(argv: string[], options?: SpawnOptions): Promise { + const exitCode = await runInherit(argv, options); + if (exitCode !== 0) throw new Error(`${describe(argv)} exited ${exitCode}`); } -export async function output(proc: $.ShellPromise): Promise { - return (await proc.quiet().nothrow().text()).trim(); +export const output = async (argv: string[]) => (await run(argv)).trim(); + +// for commands whose failure is an expected answer (a probe of something that may not exist yet) +export async function probe(argv: string[]): Promise { + const { exitCode, stdout } = await spawn(argv); + return exitCode === 0 ? stdout.trim() || undefined : undefined; } +export const succeeds = async (argv: string[]) => (await spawn(argv)).exitCode === 0; + export async function poll( attempts: number, intervalMs: number, - probe: () => Promise, + check: () => Promise, ): Promise { for (let i = 0; i < attempts; i++) { - const value = await probe(); + const value = await check(); if (value !== undefined) return value; await sleep(intervalMs); } return undefined; } -export function portOpen(host: string, port: number): Promise { - return succeeds($`nc -z -w2 ${host} ${port}`); -} +export const portOpen = (host: string, port: number) => succeeds(["nc", "-z", "-w2", host, String(port)]); export async function sudoWrite( path: string, - content: string | Uint8Array, + content: string | Uint8Array, mode = "644", owner = "root:wheel", ): Promise { - await $`sudo tee ${path} < ${new Response(content)}`.quiet(); - await $`sudo chown ${owner} ${path}`.quiet(); - await $`sudo chmod ${mode} ${path}`.quiet(); + await run(["sudo", "tee", path], { stdin: content }); + await run(["sudo", "chown", owner, path]); + await run(["sudo", "chmod", mode, path]); } -export async function sudoRead(path: string): Promise { - return (await output($`sudo cat ${path}`)) || undefined; -} +export const sudoRead = (path: string) => probe(["sudo", "cat", path]); export async function consoleUser(): Promise { - const user = await output($`stat -f %Su /dev/console`); - return user && user !== "root" ? user : undefined; + const user = await probe(["stat", "-f", "%Su", "/dev/console"]); + return user !== "root" ? user : undefined; } diff --git a/scripts/darwin-ci/lib/tart.ts b/scripts/darwin-ci/lib/tart.ts index 1445c96e2f13..2a32cab1c6e3 100644 --- a/scripts/darwin-ci/lib/tart.ts +++ b/scripts/darwin-ci/lib/tart.ts @@ -1,62 +1,95 @@ -import { $, type Subprocess } from "bun"; -import { mkdirSync, rmdirSync, rmSync } from "node:fs"; +import type { Subprocess } from "bun"; +import { readlinkSync, rmSync, symlinkSync, unlinkSync } from "node:fs"; import { config } from "./config"; -import { output, poll, portOpen, sleep, succeeds } from "./shell"; +import { poll, portOpen, probe, run, runInheritOrThrow, sleep, spawn, succeeds } from "./shell"; const bin = config.tart.bin; -const cloneLock = "/tmp/tart-clone.lock.d"; +// a symlink whose target is the holder's pid: creation is atomic and carries the owner in one syscall +const imageLock = "/tmp/tart-image.lock"; + +function lockOwner(): number | undefined { + try { + return Number(readlinkSync(imageLock)); + } catch { + return undefined; + } +} + +function processAlive(pid: number): boolean { + try { + process.kill(pid, 0); + return true; + } catch (error) { + return (error as NodeJS.ErrnoException).code === "EPERM"; + } +} export const tart = { - pull: (image: string) => $`${bin} pull ${image}`, + pull: (image: string) => runInheritOrThrow([bin, "pull", image]), - clone: (from: string, to: string) => $`${bin} clone ${from} ${to}`.quiet(), + clone: (from: string, to: string) => run([bin, "clone", from, to]), configure: (vm: string, cpu: number, memoryMb: number) => - $`${bin} set ${vm} --cpu ${cpu} --memory ${memoryMb}`.quiet(), + run([bin, "set", vm, "--cpu", String(cpu), "--memory", String(memoryMb)]), start(vm: string, logPath: string): Subprocess { const log = Bun.file(logPath); return Bun.spawn([bin, "run", vm, "--no-graphics"], { stdin: "ignore", stdout: log, stderr: log }); }, - ip: async (vm: string) => (await output($`${bin} ip ${vm}`)) || undefined, + ip: (vm: string) => probe([bin, "ip", vm]), - exec: (vm: string, script: string) => $`${bin} exec ${vm} /bin/bash -lc ${script}`.quiet(), + exec: (vm: string, script: string) => run([bin, "exec", vm, "/bin/bash", "-lc", script]), - stop: (vm: string) => $`${bin} stop ${vm} --timeout 5`.quiet().nothrow(), + async stop(vm: string): Promise { + const { exitCode, stderr } = await spawn([bin, "stop", vm, "--timeout", "5"]); + if (exitCode !== 0 && !stderr.includes("is not running")) throw new Error(`tart stop ${vm}: ${stderr.trim()}`); + }, - remove: (vm: string) => $`${bin} delete ${vm}`.quiet().nothrow(), + remove: (vm: string) => run([bin, "delete", vm]), - rename: (from: string, to: string) => $`${bin} rename ${from} ${to}`.quiet(), + rename: (from: string, to: string) => run([bin, "rename", from, to]), - exists: (vm: string) => succeeds($`${bin} get ${vm}`), + exists: (vm: string) => succeeds([bin, "get", vm]), async destroy(vm: string): Promise { + if (!(await tart.exists(vm))) return; await tart.stop(vm); - await tart.remove(vm); + // `tart stop` returns once the guest pid is gone, but `tart run` holds the vm lock a little longer and delete fails until it lets go + for (let attempt = 0; ; attempt++) { + const { exitCode, stderr } = await spawn([bin, "delete", vm]); + if (exitCode === 0) return; + if (attempt >= 10 || !stderr.includes("is running")) throw new Error(`tart delete ${vm}: ${stderr.trim()}`); + await sleep(500); + } }, - // concurrent clones of one image race; macOS has no flock(1) - async cloneLocked(from: string, to: string): Promise { - for (let waited = 0; ; waited++) { + // concurrent clones of one image race, and so does swapping the image out under a clone; macOS has no flock(1) + async withImageLock(fn: () => Promise): Promise { + for (;;) { try { - mkdirSync(cloneLock); + symlinkSync(String(process.pid), imageLock); break; - } catch { - if (waited >= 120) { - rmSync(cloneLock, { recursive: true, force: true }); - waited = 0; + } catch (error) { + if ((error as NodeJS.ErrnoException).code !== "EEXIST") throw error; + const owner = lockOwner(); + if (owner !== undefined && !processAlive(owner)) { + console.log(`${imageLock} held by dead pid ${owner}; removing`); + rmSync(imageLock, { force: true }); + continue; } await sleep(1000); } } try { - await tart.clone(from, to); + return await fn(); } finally { - rmdirSync(cloneLock); + if (lockOwner() === process.pid) unlinkSync(imageLock); } }, + cloneLocked: (from: string, to: string) => tart.withImageLock(() => tart.clone(from, to)), + waitForSsh(vm: string, attempts = 30): Promise { return poll(attempts, 4000, async () => { const ip = await tart.ip(vm); @@ -65,6 +98,6 @@ export const tart = { }, waitForAgent(vm: string): Promise { - return poll(30, 4000, async () => ((await succeeds(tart.exec(vm, "true"))) ? true : undefined)); + return poll(30, 4000, async () => ((await succeeds([bin, "exec", vm, "true"])) ? true : undefined)); }, }; diff --git a/scripts/darwin-ci/main.ts b/scripts/darwin-ci/main.ts index ea36306d9915..11f3e172957a 100644 --- a/scripts/darwin-ci/main.ts +++ b/scripts/darwin-ci/main.ts @@ -1,7 +1,6 @@ #!/usr/bin/env bun -import { $ } from "bun"; import { parseArgs } from "node:util"; -import { installBareAgent, installTartAgent } from "./lib/agent"; +import { installBareAgent, installTartAgent, type TartAgentOptions } from "./lib/agent"; import { bake } from "./lib/bake"; import { ciUserExists, enableAutoLogin, ensureCiUser } from "./lib/ci-user"; import { config } from "./lib/config"; @@ -9,6 +8,7 @@ import { bootstrapToolchain, brewInstall, disableRemoteManagement, + ensureShellProfiles, hardenSshd, installBuildkiteAgent, installSelf, @@ -16,7 +16,7 @@ import { setHostname, tailnetSummary, } from "./lib/host"; -import { consoleUser, fail, step } from "./lib/shell"; +import { consoleUser, fail, runInheritOrThrow, step } from "./lib/shell"; const usage = `usage: main.ts provision [--tags ] converge a freshly imaged host @@ -38,7 +38,6 @@ const { positionals, values } = parseArgs({ }); const [subcommand, ...args] = positionals; -const agentOptions = { release: Number(values.release), spawn: Number(values.spawn) }; switch (subcommand) { case "provision": @@ -51,7 +50,7 @@ switch (subcommand) { await bake({ base: values.base!, ref: values.ref! }); break; case "install-agent": - await installTartAgent(agentOptions); + await installTartAgent(agentOptions()); break; default: fail(usage); @@ -62,18 +61,31 @@ function parseMode(mode: string | undefined): "tart" | "bare" { return fail(usage); } +function positiveInteger(name: "release" | "spawn"): number { + const value = values[name]!; + if (!/^[1-9]\d*$/.test(value)) fail(`--${name} must be a positive integer, got ${JSON.stringify(value)}`); + return Number(value); +} + +function agentOptions(): TartAgentOptions { + return { release: positiveInteger("release"), spawn: positiveInteger("spawn") }; +} + async function setupUser(): Promise { await ensureCiUser(); await enableAutoLogin(); } async function provision(name: string, mode: "tart" | "bare"): Promise { + const agent = mode === "tart" ? agentOptions() : undefined; + step("remote management off, sshd key-only"); await disableRemoteManagement(); await hardenSshd(); step(`hostname ${name}`); await setHostname(name); + await ensureShellProfiles(); step("tailscale"); await joinTailnet(name, values.tags); @@ -84,14 +96,14 @@ async function provision(name: string, mode: "tart" | "bare"): Promise { step(`install scripts to ${config.installDir}`); await installSelf(); - if (mode === "tart") await provisionTart(); + if (agent) await provisionTart(agent); else await provisionBare(); step("done"); console.log(`tailscale: ${await tailnetSummary()}`); } -async function provisionTart(): Promise { +async function provisionTart(agent: TartAgentOptions): Promise { if (process.arch !== "arm64") fail("tart mode needs Apple Silicon; use bare on Intel"); const user = config.ciUser; const main = `${config.installDir}/main.ts`; @@ -111,10 +123,22 @@ async function provisionTart(): Promise { } step(`bake ${config.tart.image} as ${user}`); - await $`sudo -u ${user} -H /usr/local/bin/bun ${main} bake --base ${values.base!} --ref ${values.ref!}`; + await runInheritOrThrow([ + "sudo", + "-u", + user, + "-H", + "/usr/local/bin/bun", + main, + "bake", + "--base", + values.base!, + "--ref", + values.ref!, + ]); step("agent"); - await installTartAgent(agentOptions); + await installTartAgent(agent); } async function provisionBare(): Promise {