ci-images: the image's record is lines the bake script writes; the pipeline is uploaded as JSON

.buildkite/ci.ts

@@ -38,6 +38,7 @@ import {
   isPullRequest,
   parseGitUrl,
   startGroup,
+  toYaml,
   uploadArtifact,
 } from "../scripts/buildkite.ts";
 import { type ImageState, getImageState } from "../scripts/ci-image.ts";
@@ -144,67 +145,6 @@ function getGithubApiUrl(): URL {
   return new URL(process.env.GITHUB_API_URL || "https://api.github.com");
 }
 
-function toYaml(obj: object, indent = 0): string {
-  const spaces = " ".repeat(indent);
-  let result = "";
-  const entries: [string, unknown][] = Object.entries(obj);
-  for (const [key, value] of entries) {
-    if (value === undefined) {
-      continue;
-    }
-    if (value === null) {
-      result += `${spaces}${key}: null\n`;
-      continue;
-    }
-    if (Array.isArray(value)) {
-      result += `${spaces}${key}:\n`;
-      value.forEach((item: unknown) => {
-        if (typeof item === "object" && item !== null) {
-          result += `${spaces}- \n${toYaml(item, indent + 2)
-            .split("\n")
-            .map(line => `${spaces}  ${line}`)
-            .join("\n")}\n`;
-        } else {
-          result += `${spaces}- ${item}\n`;
-        }
-      });
-      continue;
-    }
-    if (typeof value === "object") {
-      result += `${spaces}${key}:\n${toYaml(value, indent + 2)}`;
-      continue;
-    }
-    if (
-      typeof value === "string" &&
-      (value.includes(":") ||
-        value.includes("#") ||
-        value.includes("'") ||
-        value.includes('"') ||
-        value.includes("\\") ||
-        value.includes("\n") ||
-        value.includes("*") ||
-        value.includes("&") ||
-        value.includes("!") ||
-        value.includes("|") ||
-        value.includes(">") ||
-        value.includes("%") ||
-        value.includes("@") ||
-        value.includes("`") ||
-        value.includes("{") ||
-        value.includes("}") ||
-        value.includes("[") ||
-        value.includes("]") ||
-        value.includes(",") ||
-        value.includes(";"))
-    ) {
-      result += `${spaces}${key}: "${value.replace(/\\/g, "\\\\").replace(/"/g, '\\"')}"\n`;
-      continue;
-    }
-    result += `${spaces}${key}: ${value}\n`;
-  }
-  return result;
-}
-
 type Emoji = keyof typeof emojiMap;
 
 const emojiMap = {

scripts/build/ci-images/CLAUDE.md

@@ -5,8 +5,8 @@ CI's build and test machines start from images baked ahead of time: AWS AMIs for
 ## How `spec.ts` is laid out
 
 1. **The data.** `files` (the repository files copied onto machines), `pins` (every version), `locations` (where a bake puts what is not the operating system's to place), `images` (each with the exact base image it starts from), the package lists, and how Packer bakes a Windows image. The build system's own LLVM, Node.js, xwin, Windows SDK, macOS SDK, Android API level and FreeBSD versions import from `pins`, and its sysroot and cache lookups from `locations`; `test/internal/source-lints/ci-image-pins.test.ts` keeps what cannot import them in step: `rust-toolchain.toml`, the GitHub workflows, `scripts/darwin-ci` (copied to its hosts), `scripts/agent.ts` (runs on the machines alone) and the Node-API test harness.
-2. **The tools.** A tool is one thing a bake sets up on the machine, whether it installs something (Bun, LLVM) or configures the system (`ulimits`, the agent's user). Each is a function of the image that returns its steps and its `identity` (how what it puts on the machine is known, for `bun-image.json`), and `tools(image)` at the end of the section is what a machine gets, in order.
-3. **The machinery.** The vocabulary the tools are written in (values and steps), its sh and PowerShell renderers, the two small programs a bake leaves on the machine, the Packer template, and `generateImage()`.
+2. **The tools.** A tool is one thing a bake sets up on the machine, whether it installs something (Bun, LLVM) or configures the system (`ulimits`, the agent's user). Each is a function of the image that returns its steps and its `identity` (how what it puts on the machine is known, for the image's record), and `tools(image)` at the end of the section is what a machine gets, in order.
+3. **The machinery.** The vocabulary the tools are written in (values and steps), its sh and PowerShell renderers, the one small program a bake leaves on the machine, the Packer template, and `generateImage()`.
 
 A tool does not contain shell, and neither does the machinery keep any in strings to call: there are no helper functions in the generated script. A step renders the lines themselves (a download is a `curl` line, putting a directory on PATH is the two lines that do it), so every line that runs is where it runs. A tool says what should be true (`download`, `unpack`, `installExecutable`, `directory`, `systemUser`, `service`, `registryValue`, `scheduledTaskAtStartup`, …) and the renderer says it in sh for Linux (as root), in sh with `sudo` where needed for macOS, or in PowerShell. One description serves every system a tool exists on: `bun` is one function for Linux, macOS and Windows.
 
@@ -39,11 +39,11 @@ bun test test/internal/source-lints/ci-images.test.ts test/internal/source-lints
 
 **Bake one image again with nothing changed.** There is deliberately no switch for it. A build bakes a name that does not exist, so removing the image in the cloud is what makes the next build bake it.
 
-**Read what is on an image.** Every bake writes `bun-image.json` on the machine (`/etc/`, `C:\`) and publishes it as the bake job's artifact `build/ci-images/<key>/bun-image.json`: the image's name, the spec's facts, every tool with how it is known, and the exact version of every distro or Scoop package the bake got. It is for keying caches of build outputs (same bytes, same machine content), so nothing in it may differ between two identical machines: no times, hostnames or instance ids. It never feeds the name.
+**Read what is on an image.** Every bake writes the image's record, `bun-image.txt`, on the machine (`/etc/`, `C:\`) and publishes it as the bake job's artifact `build/ci-images/<key>/bun-image.txt`. It is lines, so that the bake script writes it with ordinary steps in either shell (the `recordImage` tool): `name:` the image's name, `image:` the spec's facts as one line of JSON, a `tool <name>:` line for every tool saying how what it puts on the machine is known (and the value, if pinned), an `observed <name>:` line for each line an observed tool printed, and a `package` line for every entry of the distro's or Scoop's package database, sorted. It is for keying caches of build outputs (same bytes, same machine content), so nothing in it may differ between two identical machines: no times, hostnames or instance ids. It never feeds the name.
 
 ## A tool's identity
 
-The name covers everything `spec.ts` decides; `bun-image.json` adds what is only decided on the day of the bake. So that adding or removing a tool cannot leave the record wrong, `Tool` requires an `identity`, and `tsc` refuses a tool without one:
+The name covers everything `spec.ts` decides; the record adds what is only decided on the day of the bake. So that adding or removing a tool cannot leave the record wrong, `Tool` requires an `identity`, and `tsc` refuses a tool without one:
 
 - `pinned(value)`: known in this file (a version, a tag). The value is written into the record as it is.
 - `observed(step)`: only known on the machine. The step is rendered into the script right after the tool's own steps, and what it prints is the record; a tool that observed nothing fails the bake. `treeDigest(...directories)` is the step for something with no version to ask for, like the glibc and musl sysroots, whose packages are whatever the distro serves that day.
@@ -73,7 +73,7 @@ The name covers everything `spec.ts` decides; `bun-image.json` adds what is only
 
 - `bun run ci:images`, read the script, then `shellcheck -s sh -S warning build/ci-images/<key>/bootstrap.sh`; for Windows parse `bootstrap.ps1` with PowerShell's parser and run `packer validate -syntax-only` with exactly the pinned Packer version.
 - A Linux script can be run for real in a container of the same distro: `tar -C build/ci-images/<key> -c . | docker run -i debian:13 sh -c 'mkdir /bake && tar -x -C /bake && sh /bake/bootstrap.sh <commit> some-name'`. A bare container has no init system (install `systemd` or `openrc` first), cannot start services (Docker, Tailscale), and cannot run the `prefetch` tool past its clone; cut the script before it.
-- Windows and macOS scripts can only be run by a bake. After one, compare its `bun-image.json` with the previous bake's.
+- Windows and macOS scripts can only be run by a bake. After one, compare its `bun-image.txt` with the previous bake's.
 
 ## Debugging a failed bake
 

scripts/build/ci-images/spec.ts

@@ -60,8 +60,8 @@ const alpineRelease = "3.23";
  * .buildkite/ci.ts use them as they are.
  */
 export const locations = {
-  /** bun-image.json: what the bake recorded about the machine (see `Identity`). */
-  imageRecord: { linux: "/etc/bun-image.json", windows: "C:\\bun-image.json" },
+  /** The image's record: what the bake recorded about the machine (see the `recordImage` tool). */
+  imageRecord: { linux: "/etc/bun-image.txt", windows: "C:\\bun-image.txt" },
   /** `bun install`'s cache, filled while the image is baked. */
   installCache: { linux: "/var/cache/bun-install", windows: "C:\\bun-install-cache" },
   rust: { linux: "/opt/rust", darwin: "/opt/rust", windows: "C:\\Program Files\\Rust" },
@@ -330,8 +330,8 @@ const scoopPackages = {
 
 /** Where `bun run ci:images` and CI write an image's bake directory, from the repository's root. */
 export const bakeDirectory = (key: string) => `build/ci-images/${key}`;
-/** The name bun-image.json has in a bake directory, once the bake job has copied it out of the machine. */
-export const imageRecordName = "bun-image.json";
+/** The name the image's record has in a bake directory, once the bake job has copied it out of the machine. */
+export const imageRecordName = "bun-image.txt";
 
 /** Hosts more than one tool downloads from. */
 const mirrors = {
@@ -1215,17 +1215,61 @@ function agentService(): Tool {
 }
 
 /**
- * bun-image.json: what this image is, the name it was baked under, and the
- * exact packages the bake got. For keying caches of build outputs on the
- * machine that made them; it never feeds the image's name. The bake job also
- * publishes it as an artifact.
+ * The image's record: what this image is, the name it was baked under, and
+ * what arrived on the day of the bake. For keying caches of build outputs on
+ * the machine that made them (same bytes, same machine content), so nothing in
+ * it may differ between two identical machines. It never feeds the image's
+ * name. The bake job also publishes it as an artifact.
+ *
+ * It is lines, so that both shells can write it with ordinary steps:
+ *
+ *   name: <the image's name>
+ *   image: <the spec's facts, as one line of JSON>
+ *   tool <name>: <how what it puts on the machine is known, and the value if pinned>
+ *   observed <name>: <a line an `observed` tool's step printed>
+ *   package <a line of the package manager's database>
  */
 function recordImage(image: BakedImage): Tool {
-  return {
-    name: "record-image",
-    identity: configuration,
-    steps: [run("node", bakeFile("record-image.mjs"), imageName, locations.imageRecord[image.os])],
+  const record = locations.imageRecord[image.os];
+  const installedPackages: Step =
+    image.os === "windows"
+      ? pipe(
+          printLine(
+            // Out-String: one string for ConvertFrom-Json, whatever it does with several.
+            property(pipe(cmdlet("scoop", {}, "export"), cmdlet("Out-String"), cmdlet("ConvertFrom-Json")), "apps"),
+          ),
+          cmdlet("ForEach-Object", {}, expression(`{ "$($_.Name) $($_.Version)" }`)),
+        )
+      : image.distro === "alpine"
+        ? // "name-version arch {origin} (license) [installed]"
+          pipe(run("apk", "list", "--installed"), run("cut", "-d", " ", "-f1"))
+        : // binary:Package says which architecture a package of another one is for (libc6:amd64 on the arm64 build image).
+          run("dpkg-query", "--show", "--showformat", "${binary:Package} ${Version}\\n");
+  const recorded = (tool: Tool): string => {
+    const { identity } = tool;
+    const value =
+      identity.kind === "pinned" ? ` ${identity.value}` : identity.kind === "notRecorded" ? ` ${identity.reason}` : "";
+    return `tool ${tool.name}: ${identity.kind}${value}`;
+  };
+  // The list of tools includes this one, so it is only asked for when the step is rendered.
+  const steps = (): Step[] => {
+    const all = tools(image);
+    return [
+      toFile(printLine(text`name: ${imageName}`), record),
+      appendLines(record, [`image: ${JSON.stringify(image)}`, ...all.map(recorded)]),
+      ...all
+        .filter(tool => tool.identity.kind === "observed")
+        .flatMap(tool => {
+          const observation = bakeFile(`observed/${tool.name}`);
+          return [
+            failUnlessNotEmpty(output(linesOf(observation)), `nothing was observed for ${tool.name}`),
+            toFile(prefixed(`observed ${tool.name}: `, linesOf(observation)), record, { append: true }),
+          ];
+        }),
+      toFile(prefixed("package ", sorted(installedPackages)), record, { append: true }),
+    ];
   };
+  return { name: "record-image", identity: configuration, steps: [c => render(steps(), c)] };
 }
 
 function cleanup(image: LinuxImage): Tool {
@@ -1650,7 +1694,7 @@ type Context = {
 /** One or more lines of the generated script. */
 export type Step = (context: Context) => string[];
 /**
- * How what a tool puts on the machine is known, for bun-image.json. A tool
+ * How what a tool puts on the machine is known, for the image's record. A tool
  * cannot be written without saying which:
  *
  * - `pinned`: known in this file. The value is written into the record as it is.
@@ -1847,14 +1891,39 @@ const pipe =
   (...steps: Step[]): Step =>
   c => [steps.map(step => oneLine(step, c)).join(" | ")];
 
+/** What the command prints, as the file's content, or with `append` at its end. PowerShell writes ASCII: its default is UTF-16. */
 const toFile =
-  (step: Step, file: Value): Step =>
+  (step: Step, file: Value, options: { append?: boolean } = {}): Step =>
   c => [
     isPowerShell(c)
-      ? `${oneLine(step, c)} | Out-File -Encoding ascii ${renderValue(file, c)}`
-      : `${oneLine(step, c)} > ${renderValue(file, c)}`,
+      ? `${oneLine(step, c)} | Out-File ${options.append ? "-Append " : ""}-Encoding ascii ${renderValue(file, c)}`
+      : `${oneLine(step, c)} ${options.append ? ">>" : ">"} ${renderValue(file, c)}`,
   ];
 
+/** Prints the value: a line in sh, the value itself in PowerShell (which may be objects for the next command of a pipe). */
+const printLine =
+  (value: Value): Step =>
+  c => [isPowerShell(c) ? renderValue(value, c, true) : `printf '%s\\n' ${renderValue(value, c)}`];
+
+/** Prints the file's lines. */
+const linesOf =
+  (file: Value): Step =>
+  c => [`${isPowerShell(c) ? "Get-Content" : "cat"} ${renderValue(file, c)}`];
+
+/** Each line the command prints, with this before it. */
+const prefixed =
+  (prefix: string, step: Step): Step =>
+  c => [
+    isPowerShell(c)
+      ? `${oneLine(step, c)} | ForEach-Object { "${prefix.replace(/[`"$]/g, "`$&")}$_" }`
+      : `${oneLine(step, c)} | sed ${renderValue(`s/^/${prefix.replace(/[\\/&]/g, "\\$&")}/`, c)}`,
+  ];
+
+/** The lines the command prints, in an order that does not depend on the machine's locale. */
+const sorted =
+  (step: Step): Step =>
+  c => [`${oneLine(step, c)} | ${isPowerShell(c) ? "Sort-Object" : "LC_ALL=C sort"}`];
+
 /** What the command prints is not worth reading. */
 const discardOutput =
   (step: Step): Step =>
@@ -2011,7 +2080,14 @@ const writeFile =
 /** These lines, at the end of a file that may exist. */
 const appendLines =
   (path: Value, lines: string[]): Step =>
-  c => [`cat >> ${renderValue(path, c)} <<'EOF'`, ...lines, "EOF"];
+  c =>
+    isPowerShell(c)
+      ? [
+          `Add-Content -Path ${renderValue(path, c)} -Encoding ascii -Value @(`,
+          ...indent(lines.map((l, i) => `${renderValue(l, c, true)}${i < lines.length - 1 ? "," : ""}`)),
+          ")",
+        ]
+      : [`cat >> ${renderValue(path, c)} <<'EOF'`, ...lines, "EOF"];
 
 /**
  * One sha256 for the content of these directories: every file's hash and every
@@ -2411,48 +2487,6 @@ const fetchSshKeysProgram = [
   "}",
 ].join("\n");
 
-/** Run by the `record-image` tool under the Node.js the bake installed: `node record-image.mjs <image name> <output file>`. */
-const recordImageProgram = String.raw`import { spawnSync } from "node:child_process";
-import { createHash } from "node:crypto";
-import { readFileSync, writeFileSync } from "node:fs";
-import { join } from "node:path";
-
-const [name, output] = process.argv.slice(2);
-if (!name || !output) throw new Error("usage: node record-image.mjs <image name> <output file>");
-const image = JSON.parse(readFileSync(join(import.meta.dirname, "image.json"), "utf8"));
-
-function lines(command, args) {
-  const { status, stdout, stderr, error } = spawnSync(command, args, { encoding: "utf8", shell: image.os === "windows" });
-  if (error || status !== 0) throw new Error(command + " " + args.join(" ") + " failed: " + (stderr || error));
-  return stdout.split(/\r?\n/).filter(line => line.trim());
-}
-
-// "name version", one per package, sorted.
-function installedPackages() {
-  if (image.os === "windows") {
-    return JSON.parse(lines("scoop", ["export"]).join("\n")).apps.map(app => app.Name + " " + app.Version).sort();
-  }
-  if (image.distro === "alpine") {
-    // "name-version arch {origin} (license) [installed]"
-    return lines("apk", ["list", "--installed"]).map(line => line.split(" ")[0].replace(/-([^-]+-r\d+)$/, " $1")).sort();
-  }
-  return lines("dpkg-query", ["--show", "--showformat", "$" + "{Package} $" + "{Version}\\n"]).sort();
-}
-
-// What an "observed" tool printed during the bake. A tool that says it is observed and left nothing is a broken record.
-const tools = image.tools.map(tool => {
-  if (tool.identity !== "observed") return tool;
-  const value = readFileSync(join(import.meta.dirname, "observed", tool.name), "utf8").trim();
-  if (!value) throw new Error("nothing was observed for " + tool.name);
-  return { ...tool, value };
-});
-
-const packages = installedPackages();
-const packagesSha256 = createHash("sha256").update(packages.join("\n")).digest("hex");
-writeFileSync(output, JSON.stringify({ name, ...image, tools, packages, packagesSha256 }, null, 2) + "\n");
-console.log(output + ": " + name + ", " + packages.length + " packages, " + packagesSha256);
-`;
-
 // --------------------------------------------------------------------- Packer
 
 /** Every variable the template needs; scripts/ci-image.ts passes them all. */
@@ -2675,7 +2709,6 @@ function bakeFiles(image: Image): Map<string, string | Uint8Array> {
   ]);
   if (image.os !== "darwin") {
     contents.set("agent.mts", readFileSync(join(repoRoot, files["agent.mts"])));
-    contents.set("record-image.mjs", recordImageProgram);
   }
   if (image.os === "linux" && image.role === "build") {
     contents.set("xmac.mjs", readFileSync(join(repoRoot, files["xmac.mjs"])));