Skip to content
Merged
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
32 changes: 32 additions & 0 deletions scripts/ci-log-phase.mjs
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
// The header shapes runner.node.mjs writes into a CI job log, shared by the
// per-file timing parsers (scripts/ci-slowest-tests.ts and
// scripts/update-test-durations.mjs). Plain .mjs so the durations script keeps
// running under node.
//
// Per-file cost is the gap between the APC timestamps Buildkite injects into
// consecutive `[N/M] <path>` headers (ESC `_bk;t=<ms>` BEL). The runner emits
// several header shapes that must each close the open span:
// - serial test start: `--- [N/M] <path>` (startGroup)
// - retry/error label: `--- [N/M] <path> - <error>` (not a path)
// - parallel-bucket phase: `--- [A-B/M] K files in parallel`, then after the
// single `bun test --parallel` run a summary `[N/M] <path> (X.XXs)` per file
// - parallel-safe phase: `--- Running N parallel-safe tests`, then a bare
// `[N/M] <path>` per concurrent dispatch (no recoverable wall clock)
// - other startGroup phases: `--- napi prebuild: ...`, `--- End`
// A parser that only recognises the first shape folds the entire following
// phase into whichever serial test happened to precede it.
//
// pipeTestStdout sanitises `--- ` in streamed test output, but a chunk boundary
// mid-token or one of the raw-write paths (coordinator stdout, retry
// stdoutPreview) can still deliver a line that starts `--- ` without being a
// group header (unified-diff `--- a/<file>`, `--- ps ---`). The boundary check
// is therefore an allowlist of the phase headers the runner actually emits,
// not any `--- `. When runner.node.mjs grows a new phase between the serial
// tests and the next `[N/M]` header, add it here.

/**
* @param {string} body one log line with its APC timestamp and SGR codes removed
* @returns {boolean}
*/
export const isPhaseGroupHeader = body =>
/^--- (?:\[\d+-\d+\/\d+\]|napi prebuild:|Running \d+ parallel-safe|End\b|Summary\b|Received \w+, exiting)/.test(body);
282 changes: 161 additions & 121 deletions scripts/ci-slowest-tests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,8 +3,8 @@
//
// Downloads every test-bun job log from a BuildKite build, parses per-file
// wall-clock from the `_bk;t=<ms>` timestamps that prefix each
// `--- [N/TOTAL] <file>` group header, aggregates as MAX across all
// platforms, and prints the top N.
// `[N/TOTAL] <file>` header, aggregates as MAX across all platforms, and
// prints the top N.
//
// Usage:
// bun scripts/ci-slowest-tests.ts # auto-pick a recent merged-PR build, top 500
Expand All @@ -18,144 +18,184 @@ import { $, spawn } from "bun";
import { existsSync, mkdirSync, readFileSync, writeFileSync } from "fs";
import { tmpdir } from "os";
import { join } from "path";
import { isPhaseGroupHeader } from "./ci-log-phase.mjs";

const args = process.argv.slice(2);
const json = args.includes("--json");
const positional = args.filter(a => !a.startsWith("-"));
let BUILD = positional[0];
const TOP_N = parseInt(positional[1] || "500", 10);

const TOKEN = process.env.BUILDKITE_TOKEN || process.env.BUILDKITE_API_TOKEN;
if (!TOKEN) {
console.error("error: BUILDKITE_TOKEN not set");
process.exit(1);
}

// Auto-pick a build: most-recent merged PR whose branch has a finished build.
// Merged-PR builds usually report state=failed (flaky tests) — that's fine,
// we only need the timing data.
if (!BUILD) {
console.error("no build given, finding a recent merged-PR build...");
const prs = await $`gh pr list --state merged --limit 10 --json number,headRefName`.json();
for (const pr of prs) {
const builds = await $`bk build list --branch ${pr.headRefName}`
.quiet()
.json()
.catch(() => []);
const done = builds.find((b: any) => b.finished_at && b.state !== "canceled" && b.state !== "running");
if (done) {
BUILD = String(done.number);
console.error(` using build #${BUILD} (PR #${pr.number}, ${pr.headRefName}, state=${done.state})`);
break;
}
}
if (!BUILD) {
console.error("error: no finished build found among the last 10 merged PRs");
process.exit(1);
}
}

const CACHE = join(tmpdir(), `bun-ci-logs-${BUILD}`);
mkdirSync(CACHE, { recursive: true });

type Job = { id: string; name: string; raw_log_url: string; retried?: boolean };

const buildJson = JSON.parse(
await new Response(spawn({ cmd: ["bk", "build", "view", BUILD], stdout: "pipe" }).stdout).text(),
);
const jobs: Job[] = buildJson.jobs.filter(
(j: any) => j.name && j.raw_log_url && j.name.includes("test-bun") && !j.retried,
);
console.error(`build #${BUILD}: ${jobs.length} test-bun jobs across ${new Set(jobs.map(j => j.name)).size} platforms`);

const platOf = (name: string) =>
name
.replace(/ - test-bun$/, "")
.replace(/^:([a-z]+):/, "$1")
.trim();

function parseLog(text: string): Map<string, number> {
// Per-file cost is the gap between the APC timestamps Buildkite injects into
// consecutive `[N/M] <path>` headers (ESC `_bk;t=<ms>` BEL). See
// scripts/ci-log-phase.mjs for the header shapes runner.node.mjs emits and why
// each one has to close the open span.
export function parseLog(text: string): Map<string, number> {
const out = new Map<string, number>();
const startRe = /_bk;t=(\d+).*?--- .*?\[90m\[\d+\/\d+\].*?\[0m (.+)/;
let curName: string | null = null;
let curStart = 0;
let concurrent = false;
let lastTs = 0;
const close = (ts: number) => {
if (!curName) return;
const span = ts - curStart;
// Concurrent-phase gaps are inter-dispatch deltas, not per-file wall
// clock; clamp so the last-dispatched file on a shard does not absorb the
// N-wide tail drain or a sibling's 5-15 s retry backoff.
out.set(curName, (out.get(curName) ?? 0) + (concurrent ? Math.min(span, 500) : span));
curName = null;
};
for (const line of text.split("\n")) {
const m = startRe.exec(line);
if (!m) continue;
const ts = parseInt(m[1], 10);
if (curName) {
const clean = curName
.replace(/\s+\x1b\[90m\[attempt #\d+\]\x1b\[0m\r*$/, "")
.replace(/\r+$/, "")
const apc = /_bk;t=(\d+)\x07(.*)/.exec(line);
if (!apc) continue;
const ts = (lastTs = parseInt(apc[1], 10));
const body = apc[2].replace(/\x1b\[[0-9;]*m/g, "").replace(/\r+$/, "");
const hdr = /^(--- )?\[\d+\/\d+\] (.+)$/.exec(body);
if (hdr) {
close(ts);
const title = hdr[2]
.replace(/ \[attempt #\d+\]$/, "")
.replace(/\\/g, "/")
.trim();
out.set(clean, (out.get(clean) ?? 0) + (ts - curStart));
// Parallel-bucket summaries carry the authoritative wall clock inline.
const timed = !hdr[1] && /^(.+\.(?:[cm]?[jt]sx?|json)) \((\d+(?:\.\d+)?)s\)$/.exec(title);
if (timed) {
out.set(timed[1], (out.get(timed[1]) ?? 0) + Math.round(parseFloat(timed[2]) * 1000));
concurrent = false;
continue;
}
// Retry/error labels (`<path> - code 1`) are not file paths; treat them
// as a delimiter so the preceding span closes without the retry backoff
// landing on either attempt.
const isPath = /\.(?:[cm]?[jt]sx?|json)$/.test(title);
curName = isPath ? title : null;
curStart = ts;
concurrent = !hdr[1];
continue;
}
if (isPhaseGroupHeader(body)) {
close(ts);
concurrent = false;
}
curStart = ts;
curName = m[2].replace(/\r+$/, "").replace(/\\/g, "/").trim();
}
// A truncated log (job killed / timed out mid-run) has no `--- End`; charge
// the open span to the last timestamp seen so the culprit is not dropped.
close(lastTs);
return out;
}

// Do NOT use `bk job log` — it hangs indefinitely on some Windows/alpine jobs.
// Fetching raw_log_url directly with the token works for all of them.
async function fetchLog(job: Job): Promise<string> {
const path = join(CACHE, `${job.id}.log`);
if (existsSync(path)) return readFileSync(path, "utf8");
const res = await fetch(job.raw_log_url, { headers: { Authorization: `Bearer ${TOKEN}` } });
if (!res.ok) throw new Error(`${res.status} ${job.raw_log_url}`);
const out = await res.text();
writeFileSync(path, out);
return out;
}
if (import.meta.main) {
const args = process.argv.slice(2);
const json = args.includes("--json");
const positional = args.filter(a => !a.startsWith("-"));
let BUILD = positional[0];
const TOP_N = parseInt(positional[1] || "500", 10);

const TOKEN = process.env.BUILDKITE_TOKEN || process.env.BUILDKITE_API_TOKEN;
if (!TOKEN) {
console.error("error: BUILDKITE_TOKEN not set");
process.exit(1);
}

type Agg = { maxMs: number; maxPlat: string; perPlat: Map<string, number> };
const agg = new Map<string, Agg>();

let done = 0;
const queue = [...jobs];
async function worker() {
for (;;) {
const job = queue.shift();
if (!job) return;
try {
const log = await fetchLog(job);
const plat = platOf(job.name);
for (const [file, ms] of parseLog(log)) {
let a = agg.get(file);
if (!a) agg.set(file, (a = { maxMs: 0, maxPlat: "", perPlat: new Map() }));
const total = (a.perPlat.get(plat) ?? 0) + ms;
a.perPlat.set(plat, total);
if (total > a.maxMs) {
a.maxMs = total;
a.maxPlat = plat;
// Auto-pick a build: most-recent merged PR whose branch has a finished build.
// Merged-PR builds usually report state=failed (flaky tests) — that's fine,
// we only need the timing data.
if (!BUILD) {
console.error("no build given, finding a recent merged-PR build...");
const prs = await $`gh pr list --state merged --limit 10 --json number,headRefName`.json();
for (const pr of prs) {
const builds = await $`bk build list --branch ${pr.headRefName}`
.quiet()
.json()
.catch(() => []);
const done = builds.find((b: any) => b.finished_at && b.state !== "canceled" && b.state !== "running");
if (done) {
BUILD = String(done.number);
console.error(` using build #${BUILD} (PR #${pr.number}, ${pr.headRefName}, state=${done.state})`);
break;
}
}
if (!BUILD) {
console.error("error: no finished build found among the last 10 merged PRs");
process.exit(1);
}
}

const CACHE = join(tmpdir(), `bun-ci-logs-${BUILD}`);
mkdirSync(CACHE, { recursive: true });

type Job = { id: string; name: string; raw_log_url: string; retried?: boolean };

const buildJson = JSON.parse(
await new Response(spawn({ cmd: ["bk", "build", "view", BUILD], stdout: "pipe" }).stdout).text(),
);
const jobs: Job[] = buildJson.jobs.filter(
(j: any) => j.name && j.raw_log_url && j.name.includes("test-bun") && !j.retried,
);
console.error(
`build #${BUILD}: ${jobs.length} test-bun jobs across ${new Set(jobs.map(j => j.name)).size} platforms`,
);

const platOf = (name: string) =>
name
.replace(/ - test-bun$/, "")
.replace(/^:([a-z]+):/, "$1")
.trim();

// Do NOT use `bk job log` — it hangs indefinitely on some Windows/alpine jobs.
// Fetching raw_log_url directly with the token works for all of them.
async function fetchLog(job: Job): Promise<string> {
const path = join(CACHE, `${job.id}.log`);
if (existsSync(path)) return readFileSync(path, "utf8");
const res = await fetch(job.raw_log_url, { headers: { Authorization: `Bearer ${TOKEN}` } });
if (!res.ok) throw new Error(`${res.status} ${job.raw_log_url}`);
const out = await res.text();
writeFileSync(path, out);
return out;
}

type Agg = { maxMs: number; maxPlat: string; perPlat: Map<string, number> };
const agg = new Map<string, Agg>();

let done = 0;
const queue = [...jobs];
async function worker() {
for (;;) {
const job = queue.shift();
if (!job) return;
try {
const log = await fetchLog(job);
const plat = platOf(job.name);
for (const [file, ms] of parseLog(log)) {
let a = agg.get(file);
if (!a) agg.set(file, (a = { maxMs: 0, maxPlat: "", perPlat: new Map() }));
const total = (a.perPlat.get(plat) ?? 0) + ms;
a.perPlat.set(plat, total);
if (total > a.maxMs) {
a.maxMs = total;
a.maxPlat = plat;
}
}
} catch (e) {
console.error(` failed ${job.id}: ${(e as Error).message}`);
}
} catch (e) {
console.error(` failed ${job.id}: ${(e as Error).message}`);
done++;
if (done % 20 === 0 || done === jobs.length) console.error(` ${done}/${jobs.length} logs`);
}
done++;
if (done % 20 === 0 || done === jobs.length) console.error(` ${done}/${jobs.length} logs`);
}
}
await Promise.all(Array.from({ length: 16 }, worker));
await Promise.all(Array.from({ length: 16 }, worker));

// `package.json` / `test/package.json` are setup steps, not tests.
const isTest = (f: string) => /\.(m|c)?(j|t)sx?$/.test(f);
// `package.json` / `test/package.json` are setup steps, not tests.
const isTest = (f: string) => /\.(m|c)?(j|t)sx?$/.test(f);

const sorted = [...agg.entries()]
.filter(([file]) => isTest(file))
.map(([file, a]) => ({ file, maxMs: a.maxMs, maxPlat: a.maxPlat }))
.sort((a, b) => b.maxMs - a.maxMs)
.slice(0, TOP_N);
const sorted = [...agg.entries()]
.filter(([file]) => isTest(file))
.map(([file, a]) => ({ file, maxMs: a.maxMs, maxPlat: a.maxPlat }))
.sort((a, b) => b.maxMs - a.maxMs)
.slice(0, TOP_N);

console.error(`${agg.size} unique entries, ${sorted.length} test files after filtering`);
console.error(`logs cached at ${CACHE}`);
console.error(`${agg.size} unique entries, ${sorted.length} test files after filtering`);
console.error(`logs cached at ${CACHE}`);

if (json) {
console.log(JSON.stringify({ build: BUILD, count: agg.size, top: sorted }, null, 2));
} else {
console.log(`rank\tseconds\tfile\tslowest_platform`);
sorted.forEach((t, i) => console.log(`${i + 1}\t${(t.maxMs / 1000).toFixed(2)}\t${t.file}\t${t.maxPlat}`));
if (json) {
console.log(JSON.stringify({ build: BUILD, count: agg.size, top: sorted }, null, 2));
} else {
console.log(`rank\tseconds\tfile\tslowest_platform`);
sorted.forEach((t, i) => console.log(`${i + 1}\t${(t.maxMs / 1000).toFixed(2)}\t${t.file}\t${t.maxPlat}`));
}
process.exit(0);
}
process.exit(0);
18 changes: 11 additions & 7 deletions scripts/runner.node.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -2482,16 +2482,20 @@ function loadExpectedDurations(cwd) {
try {
const raw = JSON.parse(readFileSync(join(cwd, "expected-durations.json"), "utf8"));
const step = options["step"] || "";
const lane = step.includes("asan")
? "asan"
// Most specific column first, then the nearest lane, then anything.
const columns = step.includes("asan")
? ["asan"]
: step.includes("musl")
? "musl"
: isWindows || step.includes("windows")
? "windows"
: "default";
? ["musl"]
: step.includes("windows-aarch64") || (isWindows && process.arch === "arm64")
? ["windows-aarch64", "windows"]
: isWindows || step.includes("windows")
? ["windows"]
: ["default"];
columns.push("default", "asan", "musl", "windows", "windows-aarch64");
for (const [path, entry] of Object.entries(raw)) {
if (path === "_meta") continue;
const ms = entry[lane] ?? entry.default ?? entry.asan ?? entry.musl ?? entry.windows;
const ms = columns.map(column => entry[column]).find(value => typeof value === "number");
if (typeof ms === "number") durations[path] = ms;
}
} catch (e) {
Expand Down
Loading