Skip to content
Merged
Show file tree
Hide file tree
Changes from 12 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
74 changes: 74 additions & 0 deletions .buildkite/ci.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -760,6 +760,61 @@ function getVerifyBaselineStep(platform, options) {
};
}

/**
* Targets whose build lane cross-compiles (so `canTraceOrderFile()` is false)
* but whose test fleet is native. A `-trace-order` step runs there, downloads
* the cross-built `bun-profile`, traces it, and uploads the `.order` artifact
* that the next build's `inheritOrderFile()` picks up. One build of lag.
*
* linux-aarch64 is absent because its build lane runs on the aarch64 host and
* traces itself; `packageAndUpload()` is its sole publisher.
*/
const traceOrderTargets = [
{ os: "darwin", arch: "aarch64", on: { os: "darwin", arch: "aarch64", release: "26", tier: "latest" } },
{ os: "linux", arch: "x64", on: { os: "linux", arch: "x64", distro: "debian", release: "13" } },
];

/**
* Trace the symbol order file for a cross-compiled target on a native-arch
* host, so the next build's `inheritOrderFile()` has something to download.
*
* The build lane cross-compiles from the aarch64 `buildHostPlatform` and cannot
* run the binary it linked. This step runs on the target-arch test fleet,
* downloads that lane's unstripped `bun-profile`, runs it under `scripts/
* orderfile/generate.ts` (the traced binary doubles as the interpreter), and
* uploads the result.
*
* Non-PR only — `orderFileEligible()` ignores PR builds, so a trace there has
* no consumer. Soft-fail: the order file is an optimization, and a broken
* tracer must not fail a build.
* @param {Target} target
* @param {Platform} tracePlatform
* @param {PipelineOptions} options
* @returns {CommandStep}
*/
function getTraceOrderStep(target, tracePlatform, options) {
const targetKey = getTargetKey(target);
const triplet = getTargetTriplet(target);
const profileDir = `${triplet}-profile`;
return {
key: `${targetKey}-trace-order`,
label: `${getTargetLabel(target)} - trace-order`,
depends_on: [`${targetKey}-build-bun`],
agents: getTestAgent(tracePlatform, options),
retry: getRetry(),
cancel_on_build_failing: isMergeQueue(),
soft_fail: true,
timeout_in_minutes: 15,
command: [
`buildkite-agent artifact download '${profileDir}.zip' . --step ${targetKey}-build-bun`,
`unzip -o '${profileDir}.zip'`,
`chmod +x ${profileDir}/bun-profile`,
`./${profileDir}/bun-profile scripts/orderfile/generate.ts --build-dir=${profileDir} --out=${triplet}.order`,
`buildkite-agent artifact upload '${triplet}.order'`,
],
};
}

/**
* @typedef {Object} TestOptions
* @property {string} [buildId]
Expand Down Expand Up @@ -1500,6 +1555,25 @@ async function getPipeline(options = {}) {
steps.push(getStepWithDependsOn(getVerifyBaselineStep(target, options), ...verifyDeps));
}

// Seed the symbol order file for a cross-compiled target on its native
// test fleet (see getTraceOrderStep). Always on main so the inheritance
// chain stays fed, and anywhere else on commit-message opt-in so a PR
// that changes the tracer can prove the step works before merge — the
// same `[generate symbol order]` tag ci.ts already honours. Release
// profile only — usesOrderFile() is false under a sanitizer anyway.
const traceOn = traceOrderTargets.find(
t =>
t.os === target.os && t.arch === target.arch && !target.abi && (target.profile ?? "release") === "release",
);
if (traceOn && (isMainBranch() || /\[generate symbol order\]/i.test(getCommitMessage()))) {
// The trace host's image, same as verify-baseline: on the step, so
// build-cpp/build-bun don't wait for it. Darwin has no cloud image.
const traceImageKey = getImageKey(traceOn.on);
const traceDeps =
traceImageKey !== imageKey && imagePlatforms.has(traceImageKey) ? [`${traceImageKey}-build-image`] : [];
steps.push(getStepWithDependsOn(getTraceOrderStep(target, traceOn.on, options), ...traceDeps));
}

return getStepWithDependsOn(
{
key: getTargetKey(target),
Expand Down
44 changes: 24 additions & 20 deletions scripts/build/ci.ts
Original file line number Diff line number Diff line change
Expand Up @@ -424,9 +424,11 @@ export function packageAndUpload(cfg: Config, output: BunOutput): void {
zipPaths.push(makeZip(cfg, bunPath, files));

// Also upload it standalone, so the next build inherits it with a small
// download instead of pulling the whole profile zip. Named per target.
// Relative, like makeZip's return — upload() runs with cwd = buildDir.
if (hasOrderFile) {
// download instead of pulling the whole profile zip. Only when this lane
// traced the file itself — a cross-compiled lane's fresh trace comes from the
// sibling trace-order step (.buildkite/ci.mjs), and re-uploading the inherited
// copy would give inheritOrderFile() two same-named artifacts to race over.
if (hasOrderFile && canTraceOrderFile(cfg)) {
const artifact = orderFileArtifact(cfg);
cpSync(orderFilePath(cfg), resolve(buildDir, artifact));
zipPaths.push(artifact);
Expand Down Expand Up @@ -677,7 +679,6 @@ export interface OrderFileContext {
buildUrl: string | undefined;
branch: string | undefined;
buildNumber: number | undefined;
stepKey: string | undefined;
commitMessage: string;
pullRequest: boolean;
}
Expand All @@ -690,7 +691,6 @@ export function orderFileContext(): OrderFileContext {
buildUrl: process.env.BUILDKITE_BUILD_URL,
branch: process.env.BUILDKITE_BRANCH,
buildNumber: Number(process.env.BUILDKITE_BUILD_NUMBER) || undefined,
stepKey: process.env.BUILDKITE_STEP_KEY,
commitMessage: process.env.BUILDKITE_MESSAGE ?? "",
pullRequest: pr !== undefined && pr !== "" && pr !== "false",
};
Expand All @@ -709,13 +709,17 @@ export function canTraceOrderFile(cfg: Config): boolean {

/**
* An eligible lane that cannot trace (cross-compiled) and inherited nothing is
* shipping unordered with no way to recover on this host. Annotate so it is
* visible; the fix is tracing under qemu-user or on a native-arch step.
* shipping unordered. A sibling `-trace-order` step on a native-arch host seeds
* the chain (see getTraceOrderStep in .buildkite/ci.mjs), so this fires once on
* the first build and then the next build inherits that trace. If it persists,
* the trace step is failing or missing for this target.
*/
export function reportOrderFileCannotTrace(cfg: Config): void {
const msg =
`${orderFileArtifact(cfg)}: nothing to inherit and this lane cross-compiles ` +
`(target ${cfg.crossTarget}), so the binary cannot be traced here. Shipping unordered.`;
`(target ${cfg.crossTarget}), so the binary cannot be traced here. Shipping unordered. ` +
`Expected once while the native-arch trace-order step seeds the chain; if this ` +
`appears on every build, that step is failing or missing.`;
console.log(`~ symbol order: ${msg}`);
if (!isBuildkite) return;
utils.reportAnnotationToBuildKite({
Expand Down Expand Up @@ -833,22 +837,20 @@ export async function inheritOrderFile(cfg: Config, ctx: OrderFileContext): Prom
const start = Date.now();
const artifact = orderFileArtifact(cfg);

if (!ctx.stepKey) {
console.log("~ symbol order: BUILDKITE_STEP_KEY unset — linking unordered");
return false;
}

console.log(`Looking for ${artifact} published by an earlier build on ${ctx.branch}...`);
const downloaded = resolve(cfg.buildDir, artifact);
let tried = 0;

Comment thread
robobun marked this conversation as resolved.
Resolved
for await (const build of candidateBuilds(ctx)) {
if (++tried > PREVIOUS_BUILDS_TO_TRY) break;
const result = spawnSync(
"buildkite-agent",
["artifact", "download", artifact, ".", "--step", ctx.stepKey, "--build", build.id],
{ cwd: cfg.buildDir, stdio: "ignore", timeout: ARTIFACT_DOWNLOAD_TIMEOUT_MS },
);
// No --step: exactly one step per build publishes the target-unique name —
// packageAndUpload() for a lane that traced its own binary, the sibling
// trace-order step (.buildkite/ci.mjs) for a cross-compiled one.
const result = spawnSync("buildkite-agent", ["artifact", "download", artifact, ".", "--build", build.id], {
cwd: cfg.buildDir,
stdio: "ignore",
timeout: ARTIFACT_DOWNLOAD_TIMEOUT_MS,
});
if (result.status !== 0 || !existsSync(downloaded)) {
console.log(` #${build.number ?? "?"}: no ${artifact} (cancelled, failed, or too old) — looking further back`);
continue;
Expand Down Expand Up @@ -889,7 +891,7 @@ export function regenerateOrderFile(cfg: Config, ctx: OrderFileContext): void {
? "[generate symbol order] in the commit message"
: "nothing to inherit";
console.log(`Tracing ${exeName} to build a fresh order file (${why})`);
console.log("Each workload runs under an LD_PRELOAD page-fault tracer, so it is slower than a normal run.\n");
console.log("Each workload runs under an injected function-entry tracer, so it is slower than a normal run.\n");

const { count } = generateOrderFile({ buildDir: cfg.buildDir, exeName, verbose: true });

Expand Down Expand Up @@ -1028,7 +1030,9 @@ export function verifyOrderFileApplied(cfg: Config, ctx: OrderFileContext, exe:
if (control > 0 && hot > control * MAX_FRACTION_OF_CONTROL) {
fail(
`the order file had no effect: hot functions sit at ${mb(hot)}, a typical one at ${mb(control)}`,
"lld ignored it — check --symbol-ordering-file and that -ffunction-sections survived",
cfg.darwin
? "Apple ld ignored it — check -order_file and that the names match nm's"
: "lld ignored it — check --symbol-ordering-file and that -ffunction-sections survived",
);
return;
}
Expand Down
10 changes: 5 additions & 5 deletions scripts/build/configure.ts
Original file line number Diff line number Diff line change
Expand Up @@ -345,13 +345,13 @@ export async function configure(input: ConfigureInput): Promise<ConfigureResult>
mark("mkdirAll");

// Seed an empty symbol ordering file so the link flag always points at
// something. lld treats an empty file as a no-op, which is exactly the
// unordered pass-1 link; a later `generateOrderFile()` overwrites it and
// ninja relinks (linkDepends lists it). Never clobber an existing one —
// that would throw away the file a release relink or a canary download
// something. lld and Apple ld both treat an empty file as a no-op, which is
// exactly the unordered pass-1 link; a later `generateOrderFile()` overwrites
// it and ninja relinks (linkDepends lists it). Never clobber an existing one
// — that would throw away the file a release relink or a canary download
// just put there.
if (usesOrderFile(cfg) && !existsSync(orderFilePath(cfg))) {
writeIfChanged(orderFilePath(cfg), "# no order file yet — an empty file is a no-op for lld\n");
writeIfChanged(orderFilePath(cfg), "# no order file yet — an empty file is a no-op for the linker\n");
}
mark("orderFile");

Expand Down
58 changes: 44 additions & 14 deletions scripts/build/flags.ts
Original file line number Diff line number Diff line change
Expand Up @@ -1095,6 +1095,19 @@ export const linkerFlags: Flag[] = [
when: c => c.darwin && c.lto,
desc: "Persist the LTO-generated object so dsymutil can extract its DWARF into the dSYM",
},
{
// Mach-O counterpart to lld's --symbol-ordering-file below:
// <buildDir>/linker.order lists the functions bun actually executes while
// starting up, and Apple's linker sorts them to the front of __text. The
// file is a build artifact, never committed: configure seeds an empty one
// so both link passes share one build.ninja — a release build regenerates
// it from its own pass-1 binary and reruns ninja, which relinks and
// nothing else. Unknown names are silently skipped, so a stale file only
// costs part of the win.
flag: c => `-Wl,-order_file,${orderFilePath(c)}`,
when: c => c.darwin && usesOrderFile(c),
desc: "Sort startup-hot functions to the front of __text (cuts resident binary pages)",
},

// ─── Linux ───
{
Expand Down Expand Up @@ -1293,7 +1306,7 @@ export const linkerFlags: Flag[] = [
// A local `bun run build:release` therefore links unordered until you run
// `bun run orderfile` and build again.
flag: c => [`-Wl,--symbol-ordering-file=${orderFilePath(c)}`, "-Wl,--no-warn-symbol-ordering"],
when: c => usesOrderFile(c),
when: c => c.linux && usesOrderFile(c),
desc: "Sort startup-hot functions to the front of .text (cuts resident binary pages)",
},

Expand Down Expand Up @@ -1384,18 +1397,30 @@ export const linkerFlags: Flag[] = [
];

/**
* Whether this target links with an lld symbol ordering file. ELF only, and
* only where the startup win is worth a relink: release linux builds.
* Not under a sanitizer — the tracer mprotects `.text` out from under it, and
* nobody measures startup RSS on an ASAN build anyway.
* Whether this target links with a symbol ordering file (lld
* `--symbol-ordering-file` on linux, `-order_file` on darwin, which both Apple
* ld and ld64.lld take). Only where the startup win is worth a relink: release
* builds, not under a sanitizer — the tracer swaps `.text` out for a private
* copy, and nobody measures startup RSS on an ASAN build anyway.
*
* gnu only: musl links statically, so LD_PRELOAD cannot load the tracer, and
* android is cross-compiled, so the build host cannot run the binary to trace
* it. Neither can produce an order file, so link them unordered rather than
* attempt a trace that always fails and annotate every build about it.
* This says where the order file is CONSUMED, not where it is produced. A
* cross-compiled lane cannot trace its own binary (`canTraceOrderFile`), so it
* inherits an earlier build's file instead and still links ordered.
*
* linux gnu only: musl links statically, so LD_PRELOAD cannot load the tracer,
* and no musl test host exists to trace on either. android has no order-file
* linker support. Neither can produce or consume one.
*
* darwin arm64 only: the BRK-based tracer is arm64-only, so x64 has nothing to
* inherit and linking with an always-empty order file just adds noise.
*/
export function usesOrderFile(cfg: Pick<Config, "linux" | "abi" | "release" | "asan" | "valgrind">): boolean {
return cfg.linux && cfg.abi === "gnu" && cfg.release && !cfg.asan && !cfg.valgrind;
export function usesOrderFile(
cfg: Pick<Config, "linux" | "darwin" | "abi" | "arm64" | "release" | "asan" | "valgrind">,
): boolean {
if (!cfg.release || cfg.asan || cfg.valgrind) return false;
if (cfg.linux) return cfg.abi === "gnu";
if (cfg.darwin) return cfg.arm64;
return false;
Comment thread
robobun marked this conversation as resolved.
Resolved
}

/** The order file lives in the build directory — it is generated, never committed. */
Expand All @@ -1411,9 +1436,14 @@ export function orderFilePath(cfg: Pick<Config, "buildDir">): string {
export function linkDepends(cfg: Config): string[] {
if (cfg.freebsd) return [join(cfg.cwd, "src/symbols.dyn"), join(cfg.cwd, "src/linker-freebsd.lds")];
if (cfg.windows) return [join(cfg.cwd, "src/symbols.def")];
if (cfg.darwin) return [join(cfg.cwd, "src/symbols.txt")];
// linux: ELF dynamic-list + version script, plus the release symbol ordering
// file — listing it here is what makes regenerating it relink, and only relink.
// The release symbol ordering file: listing it here is what makes
// regenerating it relink, and only relink.
if (cfg.darwin) {
const darwin = [join(cfg.cwd, "src/symbols.txt")];
if (usesOrderFile(cfg)) darwin.push(orderFilePath(cfg));
return darwin;
}
// linux: ELF dynamic-list + version script.
const linux = [join(cfg.cwd, "src/symbols.dyn"), join(cfg.cwd, "src/linker.lds")];
if (usesOrderFile(cfg)) linux.push(orderFilePath(cfg));
return linux;
Expand Down
Loading
Loading