diff --git a/package.json b/package.json index 9ca95609b3bd..14ff62a7eaa5 100644 --- a/package.json +++ b/package.json @@ -56,6 +56,7 @@ "run:linux": "docker run --rm -v \"$PWD:/root/bun/\" -w /root/bun ghcr.io/oven-sh/bun-development-docker-image", "uv-posix-stubs": "bun run src/jsc/bindings/libuv/generate_uv_posix_stubs.ts", "bump": "bun ./scripts/bump.ts", + "orderfile": "bun scripts/orderfile/generate.ts", "jsc:build": "bun scripts/build.ts --profile=release-local --build-dir=build/release-local --target=WebKit", "jsc:build:debug": "bun scripts/build.ts --profile=debug-local --build-dir=build/debug-local --target=WebKit", "jsc:build:lto": "bun scripts/build.ts --profile=release-local --lto=on --build-dir=build/release-local-lto --target=WebKit", diff --git a/scripts/build.ts b/scripts/build.ts index 4d2b6141f3b8..b41de015790b 100644 --- a/scripts/build.ts +++ b/scripts/build.ts @@ -25,12 +25,18 @@ import { readFileSync } from "node:fs"; import { join } from "node:path"; import { downloadArtifacts, + inheritOrderFile, isCI, + orderFileEligible, packageAndUpload, printEnvironment, + regenerateOrderFile, + reportOrderFileFailure, + shouldGenerateOrderFile, spawnWithAnnotations, startGroup, uploadArtifacts, + verifyOrderFileApplied, } from "./build/ci.ts"; import { formatConfig, formatConfigUnchanged, type PartialConfig } from "./build/config.ts"; import { configure, type ConfigureInput, type ConfigureResult } from "./build/configure.ts"; @@ -124,9 +130,42 @@ async function main(): Promise { await startGroup("Download artifacts", () => downloadArtifacts(result.cfg)); } - await startGroup("Build", () => - spawnWithAnnotations("ninja", ninjaArgv(result.cfg), { label: "ninja", env: ninjaEnv(result.cfg, result.env) }), - ); + // Builds that don't trace their own binary link against the last successful + // build's order file. Must land before ninja: it is a link input. + if (orderFileEligible(result.cfg) && !shouldGenerateOrderFile(result.cfg)) { + await startGroup("Inherit symbol order file", () => inheritOrderFile(result.cfg)); + } + + const runNinja = () => + spawnWithAnnotations("ninja", ninjaArgv(result.cfg), { label: "ninja", env: ninjaEnv(result.cfg, result.env) }); + + await startGroup("Build", runNinja); + + // Release (and [generate symbol order]) trace the binary they just linked + // and relink against the result. Overwriting the order file dirties exactly + // one ninja edge — the link — so this second pass relinks and nothing else. + // The order file is an optimization, not correctness: a flaky trace workload + // must not fail a release build 40 minutes in. Link unordered, loudly. + if (shouldGenerateOrderFile(result.cfg)) { + let traced = true; + await startGroup("Generate symbol order file", () => { + try { + regenerateOrderFile(result.cfg); + } catch (error) { + traced = false; + reportOrderFileFailure(error as Error); + } + }); + if (traced) { + await startGroup("Relink against symbol order file", runNinja); + // We traced this exact binary: every symbol must resolve. Hard-fail. + if (result.output.exe) verifyOrderFileApplied(result.cfg, result.output.exe); + } + } else if (orderFileEligible(result.cfg) && result.output.exe) { + // Inherited: report how much of it still resolved (code moved since), but + // a stale file is a slower binary, not a broken one — never fail here. + verifyOrderFileApplied(result.cfg, result.output.exe, { strict: false }); + } // cpp-only/rust-only: upload build outputs for downstream link-only. // link-only: package + upload zips for downstream test steps. diff --git a/scripts/build/ci.ts b/scripts/build/ci.ts index 291cfb9d14fe..5311544fb6c0 100644 --- a/scripts/build/ci.ts +++ b/scripts/build/ci.ts @@ -8,15 +8,17 @@ */ import { spawn as nodeSpawn, spawnSync } from "node:child_process"; -import { cpSync, existsSync, mkdirSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; +import { cpSync, existsSync, mkdirSync, readFileSync, readdirSync, rmSync, statSync, writeFileSync } from "node:fs"; import { basename, relative, resolve } from "node:path"; import { fileURLToPath } from "node:url"; // @ts-ignore — utils.mjs has JSDoc types but no .d.ts +import { generateOrderFile } from "../orderfile/generate.ts"; import * as utils from "../utils.mjs"; import { bunExeName, shouldStrip, type BunOutput } from "./bun.ts"; import type { Config } from "./config.ts"; import { BuildError } from "./error.ts"; import { crossFeaturesJson } from "./features-json.ts"; +import { orderFilePath, usesOrderFile } from "./flags.ts"; /** True if running under any CI (env: CI, BUILDKITE, or GITHUB_ACTIONS). */ export const isCI: boolean = utils.isCI; @@ -346,7 +348,7 @@ function upload(paths: string[], cwd: string): void { * -asan) is added by the caller. Matches ci.mjs getTargetTriplet() and * cmake's bunTriplet — any drift breaks test-step downloads. */ -function computeBunTriplet(cfg: Config): string { +export function computeBunTriplet(cfg: Config): string { let t = `bun-${cfg.os}-${cfg.arch}`; if (cfg.abi === "musl") t += "-musl"; if (cfg.abi === "android") t += "-android"; @@ -412,8 +414,23 @@ export function packageAndUpload(cfg: Config, output: BunOutput): void { if (cfg.unix && !cfg.asan) { files.push(`${exeName}.linker-map`); } + // The symbol ordering file this binary was linked with, next to the linker + // map. Skip the seeded placeholder — it has no functions in it. + const hasOrderFile = usesOrderFile(cfg) && orderFileFunctionCount(cfg) > 0; + if (hasOrderFile) { + files.push(basename(orderFilePath(cfg))); + } zipPaths.push(makeZip(cfg, bunPath, files)); + // Also upload it standalone, so the next build inherits it with a small + // download instead of pulling the whole profile zip. Named per target. + // Relative, like makeZip's return — upload() runs with cwd = buildDir. + if (hasOrderFile) { + const artifact = orderFileArtifact(cfg); + cpSync(orderFilePath(cfg), resolve(buildDir, artifact)); + zipPaths.push(artifact); + } + // ─── Stripped zip ─── // Only for plain release (shouldStrip). Just the stripped `bun` binary. // cmake: bunStripPath = string(REPLACE bun ${bunTriplet} bunStripPath bun) = bunTriplet. @@ -560,3 +577,296 @@ function runAsync(argv: string[], cwd: string): Promise { }); }); } + +// ═══════════════════════════════════════════════════════════════════════════ +// Symbol ordering file +// +// The order file is a build artifact, never committed. Two ways a build gets one: +// +// generate — trace this build's own bun-profile, then relink against the +// result, so the shipped binary is ordered by exactly the +// functions it runs. Costs a second link (~15-20min on linux). +// Release builds always do this; any commit can opt in with +// [generate symbol order]. +// inherit — download the last successful build's order file for this target +// and link once. ~90% of symbols still resolve one commit later +// (100% of the hottest 1000), and lld skips the rest. Free. +// +// Canary inherits; release generates. Nothing on PRs, so a PR can neither pay +// for a relink nor publish a file the next build would inherit. +// ═══════════════════════════════════════════════════════════════════════════ + +/** + * How far back to look for a publishable order file. Builds get cancelled + * constantly, and a run of them can be long, so walk generously — this only + * costs anything when the chain really is broken that far back, and the + * alternative (giving up) means the next build publishes nothing either and + * the chain never heals until a release regenerates one. + */ +const PREVIOUS_BUILDS_TO_TRY = 50; + +/** Per-attempt cap. 50 hops × a hung agent must not blow the step's budget. */ +const ARTIFACT_DOWNLOAD_TIMEOUT_MS = 30_000; + +/** Only builds that link, on targets that use an order file, outside PRs. */ +export function orderFileEligible(cfg: Config): boolean { + if (!usesOrderFile(cfg) || !isBuildkite) return false; + if (cfg.mode !== "full" && cfg.mode !== "link-only") return false; + return !process.env.BUILDKITE_PULL_REQUEST || process.env.BUILDKITE_PULL_REQUEST === "false"; +} + +/** Artifact name for the standalone order file: `bun-linux-x64.order`. */ +function orderFileArtifact(cfg: Config): string { + return `${computeBunTriplet(cfg)}.order`; +} + +/** "1m4s" / "12s" — durations show up in every order-file log line. */ +function since(start: number): string { + const seconds = Math.round((Date.now() - start) / 1000); + return seconds >= 60 ? `${Math.floor(seconds / 60)}m${seconds % 60}s` : `${seconds}s`; +} + +/** Functions listed in the order file. 0 for the seeded placeholder or no file. */ +function orderFileFunctionCount(cfg: Config): number { + const path = orderFilePath(cfg); + if (!existsSync(path)) return 0; + return readFileSync(path, "utf8") + .split("\n") + .filter((line: string) => line && !line.startsWith("#")).length; +} + +/** + * Regenerate from this build's own binary and relink? Always for a release — + * the artifact people actually install is worth the second link. Otherwise + * opt-in per commit, so a canary can be asked for a fresh file on demand + * (e.g. after a big refactor moves a lot of code). + */ +export function shouldGenerateOrderFile(cfg: Config): boolean { + if (!orderFileEligible(cfg)) return false; + if (!cfg.canary) return true; + return /\[generate symbol order\]/i.test(process.env.BUILDKITE_MESSAGE ?? ""); +} + +/** + * Previous builds on this branch, newest first, walking `prev_branch_build`. + * + * Deliberately does NOT filter on build state, unlike `getLastSuccessfulBuild()` + * — that helper stops at the first terminal build and gives up if its `build-bun` + * steps didn't all pass, so a single cancelled build (which happens constantly) + * would break the chain. Here the artifact's *existence* is the success signal: + * only a build that linked and packaged ever uploads one. So just walk back and + * take the first build that has the file. + */ +async function previousBuilds(limit: number): Promise<{ id: string; number?: number }[]> { + const builds: { id: string; number?: number }[] = []; + let url = utils.getBuildUrl() as URL | undefined; + if (!url) return builds; + url.hash = ""; + + for (let depth = 0; url && builds.length < limit; depth++) { + const response: { error?: unknown; body?: any } = await utils.curl(`${url}.json`, { json: true, cache: true }); + const body = response.body; + if (response.error || !body) break; + if (depth > 0 && body.id) builds.push({ id: body.id, number: body.number }); + if (!body.prev_branch_build) break; + url = new URL(body.prev_branch_build["url"], url); + } + return builds; +} + +/** + * Pull a previous build's order file for this target, so a build that isn't + * generating its own still links ordered. Cheap: the standalone `.order` + * artifact, not the 170MB profile zip it also rides in. + * + * Best-effort. No previous file means an unordered link: costs pages, not a build. + */ +export async function inheritOrderFile(cfg: Config): Promise { + if (!orderFileEligible(cfg) || shouldGenerateOrderFile(cfg)) return; + const start = Date.now(); + const artifact = orderFileArtifact(cfg); + + const stepKey = process.env.BUILDKITE_STEP_KEY; + if (!stepKey) { + console.log("~ symbol order: BUILDKITE_STEP_KEY unset — linking unordered"); + return; + } + + console.log(`Looking for ${artifact} in the last ${PREVIOUS_BUILDS_TO_TRY} builds on this branch...`); + const candidates = await previousBuilds(PREVIOUS_BUILDS_TO_TRY); + if (candidates.length === 0) { + console.log(`~ symbol order: no previous build on this branch (${since(start)}) — linking unordered`); + return; + } + + const downloaded = resolve(cfg.buildDir, artifact); + for (const build of candidates) { + const result = spawnSync( + "buildkite-agent", + ["artifact", "download", artifact, ".", "--step", stepKey, "--build", build.id], + { cwd: cfg.buildDir, stdio: "ignore", timeout: ARTIFACT_DOWNLOAD_TIMEOUT_MS }, + ); + if (result.status !== 0 || !existsSync(downloaded)) { + console.log(` #${build.number ?? "?"}: no ${artifact} (cancelled, failed, or too old) — walking back`); + continue; + } + + cpSync(downloaded, orderFilePath(cfg)); + rmSync(downloaded, { force: true }); + console.log( + `+ symbol order: inherited ${artifact}, ${orderFileFunctionCount(cfg)} functions ` + + `from build #${build.number ?? "?"} in ${since(start)}`, + ); + return; + } + + console.log( + `~ symbol order: none of the last ${candidates.length} builds published ${artifact} ` + + `(${since(start)}) — linking unordered`, + ); +} + +/** + * After pass 1: trace this build's binary and overwrite the order file. The + * caller must re-run ninja, which relinks and nothing else — `linkDepends()` + * lists the order file, so it is the only edge whose input changed. + */ +export function regenerateOrderFile(cfg: Config): void { + const start = Date.now(); + const exeName = bunExeName(cfg); // bun-profile, or bun-assertions on an assertions build + const why = cfg.canary ? "[generate symbol order] in the commit message" : "release build"; + console.log(`Tracing ${exeName} to build a fresh order file (${why})`); + console.log("Each workload runs under an LD_PRELOAD page-fault tracer, so it is slower than a normal run.\n"); + + const { count } = generateOrderFile({ buildDir: cfg.buildDir, exeName, verbose: true }); + + console.log(`\n+ symbol order: traced ${count} functions in ${since(start)} — relinking against them`); +} + +/** + * The trace failed. Ship the unordered binary — it is correct, just fatter in + * resident pages — but leave a PR-visible annotation so this cannot rot into a + * permanently unordered release nobody notices. + */ +export function reportOrderFileFailure(error: Error): void { + console.error(`- symbol order: FAILED to generate — ${error.message}`); + console.error("- symbol order: linking unordered. The binary is correct; it just faults in more pages at startup."); + if (!isBuildkite) return; + utils.reportAnnotationToBuildKite({ + // Not an error: the build is fine, the binary is correct, it just faults in + // more pages. A red annotation here would read as a failed release. + style: "warning", + priority: 5, + label: "symbol order file", + content: utils.formatAnnotationToHtml({ + filename: "scripts/orderfile/generate.ts", + title: "symbol order file not generated — shipped unordered", + content: error.message, + source: "build", + level: "warning", + }), + }); +} + +/** + * Prove the relink honoured the order file. An order file lld silently ignores + * (wrong symbol spellings, `-ffunction-sections` lost, flag dropped) produces a + * binary indistinguishable from an unordered one, and we would never notice. + * + * The test is scale-free: compare where the order file's hottest functions + * landed against where a typical function landed. Ordered, the hot set clusters + * near the front of `.text` and its median offset is a small fraction of the + * median over all functions. Unordered, the two medians are the same number. + * (A fixed byte threshold would not do: `.text` grows, and a hot set that + * happens to sit in the first 40% of an unordered `.text` would sail past it.) + */ +export function verifyOrderFileApplied(cfg: Config, exe: string, { strict = true } = {}): void { + const SAMPLE = 1000; + /** Ordered, hot symbols land within a few percent of .text. Unordered, at ~100% of the control. */ + const MAX_FRACTION_OF_CONTROL = 0.4; + /** Strict mode traced this exact binary, so nearly every name must resolve. */ + const MIN_STRICT_MATCH_RATE = 0.75; + + const start = Date.now(); + if (!orderFileEligible(cfg)) return; + const wanted = readFileSync(orderFilePath(cfg), "utf8") + .split("\n") + .filter((line: string) => line && !line.startsWith("#")) + .slice(0, SAMPLE); + if (wanted.length < SAMPLE) { + console.log(`~ symbol order: only ${wanted.length} functions in the order file — nothing to verify`); + return; + } + + // Same resolution as generate.ts: honor NM, else llvm-nm, else nm. + let nm = { status: null, stdout: "" } as { status: number | null; stdout: string }; + for (const tool of [process.env.NM, "llvm-nm", "nm"].filter(Boolean) as string[]) { + nm = spawnSync(tool, ["--defined-only", exe], { encoding: "utf8", maxBuffer: 1 << 29 }); + if (nm.status === 0) break; + } + if (nm.status !== 0) { + console.log("~ symbol order: no working nm — skipping verification"); + return; + } + + const addresses = new Map(); + let textBase = Number.MAX_SAFE_INTEGER; + for (const line of nm.stdout.split("\n")) { + const m = /^([0-9a-f]+) ([tT]) (\S+)$/.exec(line); + if (!m) continue; + const address = parseInt(m[1]!, 16); + addresses.set(m[3]!, address); + if (address < textBase) textBase = address; + } + + const median = (values: number[]) => (values.length ? values[values.length >> 1]! : 0); + const sorted = (values: number[]) => values.sort((a, b) => a - b); + const offsets = sorted( + wanted + .map(name => addresses.get(name)) + .filter((address): address is number => address !== undefined) + .map(address => address - textBase), + ); + // The control: where a typical function sits. Ordering does not move this. + const control = median(sorted([...addresses.values()].map(address => address - textBase))); + + // An inherited file legitimately loses symbols to code churn (~10% one commit + // later), so a soft failure there is information, not a broken build. A file + // we just generated from this exact binary has no such excuse — and an order + // file the linker ignored is a real bug in the build, not a flaky workload, + // so unlike a failed trace it does stop the build. + const fail = (message: string, hint: string) => { + if (strict) throw new BuildError(`symbol order: ${message}`, { hint }); + console.log(`~ symbol order: ${message} — ${hint}`); + }; + + const mb = (bytes: number) => `${(bytes / 1024 / 1024).toFixed(1)}MB`; + const rate = offsets.length / wanted.length; + if (offsets.length === 0) { + fail( + "not one of the order file's symbols is in the linked binary", + "the symbol spellings do not match this link — see scripts/orderfile/generate.ts", + ); + return; + } + if (strict && rate < MIN_STRICT_MATCH_RATE) { + fail( + `only ${offsets.length}/${wanted.length} of the order file's symbols are in the binary we traced`, + "the order file and the link disagree on symbol names — most of the win is being silently lost", + ); + return; + } + + const hot = median(offsets); + if (control > 0 && hot > control * MAX_FRACTION_OF_CONTROL) { + fail( + `the order file had no effect: hot functions sit at ${mb(hot)}, a typical one at ${mb(control)}`, + "lld ignored it — check --symbol-ordering-file and that -ffunction-sections survived", + ); + return; + } + console.log( + `+ symbol order: applied — ${offsets.length}/${wanted.length} (${(rate * 100).toFixed(0)}%) of the hottest ` + + `functions resolved; median ${mb(hot)} into .text vs ${mb(control)} for a typical one (${since(start)})`, + ); +} diff --git a/scripts/build/configure.ts b/scripts/build/configure.ts index 8de9d7da99a5..334f784eeaec 100644 --- a/scripts/build/configure.ts +++ b/scripts/build/configure.ts @@ -6,7 +6,7 @@ * can configure once then run specific targets. */ -import { globSync, mkdirSync } from "node:fs"; +import { existsSync, globSync, mkdirSync } from "node:fs"; import { dirname, resolve } from "node:path"; import { globAllSources } from "../glob-sources.ts"; import { type BunOutput, bunExeName, emitBun, shouldStrip, validateBunConfig } from "./bun.ts"; @@ -21,6 +21,7 @@ import { resolveConfig, } from "./config.ts"; import { BuildError } from "./error.ts"; +import { orderFilePath, usesOrderFile } from "./flags.ts"; import { mkdirAll, writeIfChanged } from "./fs.ts"; import { ensureMacosSdk } from "./macos-sdk.ts"; import { Ninja } from "./ninja.ts"; @@ -342,6 +343,18 @@ export async function configure(input: ConfigureInput): Promise // the orchestrator already knows every .o path. mkdirAll(output.objects.map(dirname)); mark("mkdirAll"); + + // Seed an empty symbol ordering file so the link flag always points at + // something. lld treats an empty file as a no-op, which is exactly the + // unordered pass-1 link; a later `generateOrderFile()` overwrites it and + // ninja relinks (linkDepends lists it). Never clobber an existing one — + // that would throw away the file a release relink or a canary download + // just put there. + if (usesOrderFile(cfg) && !existsSync(orderFilePath(cfg))) { + writeIfChanged(orderFilePath(cfg), "# no order file yet — an empty file is a no-op for lld\n"); + } + mark("orderFile"); + const ninjaFile = resolve(cfg.buildDir, "build.ninja"); const elapsed = Math.round(performance.now() - start); diff --git a/scripts/build/flags.ts b/scripts/build/flags.ts index dc7284349acf..0be304eab7cf 100644 --- a/scripts/build/flags.ts +++ b/scripts/build/flags.ts @@ -1307,6 +1307,27 @@ export const linkerFlags: Flag[] = [ when: c => c.linux && c.release && !!c.pgoUse && !c.asan && !c.valgrind, desc: "Keep .text.hot/.text.unlikely prefixes from the PGO profile (cluster hot startup .text)", }, + { + // Same goal as keep-text-section-prefix, without needing a PGO profile: + // /linker.order lists the functions bun actually executes while + // starting up, and lld sorts their input sections to the front of `.text`. + // Starting up only touches ~8.5 MB of pages, but they are scattered over a + // ~50 MB `.text` and the kernel faults in 64 KB around each one, so the + // binary ends up with ~27 MB resident for `bun -e 'console.log(1)'`. + // Packing them together cuts that by a third for a same-size binary. + // + // The file is a build artifact, never committed: configure seeds an empty one + // (a no-op for lld) so this flag is unconditional and both link passes share + // one build.ninja — a release build regenerates it from its own pass-1 binary + // and reruns ninja, which relinks and nothing else. Symbols lld cannot find + // are skipped, so a stale file only costs part of the win. + // + // A local `bun run build:release` therefore links unordered until you run + // `bun run orderfile` and build again. + flag: c => [`-Wl,--symbol-ordering-file=${orderFilePath(c)}`, "-Wl,--no-warn-symbol-ordering"], + when: c => usesOrderFile(c), + desc: "Sort startup-hot functions to the front of .text (cuts resident binary pages)", + }, // ─── Symbols / exports ─── // These reference files on disk — linkDepends() lists the same paths @@ -1394,6 +1415,26 @@ export const linkerFlags: Flag[] = [ }, ]; +/** + * Whether this target links with an lld symbol ordering file. ELF only, and + * only where the startup win is worth a relink: release linux builds. + * Not under a sanitizer — the tracer mprotects `.text` out from under it, and + * nobody measures startup RSS on an ASAN build anyway. + * + * gnu only: musl links statically, so LD_PRELOAD cannot load the tracer, and + * android is cross-compiled, so the build host cannot run the binary to trace + * it. Neither can produce an order file, so link them unordered rather than + * attempt a trace that always fails and annotate every build about it. + */ +export function usesOrderFile(cfg: Pick): boolean { + return cfg.linux && cfg.abi === "gnu" && cfg.release && !cfg.asan && !cfg.valgrind; +} + +/** The order file lives in the build directory — it is generated, never committed. */ +export function orderFilePath(cfg: Pick): string { + return join(cfg.buildDir, "linker.order"); +} + /** * Files the linker reads via flags above. Return as implicit inputs so * ninja relinks when exported symbols / version script change. @@ -1403,8 +1444,11 @@ export function linkDepends(cfg: Config): string[] { if (cfg.freebsd) return [join(cfg.cwd, "src/symbols.dyn"), join(cfg.cwd, "src/linker-freebsd.lds")]; if (cfg.windows) return [join(cfg.cwd, "src/symbols.def")]; if (cfg.darwin) return [join(cfg.cwd, "src/symbols.txt")]; - // linux: ELF dynamic-list + version script - return [join(cfg.cwd, "src/symbols.dyn"), join(cfg.cwd, "src/linker.lds")]; + // linux: ELF dynamic-list + version script, plus the release symbol ordering + // file — listing it here is what makes regenerating it relink, and only relink. + const linux = [join(cfg.cwd, "src/symbols.dyn"), join(cfg.cwd, "src/linker.lds")]; + if (usesOrderFile(cfg)) linux.push(orderFilePath(cfg)); + return linux; } // ═══════════════════════════════════════════════════════════════════════════ diff --git a/scripts/orderfile/generate.ts b/scripts/orderfile/generate.ts new file mode 100644 index 000000000000..4fabdbc7c474 --- /dev/null +++ b/scripts/orderfile/generate.ts @@ -0,0 +1,253 @@ +#!/usr/bin/env bun +/** + * Generates the lld `--symbol-ordering-file` that packs bun's startup-hot + * functions together at the front of `.text`. + * + * Why it exists: a `bun -e 'console.log(1)'` only executes ~8.5 MB worth of + * pages, but they are scattered over a 50 MB `.text`, and Linux faults in 64 KB + * around every one of them. Sorting those functions to the front of `.text` + * cuts the resident binary pages roughly in half with no change to the binary's + * size and no change to what the code does. + * + * How: `pagetrace.c` is an LD_PRELOAD shim that mprotects the executable's own + * text+rodata to PROT_NONE and unprotects one page per SIGSEGV, so it records + * exactly the pages a run touches. We run a handful of representative + * workloads, map every touched page back to the function that owns it (`nm -S` + * on the unstripped `bun-profile`), and emit those function names in first-touch + * order. Symbols lld cannot find are ignored, so the file degrades gracefully + * as code moves. + * + * The file is never committed. Release builds generate it from their own pass-1 + * binary and relink against it; canary builds inherit the last successful + * build's file and re-publish it (scripts/build/ci.ts — inheritOrderFile / + * packageAndUpload). Locally: + * + * bun run orderfile # uses build/release, writes build/release/linker.order + * bun run orderfile -- --build-dir=build/release-lto + * + * Generating against the profile you ship is worth ~1 MB of RSS: the LTO build + * linked with a file generated from the plain release build lands at 22.6 MB, + * and at 21.6 MB with its own. + * + * Re-running against a build that already has an order file converges rather + * than drifting: a cold function only ever gets listed because it shares a page + * with a hot one, and once the hot functions are packed together those cold + * neighbours stop being touched and drop out. Hot functions always own at least + * one touched page, so they are never lost. + * + * Linux only — it is the `--symbol-ordering-file` input for the ELF link, and + * the tracer depends on /proc/self/maps. + */ +import { spawnSync } from "node:child_process"; +import { existsSync, mkdirSync, mkdtempSync, readFileSync, rmSync, writeFileSync } from "node:fs"; +import { tmpdir } from "node:os"; +import { dirname, join, resolve } from "node:path"; +import { fileURLToPath } from "node:url"; + +const HEADER_WORDS = 2; // must match pagetrace.c: page size, count + +// `import.meta.dir` is Bun-only; scripts/build/ imports this under node too. +const here = dirname(fileURLToPath(import.meta.url)); + +/** + * A trace that resolves almost nothing is worse than no file at all: it silently + * costs the win while looking like it worked. Real runs land near ~16k + * functions, so anything this low means the tracer or the symbol table broke. + */ +const MIN_FUNCTIONS = 4000; + +export interface GenerateOptions { + /** Build directory holding the unstripped binary. */ + buildDir: string; + /** Unstripped binary to trace. Defaults to `bun-profile`; an assertions build names it differently. */ + exeName?: string; + /** Where to write the order file. Defaults to `/linker.order`. */ + outPath?: string; + /** Fail if fewer than this many functions were traced. */ + minFunctions?: number; + /** Print per-workload progress. */ + verbose?: boolean; +} + +export function generateOrderFile(options: GenerateOptions): { count: number; outPath: string } { + const buildDir = resolve(options.buildDir); + const outPath = resolve(options.outPath ?? join(buildDir, "linker.order")); + const minFunctions = options.minFunctions ?? MIN_FUNCTIONS; + const log = (message: string) => options.verbose && console.log(message); + + if (process.platform !== "linux") { + throw new Error("the order file is an ELF linker input; generate it on linux"); + } + + // The unstripped binary: its symbol table is what maps pages back to functions. + const bunProfile = join(buildDir, options.exeName ?? "bun-profile"); + if (!existsSync(bunProfile)) { + throw new Error(`${bunProfile} not found — build it first (bun run build:release)`); + } + + function run(cmd: string[], env?: Record) { + const r = spawnSync(cmd[0]!, cmd.slice(1), { + env: { ...process.env, ...env }, + stdio: ["ignore", "pipe", "pipe"], + maxBuffer: 1 << 29, // nm prints ~10 MB of symbols + }); + if (r.error) throw r.error; + return r; + } + + const scratch = mkdtempSync(join(tmpdir(), "bun-orderfile-")); + try { + // ── Build the tracer ────────────────────────────────────────────────────── + const tracer = join(scratch, "pagetrace.so"); + const cc = process.env.CC || "cc"; + const build = run([cc, "-O2", "-shared", "-fPIC", "-o", tracer, join(here, "pagetrace.c"), "-ldl"]); + if (build.status !== 0) throw new Error(`failed to build the tracer with ${cc}\n${build.stderr}`); + + // ── Representative workloads ────────────────────────────────────────────── + // Order matters: earlier workloads get the densest placement, so the plain + // runtime startup path comes first. + const fixtures = join(scratch, "fixtures"); + mkdirSync(join(fixtures, "tests"), { recursive: true }); + writeFileSync( + join(fixtures, "hello.ts"), + `const greet = (name: string): string => \`hi \${name}\`;\nconsole.log(greet("world"));\n`, + ); + writeFileSync( + join(fixtures, "server.js"), + `const server = Bun.serve({ port: 0, fetch: () => new Response("ok") });\n` + + `for (let i = 0; i < 50; i++) await (await fetch(\`http://localhost:\${server.port}/\`)).text();\n` + + `server.stop(true);\n`, + ); + writeFileSync( + join(fixtures, "tests", "example.test.ts"), + `import { expect, test } from "bun:test";\ntest("passes", () => { expect(1).toBe(1); });\n`, + ); + + const workloads: { name: string; args: string[] }[] = [ + { name: "bun -e", args: ["-e", "console.log(1)"] }, + { name: "bun hello.ts", args: [join(fixtures, "hello.ts")] }, + { name: "bun server.js", args: [join(fixtures, "server.js")] }, + { name: "bun test", args: ["test", join(fixtures, "tests", "example.test.ts")] }, + ]; + + const traces: { name: string; pageSize: number; pages: BigUint64Array }[] = []; + for (const [i, workload] of workloads.entries()) { + const out = join(scratch, `trace-${i}.bin`); + const r = run([bunProfile, ...workload.args], { + LD_PRELOAD: tracer, + BUN_PAGETRACE_BIN: bunProfile, + BUN_PAGETRACE_OUT: out, + BUN_DEBUG_QUIET_LOGS: "1", + }); + if (r.status !== 0) throw new Error(`workload "${workload.name}" exited ${r.status}\n${r.stderr}`); + + const raw = readFileSync(out); + const header = new BigUint64Array(raw.buffer, raw.byteOffset, HEADER_WORDS); + const pageSize = Number(header[0]); + const capacity = raw.byteLength / 8 - HEADER_WORDS; + // The tracer's counter keeps climbing past MAX_HITS if a run ever overflows. + const count = Math.min(Number(header[1]), capacity); + if (count === 0) throw new Error(`workload "${workload.name}" recorded no faults — is LD_PRELOAD working?`); + traces.push({ + name: workload.name, + pageSize, + pages: new BigUint64Array(raw.buffer, raw.byteOffset + HEADER_WORDS * 8, count), + }); + } + + // ── Symbol table ────────────────────────────────────────────────────────── + const nm = process.env.NM || (existsSync("/usr/bin/llvm-nm") ? "llvm-nm" : "nm"); + const symbols = run([nm, "--defined-only", "-S", "--numeric-sort", bunProfile]); + if (symbols.status !== 0) throw new Error(`${nm} failed on ${bunProfile}\n${symbols.stderr}`); + + // `t`/`T` only: ordering `.rodata.*` separates constants from the mergeable + // string/constant pools they sit next to, which costs more pages than it saves. + const starts: number[] = []; + const ends: number[] = []; + const names: string[] = []; + for (const line of symbols.stdout.toString().split("\n")) { + const m = /^([0-9a-f]+) ([0-9a-f]+) (\w) (\S+)$/.exec(line); + if (!m || (m[3] !== "t" && m[3] !== "T")) continue; + const address = parseInt(m[1]!, 16); + starts.push(address); + ends.push(address + parseInt(m[2]!, 16)); + names.push(m[4]!); + } + if (!starts.length) throw new Error(`${nm} reported no sized functions — is ${bunProfile} stripped?`); + + /** Index of the last symbol starting at or before `page`, or -1. */ + function lowerBound(page: number): number { + let lo = 0; + let hi = starts.length - 1; + let ans = -1; + while (lo <= hi) { + const mid = (lo + hi) >> 1; + if (starts[mid]! <= page) { + ans = mid; + lo = mid + 1; + } else hi = mid - 1; + } + return ans; + } + + const order: string[] = []; + const seen = new Set(); + for (const trace of traces) { + const before = order.length; + const visited = new Set(); + for (const raw of trace.pages) { + const page = Number(raw); + if (visited.has(page)) continue; + visited.add(page); + // Symbols overlapping [page, page + pageSize). Walk back far enough to + // catch a function that started earlier and spans into this page. + let i = Math.max(0, lowerBound(page)); + while (i > 0 && starts[i - 1]! + 0x100000 > page) i--; + for (; i < starts.length && starts[i]! < page + trace.pageSize; i++) { + const name = names[i]!; + if (ends[i]! <= page || seen.has(name)) continue; + seen.add(name); + order.push(name); + } + } + log(` ${trace.name.padEnd(16)} ${visited.size} pages touched, +${order.length - before} functions`); + } + + if (order.length < minFunctions) { + throw new Error( + `traced only ${order.length} functions, expected at least ${minFunctions} — ` + + `the tracer or the symbol table is broken, and a near-empty order file silently costs the win`, + ); + } + + const header = [ + "# lld --symbol-ordering-file: functions bun executes while starting up,", + "# in first-touch order, so they land together at the front of .text.", + "# Generated by scripts/orderfile/generate.ts — not committed.", + `# ${order.length} functions from ${workloads.length} workloads.`, + ]; + writeFileSync(outPath, header.join("\n") + "\n" + order.join("\n") + "\n"); + return { count: order.length, outPath }; + } finally { + rmSync(scratch, { recursive: true, force: true }); + } +} + +if (import.meta.main) { + const repoRoot = resolve(here, "..", ".."); + const arg = (name: string, fallback?: string): string | undefined => { + const hit = process.argv.find(a => a.startsWith(`--${name}=`)); + return hit ? hit.slice(name.length + 3) : fallback; + }; + const buildDir = resolve(repoRoot, arg("build-dir", "build/release")!); + // Relative --out is repo-root-relative, matching --build-dir. + const out = arg("out"); + try { + const options = { buildDir, verbose: true, ...(out ? { outPath: resolve(repoRoot, out) } : {}) }; + const { count, outPath } = generateOrderFile(options); + console.log(`wrote ${outPath} (${count} functions)`); + } catch (error) { + console.error(`error: ${(error as Error).message}`); + process.exit(1); + } +} diff --git a/scripts/orderfile/pagetrace.c b/scripts/orderfile/pagetrace.c new file mode 100644 index 000000000000..1a35a467702c --- /dev/null +++ b/scripts/orderfile/pagetrace.c @@ -0,0 +1,138 @@ +// LD_PRELOAD page-fault tracer used by scripts/orderfile/generate.ts. +// +// mprotect(PROT_NONE)s the traced executable's text+rodata mappings, then +// unprotects exactly one page per SIGSEGV. The faulting pages are the pages +// the program really executes/reads, with none of the kernel's fault-around +// speculation (which maps 16 pages per fault and is what makes bun's RSS +// ~3x its true working set). +// +// The record is streamed into an mmap(MAP_SHARED) window over the output file +// so it survives whatever exit path the traced program takes — bun does not +// run atexit handlers. Layout: u64 page size, u64 count, then `count` u64 page +// addresses in first-touch order. +// +// cc -O2 -shared -fPIC -o pagetrace.so pagetrace.c -ldl +// BUN_PAGETRACE_BIN=build/release/bun-profile BUN_PAGETRACE_OUT=/tmp/trace.bin +// LD_PRELOAD=./pagetrace.so build/release/bun-profile -e 'console.log(1)' +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define MAX_REGIONS 8 +#define MAX_HITS (1 << 20) +#define HEADER_WORDS 2 // record[0] = page size, record[1] = count + +static struct { + uintptr_t start, end; +} regions[MAX_REGIONS]; +static int region_count = 0; +static uint64_t *record = NULL; +static uintptr_t page_size = 4096; +static int armed = 0; + +static int in_traced_region(uintptr_t page) +{ + for (int i = 0; i < region_count; i++) + if (page >= regions[i].start && page < regions[i].end) return 1; + return 0; +} + +static void on_fault(int sig, siginfo_t *si, void *ucontext) +{ + (void)ucontext; + uintptr_t page = (uintptr_t)si->si_addr & ~(page_size - 1); + // Not ours, or we cannot re-map it: restore the default disposition so the + // re-executed instruction produces a real crash instead of spinning here. + if (!in_traced_region(page)) { + signal(sig, SIG_DFL); + return; + } + // Faults arrive on whichever thread touched the page; bun has several. + uint64_t n = __atomic_fetch_add(&record[1], 1, __ATOMIC_RELAXED); + if (n < MAX_HITS) record[HEADER_WORDS + n] = page; + if (mprotect((void *)page, page_size, PROT_READ | PROT_EXEC) != 0) signal(sig, SIG_DFL); +} + +// Bun installs its own SIGSEGV/SIGBUS handlers (crash reporter, JIT traps). +// Swallow those registrations while tracing so ours stays in place. +typedef int (*sigaction_fn)(int, const struct sigaction *, struct sigaction *); + +int sigaction(int signum, const struct sigaction *act, struct sigaction *old) +{ + static sigaction_fn real = NULL; + if (!real) real = (sigaction_fn)dlsym(RTLD_NEXT, "sigaction"); + if (armed && (signum == SIGSEGV || signum == SIGBUS)) { + if (old) memset(old, 0, sizeof(*old)); + return 0; + } + return real(signum, act, old); +} + +__attribute__((constructor(101))) static void pagetrace_init(void) +{ + const char *binary = getenv("BUN_PAGETRACE_BIN"); + const char *out = getenv("BUN_PAGETRACE_OUT"); + if (!binary || !out) return; + + long reported = sysconf(_SC_PAGESIZE); + if (reported > 0) page_size = (uintptr_t)reported; + + size_t bytes = (size_t)(HEADER_WORDS + MAX_HITS) * 8; + int fd = open(out, O_CREAT | O_TRUNC | O_RDWR, 0644); + if (fd < 0) return; + if (ftruncate(fd, (off_t)bytes) != 0) { + close(fd); + return; + } + record = mmap(NULL, bytes, PROT_READ | PROT_WRITE, MAP_SHARED, fd, 0); + close(fd); + if (record == MAP_FAILED) { + record = NULL; + return; + } + record[0] = page_size; + + FILE *maps = fopen("/proc/self/maps", "r"); + if (!maps) return; + char line[1024]; + while (fgets(line, sizeof line, maps)) { + unsigned long start, end; + char perms[8], path[768]; + path[0] = 0; + if (sscanf(line, "%lx-%lx %7s %*s %*s %*s %767s", &start, &end, perms, path) < 4) continue; + if (!strstr(path, binary)) continue; + int executable = perms[2] == 'x'; + int read_only = perms[0] == 'r' && perms[1] == '-' && perms[2] == '-'; + if ((executable || read_only) && region_count < MAX_REGIONS) { + regions[region_count].start = start; + regions[region_count].end = end; + region_count++; + } + } + fclose(maps); + if (!region_count) return; + + static char altstack[256 * 1024]; + stack_t ss = { .ss_sp = altstack, .ss_size = sizeof altstack, .ss_flags = 0 }; + sigaltstack(&ss, NULL); + + struct sigaction sa; + memset(&sa, 0, sizeof sa); + sa.sa_sigaction = on_fault; + sa.sa_flags = SA_SIGINFO | SA_ONSTACK | SA_NODEFER; + sigemptyset(&sa.sa_mask); + sigaction_fn real = (sigaction_fn)dlsym(RTLD_NEXT, "sigaction"); + real(SIGSEGV, &sa, NULL); + real(SIGBUS, &sa, NULL); + + armed = 1; + for (int i = 0; i < region_count; i++) + mprotect((void *)regions[i].start, regions[i].end - regions[i].start, PROT_NONE); +} diff --git a/test/js/bun/perf/linker-order.test.ts b/test/js/bun/perf/linker-order.test.ts new file mode 100644 index 000000000000..53d49ca82546 --- /dev/null +++ b/test/js/bun/perf/linker-order.test.ts @@ -0,0 +1,94 @@ +import { describe, expect, it } from "bun:test"; +import { join } from "node:path"; +import type { Config } from "../../../../scripts/build/config.ts"; +import { linkDepends, linkerFlags, orderFilePath, usesOrderFile } from "../../../../scripts/build/flags.ts"; +import { generateOrderFile } from "../../../../scripts/orderfile/generate.ts"; + +/** + * `/linker.order` is the lld `--symbol-ordering-file` for the linux + * release link: it lists the functions bun executes while starting up so they + * land together at the front of `.text`, which is worth ~12 MB of resident + * binary pages on a `bun -e 'console.log(1)'` (see scripts/orderfile/generate.ts). + * + * Nothing in the build fails if this wiring rots. lld skips names it cannot + * resolve, and we pass --no-warn-symbol-ordering, so a dropped flag silently + * gives the RSS back instead of breaking the link. CI's verifyOrderFileApplied() + * catches it, but only on release builds — these checks are what notices in a PR. + */ +const cfg = (overrides: Partial = {}) => + ({ + linux: true, + abi: "gnu", + release: true, + asan: false, + valgrind: false, + darwin: false, + windows: false, + freebsd: false, + buildDir: "/tmp/build", + cwd: "/repo", + ...overrides, + }) as Config; + +describe("symbol ordering file", () => { + it("is enabled for the linux release link", () => { + expect(usesOrderFile(cfg())).toBe(true); + }); + + it("is disabled where it cannot work or is not wanted", () => { + expect(usesOrderFile(cfg({ linux: false }))).toBe(false); // ELF only + expect(usesOrderFile(cfg({ release: false }))).toBe(false); // debug: not worth a relink + expect(usesOrderFile(cfg({ asan: true }))).toBe(false); // tracer mprotects .text + expect(usesOrderFile(cfg({ valgrind: true }))).toBe(false); + // Both of these would otherwise attempt a trace that can never succeed and + // annotate every build about it. + expect(usesOrderFile(cfg({ abi: "musl" }))).toBe(false); // static: no LD_PRELOAD + expect(usesOrderFile(cfg({ abi: "android" }))).toBe(false); // cross: cannot run the binary + }); + + it("lives in the build directory, never the source tree", () => { + // A committed order file rots silently. It is a build artifact. + expect(orderFilePath(cfg())).toBe(join("/tmp/build", "linker.order")); + }); + + it("is passed to lld on the linux release link", () => { + const config = cfg(); + const applied = linkerFlags + .filter(flag => flag.when(config)) + .flatMap(flag => (typeof flag.flag === "function" ? flag.flag(config) : flag.flag)) + .flat(); + + expect(applied).toContain(`-Wl,--symbol-ordering-file=${orderFilePath(config)}`); + // Without this, a stale entry is a hard link error rather than a skipped symbol. + expect(applied).toContain("-Wl,--no-warn-symbol-ordering"); + }); + + it("is not passed on a debug or sanitizer link", () => { + for (const config of [cfg({ release: false }), cfg({ asan: true })]) { + const applied = linkerFlags + .filter(flag => flag.when(config)) + .flatMap(flag => (typeof flag.flag === "function" ? flag.flag(config) : flag.flag)) + .flat() + .join(" "); + expect(applied).not.toContain("--symbol-ordering-file"); + } + }); + + it("is a link dependency, so regenerating it relinks", () => { + // This is what makes the release two-pass work: overwrite the file, re-run + // ninja, and the link is the only edge whose input changed. + expect(linkDepends(cfg())).toContain(orderFilePath(cfg())); + expect(linkDepends(cfg({ release: false }))).not.toContain(orderFilePath(cfg({ release: false }))); + }); +}); + +describe("order file generator", () => { + it.skipIf(process.platform !== "linux")("refuses a build directory with no binary to trace", () => { + expect(() => generateOrderFile({ buildDir: "/tmp/definitely-not-a-build-dir" })).toThrow(/not found/); + }); + + it.skipIf(process.platform === "linux")("refuses to run off linux", () => { + // It is an ELF linker input and the tracer reads /proc/self/maps. + expect(() => generateOrderFile({ buildDir: "/tmp/build" })).toThrow(/linux/); + }); +});