Skip to content

test(aot): build the layers once for seven cases of require-cache-in-test-runs-3 and state what each case prints #47758

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
robobun wants to merge 2 commits into
base: main
Choose a base branch
from
Open

test(aot): build the layers once for seven cases of require-cache-in-test-runs-3 and state what each case prints #47758

Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1dc8493
test(aot): build the layers once for seven cases of require-cache-in-…
robobun Nov 2, 2026
914205b
test(aot): list the test files of a case by name in require-cache-in-…
robobun Nov 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
292 changes: 251 additions & 41 deletions test/aot/require-cache-in-test-runs-3.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import { expect } from "bun:test";
import { afterAll, expect } from "bun:test";
import { bunEnv, bunExe, tempDir } from "harness";
import { raiseDefaultTimeout } from "./harness/default-timeout";
import { run } from "./harness/run";
Expand All @@ -7,10 +7,27 @@ import { test as oneAtATime } from "./supported";
// supported.ts sets the limit for the first file of a process alone.
raiseDefaultTimeout(120_000);

// Each case builds and runs in a folder of its own. One after the other they take longer than CI gives a file on the lane with assertions,
// and at once they still take about that long there: so they are dealt out over four files, by the time that they take.
// Each case is a project that runs under plain `bun test`, compiled as one program, and compiled over two layers. The cases are dealt out over four files, by the time that they take.
// As one program a case has a folder of its own: what a build finds in one program holds for all of it, and beside the other cases a case is compiled to other code.
// Over layers nearly all of a build is the lowest layer, which holds Bun's own code and is the same under every case that does not let every file load again. Those cases
// are the folders of one project there: one run builds that layer for all of them, and `TestRunPartFiles:1` keeps each test file a program of its own.
const test = oneAtATime.concurrent;

type Case = {
name: string;
files: Record<string, string>;
/** The lines that its test files print. */
prints: string[];
/** What a run says of its test files, without the times. */
reports: string[];
/**
* Why every file of a run that holds the case may load again, as the stage log of a build says it. Such a run has another lowest layer, and each of its test files
* is another program: the case has a run in layers to itself.
*/
everyFileMayLoadAgain?: string[];
};
const cases: Case[] = [];

const looksAtFiles = {
"src/look.cjs": `
exports.look = function look(path) {
Expand All @@ -35,22 +52,9 @@ const moduleThatLooksAtItself = (name: string) => `
export const seen = look(require.resolve("./${name}"));
export const later = () => [look(require.resolve("./${name}")), whileItRan === undefined || whileItRan === require.cache[require.resolve("./${name}")]].join(" ");`;

async function expectCompiledToRunAsPlainInLayersToo(files: Record<string, string>) {
using dir = tempDir("require-cache-in-test-runs", files);
const seen = async (words?: string, ...flags: string[]) => {
const env = words ? { ...bunEnv, BUN_DEBUG_TEST_AOT_WORDS: words } : bunEnv;
const { stdout, stderr, exitCode } = await run([bunExe(), "test", "--isolate", ...flags], String(dir), env);
if (exitCode !== 0) console.error(stderr);
return { stdout, exitCode };
};
const plain = await seen();
expect(plain.exitCode).toBe(0);
for (const words of ["TestRunLayers:0", "TestRunLayers:2,TestRunPartFiles:1"])
expect(await seen(words, "--aot")).toEqual(plain);
}

test("a CommonJS file finds its own module in require.cache while it runs, whoever has begun it", async () => {
await expectCompiledToRunAsPlainInLayersToo({
cases.push({
name: "a CommonJS file finds its own module in require.cache while it runs, whoever has begun it",
files: {
...looksAtFiles,
"src/in-function.cjs": commonJSFileThatLooksAtItself,
"src/at-top.cjs": commonJSFileThatLooksAtItself,
Expand All @@ -76,11 +80,23 @@ test("a CommonJS file finds its own module in require.cache while it runs, whoev
console.log((await import("./src/imported-late.cjs")).default.seen);
console.log(require("./src/requires-file").seen);
});`,
});
},
prints: [
"in-function.cjs true,true exports [early] true true true",
"in-function.cjs true,true exports [early+seen+later] true true",
"at-top.cjs true,true exports [early] true true true",
"imported.cjs true,true exports [early] true true true",
"imported.cjs true,true exports [early+seen+later] true true",
"imported-below.cjs true,true exports [early] true true true",
"imported-late.cjs true,true exports [early] true true true",
"under-module.cjs true,true exports [early] true true true",
],
reports: ["one.test.ts:", "(pass) each way to begin it"],
});

test("an ES module is listed while it runs if a require() has begun it, and not if an import has", async () => {
await expectCompiledToRunAsPlainInLayersToo({
cases.push({
name: "an ES module is listed while it runs if a require() has begun it, and not if an import has",
files: {
...looksAtFiles,
"src/in-function.ts": moduleThatLooksAtItself("in-function.ts"),
"src/at-top.ts": moduleThatLooksAtItself("at-top.ts"),
Expand All @@ -106,11 +122,24 @@ test("an ES module is listed while it runs if a require() has begun it, and not
console.log(require("./src/file-requires.cjs").seen);
console.log(require("./src/module-requires").seen);
});`,
});
},
prints: [
"in-function.ts true,true exports [] true",
"in-function.ts true,true exports [early+later+seen] true true",
"at-top.ts true,true exports [] true",
"at-top.ts true,true exports [early+later+seen] true true",
"imported.ts false,false ",
"imported.ts true,true exports [early+later+seen] true true",
"imported-late.ts false,false ",
"under-file.ts true,true exports [] true",
"under-module.ts true,true exports [] true",
],
reports: ["one.test.ts:", "(pass) each way to begin it"],
});

test("what a required ES module imports sees that module listed, and not itself", async () => {
await expectCompiledToRunAsPlainInLayersToo({
cases.push({
name: "what a required ES module imports sees that module listed, and not itself",
files: {
...looksAtFiles,
"src/root.ts": `import { seen as below } from "./below-root";\nexport const seen = below;`,
"src/below-root.ts": `import { look } from "./look.cjs";\nexport const seen = [look(require.resolve("./below-root")), look(require.resolve("./root"))].join(" | ");`,
Expand All @@ -122,11 +151,17 @@ test("what a required ES module imports sees that module listed, and not itself"
console.log(require("./src/root").seen);
console.log(require("./src/root-with-nothing-to-run").seen);
});`,
});
},
prints: [
"below-root.ts false,false | root.ts true,true exports [] true",
"root-with-nothing-to-run.ts true,true exports [] true",
],
reports: ["one.test.ts:", "(pass) below a require()"],
});

test("files that require each other are all listed while the innermost runs", async () => {
await expectCompiledToRunAsPlainInLayersToo({
cases.push({
name: "files that require each other are all listed while the innermost runs",
files: {
...looksAtFiles,
"src/outer.ts": `const inner = require("./inner");\nexport const seen = inner.seen;`,
"src/inner.ts": `import { look } from "./look.cjs";\nexport const seen = [look(require.resolve("./outer")), look(require.resolve("./inner"))].join(" | ");`,
Expand All @@ -152,11 +187,19 @@ test("files that require each other are all listed while the innermost runs", as
console.log(require("./src/ring-first").seen);
console.log(require("./src/ring-first.cjs").seen);
});`,
});
},
prints: [
"outer.ts true,true exports [] true | inner.ts true,true exports [] true",
"outer.cjs true,true exports [before] true | inner.cjs true,true exports [] true",
" true ring-first.ts true,true exports [] true",
"first true ring-first.cjs true,true exports [first] true",
],
reports: ["one.test.ts:", "(pass) one below the other", "(pass) in a ring"],
});

test("an ES module that an import has begun is listed from the first require() of it", async () => {
await expectCompiledToRunAsPlainInLayersToo({
cases.push({
name: "an ES module that an import has begun is listed from the first require() of it",
files: {
...looksAtFiles,
"src/importer.ts": `
import { seen as below } from "./requires-importer";
Expand All @@ -175,11 +218,17 @@ test("an ES module that an import has begun is listed from the first require() o
"call.test.ts": `
import { test } from "bun:test";
test("import()", async () => { console.log((await import("./src/importer")).seen); });`,
});
},
prints: [
"importer.ts false,false | importer.ts true,true exports [ReferenceError] true | importer.ts true,true exports [ReferenceError] true",
"importer.ts false,false | importer.ts true,true exports [ReferenceError] true | importer.ts true,true exports [ReferenceError] true",
],
reports: ["call.test.ts:", "(pass) import()", "statement.test.ts:", "(pass) an import statement"],
Comment thread
robobun marked this conversation as resolved.
Resolved
});

test("a file that throws while it is required was listed until then, and is not afterwards", async () => {
await expectCompiledToRunAsPlainInLayersToo({
cases.push({
name: "a file that throws while it is required was listed until then, and is not afterwards",
files: {
...looksAtFiles,
"src/throws.cjs": `globalThis.seenByFile = require("./look.cjs").look(__filename);\nthrow new Error("from the file");`,
"src/throws.ts": `
Expand All @@ -198,11 +247,18 @@ test("a file that throws while it is required was listed until then, and is not
for (const turn of [1, 2])
console.log(turn, message(() => require("./src/throws.ts")), "|", (globalThis as any).seenByModule, "|", look(require.resolve("./src/throws.ts")));
});`,
});
},
prints: [
"from the file | throws.cjs true,true exports [] true | throws.cjs false,false ",
"1 from the module | throws.ts true,true exports [] true | throws.ts false,false ",
"2 from the module | throws.ts true,true exports [] true | throws.ts false,false ",
],
reports: ["one.test.ts:", "(pass) a CommonJS file", "(pass) an ES module"],
});

test("a file that takes itself out of require.cache while it runs: a CommonJS file runs again, an ES module does not", async () => {
await expectCompiledToRunAsPlainInLayersToo({
cases.push({
name: "a file that takes itself out of require.cache while it runs: a CommonJS file runs again, an ES module does not",
files: {
...looksAtFiles,
"src/file.cjs": `
const { look } = require("./look.cjs");
Expand All @@ -227,11 +283,23 @@ test("a file that takes itself out of require.cache while it runs: a CommonJS fi
test("an ES module", () => {
for (const turn of [1, 2]) console.log(turn, require("./src/module.ts").seen, "|", look(require.resolve("./src/module.ts")));
});`,
});
},
prints: [
"1 1 | file.cjs true,true exports [] true | file.cjs false,false | file.cjs false,false ",
"2 2 | file.cjs true,true exports [] true | file.cjs false,false | file.cjs false,false ",
"1 1 | module.ts true,true exports [] true | module.ts false,false | module.ts true,true exports [seen] true",
"2 1 | module.ts true,true exports [] true | module.ts false,false | module.ts true,true exports [seen] true",
],
reports: ["one.test.ts:", "(pass) a CommonJS file", "(pass) an ES module"],
everyFileMayLoadAgain: [
"file.cjs takes a key out of require.cache that it does not spell",
"the scan of the test files says so",
],
});

test("an ES module whose import fails behind a require() of it that has returned stays listed, and require() answers", async () => {
await expectCompiledToRunAsPlainInLayersToo({
cases.push({
name: "an ES module whose import fails behind a require() of it that has returned stays listed, and require() answers",
files: {
...looksAtFiles,
"src/fails.ts": `import "./requires-what-fails";\nthrow new Error("fails");\nexport const value = 1;`,
"src/requires-what-fails.ts": `require("./fails");\nexport const below = 1;`,
Expand All @@ -244,5 +312,147 @@ test("an ES module whose import fails behind a require() of it that has returned
try { require("./src/fails"); } catch (error) { message = error.message; }
console.log(message, "|", look(require.resolve("./src/fails")));
});`,
});
},
prints: [
"fails | fails.ts true,true exports [ReferenceError] true",
"no error | fails.ts true,true exports [ReferenceError] true",
],
reports: ["one.test.ts:", "(pass) import(), then require()"],
});

const ways = {
"plain bun test": { flags: [], env: {} },
"one program": { flags: ["--aot"], env: { BUN_DEBUG_TEST_AOT_WORDS: "TestRunLayers:0" } },
// With the stage log a build says why every file of its run may load again. The log changes no byte of a program.
"two layers": {
flags: ["--aot"],
env: { BUN_DEBUG_TEST_AOT_WORDS: "TestRunLayers:2,TestRunPartFiles:1", BUN_JSC_logAOTPhaseTimes: "1" },
},
} satisfies Record<string, { flags: string[]; env: Record<string, string> }>;

/** What a run tells of one case. */
type Told = { printed: string[]; reported: string[]; exitCode: number };
type Ran = { of: (it: Case) => Told; form: string | undefined; everyFileMayLoadAgain: string[] };

const folderOf = (it: Case) => `c${cases.indexOf(it) + 1}`;
const folders = new Map<string, ReturnType<typeof tempDir>>();
const runs = new Map<string, Promise<Ran>>();
// A run that no test waits for any longer still reads its folder.
afterAll(async () => {
await Promise.allSettled(runs.values());
folders.forEach(folder => folder[Symbol.dispose]());
});

/** Of the stage log only what it says of the run as a whole, and not the warnings that come with it: one for each `console` and `require` of a test file. */
const withoutStageLog = (stderr: string) =>
stderr
.replace(/^(?:AOT (?!test run \|)| \w+: wall ).*\n/gm, "")
.replace(/^ *\d+ \| .*\n *\^\nwarn: TS25(?:84|91): .*\n +at .*\n\n?/gm, "");

/**
* `bun test --isolate` in one project, once for each way. One case is the project as it is written. Several are the folders c1, c2 .. of one project:
* `--isolate` gives each test file a realm of its own, and the test files print to one stdout, so there a line says first which test file it is of.
*/
function ran(together: Case[], way: keyof typeof ways) {
const project = together.map(folderOf).join();
return runs.getOrInsertComputed(`${project} ${way}`, async () => {
const inFolders = together.length > 1;
const folder = folders.getOrInsertComputed(project, () => {
const files = together.flatMap(it =>
Object.entries(it.files).map(([path, text]) => {
const name = inFolders ? `${folderOf(it)}/${path}` : path;
const saysItsName = inFolders && path.endsWith(".test.ts");
return [name, saysItsName ? text.replaceAll("console.log(", `console.log("${name}|", `) : text];
}),
);
return tempDir("require-cache-in-test-runs", Object.fromEntries(files));
});
const { flags, env } = ways[way];
const cmd = [bunExe(), "test", "--isolate", ...flags];
const { stdout, stderr, exitCode } = await run(cmd, String(folder), { ...bunEnv, ...env });
if (exitCode !== 0) console.error(withoutStageLog(stderr));
// As harness/layers.ts reads it. run() asks a run in layers for its form.
const form = stderr.match(/^AOT test run \| ((?:layers|one program|programs without layers)\b.*)$/m)?.[1];
if (way === "one program") expect(form).toStartWith("one program");

// By test file. Under "": what names none, which is told of every case.
const printed = new Map<string, string[]>([["", []]]);
const lines = stdout.split("\n");
if (lines[0].startsWith("bun test v")) lines.shift();
if (lines.at(-1) === "") lines.pop();
for (const line of lines) {
const named = inFolders ? line.match(/^(c\d+\/\S+\.test\.ts)\| ?(.*)$/) : null;
printed.getOrInsertComputed(named?.[1] ?? "", () => []).push(named ? named[2] : line);
}
const reported = new Map<string, string[]>([["", []]]);
const failed = new Set<string>();
let file = "";
for (const line of stderr.split("\n")) {
const begun = line.match(/^(\S+\.test\.ts):$/)?.[1].replaceAll("\\", "/");
if (begun) file = begun;
const said = begun
? `${begun.slice(inFolders ? begun.indexOf("/") + 1 : 0)}:`
: /^(?:\((?:pass|fail|skip|todo)\) |error)/.test(line) && line.replace(/ \[[\d.]+m?s\]$/, "");
if (!said) continue;
// A test file that does not load has an error and no test.
if (/^(?:\(fail\) |error)/.test(said)) failed.add(file);
reported.getOrInsertComputed(file, () => []).push(said);
}
const why = stderr.matchAll(/^AOT test run \| every file may load again: (?:.*[\\/])?(.*)$/gm);
return {
of: (it: Case): Told => {
// Its test files by name: `bun test` runs the files of a folder under its own in the order in which the file system lists them.
const isOfCase = (file: string) => file === "" || !inFolders || file.startsWith(`${folderOf(it)}/`);
const told = (lines: Map<string, string[]>) =>
[...lines.keys()]
.filter(isOfCase)
.sort()
.flatMap(file => lines.get(file)!);
return {
printed: told(printed),
reported: told(reported),
// A run ends with 1 where a test has failed or a test file has not loaded: that is held against the case of the file alone.
exitCode: exitCode === 1 && failed.size > 0 && ![...failed].some(isOfCase) ? 0 : exitCode,
};
},
form,
everyFileMayLoadAgain: [...new Set(Array.from(why, match => match[1]))].sort(),
};
});
}

/** Plain `bun test` tells of the case what the case is written to print and to pass, and the compiled run of the same project tells the same. */
async function expectToRunAsPlain(it: Case, together: Case[], way: keyof typeof ways) {
const plain = (await ran(together, "plain bun test")).of(it);
expect(plain).toEqual({ printed: it.prints, reported: it.reports, exitCode: 0 });
const compiled = await ran(together, way);
expect(compiled.of(it)).toEqual(plain);
return compiled;
}

for (const it of cases) {
test(`${it.name}: plain bun test`, async () => {
const told = (await ran([it], "plain bun test")).of(it);
expect(told).toEqual({ printed: it.prints, reported: it.reports, exitCode: 0 });
});
}

const shareLayers = cases.filter(it => !it.everyFileMayLoadAgain);
// The runs in layers take longest, so their tests come first, and first of all that of a case with a run of its own: the builds begin together, and
// where few tests are under way at once no other build runs beside them.
for (const it of [...cases.filter(it => it.everyFileMayLoadAgain), ...shareLayers]) {
test(`${it.name}: two layers`, async () => {
const together = it.everyFileMayLoadAgain ? [it] : shareLayers;
const compiled = await expectToRunAsPlain(it, together, "two layers");
// A case that lets every file load again and does not say so would change what the others of its run are compiled to.
// A Bun that loads one program a realm builds no layers, and run() lets its runs pass: there nothing is shared.
if (compiled.form?.startsWith("layers "))
expect(compiled.everyFileMayLoadAgain).toEqual(it.everyFileMayLoadAgain ?? []);
});
}

for (const it of cases) {
test(`${it.name}: one program`, async () => {
await expectToRunAsPlain(it, [it], "one program");
});
}
Loading