From b23f0774a06849f985bad3d971b407329ac0a111 Mon Sep 17 00:00:00 2001 From: JUN Date: Tue, 22 Sep 2026 19:01:16 +0900 Subject: [PATCH] ci: isolate macOS shard subprocesses in bounded fresh batches --- .github/workflows/ci.yml | 110 ++----------- .../260922_native_tray_release/040_release.md | 12 ++ structure/ops/docs-and-release.md | 9 +- structure/overview.md | 2 +- .../ci-bun-crash-classifier.test.ts | 25 +-- tests/ci-workflows/ci-workflows.test.ts | 33 ++-- tests/ci-workflows/macos-serial-lanes.test.ts | 144 +++++++----------- 7 files changed, 108 insertions(+), 227 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 31f83d979be..e8f86a91e6e 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -545,8 +545,8 @@ jobs: # Two shards. Unsharded, this job was the critical path on every green dev # push (mean 14.9 min against a 4.7 min Linux maximum; devlog # 260905_test_modularization_and_windows/003). Two halves finish in ~7.7 and - # cost 0.6 extra macOS minutes of setup per run. The whole-pool control that - # the single job used to provide lives in macos-control below, on dispatch. + # cost 0.6 extra macOS minutes of setup per run. The full-membership control + # runs the same bounded batches without sharding on explicit dispatch. timeout-minutes: 20 strategy: fail-fast: false @@ -587,101 +587,21 @@ jobs: cd gui bun run build - # No attempt is ever repeated here. A Bun panic is the interpreter dying mid-suite, - # which is process death a user would have seen; a second execution that happens not - # to die does not un-kill the first, and a leg that reports green on it is reporting - # something that did not happen. This leg retried a crash exactly once until - # 2026-09-17, the Linux batch runner swept crashed batches into green, and the result - # was that Bun 1.4.2's preload segfault stayed invisible on every lane except Windows. - # - # `is_bun_runtime_crash` from scripts/ci/bun-crash-signatures.sh survives, and this leg, - # the Windows leg, the macOS control and the Linux batch runner all still source that one - # definition. Its job is now diagnosis only: it decides which failure message is printed, - # never whether the leg fails. - - name: Test - env: - MACOS_TEST_SHARD: ${{ matrix.shard }} + - name: Setup bounded batch utilities run: | - # GitHub Actions starts bash `run:` blocks with `-e`. Disable - # errexit so a Bun crash reaches PIPESTATUS and the classifier below, - # instead of aborting the step before either can be read. - set +e - set -uo pipefail - # One shared classifier for every lane; see scripts/ci/bun-crash-signatures.sh. - source scripts/ci/bun-crash-signatures.sh - - run_macos_suite() { - local suite_log suite_status - suite_log="$(mktemp -t ocx-macos-suite.XXXXXX)" || return $? - # The per-test ceiling applies to every invocation, including each isolated - # serial file. One attempt, whatever the outcome. - bun test --isolate --timeout 60000 "$@" 2>&1 | tee "$suite_log" - suite_status="${PIPESTATUS[0]}" - if [ "$suite_status" -eq 0 ]; then - rm -f "$suite_log" - return 0 - fi - if is_bun_runtime_crash "$suite_status" "$suite_log"; then - echo "::error::Bun runtime crash in the macOS suite (exit ${suite_status}); a crash is process death, not a test result, and it fails this leg on the first occurrence." - else - echo "::error::macOS suite failed (exit ${suite_status})." - fi - rm -f "$suite_log" - return "$suite_status" - } - - case "$MACOS_TEST_SHARD" in - 1|2) ;; - *) echo "::error::Invalid macOS test shard"; exit 64 ;; - esac - serial_manifest="$(bun -e 'import { SERIAL_FULL_SUITE_FILES } from "./scripts/test.ts"; console.log(SERIAL_FULL_SUITE_FILES.join("\n"));')" - manifest_status=$? - if [ "$manifest_status" -ne 0 ]; then - exit "$manifest_status" - fi - serial_files=() - ignore_args=() - serial_count=0 - while IFS= read -r file; do - if [[ ! "$file" =~ ^[[:alnum:]_./-]+$ || "$file" == /* || "/$file/" == *"/../"* || "/$file/" == *"/./"* ]]; then - echo "::error::Invalid serial test path" - exit 1 - fi - for ((index=0; index/dev/null 2>&1 || brew install coreutils + echo "$(brew --prefix coreutils)/libexec/gnubin" >> "$GITHUB_PATH" - run_macos_suite tests "--shard=$MACOS_TEST_SHARD/2" "${ignore_args[@]}" - suite_status=$? - if [ "$suite_status" -ne 0 ]; then exit "$suite_status"; fi - for ((index=0; index { const classifier = read("scripts", "ci", "bun-crash-signatures.sh"); const batchScript = read("scripts", "ci", "run-bun-test-batches.sh"); const workflow = read(".github", "workflows", "ci.yml"); - const lanes = { - "macos-shard": runBlockContaining(workflow, "run_macos_suite tests"), - }; + const jobs = (Bun.YAML.parse(workflow) as { jobs: Record }> }).jobs; + const lanes = Object.fromEntries(["test", "platform-macos", "macos-control", "platform-windows"].map(name => [ + name, jobs[name]!.steps.map(step => step.run ?? "").filter(run => run.includes("scripts/ci/run-bun-test-batches.sh")).join("\n"), + ])); test("the signatures exist in the classifier", () => { for (const signature of CRASH_SIGNATURES) { @@ -90,13 +79,11 @@ describe("the Bun crash classifier is shared", () => { test("every direct lane and the shared batch runner use the classifier", () => { for (const [name, text] of Object.entries(lanes)) { - expect(`${name}:sources:${text.includes(SOURCE_LINE)}`).toBe(`${name}:sources:true`); - expect(`${name}:calls:${text.includes("is_bun_runtime_crash \"$suite_status\" \"$suite_log\"")}`) - .toBe(`${name}:calls:true`); + expect(`${name}:delegates:${text.includes("scripts/ci/run-bun-test-batches.sh")}`).toBe(`${name}:delegates:true`); } expect(batchScript).toContain("bun-crash-signatures.sh"); expect(batchScript).toContain('is_bun_runtime_crash "$status" "$log_file"'); - expect(workflow.match(/run: bash scripts\/ci\/run-bun-test-batches\.sh/g)).toHaveLength(3); + expect(workflow.match(/run: bash scripts\/ci\/run-bun-test-batches\.sh/g)).toHaveLength(4); }); test("the thread-numbered panic form is the anchor nowhere", () => { diff --git a/tests/ci-workflows/ci-workflows.test.ts b/tests/ci-workflows/ci-workflows.test.ts index f345e16c515..6d3d2841baa 100644 --- a/tests/ci-workflows/ci-workflows.test.ts +++ b/tests/ci-workflows/ci-workflows.test.ts @@ -258,35 +258,22 @@ describe("GitHub Actions hardening", () => { const macosSteps = (ci.jobs?.["platform-macos"] as { steps?: { name?: string; env?: Record; run?: string }[] })?.steps ?? []; // The 60s per-test ceiling is part of the pinned shape: dropping it silently // restores the timing-flake class this lane kept surfacing. - const macosTestStep = macosSteps.find(step => step.name === "Test"); - expect(macosTestStep?.env?.MACOS_TEST_SHARD).toBe("${{ matrix.shard }}"); - expect(hasShellCommandHead(macosTestStep?.run, 'bun test --isolate --timeout 60000 "$@"')).toBe(true); - expect(hasExactShellCommand(macosTestStep?.run, 'run_macos_suite tests "--shard=$MACOS_TEST_SHARD/2" "${ignore_args[@]}"')).toBe(true); - expect(hasExactShellCommand(macosTestStep?.run, 'run_macos_suite --parallel=1 "./tests/$file"')).toBe(true); - expect(macosTestStep?.run).toContain('import { SERIAL_FULL_SUITE_FILES } from "./scripts/test.ts"'); + const macosTestStep = macosSteps.find(step => step.name === "Test in fresh-process batches"); + expect(macosTestStep?.env).toMatchObject({ + TEST_SHARD: "${{ matrix.shard }}/2", BUN_TEST_FILE_SCOPE: "all", BUN_TEST_BATCH_SIZE: "12", + BUN_TEST_PARALLEL: "1", BUN_TEST_BATCH_TIMEOUT_SECONDS: "300", OCX_TEST_NO_QUEUE: "1", OCX_TEST_FULL_SUITE: "1", + }); + expect(macosTestStep?.run).toBe('bash scripts/ci/run-bun-test-batches.sh "$TEST_SHARD"'); + expect(macosSteps.some(step => step.run?.includes("coreutils") && step.run.includes("GITHUB_PATH"))).toBe(true); const macosShards = (ci.jobs?.["platform-macos"] as { strategy?: { "fail-fast"?: boolean; matrix?: { shard?: number[] } }; })?.strategy; expect(macosShards?.["fail-fast"]).toBe(false); expect(macosShards?.matrix?.shard).toEqual([1, 2]); - // The macOS leg retries NOTHING. It carried a crash-only retry until - // 2026-09-17, on the reasoning that a Bun panic is a runtime defect rather than a - // test result. Both halves of that are true and the conclusion still does not - // follow: a panic is process death a user would have seen, and a second execution - // that happens not to die does not un-kill the first. Pin the absence of the loop - // and of its vocabulary, so it cannot return in a renamed form. - const macosTestRun = macosTestStep?.run ?? ""; - // Actions invokes multiline `run:` blocks with `bash -e`. The retry loop - // is gone but errexit must still be disabled before the crash-prone command: - // otherwise exit 133 aborts the step before PIPESTATUS can be inspected and the - // failure is reported without saying what kind it was. - expect(hasExactShellCommand(macosTestRun, "set +e")).toBe(true); - // The crash signatures themselves moved to scripts/ci/bun-crash-signatures.sh; that one - // definition and every lane that sources it are pinned by ci-bun-crash-classifier.test.ts. - expect(macosTestRun).toContain("it fails this leg on the first occurrence"); - expect(macosTestRun).not.toContain("for attempt in"); - expect(macosTestRun).not.toContain("while true"); + // Failure disposition belongs to the shared runner; the workflow has no recovery loop. + expect(macosTestStep?.run).not.toContain("for attempt in"); + expect(macosTestStep?.run).not.toContain("while true"); expect((ci.jobs?.["platform-macos"] as { needs?: string; if?: string })?.needs).toBe("changes"); expect((ci.jobs?.["platform-macos"] as { if?: string })?.if) .toBe("github.event_name != 'pull_request' || needs.changes.outputs.ci == 'true'"); diff --git a/tests/ci-workflows/macos-serial-lanes.test.ts b/tests/ci-workflows/macos-serial-lanes.test.ts index f0c70fa63bb..7c331c682f0 100644 --- a/tests/ci-workflows/macos-serial-lanes.test.ts +++ b/tests/ci-workflows/macos-serial-lanes.test.ts @@ -40,6 +40,7 @@ type FixtureOptions = { target?: "main" | string; outcomes?: Array<"assert" | "crash">; crashSignature?: string; + crashStatus?: number; }; function shellQuote(value: string): string { @@ -54,15 +55,15 @@ function macosTestBlock(shard: number): string { const workflow = Bun.YAML.parse(readFileSync(repoPath(".github/workflows/ci.yml"), "utf8")) as { jobs: Record }>; }; - const run = workflow.jobs["platform-macos"]?.steps.find(step => step.name === "Test")?.run; - if (!run) throw new Error("platform-macos must contain the executable Test step"); + const run = workflow.jobs["platform-macos"]?.steps.find(step => step.name === "Test in fresh-process batches")?.run; + if (!run) throw new Error("platform-macos must contain its executable batch step"); // Render the existing Actions expression too, so the old workflow reaches // the ownership assertions instead of failing with Bash's 'bad substitution'. return run.replace(/\$\{\{\s*matrix\.shard\s*\}\}/g, String(shard)); } -// Only the Bun CLI is replaced. Bash, arrays, find, pipes, PIPESTATUS, and -// filesystem validation all execute unchanged from the actual YAML run block. +// Bun and the outer timeout executable are fixture shims. The actual workflow +// command, batch runner, Bash selection, pipes and filesystem validation execute unchanged. const FAKE_BUN = String.raw` import { appendFileSync, readFileSync } from "node:fs"; const config = JSON.parse(readFileSync(process.env.MACOS_FIXTURE_CONFIG, "utf8")); @@ -79,7 +80,7 @@ if (argv[0] !== "test") { process.exit(97); } record("test"); -const matches = args => config.target === "main" ? args.includes("tests") +const matches = args => config.target === "main" ? args.some(arg => arg.includes("tests/general/")) : args.some(arg => arg.replace(/^\.\//, "") === "tests/" + config.target); if (!matches(argv)) process.exit(0); const attempts = readFileSync(log, "utf8").trim().split("\n").map(line => JSON.parse(line)) @@ -108,6 +109,9 @@ function createFixture(directory: string, options: FixtureOptions): void { mkdirSync(join(directory, "scripts", "ci"), { recursive: true }); copyFileSync(repoPath("scripts", "ci", "bun-crash-signatures.sh"), join(directory, "scripts", "ci", "bun-crash-signatures.sh")); + copyFileSync(repoPath("scripts", "ci", "run-bun-test-batches.sh"), + join(directory, "scripts", "ci", "run-bun-test-batches.sh")); + writeFileSync(join(directory, "bin/timeout"), '#!/bin/sh\nwhile [ "${1#--}" != "$1" ]; do shift; done\nshift\nexec "$@"\n', { mode: 0o755 }); for (const file of [...SERIAL_FILES, ...GENERAL_FILES]) { if (file === options.missing) continue; mkdirSync(dirname(join(directory, "tests", file)), { recursive: true }); @@ -144,7 +148,8 @@ function runShell(directory: string, shard: number): Promise<{ status: number | env: { PATH: `${join(directory, "bin")}:/usr/bin:/bin`, HOME: directory, TMPDIR: join(directory, "tmp"), RUNNER_TEMP: join(directory, "tmp"), CI: "true", - MACOS_TEST_SHARD: String(shard), + TEST_SHARD: `${shard}/2`, BUN_TEST_FILE_SCOPE: "all", BUN_TEST_BATCH_SIZE: "12", + BUN_TEST_PARALLEL: "1", BUN_TEST_BATCH_TIMEOUT_SECONDS: "300", OCX_TEST_NO_QUEUE: "1", MACOS_FIXTURE_CONFIG: join(directory, "config.json"), MACOS_FIXTURE_LOG: join(directory, "invocations.jsonl"), }, @@ -247,7 +252,7 @@ function testPaths(call: Invocation): string[] { } function targets(call: Invocation, target: string): boolean { - return testPaths(call).includes(target === "main" ? "tests" : `tests/${target}`); + return target === "main" ? testPaths(call).some(path => path.startsWith("tests/general/")) : testPaths(call).includes(`tests/${target}`); } function optionValues(argv: string[], option: string): string[] { @@ -255,107 +260,77 @@ function optionValues(argv: string[], option: string): string[] { : arg.startsWith(`${option}=`) ? [arg.slice(option.length + 1)] : []); } -function expectGeneralCall(call: Invocation, shard: number): void { - const ignores = optionValues(call.argv, "--path-ignore-patterns"); - expect(ignores.toSorted()).toEqual(SERIAL_FILES.map(file => `**/${basename(file)}`).toSorted()); - expect(optionValues(call.argv, "--shard")).toEqual([`${shard}/2`]); +function selectedFiles(shard: number, collision = false): string[] { + return [...SERIAL_FILES, ...GENERAL_FILES, ...(collision ? [`collision/${basename(SERIAL_FILES[0]!)}`] : [])] + .map(file => `tests/${file}`).sort().filter((_, index) => index % 2 === shard - 1); +} + +function expectBatchArguments(call: Invocation): void { + expect(optionValues(call.argv, "--parallel")).toEqual(["1"]); expect(optionValues(call.argv, "--timeout")).toEqual(["60000"]); expect(call.argv).toContain("--isolate"); - expect(testPaths(call)).toEqual(["tests"]); - // Account for every CLI argument: a name filter or extra exclusion could - // silently drop ordinary files even while the serial ownership oracle passes. - expect(call.argv.toSorted()).toEqual([ - "test", "--isolate", "--timeout", "60000", "tests", `--shard=${shard}/2`, - ...SERIAL_FILES.flatMap(file => ["--path-ignore-patterns", `**/${basename(file)}`]), - ].toSorted()); - // Exact exclusions above plus the unrestricted tests root leave these files - // in the main pool. Similar basenames must not become accidental exclusions. - for (const file of GENERAL_FILES) expect(ignores).not.toContain(`**/${basename(file)}`); + expect(optionValues(call.argv, "--shard")).toEqual([]); + expect(optionValues(call.argv, "--path-ignore-patterns")).toEqual([]); + expect(testPaths(call).length).toBeLessThanOrEqual(12); } -// These are explicitly Unix Bash integration tests; Windows still runs the -// existing cross-platform workflow source/layout contracts unchanged. -describe.skipIf(process.platform === "win32")("macOS serial lane shell ownership", () => { - test("both shards own each canonical file exactly once in a fresh isolated process", async () => { +describe.skipIf(process.platform === "win32")("macOS bounded shard shell ownership", () => { + test("both actual workflow shards own every file exactly once", async () => { const runs = [await runShard(1), await runShard(2)]; for (const [index, run] of runs.entries()) { expect(run.status, run.output).toBe(0); const calls = testCalls(run); - const serial = calls.filter(call => !call.argv.includes("tests")); - const owned = SERIAL_FILES.filter((_, fileIndex) => fileIndex % 2 === index); - // First oracle deliberately fails old CI for missing isolated ownership. - expect(serial.length, "missing isolated ownership of canonical serial files").toBe(owned.length); - expect(calls.filter(call => call.argv.includes("tests"))).toHaveLength(1); - expectGeneralCall(calls[0]!, index + 1); - expect(serial.map(testPaths)).toEqual(owned.map(file => [`tests/${file}`])); - for (const call of serial) { - expect(call.argv).toContain("--parallel=1"); - expect(call.argv).toContain("--isolate"); - expect(optionValues(call.argv, "--timeout")).toEqual(["60000"]); - expect(optionValues(call.argv, "--shard")).toEqual([]); - expect(optionValues(call.argv, "--path-ignore-patterns")).toEqual([]); + expect(calls.flatMap(testPaths)).toEqual(selectedFiles(index + 1)); + for (const call of calls) { + expectBatchArguments(call); + if (testPaths(call).some(path => SERIAL_FILES.some(file => path === `tests/${file}`))) expect(testPaths(call)).toHaveLength(1); } - const manifests = run.invocations.filter(call => call.kind === "manifest"); - expect(manifests).toHaveLength(1); - expect(manifests[0]!.argv[1]).toContain("SERIAL_FULL_SUITE_FILES"); + expect(run.invocations.filter(call => call.kind === "manifest")).toHaveLength(1); + expect(new Set(calls.map(call => call.pid)).size).toBe(calls.length); } - const calls = runs.flatMap(testCalls); - expect(new Set(calls.map(call => call.pid)).size).toBe(calls.length); + const all = runs.flatMap(testCalls).flatMap(testPaths); + expect(all.toSorted()).toEqual([...SERIAL_FILES, ...GENERAL_FILES].map(file => `tests/${file}`).sort()); + expect(new Set(all).size).toBe(all.length); }, SPAWN_BUDGET_MS); - for (const target of ["main", SERIAL_FILES[0]!] as const) { - test(`${target}: assertion failure propagates without retry or later files`, async () => { + test("same basenames at distinct exact paths are not silently excluded", async () => { + const runs = [await runShard(1, { collision: true }), await runShard(2, { collision: true })]; + for (const [index, run] of runs.entries()) { + expect(run.status, run.output).toBe(0); + expect(testCalls(run).flatMap(testPaths)).toEqual(selectedFiles(index + 1, true)); + } + }, SPAWN_BUDGET_MS); + + for (const target of ["main", SERIAL_FILES[1]!] as const) { + const targetPath = target === "main" ? selectedFiles(1)[0]! : `tests/${target}`; + const primaryCount = selectedFiles(1).indexOf(targetPath) + 1; + test(`${target}: assertion failure stops all later primary files`, async () => { const run = await runShard(1, { target, outcomes: ["assert"] }); expect(run.status, run.output).toBe(ASSERTION_STATUS); const calls = testCalls(run); - expect(calls).toHaveLength(target === "main" ? 1 : 2); + expect(calls).toHaveLength(primaryCount); expect(targets(calls.at(-1)!, target)).toBe(true); + expect(run.output).toContain("not retrying assertion/test failures"); + expect(run.output).not.toContain("Attribution:"); }, SPAWN_BUDGET_MS); for (const [caseIndex, signature] of CRASH_SIGNATURES.entries()) { - test(`${target}: fails on the first runtime crash (case ${caseIndex + 1})`, async () => { - // Exit 3, not 139, so the SIGNATURE arm of the shared classifier is what is under test. - // With 139 the status arm matches first and this case would pass even if the signature - // list were empty -- which is how a lane can carry a broken list and look covered (#2152). - // Exit 3 is also the real Windows shape: Bun prints the panic banner and returns 3, - // and a bare 3 must NOT be treated as a crash, so the banner is doing the work here. - const run = await runShard(1, { - target, outcomes: ["crash"], crashSignature: signature, crashStatus: SIGNATURE_ONLY_CRASH_STATUS, - }); - // This case returned 0 until 2026-09-17: the leg ran the identical command a second - // time and reported the crash as recovered. The classifier still runs -- it decides - // the message -- but it no longer decides the outcome. + test(`${target}: signature crash stays red after clean diagnostic attribution (${caseIndex})`, async () => { + const run = await runShard(1, { target, outcomes: ["crash"], crashSignature: signature, crashStatus: SIGNATURE_ONLY_CRASH_STATUS }); expect(run.status, run.output).toBe(SIGNATURE_ONLY_CRASH_STATUS); const calls = testCalls(run); - const attempts = calls.filter(call => targets(call, target)); - expect(attempts).toHaveLength(1); - expect(new Set(calls.map(call => call.pid)).size).toBe(calls.length); - // The leg stops where it crashed: the main pool alone, or the main pool plus the - // first owned serial file. The second owned serial file never starts. - expect(calls).toHaveLength(target === "main" ? 1 : 2); - expect(calls.some(call => targets(call, SERIAL_FILES[2]!))).toBe(false); - expect(run.output).toContain("it fails this leg on the first occurrence"); + expect(calls).toHaveLength(primaryCount + 1); + expect(calls.filter(call => targets(call, target))).toHaveLength(2); + expect(calls.slice(primaryCount).flatMap(testPaths)).toEqual([targetPath]); + expect(run.output).toContain("has already failed this shard"); }, SPAWN_BUDGET_MS); } - test(`${target}: a crash the classifier reads from the status alone is not retried either`, async () => { - // 139 takes the status arm rather than the signature arm. The fixture is armed to - // crash twice; a surviving retry would show up as a second attempt here. + test(`${target}: status-only crash cannot recover after repeated failure`, async () => { const run = await runShard(1, { target, outcomes: ["crash", "crash"] }); expect(run.status, run.output).toBe(CRASH_STATUS); - const calls = testCalls(run); - expect(calls).toHaveLength(target === "main" ? 1 : 2); - expect(calls.filter(call => targets(call, target))).toHaveLength(1); - }, SPAWN_BUDGET_MS); - - test(`${target}: an assertion failure is still distinguished from a crash`, async () => { - // Both fail the leg now, so the only thing separating them is what the log says. A - // classifier that matched everything would report every assertion failure as a crash - // and send the next reader hunting an interpreter bug that is not there. - const run = await runShard(1, { target, outcomes: ["assert"] }); - expect(run.status, run.output).toBe(ASSERTION_STATUS); - expect(run.output).toContain(`macOS suite failed (exit ${ASSERTION_STATUS})`); - expect(run.output).not.toContain("Bun runtime crash"); + expect(testCalls(run)).toHaveLength(primaryCount + 1); + expect(run.output).toContain("reproduces alone"); }, SPAWN_BUDGET_MS); } @@ -364,16 +339,15 @@ describe.skipIf(process.platform === "win32")("macOS serial lane shell ownership ["empty manifest", { manifest: [] }], ["duplicate entry", { manifest: [...SERIAL_FILES, SERIAL_FILES[0]!] }], ["missing file", { missing: SERIAL_FILES[3] }], - ["basename collision", { collision: true }], ["basename without its full relative path", { manifest: [basename(SERIAL_FILES[0]!)] }], ["absolute path", { manifest: [`/${SERIAL_FILES[0]}`] }], ["parent traversal", { manifest: ["serial/../serial/falcon.test.ts"] }], ]; - test.each(invalidManifests)("rejects %s before any tests start", async (_name, options) => { + test.each(invalidManifests)("rejects %s before any test starts", async (_name, options) => { for (const shard of [1, 2]) { const run = await runShard(shard, options); expect(run.status, run.output).not.toBe(0); - expect(testCalls(run), run.output).toEqual([]); + expect(testCalls(run)).toEqual([]); expect(run.invocations.filter(call => call.kind === "manifest")).toHaveLength(1); } }, SPAWN_BUDGET_MS);