From 16f8559965e47964efd019584ccbafa1ec03ed88 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 10 Jul 2026 18:37:51 -0700 Subject: [PATCH 1/3] test: stop quarantining whole files for one broken case MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit An entry in test/expectations.txt removes the FILE from the run — the kind (FAIL/SKIP/CRASH/FLAKY) is parsed but never read, so there is no "expect failure" mode. Quarantining one case deletes the file's whole coverage, and nobody finds out, because a skipped file reports nothing. Verified with the debug (ASAN) build, per file: native-plugin.test.ts 18 tests dark for 1 real failure. That failure is ASAN-only: the plugin null-derefs on purpose and ASAN traps the SEGV before the crash handler can print the name. Skip that case under ASAN (it already skips on Windows) -> 18 pass, 1 skip. run-crash-handler.test.ts 9 pass, 0 fail. Entry was stale. reportError.test.ts 2 pass under ASAN. Entry was stale. child_process.test.ts 43 pass under ASAN. Entry was stale. bunshell.test.ts 388 pass under ASAN. Entry was stale. bundler/cli.test.ts no debug-log failures under ASAN. Entry was stale. create-jsx.test.ts 7 pass locally; the entry names a CI-only build failure that no longer reproduces. Removed - if CI disagrees it will say so, which is the point. Also drop 5 entries naming files that do not exist, and rewrite the header to say what the runner actually does, so the next person quarantines the broken TEST rather than the file. spawn-maxbuf.test.ts stays quarantined: it is not flaky, it fails deterministically, and the cause is a real behavior question (below). --- test/bundler/native-plugin.test.ts | 8 ++++--- test/expectations.txt | 36 ++++++++++++++---------------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/test/bundler/native-plugin.test.ts b/test/bundler/native-plugin.test.ts index 6ca832691911..1472fe274588 100644 --- a/test/bundler/native-plugin.test.ts +++ b/test/bundler/native-plugin.test.ts @@ -1,6 +1,6 @@ import { BunFile, Loader } from "bun"; import { afterEach, beforeAll, beforeEach, describe, expect, it } from "bun:test"; -import { bunEnv, bunExe, makeTree, tempDirWithFiles } from "harness"; +import { bunEnv, bunExe, isASAN, makeTree, tempDirWithFiles } from "harness"; import path from "path"; import bundlerPluginHeader from "../../packages/bun-native-bundler-plugin-api/bundler_plugin.h" with { type: "file" }; import source from "./native_plugin.cc" with { type: "file" }; @@ -407,8 +407,10 @@ const many_foo = ["foo","foo","foo","foo","foo","foo","foo"] expect.unreachable("Should have caught an error"); }); - // don't know how to reliably test this on windows - it.skipIf(process.platform === "win32")("prints name when plugin crashes", async () => { + // don't know how to reliably test this on windows. + // Skipped under ASAN: the plugin dereferences null on purpose, and ASAN traps + // the SEGV and aborts the process before Bun's crash handler can print. + it.skipIf(process.platform === "win32" || isASAN)("prints name when plugin crashes", async () => { const prelude = /* ts */ `import values from "./stuff.ts" const many_foo = ["foo","foo","foo","foo","foo","foo","foo"] `; diff --git a/test/expectations.txt b/test/expectations.txt index 24e09b7de2ad..4e7bb80e16ec 100644 --- a/test/expectations.txt +++ b/test/expectations.txt @@ -1,16 +1,25 @@ -# Documentation: http://trac.webkit.org/wiki/TestExpectations # Format: [modifier] test-name [[ expectations ]] [# comment] +# +# READ THIS BEFORE ADDING AN ENTRY. +# +# An entry does ONE thing: it removes the file from the run on every platform +# the modifier matches. The runner parses the expectation kind (FAIL / SKIP / +# CRASH / FLAKY / LEAK / TIMEOUT) but never reads it — see getRelevantTests in +# scripts/runner.node.mjs. They are all the same thing, and there is no +# "run it and expect failure" mode. +# +# So: quarantining ONE broken case here silently deletes the whole file's +# coverage. If a single test is broken, skip THAT test in the file +# (test.skip / test.skipIf(isASAN) / test.todo, with a comment naming the +# failure) and do not add an entry here. Reserve this file for a file that +# cannot be run at all — it hangs, crashes the runner, or trips LeakSanitizer. +# +# Entries match by SUBSTRING, so a path prefix quarantines an entire family. +# A stale entry is invisible: the file simply never runs and nobody finds out. # Tests that are broken -test/cli/create/create-jsx.test.ts [ FAIL ] # false > react spa (no tailwind) > build test/cli/install/bunx.test.ts [ FAIL ] # should handle package that requires node 24: @angular/cli@latest needs a newer Node.js than Bun reports (24.3.0), unskip after the Node.js version bump [ WINDOWS-AARCH64 ] test/js/node/test/parallel/test-repl-close.js [ FAIL ] # EPIPE on stdin.write to closed child process -test/bundler/native-plugin.test.ts [ FAIL ] # prints name when plugin crashes -test/cli/run/run-crash-handler.test.ts [ FAIL ] # automatic crash reporter > segfault should report - -# Exposed by the bun -e/--eval fix that stopped swallowing uncaught exceptions (see PR #31826): -# the spawned child needs process.binding('inspector'), which is not implemented -test/js/node/test/parallel/test-inspector-enabled.js [ FAIL ] # Verbatim node v26.3.0 test asserting a FinalizationRegistry callback fires # within ONE globalThis.gc() + ONE setImmediate after the connect callback's @@ -46,12 +55,6 @@ test/js/node/test/parallel/test-stream-wrap-encoding.js [ FAIL ] # needs interna # Tests that are flaky test/js/bun/spawn/spawn-maxbuf.test.ts [ FLAKY ] -# Tests skipped due to different log/line outputs -[ ASAN ] test/js/bun/util/reportError.test.ts [ SKIP ] # log line mismatch -[ ASAN ] test/js/node/child_process/child_process.test.ts [ SKIP ] # Unexpected identifier "WARNING" -[ ASAN ] test/js/bun/shell/bunshell.test.ts [ SKIP ] # bunshell > quiet > basic -[ ASAN ] test/bundler/cli.test.ts [ SKIP ] # debug logs - # Tests failed due to ASAN: attempting free on address which was not malloc()-ed [ ASAN ] test/js/node/worker_threads/worker_threads.test.ts [ CRASH ] # After: threadId module and worker property is consistent [ ASAN ] test/js/node/worker_threads/worker_destruction.test.ts [ CRASH ] # After: bun closes cleanly when Bun.connect is used in a Worker that is terminating @@ -204,8 +207,6 @@ test/js/node/test/parallel/test-http2-util-asserts.js [ FAIL ] # not yet passing test/js/node/test/parallel/test-http2-util-headers-list.js [ FAIL ] # not yet passing test/js/node/test/parallel/test-http2-util-nghttp2error.js [ FAIL ] # not yet passing test/js/node/test/parallel/test-http2-util-update-options-buffer.js [ FAIL ] # not yet passing -test/js/node/test/sequential/test-http2-timeout-large-write-file.js [ FAIL ] # not yet passing -test/js/node/test/sequential/test-http2-timeout-large-write.js [ FAIL ] # not yet passing test/js/node/test/parallel/test-http2-autoselect-protocol.js [ SKIP ] # hangs; skip until the underlying feature lands test/js/node/test/parallel/test-http2-client-connection-tunnelling.js [ SKIP ] # hangs; skip until the underlying feature lands test/js/node/test/parallel/test-http2-client-jsstream-destroy.js [ SKIP ] # hangs; skip until the underlying feature lands @@ -214,9 +215,6 @@ test/js/node/test/parallel/test-http2-max-invalid-frames.js [ SKIP ] # hangs; sk test/js/node/test/parallel/test-http2-pack-end-stream-flag.js [ SKIP ] # hangs; skip until the underlying feature lands test/js/node/test/parallel/test-http2-ping-unsolicited-ack.js [ SKIP ] # hangs; skip until the underlying feature lands test/js/node/test/parallel/test-http2-session-cleanup-on-nghttp2-goaway.js [ SKIP ] # hangs; skip until the underlying feature lands -test/js/node/test/sequential/test-http2-max-session-memory.js [ SKIP ] # hangs; skip until the underlying feature lands -test/js/node/test/sequential/test-http2-ping-flood.js [ SKIP ] # hangs; skip until the underlying feature lands -test/js/node/test/sequential/test-http2-settings-flood.js [ SKIP ] # hangs; skip until the underlying feature lands # AddressSanitizer findings under investigation on the HTTP/2 inbound path [ ASAN ] test/js/node/test/parallel/test-diagnostics-channel-http2-client-stream-close-error.js [ SKIP ] # ASan report under investigation From aaace0dba52df56db9a7965d779826e78cc6c51c Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 10 Jul 2026 19:13:57 -0700 Subject: [PATCH 2/3] test: address what CI surfaced from the un-quarantined files Build 71732 ran these files for the first time. What the blanket entries were hiding, and what each needs: native-plugin.test.ts ASAN Its napi addon's Init() calls napi_create_function and then napi_set_named_property without checking the status in between, like most real addons. validateExceptionChecks simulates a throw in every scope, so the second call trips at the napi boundary. Added to no-validate-exceptions.txt, which keeps the file RUNNING on the ASAN lane instead of skipped. musl Every test passes (19/19). The job fails only because the "plugin crashes" test segfaults on purpose and the agent writes a core, which the runner counts as a failure. Skip that one test. Windows beforeAll builds the addon with node-gyp and the agents have no ClangCL toolset (MSB8020), so nothing in the file can run. This one is a real whole-file skip, with the reason written down. create-jsx.test.ts Windows 12 of 13 tests are already todo there; the last one asserts on an "--only-missing install" line `bun create` never prints on Windows. Marked todoIf(isWindows). Net: native-plugin runs on darwin, glibc linux, and ASAN (18 pass, 1 skip); create-jsx runs everywhere. --- test/bundler/native-plugin.test.ts | 11 ++++++----- test/cli/create/create-jsx.test.ts | 4 +++- test/expectations.txt | 6 ++++++ test/no-validate-exceptions.txt | 6 ++++++ 4 files changed, 21 insertions(+), 6 deletions(-) diff --git a/test/bundler/native-plugin.test.ts b/test/bundler/native-plugin.test.ts index 1472fe274588..732c3ca4f534 100644 --- a/test/bundler/native-plugin.test.ts +++ b/test/bundler/native-plugin.test.ts @@ -1,6 +1,6 @@ import { BunFile, Loader } from "bun"; import { afterEach, beforeAll, beforeEach, describe, expect, it } from "bun:test"; -import { bunEnv, bunExe, isASAN, makeTree, tempDirWithFiles } from "harness"; +import { bunEnv, bunExe, isASAN, isMusl, makeTree, tempDirWithFiles } from "harness"; import path from "path"; import bundlerPluginHeader from "../../packages/bun-native-bundler-plugin-api/bundler_plugin.h" with { type: "file" }; import source from "./native_plugin.cc" with { type: "file" }; @@ -407,10 +407,11 @@ const many_foo = ["foo","foo","foo","foo","foo","foo","foo"] expect.unreachable("Should have caught an error"); }); - // don't know how to reliably test this on windows. - // Skipped under ASAN: the plugin dereferences null on purpose, and ASAN traps - // the SEGV and aborts the process before Bun's crash handler can print. - it.skipIf(process.platform === "win32" || isASAN)("prints name when plugin crashes", async () => { + // This test segfaults on purpose. Windows: never worked. ASAN: traps the SEGV + // and aborts before the crash handler can print the name. musl: the crash + // handler re-raises and the agent writes a core, which the runner counts as a + // failed job even though every test passed. + it.skipIf(process.platform === "win32" || isASAN || isMusl)("prints name when plugin crashes", async () => { const prelude = /* ts */ `import values from "./stuff.ts" const many_foo = ["foo","foo","foo","foo","foo","foo","foo"] `; diff --git a/test/cli/create/create-jsx.test.ts b/test/cli/create/create-jsx.test.ts index 8612ea744695..bdd53cae300b 100644 --- a/test/cli/create/create-jsx.test.ts +++ b/test/cli/create/create-jsx.test.ts @@ -323,7 +323,9 @@ for (const development of [true, false]) { }); } -test("auto-install passes detected dependencies as positionals", async () => { +// Windows: `bun create` never prints the "--only-missing install" line this +// asserts on, so the dependency detection cannot be observed there. +test.todoIf(isWindows)("auto-install passes detected dependencies as positionals", async () => { using dir = tempDir("create-arg-separator", { "Component.tsx": `import "--trust"; diff --git a/test/expectations.txt b/test/expectations.txt index 4e7bb80e16ec..f9dfd7b9f6f0 100644 --- a/test/expectations.txt +++ b/test/expectations.txt @@ -21,6 +21,12 @@ test/cli/install/bunx.test.ts [ FAIL ] # should handle package that requires node 24: @angular/cli@latest needs a newer Node.js than Bun reports (24.3.0), unskip after the Node.js version bump [ WINDOWS-AARCH64 ] test/js/node/test/parallel/test-repl-close.js [ FAIL ] # EPIPE on stdin.write to closed child process +# The whole file's beforeAll builds a napi addon with node-gyp, and the Windows +# agents have no ClangCL toolset ("MSB8020: The build tools for ClangCL cannot +# be found"), so nothing in the file can run there. Remove once the agent image +# ships ClangCL. Runs everywhere else. +[ WINDOWS ] test/bundler/native-plugin.test.ts [ SKIP ] # node-gyp needs a ClangCL toolset the Windows agents do not have + # Verbatim node v26.3.0 test asserting a FinalizationRegistry callback fires # within ONE globalThis.gc() + ONE setImmediate after the connect callback's # closure is unreferenced. The FinalizationRegistry spec gives no timing diff --git a/test/no-validate-exceptions.txt b/test/no-validate-exceptions.txt index 4bece98c3b89..6f4087221b33 100644 --- a/test/no-validate-exceptions.txt +++ b/test/no-validate-exceptions.txt @@ -165,3 +165,9 @@ test/regression/issue/isArray-proxy-crash.test.ts # Third-party SDK with unchecked exception path in JSArray pushInline test/js/third_party/@azure/service-bus/azure-service-bus.test.ts + +# The napi addon fixture's Init() calls napi_create_function then +# napi_set_named_property without checking the status in between, like most +# real addons. Under validateExceptionChecks every scope simulates a throw, so +# the second call trips the assertion at the napi boundary. +test/bundler/native-plugin.test.ts From 54c2a15d2c7f2eb22d1b6a5dc584d1e66e132a8e Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 10 Jul 2026 19:29:06 -0700 Subject: [PATCH 3/3] test: exempt native-plugin from LSAN over a known 8-byte dlopen leak With the exception-check exemption in place the file now passes on the ASAN lane (18 pass, 0 fail) and then aborts at exit: LeakSanitizer reports a direct 8-byte leak from Bun's own dlopen path. BunProcess.cpp allocates a NapiModuleMeta and hands it to a NapiExternal with no finalizer, right under a comment saying "TODO: think about the finalizer here / currently we do not dealloc napi modules". So the meta is never freed. That is a real leak, but fixing it means changing napi module lifetime, which does not belong in a test-quarantine cleanup. Exempt the file from the leak check so its 18 tests run on ASAN, and leave the leak reported. --- test/no-validate-leaksan.txt | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/test/no-validate-leaksan.txt b/test/no-validate-leaksan.txt index 873cc966375f..334b94e568ce 100644 --- a/test/no-validate-leaksan.txt +++ b/test/no-validate-leaksan.txt @@ -453,3 +453,9 @@ test/js/third_party/prisma/prisma.test.ts # upgradeDuplexToTLS protos/server_name Box<[u8]> are owned by the JS-held # socket cell; tests that exit before a final GC report them as indirect leaks. test/js/node/tls/node-tls-connect.test.ts + +# LSAN aborts the process at exit over an 8-byte leak in Bun's own dlopen path: +# BunProcess.cpp gives the NapiExternal holding NapiModuleMeta no finalizer +# ("TODO: think about the finalizer here"), so the meta is never freed. All 18 +# tests pass; only the exit check fails. Remove once that TODO is resolved. +test/bundler/native-plugin.test.ts