From a8e5dc7def4755125f5e628c050a7667517ed53f Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 1 Oct 2026 05:21:47 +0000 Subject: [PATCH 1/8] test(HTMLRewriter): check the handler allocation leak with LeakSanitizer on ASAN builds The RSS test for the handler structs makes 1.79 million on() and onDocument() calls. On the ASAN lane that takes 11.8 to 14.5 s of the 15 s limit on the requested instance type, and a slower machine times out on every attempt. ASAN builds now run a LeakSanitizer test for the same regression: 4096 registrations from a macrotask, then an exit with detect_leaks=1. The RSS test is unchanged on release builds and is skipped on ASAN builds. --- test/js/workerd/html-rewriter-leak.test.ts | 91 ++++++++++++++++++---- 1 file changed, 77 insertions(+), 14 deletions(-) diff --git a/test/js/workerd/html-rewriter-leak.test.ts b/test/js/workerd/html-rewriter-leak.test.ts index 7c6ff001fc86..48c7b98fbde4 100644 --- a/test/js/workerd/html-rewriter-leak.test.ts +++ b/test/js/workerd/html-rewriter-leak.test.ts @@ -1,6 +1,6 @@ import { heapStats } from "bun:jsc"; import { describe, expect, test } from "bun:test"; -import { bunEnv, bunExe, expectRssDeltaBelow, isASAN, isDebug, tempDir } from "harness"; +import { bunEnv, bunExe, expectRssDeltaBelow, isASAN, isDebug, isWindows, tempDir } from "harness"; import { join } from "node:path"; // `wire_input`'s materialized-body path transfers the body's `+1` (a @@ -175,8 +175,12 @@ test("onEndTag callbacks are released after the rewrite", () => { // // Skipped in debug: at this N a debug pass is ~40s and the extra debug-build // allocation tracking adds enough RSS noise to drown the signal. CI has no -// debug test lane; release + ASAN cover the regression. -test.skipIf(isDebug)( +// debug test lane. +// +// Skipped on sanitizer builds: the 1.8 million registrations below cost about +// 7 µs each on the ASAN lane, most of the 15 s limit. The LeakSanitizer test +// after this one counts the same allocations exactly with a few thousand. +test.skipIf(isDebug || isASAN)( "HTMLRewriter does not leak element/document handler allocations", async () => { const code = /* js */ ` @@ -213,11 +217,6 @@ test.skipIf(isDebug)( ...bunEnv, // Don't inherit the runner's GC_LEVEL=1 — it changes the per-pass live set. BUN_GARBAGE_COLLECTOR_LEVEL: "0", - // ASAN's freed-block quarantine is exactly the thing that pins RSS at - // peak; disable it so freed lol-html builders get reused across passes. - ASAN_OPTIONS: [bunEnv.ASAN_OPTIONS, "quarantine_size_mb=0", "thread_local_quarantine_size_kb=0"] - .filter(Boolean) - .join(":"), }, stdout: "pipe", stderr: "pipe", @@ -225,12 +224,7 @@ test.skipIf(isDebug)( const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - const filteredStderr = stderr - .split("\n") - .filter(line => !line.startsWith("WARNING: ASAN interferes")) - .join("\n") - .trim(); - expect(filteredStderr).toBe(""); + expect(stderr.trim()).toBe(""); const { deltaMB } = JSON.parse(stdout.trim()); @@ -243,6 +237,75 @@ test.skipIf(isDebug)( 15_000, ); +// The same regression on sanitizer builds. LeakSanitizer fails the run for +// every handler struct that is still allocated at exit with nothing pointing +// to it, so one leaked struct is enough: no RSS threshold, no warmup. +test.skipIf(!isASAN || isWindows)( + "HTMLRewriter does not leak element/document handler allocations (LeakSanitizer)", + async () => { + const ROUNDS = 4; + const REWRITERS_PER_ROUND = 16; + const code = /* js */ ` + const { heapStats } = require("bun:jsc"); + const noop = { element() {}, comments() {}, text() {} }; + const docNoop = { doctype() {}, comments() {}, text() {}, end() {} }; + let handlers = 0; + + function once() { + const rw = new HTMLRewriter(); + for (let i = 0; i < 32; i++) rw.on("div", noop); + for (let i = 0; i < 32; i++) rw.onDocument(docNoop); + handlers += 64; + } + + // From a macrotask on purpose: leaksan.supp has entries for module + // evaluation, and they hide every allocation made while the module + // body is on the stack. + setImmediate(() => { + for (let round = 0; round < ${ROUNDS}; round++) { + for (let i = 0; i < ${REWRITERS_PER_ROUND}; i++) once(); + Bun.gc(true); + } + const rewriters = heapStats().objectTypeCounts.HTMLRewriter ?? 0; + process.stdout.write(JSON.stringify({ handlers, rewriters })); + }); + `; + + await using proc = Bun.spawn({ + cmd: [bunExe(), "-e", code], + env: { + ...bunEnv, + // Every cell that is still alive at exit gets finalized, so what + // LeakSanitizer then finds is a struct that a finalizer did not free. + BUN_DESTRUCT_VM_ON_EXIT: "1", + ASAN_OPTIONS: [bunEnv.ASAN_OPTIONS, "detect_leaks=1"].filter(Boolean).join(":"), + LSAN_OPTIONS: `print_suppressions=0:suppressions=${join(import.meta.dirname, "../../leaksan.supp")}`, + }, + stdout: "pipe", + stderr: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + // Unfixed: "SUMMARY: AddressSanitizer: 180224 byte(s) leaked in 4096 + // allocation(s)" on stderr and a non-zero exit code. + expect({ stdout, stderr: withoutAsanWarning(stderr), exitCode }).toEqual({ + stdout: expect.stringMatching(/^\{"handlers":/), + stderr: "", + exitCode: 0, + }); + + const { handlers, rewriters } = JSON.parse(stdout); + expect(handlers).toBe(ROUNDS * REWRITERS_PER_ROUND * 64); + // The collector freed them: they did not wait for the VM to be torn down. + expect(rewriters).toBeLessThan((ROUNDS * REWRITERS_PER_ROUND) / 4); + }, + // A pass takes about 0.3 s on a release ASAN build and 2 s on a debug build. + // A failure takes over 6 s, more than the default limit: LeakSanitizer + // symbolizes its report before the child exits. + 90_000, +); + // `fail()` / `cancel_from_output()` on a native ByteStream/FileReader input // must detach the source's sink backref and let the Transform cell become // unreachable, or every rewrite pins its output Response. Covers the From 40ffc053117ee8ae4e817a7b4da559dc4df275f4 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 1 Oct 2026 07:25:37 +0000 Subject: [PATCH 2/8] test(HTMLRewriter): keep the RSS leak test on ASAN builds at a quarter of the count LeakSanitizer cannot see memory that something still points to at exit, so the RSS delta stays on the ASAN lane. ASAN builds run 1000 rewriters per pass in place of 4000, with a bound of 6 MB: the unfixed leak measures 11 to 14 MB there and a clean build -1.5 to 2 MB, because the quarantine is off. The LeakSanitizer test no longer asserts on a counter that the child script maintains by itself. --- test/js/workerd/html-rewriter-leak.test.ts | 49 +++++++++++++--------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/test/js/workerd/html-rewriter-leak.test.ts b/test/js/workerd/html-rewriter-leak.test.ts index 48c7b98fbde4..b3bc87adc5e6 100644 --- a/test/js/workerd/html-rewriter-leak.test.ts +++ b/test/js/workerd/html-rewriter-leak.test.ts @@ -175,12 +175,11 @@ test("onEndTag callbacks are released after the rewrite", () => { // // Skipped in debug: at this N a debug pass is ~40s and the extra debug-build // allocation tracking adds enough RSS noise to drown the signal. CI has no -// debug test lane. +// debug test lane; release + ASAN cover the regression. // -// Skipped on sanitizer builds: the 1.8 million registrations below cost about -// 7 µs each on the ASAN lane, most of the 15 s limit. The LeakSanitizer test -// after this one counts the same allocations exactly with a few thousand. -test.skipIf(isDebug || isASAN)( +// ASAN builds run a quarter of the registrations. One costs about 7 µs on the +// ASAN lane, so the full count took 11.8 to 14.5 s of the 15 s limit there. +test.skipIf(isDebug)( "HTMLRewriter does not leak element/document handler allocations", async () => { const code = /* js */ ` @@ -194,7 +193,7 @@ test.skipIf(isDebug || isASAN)( for (let i = 0; i < 32; i++) rw.onDocument(docNoop); } - const N = 4000; + const N = ${isASAN ? 1000 : 4000}; function pass() { for (let i = 0; i < N; i++) once(); Bun.gc(true); @@ -217,6 +216,11 @@ test.skipIf(isDebug || isASAN)( ...bunEnv, // Don't inherit the runner's GC_LEVEL=1 — it changes the per-pass live set. BUN_GARBAGE_COLLECTOR_LEVEL: "0", + // ASAN's freed-block quarantine is exactly the thing that pins RSS at + // peak; disable it so freed lol-html builders get reused across passes. + ASAN_OPTIONS: [bunEnv.ASAN_OPTIONS, "quarantine_size_mb=0", "thread_local_quarantine_size_kb=0"] + .filter(Boolean) + .join(":"), }, stdout: "pipe", stderr: "pipe", @@ -224,22 +228,31 @@ test.skipIf(isDebug || isASAN)( const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - expect(stderr.trim()).toBe(""); + const filteredStderr = stderr + .split("\n") + .filter(line => !line.startsWith("WARNING: ASAN interferes")) + .join("\n") + .trim(); + expect(filteredStderr).toBe(""); const { deltaMB } = JSON.parse(stdout.trim()); // Unfixed: ~50 MB over 3 measured passes. Fixed: a plateau, but RSS is a // high-water mark and allocator jitter has been observed to reach ~30 MB // on release lanes, so the bound sits between that and the unfixed signal. - expect(deltaMB).toBeLessThan(35); + // + // ASAN, at a quarter of the count and with the quarantine off: 11 to 14 MB + // unfixed, -1.5 to 2 MB fixed. + expect(deltaMB).toBeLessThan(isASAN ? 6 : 35); expect(exitCode).toBe(0); }, 15_000, ); -// The same regression on sanitizer builds. LeakSanitizer fails the run for -// every handler struct that is still allocated at exit with nothing pointing -// to it, so one leaked struct is enough: no RSS threshold, no warmup. +// The same regression, counted exactly on sanitizer builds. That includes +// debug builds, which skip the RSS test. LeakSanitizer fails the run for every +// handler struct that is still allocated at exit with nothing pointing to it, +// so one leaked struct is enough. test.skipIf(!isASAN || isWindows)( "HTMLRewriter does not leak element/document handler allocations (LeakSanitizer)", async () => { @@ -249,13 +262,11 @@ test.skipIf(!isASAN || isWindows)( const { heapStats } = require("bun:jsc"); const noop = { element() {}, comments() {}, text() {} }; const docNoop = { doctype() {}, comments() {}, text() {}, end() {} }; - let handlers = 0; function once() { const rw = new HTMLRewriter(); for (let i = 0; i < 32; i++) rw.on("div", noop); for (let i = 0; i < 32; i++) rw.onDocument(docNoop); - handlers += 64; } // From a macrotask on purpose: leaksan.supp has entries for module @@ -266,8 +277,7 @@ test.skipIf(!isASAN || isWindows)( for (let i = 0; i < ${REWRITERS_PER_ROUND}; i++) once(); Bun.gc(true); } - const rewriters = heapStats().objectTypeCounts.HTMLRewriter ?? 0; - process.stdout.write(JSON.stringify({ handlers, rewriters })); + process.stdout.write(String(heapStats().objectTypeCounts.HTMLRewriter ?? 0)); }); `; @@ -290,15 +300,14 @@ test.skipIf(!isASAN || isWindows)( // Unfixed: "SUMMARY: AddressSanitizer: 180224 byte(s) leaked in 4096 // allocation(s)" on stderr and a non-zero exit code. expect({ stdout, stderr: withoutAsanWarning(stderr), exitCode }).toEqual({ - stdout: expect.stringMatching(/^\{"handlers":/), + stdout: expect.stringMatching(/^\d+$/), stderr: "", exitCode: 0, }); - const { handlers, rewriters } = JSON.parse(stdout); - expect(handlers).toBe(ROUNDS * REWRITERS_PER_ROUND * 64); - // The collector freed them: they did not wait for the VM to be torn down. - expect(rewriters).toBeLessThan((ROUNDS * REWRITERS_PER_ROUND) / 4); + // stdout is the count of rewriters that are still alive. The collector + // freed the others: they did not wait for the VM to be torn down. + expect(Number(stdout)).toBeLessThan((ROUNDS * REWRITERS_PER_ROUND) / 4); }, // A pass takes about 0.3 s on a release ASAN build and 2 s on a debug build. // A failure takes over 6 s, more than the default limit: LeakSanitizer From d83a34ce83fdca65f3561aa8fab431678dcfc5d1 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 1 Oct 2026 10:52:40 +0000 Subject: [PATCH 3/8] test(HTMLRewriter): count live allocator blocks for the handler leak on release builds On release builds the resident memory test for the handler structs of on() and onDocument() could pass with the leak. With the leak of #29879 put back, a release build measures 33 to 36 MB against the bound of 35 MB, and the test passes 5 of 12 runs. Since #43210 both structs fit a 48-byte block. Release builds now read mimalloc's count of live blocks (heapStats().mimalloc.malloc_bins). A round makes 1000 rewriters with 64 registrations each. The test checks that the count rises by at least one block for each registration while the rewriters are alive, and that it is back after they are collected. With the leak, the count stays 127,400 to 128,000 above the baseline, against a bound of 32,000. Without it, it ends -600 to 40 from the baseline. The resident memory test now runs on ASAN builds only. Their allocator is ASAN's, and mimalloc counts none of these structs there. --- test/js/workerd/html-rewriter-leak.test.ts | 101 ++++++++++++++++++--- 1 file changed, 86 insertions(+), 15 deletions(-) diff --git a/test/js/workerd/html-rewriter-leak.test.ts b/test/js/workerd/html-rewriter-leak.test.ts index b3bc87adc5e6..dbde70c6b479 100644 --- a/test/js/workerd/html-rewriter-leak.test.ts +++ b/test/js/workerd/html-rewriter-leak.test.ts @@ -167,20 +167,96 @@ test("onEndTag callbacks are released after the rewrite", () => { // LOLHTMLContext.deinit() must destroy those allocations. Previously it only // unprotected the held JSValues and leaked the struct memory. // +// The allocator counts the blocks it has handed out and not got back, in +// release builds too. A leaked struct is one block more for every +// registration, whatever its size and whatever the OS does with freed pages. +// +// Skipped in debug: too slow for this many registrations, and CI has no debug +// lane. +test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document handler allocations", async () => { + const ROUNDS = 2; + const REWRITERS_PER_ROUND = 1000; + // Each rewriter gets 32 on() and 32 onDocument() calls. + const REGISTRATIONS_PER_ROUND = REWRITERS_PER_ROUND * 64; + const code = /* js */ ` + const { heapStats } = require("bun:jsc"); + const noop = { element() {}, comments() {}, text() {} }; + const docNoop = { doctype() {}, comments() {}, text() {}, end() {} }; + + function liveBlocks() { + return heapStats().mimalloc.malloc_bins.reduce((sum, bin) => sum + bin.current, 0); + } + + // Counts while the rewriters of the round are alive, then lets the + // collector have them. + function round() { + const rewriters = []; + for (let i = 0; i < ${REWRITERS_PER_ROUND}; i++) { + const rewriter = new HTMLRewriter(); + for (let j = 0; j < 32; j++) rewriter.on("div", noop); + for (let j = 0; j < 32; j++) rewriter.onDocument(docNoop); + rewriters.push(rewriter); + } + const held = liveBlocks(); + rewriters.length = 0; + Bun.gc(true); + return held; + } + + round(); + const before = liveBlocks(); + let held; + for (let i = 0; i < ${ROUNDS}; i++) held = round(); + const after = liveBlocks(); + + process.stdout.write(JSON.stringify({ before, held, after })); + `; + + await using proc = Bun.spawn({ + cmd: [bunExe(), "-e", code], + env: { + ...bunEnv, + // The runner's GC_LEVEL=1 adds collections of its own. + BUN_GARBAGE_COLLECTOR_LEVEL: "0", + }, + stdout: "pipe", + stderr: "pipe", + }); + + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + + expect({ stdout, stderr, exitCode }).toEqual({ + stdout: expect.stringMatching(/^\{"before":-?\d+,"held":-?\d+,"after":-?\d+\}$/), + stderr: "", + exitCode: 0, + }); + const { before, held, after } = JSON.parse(stdout); + + // The count sees the handler structs: while the rewriters of a round are + // alive, it has at least one block for each registration. + expect(held - before, stdout).toBeGreaterThanOrEqual(REGISTRATIONS_PER_ROUND); + // Unfixed: the struct of every registration of both rounds is still there, + // 128,000 blocks. Fixed: within a few hundred of zero. + expect(after - before, stdout).toBeLessThan((ROUNDS * REGISTRATIONS_PER_ROUND) / 4); +}); + +// ASAN builds cannot read that count: their allocator is ASAN's. They measure +// resident memory. +// // RSS is a high-water mark — Bun.gc(true) collects every wrapper and its // lol-html builder, but the allocators don't promptly hand pages back to the // OS. So warmup runs the *same* workload as the measured phase: the allocator // footprint is established before the baseline, and any growth past that is // what's actually retained. // -// Skipped in debug: at this N a debug pass is ~40s and the extra debug-build -// allocation tracking adds enough RSS noise to drown the signal. CI has no -// debug test lane; release + ASAN cover the regression. +// Skipped in debug: at this N a debug pass is ~8s and the extra debug-build +// allocation tracking adds enough RSS noise to drown the signal. The +// LeakSanitizer test below runs there. // -// ASAN builds run a quarter of the registrations. One costs about 7 µs on the -// ASAN lane, so the full count took 11.8 to 14.5 s of the 15 s limit there. -test.skipIf(isDebug)( - "HTMLRewriter does not leak element/document handler allocations", +// One registration costs about 7 µs on the ASAN lane: four times this N took +// 11.8 to 14.5 s of the 15 s limit there. +test.skipIf(isDebug || !isASAN)( + "HTMLRewriter does not leak element/document handler allocations (resident memory)", async () => { const code = /* js */ ` const rss = process.memoryUsage.rss; @@ -193,7 +269,7 @@ test.skipIf(isDebug)( for (let i = 0; i < 32; i++) rw.onDocument(docNoop); } - const N = ${isASAN ? 1000 : 4000}; + const N = 1000; function pass() { for (let i = 0; i < N; i++) once(); Bun.gc(true); @@ -237,13 +313,8 @@ test.skipIf(isDebug)( const { deltaMB } = JSON.parse(stdout.trim()); - // Unfixed: ~50 MB over 3 measured passes. Fixed: a plateau, but RSS is a - // high-water mark and allocator jitter has been observed to reach ~30 MB - // on release lanes, so the bound sits between that and the unfixed signal. - // - // ASAN, at a quarter of the count and with the quarantine off: 11 to 14 MB - // unfixed, -1.5 to 2 MB fixed. - expect(deltaMB).toBeLessThan(isASAN ? 6 : 35); + // With the quarantine off: 11 to 14 MB unfixed, -1.5 to 2 MB fixed. + expect(deltaMB).toBeLessThan(6); expect(exitCode).toBe(0); }, 15_000, From 499638857bc5382dccdc6786ce96e5d4b051a2a0 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 1 Oct 2026 11:01:33 +0000 Subject: [PATCH 4/8] test(HTMLRewriter): print the handler allocation numbers of each CI lane (temporary) Prints the block counts and the resident memory of the handler leak workload on every release lane, to check the bounds against the platforms that only CI runs. This commit is removed before the pull request leaves draft. --- test/js/workerd/html-rewriter-leak.test.ts | 91 ++++++++++++++++++++++ 1 file changed, 91 insertions(+) diff --git a/test/js/workerd/html-rewriter-leak.test.ts b/test/js/workerd/html-rewriter-leak.test.ts index dbde70c6b479..f18a896b4603 100644 --- a/test/js/workerd/html-rewriter-leak.test.ts +++ b/test/js/workerd/html-rewriter-leak.test.ts @@ -240,6 +240,97 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand expect(after - before, stdout).toBeLessThan((ROUNDS * REGISTRATIONS_PER_ROUND) / 4); }); +// TEMPORARY, removed before this PR leaves draft: prints the numbers of this +// lane (the block counts of the test above, and the resident memory that the +// test below used to assert on release builds), so that the bounds can be +// checked against every platform CI runs. +test.skipIf(isDebug || isASAN)( + "diagnostic: handler allocation numbers of this lane", + async () => { + const run = async (args: string[], env: Record = {}) => { + await using proc = Bun.spawn({ + cmd: [bunExe(), ...args], + env: { ...bunEnv, BUN_GARBAGE_COLLECTOR_LEVEL: "0", ...env }, + stdout: "pipe", + stderr: "pipe", + }); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + return exitCode === 0 && stderr === "" ? stdout.trim() : JSON.stringify({ exitCode, stderr, stdout }); + }; + const lane = `${process.platform}-${process.arch}`; + const prelude = /* js */ ` + const { heapStats } = require("bun:jsc"); + const noop = { element() {}, comments() {}, text() {} }; + const docNoop = { doctype() {}, comments() {}, text() {}, end() {} }; + function rewriter() { + const rw = new HTMLRewriter(); + for (let j = 0; j < 32; j++) rw.on("div", noop); + for (let j = 0; j < 32; j++) rw.onDocument(docNoop); + return rw; + } + `; + + // 1. The block counts of the test above, per round, several runs. + const blocks = /* js */ `${prelude} + const liveBlocks = () => heapStats().mimalloc.malloc_bins.reduce((sum, bin) => sum + bin.current, 0); + function round() { + const rewriters = []; + for (let i = 0; i < 1000; i++) rewriters.push(rewriter()); + const held = liveBlocks(); + rewriters.length = 0; + Bun.gc(true); + return held; + } + const started = performance.now(); + round(); + const before = liveBlocks(); + const held = [], after = []; + for (let i = 0; i < 6; i++) { held.push(round() - before); after.push(liveBlocks() - before); } + process.stdout.write(JSON.stringify({ before, held, after, ms: Math.round(performance.now() - started) })); + `; + for (let i = 0; i < 6; i++) console.log(`HRDIAG ${lane} blocks ${await run(["-e", blocks])}`); + + // 2. The same by size class, one run. + const bins = /* js */ `${prelude} + const sizes = () => Object.fromEntries(heapStats().mimalloc.malloc_bins.filter(b => b.block_size).map(b => [b.block_size, b.current])); + const diff = (a, b) => Object.fromEntries(Object.keys({ ...a, ...b }).map(k => [k, (a[k] ?? 0) - (b[k] ?? 0)]).filter(([, v]) => Math.abs(v) >= 20)); + function round() { + const rewriters = []; + for (let i = 0; i < 1000; i++) rewriters.push(rewriter()); + const held = sizes(); + rewriters.length = 0; + Bun.gc(true); + return held; + } + round(); + const before = sizes(); + const held = round(); + round(); + const after = sizes(); + const total = Object.values(before).reduce((a, b) => a + b, 0); + process.stdout.write(JSON.stringify({ total, held: diff(held, before), after: diff(after, before), negative: Object.entries(after).filter(([, v]) => v < 0) })); + `; + console.log(`HRDIAG ${lane} bins ${await run(["-e", bins])}`); + + // 3. Resident memory after each pass of the old release workload (N 4000), + // and the kernel's peak, with 3 and with 6 measured passes in one run. + const rss = /* js */ `${prelude} + const mb = bytes => +(bytes / 1024 / 1024).toFixed(1); + const samples = [], peaks = []; + const started = performance.now(); + for (let pass = 0; pass < 10; pass++) { + for (let i = 0; i < 4000; i++) rewriter(); + Bun.gc(true); + samples.push(mb(process.memoryUsage.rss())); + peaks.push(process.resourceUsage().maxRSS); + } + process.stdout.write(JSON.stringify({ samples, peaks, ms: Math.round(performance.now() - started) })); + `; + for (let i = 0; i < 4; i++) console.log(`HRDIAG ${lane} rss ${await run(["--smol", "-e", rss])}`); + }, + 120_000, +); + // ASAN builds cannot read that count: their allocator is ASAN's. They measure // resident memory. // From 20b7327d62a7bc30e7c6518f483b38092766d6b8 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 1 Oct 2026 12:48:22 +0000 Subject: [PATCH 5/8] test(HTMLRewriter): remove the lane diagnostics Build 122356 printed the block counts and the resident memory of every release lane. The numbers are in the pull request. --- test/js/workerd/html-rewriter-leak.test.ts | 91 ---------------------- 1 file changed, 91 deletions(-) diff --git a/test/js/workerd/html-rewriter-leak.test.ts b/test/js/workerd/html-rewriter-leak.test.ts index f18a896b4603..dbde70c6b479 100644 --- a/test/js/workerd/html-rewriter-leak.test.ts +++ b/test/js/workerd/html-rewriter-leak.test.ts @@ -240,97 +240,6 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand expect(after - before, stdout).toBeLessThan((ROUNDS * REGISTRATIONS_PER_ROUND) / 4); }); -// TEMPORARY, removed before this PR leaves draft: prints the numbers of this -// lane (the block counts of the test above, and the resident memory that the -// test below used to assert on release builds), so that the bounds can be -// checked against every platform CI runs. -test.skipIf(isDebug || isASAN)( - "diagnostic: handler allocation numbers of this lane", - async () => { - const run = async (args: string[], env: Record = {}) => { - await using proc = Bun.spawn({ - cmd: [bunExe(), ...args], - env: { ...bunEnv, BUN_GARBAGE_COLLECTOR_LEVEL: "0", ...env }, - stdout: "pipe", - stderr: "pipe", - }); - const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - return exitCode === 0 && stderr === "" ? stdout.trim() : JSON.stringify({ exitCode, stderr, stdout }); - }; - const lane = `${process.platform}-${process.arch}`; - const prelude = /* js */ ` - const { heapStats } = require("bun:jsc"); - const noop = { element() {}, comments() {}, text() {} }; - const docNoop = { doctype() {}, comments() {}, text() {}, end() {} }; - function rewriter() { - const rw = new HTMLRewriter(); - for (let j = 0; j < 32; j++) rw.on("div", noop); - for (let j = 0; j < 32; j++) rw.onDocument(docNoop); - return rw; - } - `; - - // 1. The block counts of the test above, per round, several runs. - const blocks = /* js */ `${prelude} - const liveBlocks = () => heapStats().mimalloc.malloc_bins.reduce((sum, bin) => sum + bin.current, 0); - function round() { - const rewriters = []; - for (let i = 0; i < 1000; i++) rewriters.push(rewriter()); - const held = liveBlocks(); - rewriters.length = 0; - Bun.gc(true); - return held; - } - const started = performance.now(); - round(); - const before = liveBlocks(); - const held = [], after = []; - for (let i = 0; i < 6; i++) { held.push(round() - before); after.push(liveBlocks() - before); } - process.stdout.write(JSON.stringify({ before, held, after, ms: Math.round(performance.now() - started) })); - `; - for (let i = 0; i < 6; i++) console.log(`HRDIAG ${lane} blocks ${await run(["-e", blocks])}`); - - // 2. The same by size class, one run. - const bins = /* js */ `${prelude} - const sizes = () => Object.fromEntries(heapStats().mimalloc.malloc_bins.filter(b => b.block_size).map(b => [b.block_size, b.current])); - const diff = (a, b) => Object.fromEntries(Object.keys({ ...a, ...b }).map(k => [k, (a[k] ?? 0) - (b[k] ?? 0)]).filter(([, v]) => Math.abs(v) >= 20)); - function round() { - const rewriters = []; - for (let i = 0; i < 1000; i++) rewriters.push(rewriter()); - const held = sizes(); - rewriters.length = 0; - Bun.gc(true); - return held; - } - round(); - const before = sizes(); - const held = round(); - round(); - const after = sizes(); - const total = Object.values(before).reduce((a, b) => a + b, 0); - process.stdout.write(JSON.stringify({ total, held: diff(held, before), after: diff(after, before), negative: Object.entries(after).filter(([, v]) => v < 0) })); - `; - console.log(`HRDIAG ${lane} bins ${await run(["-e", bins])}`); - - // 3. Resident memory after each pass of the old release workload (N 4000), - // and the kernel's peak, with 3 and with 6 measured passes in one run. - const rss = /* js */ `${prelude} - const mb = bytes => +(bytes / 1024 / 1024).toFixed(1); - const samples = [], peaks = []; - const started = performance.now(); - for (let pass = 0; pass < 10; pass++) { - for (let i = 0; i < 4000; i++) rewriter(); - Bun.gc(true); - samples.push(mb(process.memoryUsage.rss())); - peaks.push(process.resourceUsage().maxRSS); - } - process.stdout.write(JSON.stringify({ samples, peaks, ms: Math.round(performance.now() - started) })); - `; - for (let i = 0; i < 4; i++) console.log(`HRDIAG ${lane} rss ${await run(["--smol", "-e", rss])}`); - }, - 120_000, -); - // ASAN builds cannot read that count: their allocator is ASAN's. They measure // resident memory. // From 2fbb9cf01c8c84122d962a576f3e6e526d27c83a Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 1 Oct 2026 13:09:14 +0000 Subject: [PATCH 6/8] test(HTMLRewriter): say what the block count reads --- test/js/workerd/html-rewriter-leak.test.ts | 3 +++ 1 file changed, 3 insertions(+) diff --git a/test/js/workerd/html-rewriter-leak.test.ts b/test/js/workerd/html-rewriter-leak.test.ts index dbde70c6b479..04fc0757cfbe 100644 --- a/test/js/workerd/html-rewriter-leak.test.ts +++ b/test/js/workerd/html-rewriter-leak.test.ts @@ -183,6 +183,8 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand const noop = { element() {}, comments() {}, text() {} }; const docNoop = { doctype() {}, comments() {}, text() {}, end() {} }; + // malloc_bins has an entry for each size class. Its "current" is the + // count of live blocks of that size. function liveBlocks() { return heapStats().mimalloc.malloc_bins.reduce((sum, bin) => sum + bin.current, 0); } @@ -203,6 +205,7 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand return held; } + // The first round pays for what is allocated once. round(); const before = liveBlocks(); let held; From b1948d85bd0e91d60fcad1645d6a073819dedab2 Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 1 Oct 2026 15:32:16 +0000 Subject: [PATCH 7/8] test(HTMLRewriter): count only the size class of the handler structs The sum over every size class let the guard pass on the blocks of the selectors alone: about 100,000 of the 164,000 blocks of a round are not handler structs. The count is now the 48-byte class, where mimalloc puts both structs. It is up by 64,002 while the 1000 rewriters of a round are alive. One kind of struct alone gives 32,002, so the guard is 48,000. With the leak, the count stays up by 128,003 after two rounds. Without it, the count ends within 100 of the baseline. --- test/js/workerd/html-rewriter-leak.test.ts | 29 +++++++++++++--------- 1 file changed, 17 insertions(+), 12 deletions(-) diff --git a/test/js/workerd/html-rewriter-leak.test.ts b/test/js/workerd/html-rewriter-leak.test.ts index 04fc0757cfbe..6de2bcbc06b6 100644 --- a/test/js/workerd/html-rewriter-leak.test.ts +++ b/test/js/workerd/html-rewriter-leak.test.ts @@ -167,9 +167,10 @@ test("onEndTag callbacks are released after the rewrite", () => { // LOLHTMLContext.deinit() must destroy those allocations. Previously it only // unprotected the held JSValues and leaked the struct memory. // -// The allocator counts the blocks it has handed out and not got back, in -// release builds too. A leaked struct is one block more for every -// registration, whatever its size and whatever the OS does with freed pages. +// mimalloc counts, for each size class, the blocks it has handed out and not +// got back, in release builds too. Next to nothing else is in the size class +// of the handler structs, so its count is the count of live structs, whatever +// the OS does with freed pages. // // Skipped in debug: too slow for this many registrations, and CI has no debug // lane. @@ -178,6 +179,9 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand const REWRITERS_PER_ROUND = 1000; // Each rewriter gets 32 on() and 32 onDocument() calls. const REGISTRATIONS_PER_ROUND = REWRITERS_PER_ROUND * 64; + // ElementHandler is 40 bytes and DocumentHandler is 48. mimalloc serves both + // from its 48-byte size class. + const HANDLER_BLOCK_SIZE = 48; const code = /* js */ ` const { heapStats } = require("bun:jsc"); const noop = { element() {}, comments() {}, text() {} }; @@ -185,8 +189,8 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand // malloc_bins has an entry for each size class. Its "current" is the // count of live blocks of that size. - function liveBlocks() { - return heapStats().mimalloc.malloc_bins.reduce((sum, bin) => sum + bin.current, 0); + function liveHandlerBlocks() { + return heapStats().mimalloc.malloc_bins.find(bin => bin.block_size === ${HANDLER_BLOCK_SIZE}).current; } // Counts while the rewriters of the round are alive, then lets the @@ -199,7 +203,7 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand for (let j = 0; j < 32; j++) rewriter.onDocument(docNoop); rewriters.push(rewriter); } - const held = liveBlocks(); + const held = liveHandlerBlocks(); rewriters.length = 0; Bun.gc(true); return held; @@ -207,10 +211,10 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand // The first round pays for what is allocated once. round(); - const before = liveBlocks(); + const before = liveHandlerBlocks(); let held; for (let i = 0; i < ${ROUNDS}; i++) held = round(); - const after = liveBlocks(); + const after = liveHandlerBlocks(); process.stdout.write(JSON.stringify({ before, held, after })); `; @@ -235,11 +239,12 @@ test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document hand }); const { before, held, after } = JSON.parse(stdout); - // The count sees the handler structs: while the rewriters of a round are - // alive, it has at least one block for each registration. - expect(held - before, stdout).toBeGreaterThanOrEqual(REGISTRATIONS_PER_ROUND); + // While the rewriters of a round are alive, the count is up by one for each + // registration. One kind of struct alone is half of that: a struct that + // leaves this size class, or that mimalloc does not count, fails here. + expect(held - before, stdout).toBeGreaterThan((REGISTRATIONS_PER_ROUND * 3) / 4); // Unfixed: the struct of every registration of both rounds is still there, - // 128,000 blocks. Fixed: within a few hundred of zero. + // 128,000 blocks. Fixed: within 100 of zero. expect(after - before, stdout).toBeLessThan((ROUNDS * REGISTRATIONS_PER_ROUND) / 4); }); From 285784f97250dc2321b8287de90904b56e59de8e Mon Sep 17 00:00:00 2001 From: robobun <117481402+robobun@users.noreply.github.com> Date: Thu, 1 Oct 2026 17:12:13 +0000 Subject: [PATCH 8/8] test(HTMLRewriter): bound the handler count at half a struct for each rewriter Without the JIT the count of the class is exact: 64,000 above the baseline while the rewriters of a round are alive and 0 after two rounds, in each of 200 runs. With the JIT it ends -62 to 66 from the baseline. The child now runs without the JIT, and the bound is 1000 blocks where it was 32,000. One leaked struct for each rewriter would be 2,000 blocks after two rounds. The test is now a concurrent test. --- test/js/workerd/html-rewriter-leak.test.ts | 135 +++++++++++---------- 1 file changed, 70 insertions(+), 65 deletions(-) diff --git a/test/js/workerd/html-rewriter-leak.test.ts b/test/js/workerd/html-rewriter-leak.test.ts index 6de2bcbc06b6..b25844a8b6bd 100644 --- a/test/js/workerd/html-rewriter-leak.test.ts +++ b/test/js/workerd/html-rewriter-leak.test.ts @@ -174,79 +174,84 @@ test("onEndTag callbacks are released after the rewrite", () => { // // Skipped in debug: too slow for this many registrations, and CI has no debug // lane. -test.skipIf(isDebug || isASAN)("HTMLRewriter does not leak element/document handler allocations", async () => { - const ROUNDS = 2; - const REWRITERS_PER_ROUND = 1000; - // Each rewriter gets 32 on() and 32 onDocument() calls. - const REGISTRATIONS_PER_ROUND = REWRITERS_PER_ROUND * 64; - // ElementHandler is 40 bytes and DocumentHandler is 48. mimalloc serves both - // from its 48-byte size class. - const HANDLER_BLOCK_SIZE = 48; - const code = /* js */ ` - const { heapStats } = require("bun:jsc"); - const noop = { element() {}, comments() {}, text() {} }; - const docNoop = { doctype() {}, comments() {}, text() {}, end() {} }; +test.concurrent.skipIf(isDebug || isASAN)( + "HTMLRewriter does not leak element/document handler allocations", + async () => { + const ROUNDS = 2; + const REWRITERS_PER_ROUND = 1000; + // Each rewriter gets 32 on() and 32 onDocument() calls. + const REGISTRATIONS_PER_ROUND = REWRITERS_PER_ROUND * 64; + // ElementHandler is 40 bytes and DocumentHandler is 48. mimalloc serves both + // from its 48-byte size class. + const HANDLER_BLOCK_SIZE = 48; + const code = /* js */ ` + const { heapStats } = require("bun:jsc"); + const noop = { element() {}, comments() {}, text() {} }; + const docNoop = { doctype() {}, comments() {}, text() {}, end() {} }; - // malloc_bins has an entry for each size class. Its "current" is the - // count of live blocks of that size. - function liveHandlerBlocks() { - return heapStats().mimalloc.malloc_bins.find(bin => bin.block_size === ${HANDLER_BLOCK_SIZE}).current; - } + // malloc_bins has an entry for each size class. Its "current" is the + // count of live blocks of that size. + function liveHandlerBlocks() { + return heapStats().mimalloc.malloc_bins.find(bin => bin.block_size === ${HANDLER_BLOCK_SIZE}).current; + } - // Counts while the rewriters of the round are alive, then lets the - // collector have them. - function round() { - const rewriters = []; - for (let i = 0; i < ${REWRITERS_PER_ROUND}; i++) { - const rewriter = new HTMLRewriter(); - for (let j = 0; j < 32; j++) rewriter.on("div", noop); - for (let j = 0; j < 32; j++) rewriter.onDocument(docNoop); - rewriters.push(rewriter); + // Counts while the rewriters of the round are alive, then lets the + // collector have them. + function round() { + const rewriters = []; + for (let i = 0; i < ${REWRITERS_PER_ROUND}; i++) { + const rewriter = new HTMLRewriter(); + for (let j = 0; j < 32; j++) rewriter.on("div", noop); + for (let j = 0; j < 32; j++) rewriter.onDocument(docNoop); + rewriters.push(rewriter); + } + const held = liveHandlerBlocks(); + rewriters.length = 0; + Bun.gc(true); + return held; } - const held = liveHandlerBlocks(); - rewriters.length = 0; - Bun.gc(true); - return held; - } - // The first round pays for what is allocated once. - round(); - const before = liveHandlerBlocks(); - let held; - for (let i = 0; i < ${ROUNDS}; i++) held = round(); - const after = liveHandlerBlocks(); + // The first round pays for what is allocated once. + round(); + const before = liveHandlerBlocks(); + let held; + for (let i = 0; i < ${ROUNDS}; i++) held = round(); + const after = liveHandlerBlocks(); - process.stdout.write(JSON.stringify({ before, held, after })); - `; + process.stdout.write(JSON.stringify({ before, held, after })); + `; - await using proc = Bun.spawn({ - cmd: [bunExe(), "-e", code], - env: { - ...bunEnv, - // The runner's GC_LEVEL=1 adds collections of its own. - BUN_GARBAGE_COLLECTOR_LEVEL: "0", - }, - stdout: "pipe", - stderr: "pipe", - }); + await using proc = Bun.spawn({ + cmd: [bunExe(), "-e", code], + env: { + ...bunEnv, + // The runner's GC_LEVEL=1 adds collections of its own. + BUN_GARBAGE_COLLECTOR_LEVEL: "0", + // The JIT allocates blocks of this size class while it compiles. + BUN_JSC_useJIT: "0", + }, + stdout: "pipe", + stderr: "pipe", + }); - const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); + const [stdout, stderr, exitCode] = await Promise.all([proc.stdout.text(), proc.stderr.text(), proc.exited]); - expect({ stdout, stderr, exitCode }).toEqual({ - stdout: expect.stringMatching(/^\{"before":-?\d+,"held":-?\d+,"after":-?\d+\}$/), - stderr: "", - exitCode: 0, - }); - const { before, held, after } = JSON.parse(stdout); - - // While the rewriters of a round are alive, the count is up by one for each - // registration. One kind of struct alone is half of that: a struct that - // leaves this size class, or that mimalloc does not count, fails here. - expect(held - before, stdout).toBeGreaterThan((REGISTRATIONS_PER_ROUND * 3) / 4); - // Unfixed: the struct of every registration of both rounds is still there, - // 128,000 blocks. Fixed: within 100 of zero. - expect(after - before, stdout).toBeLessThan((ROUNDS * REGISTRATIONS_PER_ROUND) / 4); -}); + expect({ stdout, stderr, exitCode }).toEqual({ + stdout: expect.stringMatching(/^\{"before":-?\d+,"held":-?\d+,"after":-?\d+\}$/), + stderr: "", + exitCode: 0, + }); + const { before, held, after } = JSON.parse(stdout); + + // While the rewriters of a round are alive, the count is up by one for each + // registration. One kind of struct alone is half of that: a struct that + // leaves this size class, or that mimalloc does not count, fails here. + expect(held - before, stdout).toBeGreaterThan((REGISTRATIONS_PER_ROUND * 3) / 4); + // Unfixed: the struct of every registration of both rounds is still there, + // 128,000 blocks. One struct for each rewriter would be 2,000. Fixed: about 0. + expect(after - before, stdout).toBeLessThan(REWRITERS_PER_ROUND); + }, +); // ASAN builds cannot read that count: their allocator is ASAN's. They measure // resident memory.