gc: give back the large pages mimalloc keeps at the idle full collection, and test oven-sh/mimalloc#52

src/jsc/GarbageCollectionController.rs

@@ -1,4 +1,4 @@
-//! Idle GC timer: JSC's own `GCActivityCallback` (via `WTFTimer`) paces eden/full against allocation rate; this adds a 1 s / 30 s idle `collect_async()` so a process that stops allocating still releases memory, and once the heap has been quiet for `BUN_IDLE_GC_SECONDS` (default "10,65,65": first after 10 s of quiet, then one per CodeBlock-aging lease; 0 = off; main thread only) full collections so JSC can age out code that no longer runs, plus a page-out of a standalone executable's embedded module graph. Knobs: `BUN_GC_TIMER_INTERVAL` (ms), `BUN_GC_TIMER_DISABLE`. One per JS thread, not thread-safe.
+//! Idle GC timer: JSC's own `GCActivityCallback` (via `WTFTimer`) paces eden/full against allocation rate; this adds a 1 s / 30 s idle `collect_async()` so a process that stops allocating still releases memory, and once the heap has been quiet for `BUN_IDLE_GC_SECONDS` (default "10,65,65": first after 10 s of quiet, then one per CodeBlock-aging lease; 0 = off; main thread only) full collections so JSC can age out code that no longer runs, each with a forced allocator collect then and at the next tick, plus a page-out of a standalone executable's embedded module graph. Knobs: `BUN_GC_TIMER_INTERVAL` (ms), `BUN_GC_TIMER_DISABLE`. One per JS thread, not thread-safe.
 
 use core::cell::Cell;
 use core::ffi::c_int;
@@ -24,6 +24,8 @@ pub struct GarbageCollectionController {
     /// nominal time (from tick intervals) since the JS heap last grew.
     idle_gc_at_ms: Cell<[u32; 3]>,
     idle_quiet_ms: Cell<u32>,
+    /// Set when an idle full collection is requested; the next fire runs the forced allocator collect again.
+    idle_release_pending: Cell<bool>,
 }
 
 bun_event_loop::impl_timer_owner!(
@@ -42,6 +44,7 @@ impl Default for GarbageCollectionController {
             disabled: Cell::new(false),
             idle_gc_at_ms: Cell::new([0; 3]),
             idle_quiet_ms: Cell::new(0),
+            idle_release_pending: Cell::new(false),
         }
     }
 }
@@ -196,6 +199,9 @@ impl GarbageCollectionController {
         let vm = VirtualMachine::get().jsc_vm();
         if idle_full {
             vm.collect_async_idle();
+            // Only a forced collect returns the large pages mimalloc keeps for the next burst (oven-sh/mimalloc#52).
+            bun_core::Global::mimalloc_cleanup(true);
+            self.idle_release_pending.set(true);
         } else {
             vm.collect_async(false);
         }
@@ -214,6 +220,10 @@ impl GarbageCollectionController {
         if this.disabled.get() {
             return;
         }
+        if this.idle_release_pending.replace(false) {
+            // ..and once more for what that collection has freed since.
+            bun_core::Global::mimalloc_cleanup(true);
+        }
         // Timer chatter in a parked app churns a few blocks per tick; real work grows the heap by far more.
         const IDLE_GROWTH_SLACK: usize = 2 * 1024 * 1024;
         let prev_heap_size = this.gc_last_heap_size.get();

test/harness.ts

@@ -1717,6 +1717,19 @@ export function isGlibcVersionAtLeast(version: string): boolean {
   return Bun.semver.satisfies(glibcVersion, `>=${version}`);
 }
 
+/** The size of a page of memory of this process in bytes, from the auxiliary vector (Linux; 0 elsewhere). */
+export function linuxPageSize(): number {
+  if (!isLinux) return 0;
+  const AT_PAGESZ = 6n;
+  try {
+    const auxv = fs.readFileSync("/proc/self/auxv");
+    for (let i = 0; i + 16 <= auxv.length; i += 16) {
+      if (auxv.readBigUInt64LE(i) === AT_PAGESZ) return Number(auxv.readBigUInt64LE(i + 8));
+    }
+  } catch {}
+  return 0;
+}
+
 let macOSVersion: string | undefined;
 
 export function getMacOSVersion(): string | undefined {

test/js/bun/gc/gc-controller-cadence.test.ts

@@ -1,5 +1,5 @@
 import { describe, expect, test } from "bun:test";
-import { bunEnv, bunExe, isASAN, isDebug, isLinux, tempDir } from "harness";
+import { bunEnv, bunExe, isASAN, isDebug, isLinux, linuxPageSize, tempDir } from "harness";
 
 // Bun's GarbageCollectionController used to sample `blockBytesAllocated +
 // extraMemorySize` on every event-loop tick and arm a 16 ms one-shot whenever
@@ -195,6 +195,71 @@ describe("idle release", () => {
   });
 });
 
+// The allocator leaves up to 4 MiB of the 4 MiB pages that a thread used last alone, for a thread that takes its
+// buffers again soon, and before such a page has aged only a forced collect takes it (oven-sh/mimalloc#52). A heap that
+// has been quiet for BUN_IDLE_GC_SECONDS is not coming back soon: the tick after the idle collection gives those pages
+// back. While the process is active they stay, also across a collect that is not forced (JavaScriptCore runs one after
+// each full collection, and Bun.gc(false) is one).
+// Linux only: reads the faults (counted in pages of 4 KiB) and the resident memory of the process from /proc. Not ASAN:
+// malloc is not mimalloc there.
+test.skipIf(!isLinux || isASAN || linuxPageSize() !== 4096)(
+  "the large pages that the allocator keeps for the next burst stay while the process is active and go when it is idle",
+  async () => {
+    await using proc = Bun.spawn({
+      cmd: [
+        bunExe(),
+        "-e",
+        `
+        const { readFileSync } = require("node:fs");
+        const faults = () => {
+          const stat = readFileSync("/proc/self/stat", "utf8");
+          const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
+          return Number(fields[7]) + Number(fields[9]);
+        };
+        const rssAnonKiB = () => Number(/RssAnon:\\s+(\\d+)/.exec(readFileSync("/proc/self/status", "utf8"))[1]);
+        const burst = () => {
+          const buffers = [];
+          for (let i = 0; i < 6; i++) buffers.push(new Uint8Array(512 * 1024).fill(1));
+          for (const buffer of buffers) buffer.buffer.transfer(0); // frees it now: no collector in this
+        };
+        const faultsOfABurst = () => { const before = faults(); burst(); return faults() - before; };
+        // The pauses are input: the thread goes idle and the allocator's sweeps decide what stays, then it wakes for a
+        // collect that is not forced, and goes idle again.
+        for (let i = 0; i < 3; i++) { burst(); await Bun.sleep(60); Bun.gc(false); await Bun.sleep(40); }
+        const whileActive = faultsOfABurst();
+        // The heap grows by more than the controller's slack, so the quiet time starts over at its next tick, here
+        // and not when the process started.
+        globalThis.growth = Array.from({ length: 100000 }, (_, i) => ({ i, s: "x" + i }));
+        burst();
+        await Bun.sleep(60);
+        // Quiet from here on. The 3 MiB of the six buffers leave the resident set when an idle collection frees their page.
+        const resident = rssAnonKiB();
+        const deadline = Date.now() + 3000;
+        while (rssAnonKiB() > resident - 2048 && Date.now() < deadline) await Bun.sleep(100);
+        console.log(JSON.stringify({ whileActive, afterIdle: faultsOfABurst() }));
+        `,
+      ],
+      env: {
+        ...bunEnv,
+        BUN_IDLE_GC_SECONDS: "1,1,1",
+        BUN_GC_TIMER_DISABLE: undefined,
+        BUN_GC_TIMER_INTERVAL: "100",
+        // an epoch of the allocator's sweep lasts 10 ms instead of 100: it has decided 60 ms after a burst
+        MIMALLOC_PURGE_HOLES_MIN_INTERVAL: "10",
+      },
+      stdout: "pipe",
+      stderr: "inherit",
+    });
+    const [stdout, exitCode] = await Promise.all([proc.stdout.text(), proc.exited]);
+    const { whileActive, afterIdle } = JSON.parse(stdout) as { whileActive: number; afterIdle: number };
+    // The six buffers of a burst are 768 pages of 4 KiB: a burst that finds its page there faults a few pages of other
+    // things, one that finds it gone faults all of them in.
+    expect(whileActive, stdout).toBeLessThan(384);
+    expect(afterIdle, stdout).toBeGreaterThan(384);
+    expect(exitCode).toBe(0);
+  },
+);
+
 // Those idle full collections are tagged (GCRequest::isIdle) so JSC may also let idle FTL code — which has no execution
 // counter of its own and pins every baseline CodeBlock it inlined — age out in them, and only in them: a program that
 // forces collections itself while running hot code must not lose that code. Eager JIT TTLs make it observable in seconds.

test/js/bun/jsc/heapStats-mimalloc.test.ts

@@ -1,6 +1,6 @@
 import { heapStats } from "bun:jsc";
 import { describe, expect, test } from "bun:test";
-import { bunEnv, bunExe, isASAN, isLinux, isMacOS, tempDir } from "harness";
+import { bunEnv, bunExe, isASAN, isLinux, isMacOS, linuxPageSize, tempDir } from "harness";
 
 describe("heapStats() mimalloc integration", () => {
   test("mimalloc aggregate stats are present", () => {
@@ -293,4 +293,60 @@ describe("heapStats() mimalloc integration", () => {
       expect(perRequest).toBeLessThan(20);
     },
   );
+
+  // What the sweeps leave is not for a collect that is not forced to take. JavaScriptCore does one on the JS thread at
+  // the end of every full collection (`mi_theap_collect(theap, false)`), the allocator does one every 10000 slow-path
+  // allocations, and `Bun.gc(false)` does one: each freed the 4 MiB page that the sweep before it had just decided to
+  // keep whenever no buffer in the page was in use, and the next burst took all of its buffers out of new memory.
+  // 512 KiB buffers: nothing in the process keeps a block of that size, so their page has no block in use between bursts.
+  // Linux only: reads the faults of the process from /proc, and counts them in pages of 4 KiB. Not ASAN: malloc is not
+  // mimalloc there.
+  test.skipIf(!isLinux || isASAN || linuxPageSize() !== 4096)(
+    "a collect that is not forced leaves the large page that the idle sweep kept",
+    async () => {
+      await using proc = Bun.spawn({
+        cmd: [
+          bunExe(),
+          "-e",
+          `
+          const { readFileSync } = require("node:fs");
+          const faults = () => {
+            const stat = readFileSync("/proc/self/stat", "utf8");
+            const fields = stat.slice(stat.lastIndexOf(")") + 2).split(" ");
+            return Number(fields[7]) + Number(fields[9]);
+          };
+          const burst = () => {
+            const buffers = [];
+            for (let i = 0; i < 4; i++) buffers.push(new Uint8Array(512 * 1024).fill(1));
+            for (const buffer of buffers) buffer.buffer.transfer(0); // frees it now: no collector in this
+          };
+          for (let i = 0; i < 3; i++) { burst(); await Bun.sleep(30); }
+          const perBurst = [];
+          for (let i = 0; i < 12; i++) {
+            const before = faults();
+            burst();
+            perBurst.push(faults() - before);
+            // The pauses are the input of this test and not a wait for something: the thread goes idle and is swept,
+            // wakes for a collect that is not forced, and goes idle again.
+            await Bun.sleep(60);
+            Bun.gc(false);
+            await Bun.sleep(40);
+          }
+          console.log(JSON.stringify(perBurst));
+          `,
+        ],
+        // An epoch of the sweep lasts 10 ms here instead of 100: the sweeps have decided about the page 60 ms after a burst.
+        env: { ...bunEnv, MIMALLOC_PURGE_HOLES_MIN_INTERVAL: "10" },
+        stdout: "pipe",
+        stderr: "inherit",
+      });
+      const [stdout, exitCode] = await Promise.all([proc.stdout.text(), proc.exited]);
+      const perBurst: number[] = JSON.parse(stdout);
+      expect(perBurst).toHaveLength(12);
+      // The four buffers of a burst are 512 pages of 4 KiB. A burst that finds its page gone faults all of them in
+      // (every burst but the first did); one that finds it there faults a few pages of other things.
+      expect(perBurst.filter(n => n > 256)).toEqual([]);
+      expect(exitCode).toBe(0);
+    },
+  );
 });