arena: a zero-claim purge debits the committed stat the next allocation credits again (re-land of 74086412)

CMakeLists.txt

@@ -1147,7 +1147,7 @@ if (MI_BUILD_TESTS)
   enable_testing()
 
   # static link tests
-  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot purge-zero park-handoff forced-purge purge-behind-pass free-before-init)
+  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot purge-zero purge-committed park-handoff forced-purge purge-behind-pass free-before-init)
   if(NOT (MI_DEBUG_TSAN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_UBSAN))
     list(APPEND mi_static_tests thp-optout)  # counts madvise calls by interposing it, which a sanitizer runtime does first
   endif()
@@ -1203,6 +1203,11 @@ if (MI_BUILD_TESTS)
   # and sweep every park, so the reclaim path is hit as hard as possible
   add_test(NAME test-park-handoff-eager COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES_MIN_INTERVAL=0 $<TARGET_FILE:mimalloc-test-park-handoff>)
 
+  # an allocation that does not ask for commit has to count its slices as the committing one does
+  add_test(NAME test-purge-committed-on-demand COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PAGE_COMMIT_ON_DEMAND=1 $<TARGET_FILE:mimalloc-test-purge-committed>)
+  # and an arena that is not committed up front is counted on its commit bits alone
+  add_test(NAME test-purge-committed-no-eager-commit COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 $<TARGET_FILE:mimalloc-test-purge-committed>)
+
   # The allocator must never touch one of its own `__thread` variables from inside an allocation:
   # where `__thread` is emulated (Android before API 29) the first access on a thread mallocs, and
   # with the allocator being malloc that is a re-entrant allocation (oven-sh/bun#38051). Build the

src/arena.c

@@ -241,6 +241,22 @@ static size_t mi_page_full_size(mi_page_t* page) {
   Arena Allocation
 ----------------------------------------------------------- */
 
+// On an OS with overcommit, `mi_arena_reserve` does not count an eagerly committed arena as
+// committed: the OS backs a slice the first time it is written. So a slice of such an arena counts
+// as committed while its dirty bit is set: it is credited here when it is handed out for the first
+// time, and debited in `mi_arena_purge` when a zero-claim purge clears the dirty bit again.
+// An arena that was not committed up front is counted on its commit bits alone (`_mi_os_commit_ex`):
+// a purge keeps those set, so a slice of it is counted once and stays counted.
+static bool mi_arena_stat_on_dirty(const mi_arena_t* arena) {
+  return (arena->memid.initially_committed && _mi_os_has_overcommit() && !arena->memid.is_pinned /* huge pages, issue #1236 */);
+}
+
+static void mi_arena_stat_touched(mi_arena_t* arena, size_t touched_slices) {
+  if (touched_slices > 0 && mi_arena_stat_on_dirty(arena)) {
+    mi_subproc_stat_increase(arena->subproc, committed, mi_size_of_slices(touched_slices));
+  }
+}
+
 static mi_decl_noinline void* mi_arena_try_alloc_at(
   mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid)
 {
@@ -255,12 +271,22 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
   memid->is_pinned = arena->memid.is_pinned;
 
   // set the dirty bits and track which slices become accessible
-  size_t touched_slices = slice_count;
+  const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count);
+  size_t touched_slices = slice_count;   // slices whose dirty bit was clear
+  size_t touched_committed = 0;          // of those, the ones whose commit bit is set: counted by `mi_arena_stat_touched`
   if (arena->memid.initially_zero) {
+    if (already_committed < slice_count && mi_arena_stat_on_dirty(arena)) {
+      // committed in part (a purge decommitted some of it): the commit below counts the slices it commits,
+      // so count the clean slices among the committed ones here, before the dirty bits are set
+      for (size_t i = slice_index; i < slice_index + slice_count; i++) {
+        if (mi_bitmap_is_set(arena->slices_committed, i) && !mi_bitmap_is_set(arena->slices_dirty, i)) { touched_committed++; }
+      }
+    }
     size_t already_dirty = 0;
     memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, &already_dirty);
     mi_assert_internal(already_dirty <= touched_slices);
     touched_slices -= already_dirty;
+    if (already_committed == slice_count) { touched_committed = touched_slices; }
   }
   else {
     // todo: properly count touched pages with a separate bitmap?
@@ -270,7 +296,6 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
   // set commit state
   if (commit) {
     // commit requested, but the range may not be committed as a whole: ensure it is committed now
-    const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count);
     if (already_committed < slice_count) {
       // not all committed, try to commit now
       bool commit_zero = false;
@@ -286,6 +311,8 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
 
       // set the commit bits
       mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL);
+      // the commit counted what it committed; the slices that were committed already count now
+      mi_arena_stat_touched(arena, touched_committed);
 
       // committed
       #if MI_DEBUG > 1
@@ -300,11 +327,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
     else {
       // already fully committed.
       _mi_os_reuse(arena->subproc, p, mi_size_of_slices(slice_count));
-      // if the OS has overcommit, and this is the first time we access these pages, then
-      // count the commit now (as at arena reserve we didn't count those commits as these are on-demand)
-      if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) {
-        mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices));
-      }
+      mi_arena_stat_touched(arena, touched_committed);
     }
 
     mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count));
@@ -321,12 +344,17 @@ static mi_decl_noinline void* mi_arena_try_alloc_at(
   else {
     // no need to commit, but check if it is already fully committed
     memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count);
-    if (!memid->initially_committed) {
-      // partly committed.. adjust stats
-      size_t already_committed_count = 0;
-      mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count);
+    if (memid->initially_committed) {
+      // nothing commits this range on demand later: count the slices it touches now
+      mi_arena_stat_touched(arena, touched_committed);
+    }
+    else {
+      // partly committed.. adjust stats: the range is committed on demand from here on, so the
+      // slices that were committed (and counted) are not counted any longer
       mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count);
-      mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count));
+      // a slice counted on its dirty bit was not counted while that bit was clear
+      const size_t counted = (mi_arena_stat_on_dirty(arena) ? already_committed - touched_committed : already_committed);
+      mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(counted));
     }
   }
 
@@ -2566,14 +2594,15 @@ static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_c
       // The OS guarantees this range reads back zero, so forget that it was ever dirty: the
       // next `mi_arenas_alloc` hands it out with `initially_zero`, `mi_zalloc` skips the
       // memset, and pages the caller never writes are never made resident again.
+      const size_t dirty_slices = mi_bitmap_popcountN(arena->slices_dirty, slice_index, slice_count);
       mi_bitmap_clearN(arena->slices_dirty, slice_index, slice_count);
-      // Do NOT touch the `committed` stat here. `committed` is keyed on the COMMIT bits, not the
-      // dirty bits: it is credited in `mi_arena_try_alloc_at` only for slices whose commit bit was
-      // clear (`slice_count - already_committed`, and the whole block is skipped when the range is
-      // already fully committed). A zero-claim purge deliberately KEEPS the commit bits set, so the
-      // next allocation credits nothing -- and debiting here would be an unmatched debit that walks
-      // `committed` down without bound (it is an int64; `mi_process_info` casts it to size_t, so it
-      // wraps). The real ratchet is the partial-commit branch below, which *clears* commit bits.
+      // A range that needs no recommit keeps its commit bits, so `_mi_os_purge_zero` debited
+      // nothing, and the next `mi_arena_try_alloc_at` credits `committed` again for every slice
+      // that was just un-dirtied (`mi_arena_stat_touched`). Debit those slices here, under the
+      // same condition, or `committed` grows by the size of every purge/reuse cycle, forever.
+      if (!needs_recommit && all_committed && dirty_slices > 0 && mi_arena_stat_on_dirty(arena)) {
+        mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(dirty_slices));
+      }
     }
   }
 

test/test-purge-committed.c

@@ -0,0 +1,134 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// The `committed` stat comes back down when a purge returns a range to the OS, and reads the same
+// after every purge/reuse cycle.
+//
+// On an OS with overcommit (Linux, macOS) an arena is committed when it is reserved and the stat
+// does not count that: a slice is counted the first time it is handed out. A purge on those
+// systems (`MADV_DONTNEED`) leaves the range committed and the arena clears its dirty bits, so the
+// next allocation of the range counts its slices as handed out for the first time, again. The purge
+// has to debit what that allocation credits, or `committed` grows by the size of every cycle and
+// never comes down: bun saw 28 GB "committed" against 0.6 GB resident after 8 days.
+//
+// `MIMALLOC_PAGE_COMMIT_ON_DEMAND=1` covers the other direction: an allocation that does not ask
+// for commit, on a range that is committed already, has to credit the same slices, or the debit at
+// the purge walks the stat below zero. `MIMALLOC_ARENA_EAGER_COMMIT=0` covers an arena that is
+// not committed up front: its slices are counted when they are committed, and a purge must not
+// debit what no allocation credited.
+
+#include "mimalloc.h"
+#include "mimalloc-stats.h"
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+
+static int failures = 0;
+
+static void check(const char* name, bool ok) {
+  fprintf(stderr, "test: %s...  %s\n", name, ok ? "ok." : "FAILED");
+  if (!ok) failures++;
+}
+
+#define MiB         ((size_t)1024 * 1024)
+#define ROUNDS      (8)
+#define BLOCKS      (16)
+#define SLACK       (2 * MiB)   // page meta data and the blocks the allocator keeps for itself
+
+// Whether a purge brings `committed` back down. Only a purge that zero-fills on the next access
+// (Linux `MADV_DONTNEED`; `MADV_ZERO` on a recent macOS) un-dirties the range, and only an arena
+// that was committed up front is counted on its dirty bits. Elsewhere the stat is a high-water mark
+// by design: it does not come down after the purge, but it does not grow either.
+static bool purge_counts_down(void) {
+  #if defined(__linux__)
+  return (mi_option_get(mi_option_arena_eager_commit) != 0);
+  #else
+  return false;
+  #endif
+}
+
+static int64_t committed(void) {
+  mi_stats_t_decl(stats);
+  mi_subproc_stats_get_exclusive(mi_subproc_main(), &stats);
+  return stats.committed.current;
+}
+
+// allocate `BLOCKS` blocks of `size`, write them, free them, and purge: `committed` has to be back
+// where it started, and the same in every round
+static void cycle(const char* name, size_t size) {
+  void* blocks[BLOCKS];
+  // let the first round take its pages and meta data: the rounds after it reuse them
+  for (int i = 0; i < BLOCKS; i++) { blocks[i] = mi_malloc(size); memset(blocks[i], 1, size); }
+  for (int i = 0; i < BLOCKS; i++) { mi_free(blocks[i]); }
+  mi_collect(true);
+  const int64_t base = committed();
+
+  bool flat = true;
+  bool down = true;
+  bool nonneg = (base >= 0);
+  int64_t held0 = 0;
+  for (int round = 0; round < ROUNDS; round++) {
+    for (int i = 0; i < BLOCKS; i++) { blocks[i] = mi_malloc(size); memset(blocks[i], 1, size); }
+    const int64_t held = committed();
+    if (round == 0) { held0 = held; }
+    for (int i = 0; i < BLOCKS; i++) { mi_free(blocks[i]); }
+    mi_collect(true);
+    const int64_t after = committed();
+    fprintf(stderr, "  %s round %d: held %lld KiB, after purge %lld KiB (base %lld KiB)\n",
+            name, round, (long long)(held / 1024), (long long)(after / 1024), (long long)(base / 1024));
+    if (held > held0 + (int64_t)SLACK) { flat = false; }   // grows with every cycle
+    if (after > base + (int64_t)SLACK) { down = false; }   // the purge debited nothing
+    if (held < 0 || after < 0) { nonneg = false; }         // debited more than was credited
+  }
+  if (!purge_counts_down()) { down = true; }
+  check(name, flat && down && nonneg);
+}
+
+// Each round takes a block twice the size of the one before it, so every round's range covers slices
+// the round before it took and purged, plus slices nothing took yet. In an arena that is not committed
+// up front (`MIMALLOC_ARENA_EAGER_COMMIT=0`) the commit of such a range is partial. What the purge
+// debits for it has to be what its allocation credited, or the stat drifts below zero, a little
+// more with every round.
+static void growing(const char* name) {
+  void* block = mi_malloc(64 * 1024); memset(block, 1, 64 * 1024); mi_free(block);
+  mi_collect(true);
+  const int64_t base = committed();
+  bool nonneg = (base >= 0);
+  bool down = true;
+  int64_t after = base;
+  for (size_t size = 1 * MiB; size <= 32 * MiB; size *= 2) {
+    block = mi_malloc(size); memset(block, 1, size);
+    const int64_t held = committed();
+    mi_free(block);
+    mi_collect(true);
+    after = committed();
+    fprintf(stderr, "  %s %zu MiB: held %lld KiB, after purge %lld KiB (base %lld KiB)\n",
+            name, size / MiB, (long long)(held / 1024), (long long)(after / 1024), (long long)(base / 1024));
+    if (held < 0 || after < 0) { nonneg = false; }
+  }
+  if (after > base + (int64_t)SLACK && purge_counts_down()) { down = false; }
+  check(name, nonneg && down);
+}
+
+int main(void) {
+  if (mi_option_get(mi_option_purge_delay) < 0) {
+    fprintf(stderr, "test-purge-committed: skipped (purging is off)\n");
+    return 0;
+  }
+  mi_option_set(mi_option_purge_delay, 0);                    // a free purges at once
+  mi_option_set(mi_option_minimal_purge_size, 4 /* KiB */);   // and purges the edges of a run too, not 2 MiB units only
+
+  // first, while the arena is fresh: every block of the growing run has to reach slices nothing took yet
+  growing("committed stays at or above zero across blocks of growing size");
+  cycle("committed is flat across 16 x 1 MiB purge/reuse cycles", 1 * MiB);
+  cycle("committed is flat across 16 x 64 KiB purge/reuse cycles", 64 * 1024);
+  cycle("committed is flat across 16 x 4 MiB purge/reuse cycles", 4 * MiB);
+
+  if (failures == 0) { fprintf(stderr, "test-purge-committed: all ok.\n"); }
+  else { fprintf(stderr, "test-purge-committed: %d FAILED\n", failures); }
+  return (failures == 0 ? 0 : 1);
+}