arena: what is freed during a purge pass is purged by the next pass

CMakeLists.txt

@@ -878,7 +878,7 @@ if (MI_BUILD_TESTS)
   enable_testing()
 
   # static link tests
-  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init)
+  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff forced-purge purge-behind-pass free-before-init)
   if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN))
     list(APPEND mi_static_tests thp-optout)  # counts madvise calls by interposing it, which a sanitizer runtime does first
   endif()

include/mimalloc/internal.h

@@ -268,7 +268,8 @@ void*         _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alig
 void          _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid);
 void          _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld);
 void          _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld);
-void          _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq);
+bool          _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq);   // false: dropped, another thread is purging
+bool          _mi_arenas_purge_guard_reset(void);   // true: a purge pass was cut off (by a fork, or by the end of the process)
 void          _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc);
 
 mi_page_t*    _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment);

src/arena.c

@@ -1544,7 +1544,15 @@ void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t mem
 
 // Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired
 void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) {
-  _mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq);
+  // A forced purge is `mi_collect(true)`, and whoever asks for that reads the footprint next. Only one thread purges at
+  // a time, and the pass that holds the guard does not stand in for ours: the scavenger's is never forced, so it leaves
+  // every arena whose delay has not passed, and what was freed behind it is for its next pass. So take our turn
+  // after it. The holder takes no lock and waits for no one; it is in there for its madvise calls.
+  size_t spin = 0;
+  while (!_mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq) && force_purge) {
+    if (spin < 256) { mi_atomic_pause(); spin++; }
+    else { _mi_prim_thread_yield(); }
+  }
 }
 
 
@@ -2545,59 +2553,88 @@ void _mi_arenas_purge_now(mi_subproc_t* subproc) {
   }
 }
 
-void _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq)
+// allow only one thread to purge at a time (todo: allow concurrent purging?)
+static mi_atomic_guard_t mi_arenas_purge_guard;
+
+// Release the guard for a thread that is gone: the one that held it across fork() is not in the child, and at process exit
+// on Windows every other thread is terminated before the detach callback runs. Nothing else would ever release it, and a
+// forced purge waits for it. Returns true if it was held: that pass had reset the `purge_expire` of a sub-process, and
+// it is not there to put back what was still pending (see `_mi_arenas_try_purge`).
+bool _mi_arenas_purge_guard_reset(void) {
+  return (mi_atomic_exchange_acq_rel(&mi_arenas_purge_guard, (uintptr_t)0) != 0);
+}
+
+// Make a purge pass over the arenas of `subproc` due at `expire`, unless one is due before that already. The scavenger
+// sleeps until the time it last read here (or for its safety timeout if that was 0), so wake it to read the new one.
+static void mi_subproc_schedule_purge(mi_subproc_t* subproc, mi_msecs_t expire) {
+  mi_assert_internal(expire != 0);
+  mi_msecs_t current = mi_atomic_loadi64_relaxed(&subproc->purge_expire);
+  while (current == 0 || current > expire) {
+    if (mi_atomic_casi64_strong_acq_rel(&subproc->purge_expire, &current, expire)) {
+      _mi_scavenger_wake(subproc);
+      return;
+    }
+  }
+}
+
+// Returns false if another thread was purging and this pass was dropped because of it.
+bool _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq)
 {
   // try purge can be called often so try to only run when needed
   const long delay = mi_arena_purge_delay();
-  if (_mi_preloading() || delay <= 0) return;  // nothing will be scheduled
+  if (_mi_preloading() || delay <= 0) return true;  // nothing will be scheduled
 
   // check if any arena needs purging?
   const mi_msecs_t now = _mi_clock_now();
   const mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&subproc->purge_expire);
-  if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return;
+  if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return true;
 
   const size_t max_arena = mi_arenas_get_count(subproc);
-  if (max_arena == 0) return;
+  if (max_arena == 0) return true;
 
-  // allow only one thread to purge at a time (todo: allow concurrent purging?)
-  static mi_atomic_guard_t purge_guard;
-  mi_atomic_guard(&purge_guard)
+  bool entered = false;
+  mi_atomic_guard(&mi_arenas_purge_guard)
   {
-    // increase global expire: at most one purge per delay cycle
-    if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); }
+    entered = true;
+    // Reset the subproc expire before the pass and not after it, as `mi_arena_try_purge` does with the expire of an
+    // arena. The first free into an arena after the pass went into it finds the arena expire at 0 and sets it, and
+    // only then looks at the subproc expire: it must find 0 there to set that too and wake the scavenger. With our old
+    // value still in place it would arm the arena alone, and no later free into an armed arena looks further.
+    // What is still pending when we are done is put back below.
+    mi_msecs_t expected = arenas_expire;
+    while (!mi_atomic_casi64_strong_acq_rel(&subproc->purge_expire, &expected, (mi_msecs_t)0)) { /* retry with the updated `expected` */ }
     const size_t arena_start = tseq % max_arena;
     size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1);
     bool all_visited = true;
-    bool any_purged = false;
     mi_msecs_t next_expire = 0;   // earliest still-pending per-arena expire
     for (size_t _i = 0; _i < max_arena; _i++) {
       size_t i = _i + arena_start;
       if (i >= max_arena) { i -= max_arena; }
       mi_arena_t* arena = mi_arena_from_index(subproc,i);
       if (arena != NULL) {
         const int purged = mi_arena_try_purge(arena, now, force);
-        if (purged >= 0) {      // purged, or arena expire is not yet reached
-          any_purged = true;
-          if (purged >= 1) {    // purged
-            if (max_purge_count <= 1) {
-              all_visited = false;
-              break;
-            }
-            max_purge_count--;
+        // Its expire is set if it is not yet due, or if a free came in behind the pass over it. Read it with a CAS
+        // (0 -> 0), not a load: that orders us with the CAS in `mi_arena_schedule_purge`, so either we see the expire
+        // that a free sets, or that free sees our reset above. With a load both can miss (store buffering).
+        mi_msecs_t aexpire = 0;
+        mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &aexpire, (mi_msecs_t)0);
+        if (aexpire != 0 && (next_expire == 0 || aexpire < next_expire)) { next_expire = aexpire; }
+        if (purged >= 1) {    // purged
+          if (max_purge_count <= 1 && _i + 1 < max_arena) {   // (after the last arena we did visit all)
+            all_visited = false;
+            break;
           }
+          max_purge_count--;
         }
-        const mi_msecs_t aexpire = mi_atomic_loadi64_relaxed(&arena->purge_expire);
-        if (aexpire != 0 && (next_expire == 0 || aexpire < next_expire)) { next_expire = aexpire; }
       }
     }
-    MI_UNUSED(any_purged);
-    if (all_visited) {
-      // we saw every arena: subproc->purge_expire becomes the earliest pending
-      // per-arena expire (0 if none) so the scavenger's next wait is exact.
-      mi_msecs_t expected = arenas_expire;
-      mi_atomic_casi64_strong_acq_rel(&subproc->purge_expire, &expected, next_expire);
-    }
+    // the arenas we did not get to are for the next call: keep that one due
+    if (!all_visited && (next_expire == 0 || next_expire > now)) { next_expire = now; }
+    // subproc->purge_expire becomes the earliest pending per-arena expire (it stays 0 if there is none), unless a
+    // free set an earlier one in the meantime, so the scavenger's next wait is exact.
+    if (next_expire != 0) { mi_subproc_schedule_purge(subproc, next_expire); }
   }
+  return entered;
 }
 
 

src/init.c

@@ -666,6 +666,11 @@ void mi_process_init(void) mi_attr_noexcept {
 // Called when the process is done
 static void mi_process_done_once(void) {
   _mi_scavenger_stop();          // fork: stop the background scavenger before any teardown
+  #if defined(_WIN32)
+  // This runs in the process detach callback, after the system terminated every other thread. One that was in a purge
+  // pass then (the scavenger, as a rule) holds the purge guard for good, and the forced collect below would wait for it.
+  _mi_arenas_purge_guard_reset();
+  #endif
   _mi_heap_snapshot_on_exit();   // fork: heap snapshot / profile at exit, before anything is torn down
   _mi_prof_on_exit();
   // only shutdown if we were initialized

src/scavenger.c

@@ -205,12 +205,12 @@ static void mi_scavenger_run(void) {
     // Do the idle work of any thread that parked and handed us its theaps. This is the expensive
     // part (the hole punch is ~99% madvise) and it is why the owner gets to skip it.
     const mi_msecs_t park_due = _mi_theap_sweep_parked(subproc);
-    mi_msecs_t expire = mi_atomic_loadi64_acquire(&subproc->purge_expire);
+    const mi_msecs_t expire = mi_atomic_loadi64_acquire(&subproc->purge_expire);
     mi_msecs_t timeout_ms;
     if (expire == 0) {
       // Nothing scheduled: park until woken. The 30s bound is a pure safety
       // net so stop() is guaranteed to take effect and any per-arena expiry
-      // that did not propagate to subproc is still eventually purged.
+      // that did not propagate to subproc is still eventually purged (below).
       timeout_ms = 30000;
     }
     else {
@@ -220,20 +220,28 @@ static void mi_scavenger_run(void) {
         if (timeout_ms > 30000) timeout_ms = 30000;
       }
       else {
-        _mi_arenas_try_purge(false /* force */, true /* visit_all */, subproc, 0 /* tseq */);
-        // _mi_arenas_try_purge clears subproc->purge_expire to 0 once every
-        // arena is done. If it left the stale past value (some arena's own
-        // expire is still in the future), clear it so the next iteration parks
-        // on the 30s safety net instead of spinning. CAS so a concurrently
-        // scheduled future expire is never clobbered.
-        mi_atomic_casi64_strong_acq_rel(&subproc->purge_expire, &expire, (mi_msecs_t)0);
-        continue;
+        const bool dropped = !_mi_arenas_try_purge(false /* force */, true /* visit_all */, subproc, 0 /* tseq */);
+        // A pass resets subproc->purge_expire before it looks at the arenas, and
+        // leaves the earliest per-arena expire that is still pending (0 if none),
+        // or what a free set since, so the next wait is exact.
+        if (mi_atomic_loadi64_acquire(&subproc->purge_expire) != expire) continue;
+        // Not reset. Do not clear it from here: if it was set to this same value
+        // again since (`_mi_arenas_purge_now` stores the current time), that pass
+        // would be lost, and no free into an armed arena asks for another.
+        // Dropped: another thread is in a pass. What was set since that pass reset
+        // it, the pass leaves as it is, without a wake: come back for it.
+        // Not dropped: purging got switched off, so nothing is scheduled.
+        timeout_ms = (dropped ? 1 : 30000);
       }
     }
     // a park passed over for `purge_holes_min_interval` is swept when its window ends, not at the safety timeout
     if (park_due > 0 && park_due < timeout_ms) { timeout_ms = park_due; }
     if (mi_atomic_load_acquire(&_mi_scavenger_running) == 0) break;
     mi_scav_wait(&subproc->scavenger_wake, timeout_ms);
+    // The safety net: nothing was scheduled and nothing woke us. Look at the arenas all the same.
+    if (expire == 0 && mi_atomic_load_relaxed(&subproc->scavenger_wake) == 0) {
+      _mi_arenas_try_purge(false /* force */, true /* visit_all */, subproc, 0 /* tseq */);
+    }
   }
 }
 

src/subproc.c

@@ -410,8 +410,11 @@ void _mi_process_fork_child(void) {
   if (mi_atomic_exchange_acq_rel(&mi_fork_depth, 0) == 0) return;
   _mi_process_is_forked_child = true;
   _mi_scavenger_forked_child();   // the scavenger thread did not survive the fork; clear the state that says it did
+  const bool purge_cut_off = _mi_arenas_purge_guard_reset();   // nor did a thread that was purging; release the guard it held
   mi_lock_init(&mi_subprocs_lock);
   for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) {
+    // the purge pass that was cut off may have been over this one: make a pass due, for the arenas whose expire is still set
+    if (purge_cut_off) { mi_atomic_storei64_relaxed(&sp->purge_expire, (mi_msecs_t)1); }
     mi_lock_init(&sp->arena_reserve_lock);
     mi_lock_init(&sp->heaps_lock);
     mi_lock_init(&sp->tlds_lock);

test/test-forced-purge.c

@@ -0,0 +1,86 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license. A copy of the license can be found in the file
+"LICENSE" at the root of this distribution.
+-----------------------------------------------------------------------------*/
+
+// `mi_collect(true)` purges every free arena range before it returns.
+//
+// Only one thread runs a purge pass at a time, and with a scavenger thread there often is one in a
+// pass: it takes the freed ranges `purge_delay` after they were freed. Its pass is not forced, so it
+// is no substitute for the caller's, and a caller of `mi_collect(true)` that found the scavenger in
+// there used to return with nothing purged at all. Whoever forces a collect reads the footprint next.
+//
+// Each round frees a few hundred MiB, waits for the scavenger to start on them (`arena_purges` counts
+// the arenas a pass went into), and forces a collect while the scavenger is in the middle of that.
+// Everything the round freed has to be counted in `purged` when the collect returns. Counters, not
+// RSS: they read the same on every platform and in every build.
+
+#include "mimalloc.h"
+#include "mimalloc-stats.h"
+#include <stdbool.h>
+#include <stdio.h>
+#include <string.h>
+#include <time.h>
+
+static int failures = 0;
+
+static void check(const char* name, bool ok) {
+  fprintf(stderr, "test: %s...  %s\n", name, ok ? "ok." : "FAILED");
+  if (!ok) failures++;
+}
+
+#define MAX_BLOCKS  (32)
+#define BLOCK_SIZE  ((size_t)8 * 1024 * 1024)
+#define ROUNDS      (4)
+
+// the counters of the subprocess itself: read without a lock, and without touching our theap
+static void counters(size_t* purged, size_t* arena_purges) {
+  mi_stats_t_decl(stats);
+  mi_subproc_stats_get_exclusive(mi_subproc_main(), &stats);
+  *purged = (size_t)stats.purged.total;
+  *arena_purges = (size_t)stats.arena_purges.total;
+}
+
+int main(void) {
+  if (!mi_option_is_enabled(mi_option_scavenger) || mi_option_get(mi_option_purge_delay) <= 0) {
+    fprintf(stderr, "test-forced-purge: skipped (no scavenger, or no purge delay)\n");
+    return 0;
+  }
+  // the scavenger starts when a second thread initializes or a thread first parks
+  void* warm = mi_malloc(64); mi_free(warm);
+  if (mi_on_thread_idle_start()) { mi_on_thread_idle_end(); }
+
+  // 256 MiB a round, so that a pass over it holds the purge guard for milliseconds (64 MiB where address space is scarce)
+  const int BLOCKS = (sizeof(void*) >= 8 ? MAX_BLOCKS : MAX_BLOCKS / 4);
+  static void* blocks[MAX_BLOCKS];
+  int in_a_pass = 0;
+  for (int round = 0; round < ROUNDS; round++) {
+    for (int i = 0; i < BLOCKS; i++) {
+      blocks[i] = mi_malloc(BLOCK_SIZE);
+      if (blocks[i] == NULL) { fprintf(stderr, "test-forced-purge: out of memory\n"); return 1; }
+      memset(blocks[i], 1, BLOCK_SIZE);   // resident, so purging it takes the scavenger a while
+    }
+    size_t purged0, passes0;
+    counters(&purged0, &passes0);
+    for (int i = 0; i < BLOCKS; i++) { mi_free(blocks[i]); }   // back to the arena, purge scheduled
+
+    // Wait for the scavenger to go into an arena. If it never does (bounded), the forced collect
+    // below has all of it to purge by itself, which has to work just as well.
+    size_t purged1, passes1;
+    const time_t deadline = time(NULL) + 10;
+    do { counters(&purged1, &passes1); } while (passes1 == passes0 && time(NULL) < deadline);
+    if (passes1 != passes0) { in_a_pass++; }
+
+    mi_collect(true);
+
+    counters(&purged1, &passes1);
+    char name[128];
+    snprintf(name, sizeof(name), "round %d: a forced collect leaves nothing that was freed unpurged (%zu of %zu MiB)",
+             round, (purged1 - purged0) / (1024 * 1024), ((size_t)BLOCKS * BLOCK_SIZE) / (1024 * 1024));
+    check(name, purged1 - purged0 >= (size_t)BLOCKS * BLOCK_SIZE);
+  }
+  fprintf(stderr, "test-forced-purge: %d of %d forced collects came while the scavenger was purging\n", in_a_pass, ROUNDS);
+  return failures;
+}