diff --git a/CMakeLists.txt b/CMakeLists.txt index 28d699352..b6a5c9de9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -878,7 +878,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff free-before-init) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot prof prof-adversarial purge-zero park-handoff forced-purge purge-behind-pass free-before-init) if(NOT (MI_DEBUG_TSAN OR MI_TRACK_ASAN OR MI_DEBUG_UBSAN)) list(APPEND mi_static_tests thp-optout) # counts madvise calls by interposing it, which a sanitizer runtime does first endif() diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index f39b1c3df..29433a8e7 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -268,7 +268,8 @@ void* _mi_arenas_alloc_aligned(mi_heap_t* heap, size_t size, size_t alig void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t memid); void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld); void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld); -void _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); +bool _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); // false: dropped, another thread is purging +bool _mi_arenas_purge_guard_reset(void); // true: a purge pass was cut off (by a fork, or by the end of the process) void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc); mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment); diff --git a/src/arena.c b/src/arena.c index 29f83faef..5dc14ffad 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1544,7 +1544,15 @@ void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_memid_t mem // Purge the arenas; if `force_purge` is true, amenable parts are purged even if not yet expired void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld) { - _mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq); + // A forced purge is `mi_collect(true)`, and whoever asks for that reads the footprint next. Only one thread purges at + // a time, and the pass that holds the guard does not stand in for ours: the scavenger's is never forced, so it leaves + // every arena whose delay has not passed, and what was freed behind it is for its next pass. So take our turn + // after it. The holder takes no lock and waits for no one; it is in there for its madvise calls. + size_t spin = 0; + while (!_mi_arenas_try_purge(force_purge, visit_all, tld->subproc, tld->thread_seq) && force_purge) { + if (spin < 256) { mi_atomic_pause(); spin++; } + else { _mi_prim_thread_yield(); } + } } @@ -2545,30 +2553,59 @@ void _mi_arenas_purge_now(mi_subproc_t* subproc) { } } -void _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq) +// allow only one thread to purge at a time (todo: allow concurrent purging?) +static mi_atomic_guard_t mi_arenas_purge_guard; + +// Release the guard for a thread that is gone: the one that held it across fork() is not in the child, and at process exit +// on Windows every other thread is terminated before the detach callback runs. Nothing else would ever release it, and a +// forced purge waits for it. Returns true if it was held: that pass had reset the `purge_expire` of a sub-process, and +// it is not there to put back what was still pending (see `_mi_arenas_try_purge`). +bool _mi_arenas_purge_guard_reset(void) { + return (mi_atomic_exchange_acq_rel(&mi_arenas_purge_guard, (uintptr_t)0) != 0); +} + +// Make a purge pass over the arenas of `subproc` due at `expire`, unless one is due before that already. The scavenger +// sleeps until the time it last read here (or for its safety timeout if that was 0), so wake it to read the new one. +static void mi_subproc_schedule_purge(mi_subproc_t* subproc, mi_msecs_t expire) { + mi_assert_internal(expire != 0); + mi_msecs_t current = mi_atomic_loadi64_relaxed(&subproc->purge_expire); + while (current == 0 || current > expire) { + if (mi_atomic_casi64_strong_acq_rel(&subproc->purge_expire, ¤t, expire)) { + _mi_scavenger_wake(subproc); + return; + } + } +} + +// Returns false if another thread was purging and this pass was dropped because of it. +bool _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq) { // try purge can be called often so try to only run when needed const long delay = mi_arena_purge_delay(); - if (_mi_preloading() || delay <= 0) return; // nothing will be scheduled + if (_mi_preloading() || delay <= 0) return true; // nothing will be scheduled // check if any arena needs purging? const mi_msecs_t now = _mi_clock_now(); const mi_msecs_t arenas_expire = mi_atomic_loadi64_acquire(&subproc->purge_expire); - if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return; + if (!visit_all && !force && (arenas_expire == 0 || arenas_expire > now)) return true; const size_t max_arena = mi_arenas_get_count(subproc); - if (max_arena == 0) return; + if (max_arena == 0) return true; - // allow only one thread to purge at a time (todo: allow concurrent purging?) - static mi_atomic_guard_t purge_guard; - mi_atomic_guard(&purge_guard) + bool entered = false; + mi_atomic_guard(&mi_arenas_purge_guard) { - // increase global expire: at most one purge per delay cycle - if (arenas_expire > now) { mi_atomic_storei64_release(&subproc->purge_expire, now + (delay/10)); } + entered = true; + // Reset the subproc expire before the pass and not after it, as `mi_arena_try_purge` does with the expire of an + // arena. The first free into an arena after the pass went into it finds the arena expire at 0 and sets it, and + // only then looks at the subproc expire: it must find 0 there to set that too and wake the scavenger. With our old + // value still in place it would arm the arena alone, and no later free into an armed arena looks further. + // What is still pending when we are done is put back below. + mi_msecs_t expected = arenas_expire; + while (!mi_atomic_casi64_strong_acq_rel(&subproc->purge_expire, &expected, (mi_msecs_t)0)) { /* retry with the updated `expected` */ } const size_t arena_start = tseq % max_arena; size_t max_purge_count = (visit_all ? max_arena : (max_arena/4)+1); bool all_visited = true; - bool any_purged = false; mi_msecs_t next_expire = 0; // earliest still-pending per-arena expire for (size_t _i = 0; _i < max_arena; _i++) { size_t i = _i + arena_start; @@ -2576,28 +2613,28 @@ void _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, siz mi_arena_t* arena = mi_arena_from_index(subproc,i); if (arena != NULL) { const int purged = mi_arena_try_purge(arena, now, force); - if (purged >= 0) { // purged, or arena expire is not yet reached - any_purged = true; - if (purged >= 1) { // purged - if (max_purge_count <= 1) { - all_visited = false; - break; - } - max_purge_count--; + // Its expire is set if it is not yet due, or if a free came in behind the pass over it. Read it with a CAS + // (0 -> 0), not a load: that orders us with the CAS in `mi_arena_schedule_purge`, so either we see the expire + // that a free sets, or that free sees our reset above. With a load both can miss (store buffering). + mi_msecs_t aexpire = 0; + mi_atomic_casi64_strong_acq_rel(&arena->purge_expire, &aexpire, (mi_msecs_t)0); + if (aexpire != 0 && (next_expire == 0 || aexpire < next_expire)) { next_expire = aexpire; } + if (purged >= 1) { // purged + if (max_purge_count <= 1 && _i + 1 < max_arena) { // (after the last arena we did visit all) + all_visited = false; + break; } + max_purge_count--; } - const mi_msecs_t aexpire = mi_atomic_loadi64_relaxed(&arena->purge_expire); - if (aexpire != 0 && (next_expire == 0 || aexpire < next_expire)) { next_expire = aexpire; } } } - MI_UNUSED(any_purged); - if (all_visited) { - // we saw every arena: subproc->purge_expire becomes the earliest pending - // per-arena expire (0 if none) so the scavenger's next wait is exact. - mi_msecs_t expected = arenas_expire; - mi_atomic_casi64_strong_acq_rel(&subproc->purge_expire, &expected, next_expire); - } + // the arenas we did not get to are for the next call: keep that one due + if (!all_visited && (next_expire == 0 || next_expire > now)) { next_expire = now; } + // subproc->purge_expire becomes the earliest pending per-arena expire (it stays 0 if there is none), unless a + // free set an earlier one in the meantime, so the scavenger's next wait is exact. + if (next_expire != 0) { mi_subproc_schedule_purge(subproc, next_expire); } } + return entered; } diff --git a/src/init.c b/src/init.c index e9ad45d81..8ab8dcd8e 100644 --- a/src/init.c +++ b/src/init.c @@ -666,6 +666,11 @@ void mi_process_init(void) mi_attr_noexcept { // Called when the process is done static void mi_process_done_once(void) { _mi_scavenger_stop(); // fork: stop the background scavenger before any teardown + #if defined(_WIN32) + // This runs in the process detach callback, after the system terminated every other thread. One that was in a purge + // pass then (the scavenger, as a rule) holds the purge guard for good, and the forced collect below would wait for it. + _mi_arenas_purge_guard_reset(); + #endif _mi_heap_snapshot_on_exit(); // fork: heap snapshot / profile at exit, before anything is torn down _mi_prof_on_exit(); // only shutdown if we were initialized diff --git a/src/scavenger.c b/src/scavenger.c index 9c3ae498b..f0d3e12d7 100644 --- a/src/scavenger.c +++ b/src/scavenger.c @@ -205,12 +205,12 @@ static void mi_scavenger_run(void) { // Do the idle work of any thread that parked and handed us its theaps. This is the expensive // part (the hole punch is ~99% madvise) and it is why the owner gets to skip it. const mi_msecs_t park_due = _mi_theap_sweep_parked(subproc); - mi_msecs_t expire = mi_atomic_loadi64_acquire(&subproc->purge_expire); + const mi_msecs_t expire = mi_atomic_loadi64_acquire(&subproc->purge_expire); mi_msecs_t timeout_ms; if (expire == 0) { // Nothing scheduled: park until woken. The 30s bound is a pure safety // net so stop() is guaranteed to take effect and any per-arena expiry - // that did not propagate to subproc is still eventually purged. + // that did not propagate to subproc is still eventually purged (below). timeout_ms = 30000; } else { @@ -220,20 +220,28 @@ static void mi_scavenger_run(void) { if (timeout_ms > 30000) timeout_ms = 30000; } else { - _mi_arenas_try_purge(false /* force */, true /* visit_all */, subproc, 0 /* tseq */); - // _mi_arenas_try_purge clears subproc->purge_expire to 0 once every - // arena is done. If it left the stale past value (some arena's own - // expire is still in the future), clear it so the next iteration parks - // on the 30s safety net instead of spinning. CAS so a concurrently - // scheduled future expire is never clobbered. - mi_atomic_casi64_strong_acq_rel(&subproc->purge_expire, &expire, (mi_msecs_t)0); - continue; + const bool dropped = !_mi_arenas_try_purge(false /* force */, true /* visit_all */, subproc, 0 /* tseq */); + // A pass resets subproc->purge_expire before it looks at the arenas, and + // leaves the earliest per-arena expire that is still pending (0 if none), + // or what a free set since, so the next wait is exact. + if (mi_atomic_loadi64_acquire(&subproc->purge_expire) != expire) continue; + // Not reset. Do not clear it from here: if it was set to this same value + // again since (`_mi_arenas_purge_now` stores the current time), that pass + // would be lost, and no free into an armed arena asks for another. + // Dropped: another thread is in a pass. What was set since that pass reset + // it, the pass leaves as it is, without a wake: come back for it. + // Not dropped: purging got switched off, so nothing is scheduled. + timeout_ms = (dropped ? 1 : 30000); } } // a park passed over for `purge_holes_min_interval` is swept when its window ends, not at the safety timeout if (park_due > 0 && park_due < timeout_ms) { timeout_ms = park_due; } if (mi_atomic_load_acquire(&_mi_scavenger_running) == 0) break; mi_scav_wait(&subproc->scavenger_wake, timeout_ms); + // The safety net: nothing was scheduled and nothing woke us. Look at the arenas all the same. + if (expire == 0 && mi_atomic_load_relaxed(&subproc->scavenger_wake) == 0) { + _mi_arenas_try_purge(false /* force */, true /* visit_all */, subproc, 0 /* tseq */); + } } } diff --git a/src/subproc.c b/src/subproc.c index d16d9fe5f..8fa23f3f0 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -410,8 +410,11 @@ void _mi_process_fork_child(void) { if (mi_atomic_exchange_acq_rel(&mi_fork_depth, 0) == 0) return; _mi_process_is_forked_child = true; _mi_scavenger_forked_child(); // the scavenger thread did not survive the fork; clear the state that says it did + const bool purge_cut_off = _mi_arenas_purge_guard_reset(); // nor did a thread that was purging; release the guard it held mi_lock_init(&mi_subprocs_lock); for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { + // the purge pass that was cut off may have been over this one: make a pass due, for the arenas whose expire is still set + if (purge_cut_off) { mi_atomic_storei64_relaxed(&sp->purge_expire, (mi_msecs_t)1); } mi_lock_init(&sp->arena_reserve_lock); mi_lock_init(&sp->heaps_lock); mi_lock_init(&sp->tlds_lock); diff --git a/test/test-forced-purge.c b/test/test-forced-purge.c new file mode 100644 index 000000000..e2555ce24 --- /dev/null +++ b/test/test-forced-purge.c @@ -0,0 +1,86 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// `mi_collect(true)` purges every free arena range before it returns. +// +// Only one thread runs a purge pass at a time, and with a scavenger thread there often is one in a +// pass: it takes the freed ranges `purge_delay` after they were freed. Its pass is not forced, so it +// is no substitute for the caller's, and a caller of `mi_collect(true)` that found the scavenger in +// there used to return with nothing purged at all. Whoever forces a collect reads the footprint next. +// +// Each round frees a few hundred MiB, waits for the scavenger to start on them (`arena_purges` counts +// the arenas a pass went into), and forces a collect while the scavenger is in the middle of that. +// Everything the round freed has to be counted in `purged` when the collect returns. Counters, not +// RSS: they read the same on every platform and in every build. + +#include "mimalloc.h" +#include "mimalloc-stats.h" +#include +#include +#include +#include + +static int failures = 0; + +static void check(const char* name, bool ok) { + fprintf(stderr, "test: %s... %s\n", name, ok ? "ok." : "FAILED"); + if (!ok) failures++; +} + +#define MAX_BLOCKS (32) +#define BLOCK_SIZE ((size_t)8 * 1024 * 1024) +#define ROUNDS (4) + +// the counters of the subprocess itself: read without a lock, and without touching our theap +static void counters(size_t* purged, size_t* arena_purges) { + mi_stats_t_decl(stats); + mi_subproc_stats_get_exclusive(mi_subproc_main(), &stats); + *purged = (size_t)stats.purged.total; + *arena_purges = (size_t)stats.arena_purges.total; +} + +int main(void) { + if (!mi_option_is_enabled(mi_option_scavenger) || mi_option_get(mi_option_purge_delay) <= 0) { + fprintf(stderr, "test-forced-purge: skipped (no scavenger, or no purge delay)\n"); + return 0; + } + // the scavenger starts when a second thread initializes or a thread first parks + void* warm = mi_malloc(64); mi_free(warm); + if (mi_on_thread_idle_start()) { mi_on_thread_idle_end(); } + + // 256 MiB a round, so that a pass over it holds the purge guard for milliseconds (64 MiB where address space is scarce) + const int BLOCKS = (sizeof(void*) >= 8 ? MAX_BLOCKS : MAX_BLOCKS / 4); + static void* blocks[MAX_BLOCKS]; + int in_a_pass = 0; + for (int round = 0; round < ROUNDS; round++) { + for (int i = 0; i < BLOCKS; i++) { + blocks[i] = mi_malloc(BLOCK_SIZE); + if (blocks[i] == NULL) { fprintf(stderr, "test-forced-purge: out of memory\n"); return 1; } + memset(blocks[i], 1, BLOCK_SIZE); // resident, so purging it takes the scavenger a while + } + size_t purged0, passes0; + counters(&purged0, &passes0); + for (int i = 0; i < BLOCKS; i++) { mi_free(blocks[i]); } // back to the arena, purge scheduled + + // Wait for the scavenger to go into an arena. If it never does (bounded), the forced collect + // below has all of it to purge by itself, which has to work just as well. + size_t purged1, passes1; + const time_t deadline = time(NULL) + 10; + do { counters(&purged1, &passes1); } while (passes1 == passes0 && time(NULL) < deadline); + if (passes1 != passes0) { in_a_pass++; } + + mi_collect(true); + + counters(&purged1, &passes1); + char name[128]; + snprintf(name, sizeof(name), "round %d: a forced collect leaves nothing that was freed unpurged (%zu of %zu MiB)", + round, (purged1 - purged0) / (1024 * 1024), ((size_t)BLOCKS * BLOCK_SIZE) / (1024 * 1024)); + check(name, purged1 - purged0 >= (size_t)BLOCKS * BLOCK_SIZE); + } + fprintf(stderr, "test-forced-purge: %d of %d forced collects came while the scavenger was purging\n", in_a_pass, ROUNDS); + return failures; +} diff --git a/test/test-purge-behind-pass.c b/test/test-purge-behind-pass.c new file mode 100644 index 000000000..4c15f6a23 --- /dev/null +++ b/test/test-purge-behind-pass.c @@ -0,0 +1,168 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// What a purge pass leaves behind is purged by a later pass, without a thread that parks or forces a collect. +// +// An arena has an expire and so has the subprocess. A free arms the arena if it is not armed yet, and only then +// arms the subprocess, which is what the scavenger sleeps on (and what an allocating thread polls without one). +// A pass resets the expire of an arena when it goes into it. So an arena can be armed when a pass ends: a free +// came in behind the pass, or the arena was not yet due. The subprocess has to be armed as well then. If it is +// not, the arena stays as it is: every later free into it sees an armed arena and does not look further. +// +// 1. A free behind a pass. The pass used to keep the old expire of the subprocess until its end (and the scavenger +// cleared it after that), so a free during the pass found it set and armed the arena alone. Each round frees a +// first batch, waits for the scavenger to go into an arena for it (`arena_purges` counts those), and frees a +// second batch while the scavenger is in there. All of both has to be counted in `purged` within the bound. +// 2. A fork in the middle of a pass. The child has no thread that ends the pass, so an arena that the pass had yet +// to come to stays armed there with the expire of the subprocess reset. An arena of its own is freed into a +// little later than the main one, so that the pass over the main one finds it not yet due. +// +// Counters, not RSS: they read the same on every platform and in every build. + +#include "mimalloc.h" +#include "mimalloc-stats.h" +#include +#include +#include +#include +#if !defined(_WIN32) +#include +#include +#endif + +static int failures = 0; + +static void check(const char* name, bool ok) { + fprintf(stderr, "test: %s... %s\n", name, ok ? "ok." : "FAILED"); + if (!ok) failures++; +} + +#define MAX_FIRST (32) +#define MAX_SECOND (8) +#define BLOCK_SIZE ((size_t)8 * 1024 * 1024) +#define ROUNDS (3) +#define BOUND_SECS (10) // a pass comes `purge_delay` (100ms) after a free + +// the counters of the subprocess itself: read without a lock, and without touching our theap +static void counters(size_t* purged, size_t* arena_purges) { + mi_stats_t_decl(stats); + mi_subproc_stats_get_exclusive(mi_subproc_main(), &stats); + *purged = (size_t)stats.purged.total; + *arena_purges = (size_t)stats.arena_purges.total; +} + +static bool alloc_blocks(void** blocks, int count) { + for (int i = 0; i < count; i++) { + blocks[i] = mi_malloc(BLOCK_SIZE); + if (blocks[i] == NULL) return false; + memset(blocks[i], 1, BLOCK_SIZE); // resident, so purging it takes the scavenger a while + } + return true; +} + +// Spin until the scavenger goes into an arena. Only spin: a park or a collect would purge as well. +static bool wait_for_pass(size_t passes0) { + size_t purged, passes; + const time_t deadline = time(NULL) + BOUND_SECS; + do { counters(&purged, &passes); } while (passes == passes0 && time(NULL) < deadline); + return (passes != passes0); +} + +static void test_free_behind_pass(int first_count, int second_count) { + static void* first[MAX_FIRST]; + static void* second[MAX_SECOND]; + const size_t total = (size_t)(first_count + second_count) * BLOCK_SIZE; + for (int round = 0; round < ROUNDS && failures == 0; round++) { + if (!alloc_blocks(first, first_count) || !alloc_blocks(second, second_count)) { + check("out of memory", false); + return; + } + size_t purged0, passes0; + counters(&purged0, &passes0); + for (int i = 0; i < first_count; i++) { mi_free(first[i]); } // back to the arena, purge scheduled + + char name[160]; + snprintf(name, sizeof(name), "round %d: the scavenger starts on what was freed", round); + check(name, wait_for_pass(passes0)); + + for (int i = 0; i < second_count; i++) { mi_free(second[i]); } // behind the pass, or in front of it + + size_t purged1, passes1; + const time_t deadline = time(NULL) + BOUND_SECS; + do { counters(&purged1, &passes1); } while (purged1 - purged0 < total && time(NULL) < deadline); + snprintf(name, sizeof(name), "round %d: what was freed during a pass is purged as well (%zu of %zu MiB)", + round, (purged1 - purged0) / (1024 * 1024), total / (1024 * 1024)); + check(name, purged1 - purged0 >= total); + } +} + +#if defined(_WIN32) +static void test_fork_in_pass(int first_count) { (void)first_count; } // no fork +#else +static void test_fork_in_pass(int first_count) { + // an arena of our own (the second one, so a pass comes to it after the main one), and a heap that allocates in it only + mi_arena_id_t arena_id; + if (mi_reserve_os_memory_ex(4 * BLOCK_SIZE, false /* commit */, false /* allow large */, true /* exclusive */, &arena_id) != 0) { + fprintf(stderr, "test-purge-behind-pass: no fork test (could not reserve an arena)\n"); + return; + } + mi_heap_t* const heap = mi_heap_new_in_arena(arena_id); + void* const late = (heap == NULL ? NULL : mi_heap_malloc(heap, BLOCK_SIZE)); + static void* first[MAX_FIRST]; + if (late == NULL || !alloc_blocks(first, first_count)) { + check("out of memory", false); + return; + } + memset(late, 1, BLOCK_SIZE); + mi_collect(false); // nothing is left for the child to free into the main arena: that would arm it, and the subprocess with it + + size_t purged0, passes0; + counters(&purged0, &passes0); + for (int i = 0; i < first_count; i++) { mi_free(first[i]); } // arms the main arena and the subprocess + // ..and our arena about half a purge delay later: a pass that is due for the main arena finds ours not yet due + const long half_delay_ms = mi_option_get(mi_option_purge_delay) / 2; + struct timespec ts = { (time_t)(half_delay_ms / 1000), (long)((half_delay_ms % 1000) * 1000000L) }; + nanosleep(&ts, NULL); + mi_free(late); + const bool in_pass = wait_for_pass(passes0); // the scavenger is in the main arena; it has yet to come to ours + size_t purged_fork, passes_fork; + counters(&purged_fork, &passes_fork); + + const pid_t pid = fork(); + if (pid == 0) { + // Poll as an allocating thread does (there is no scavenger here): a pass goes into our arena once it is due. + // The main arena had its expire reset by the pass that the fork cut off, so it is not counted again. + size_t purged, passes; + const time_t deadline = time(NULL) + BOUND_SECS; + do { mi_heap_collect(heap, false); counters(&purged, &passes); } while (passes == passes_fork && time(NULL) < deadline); + _exit(passes != passes_fork ? 0 : 1); + } + int status = -1; + if (pid > 0) { waitpid(pid, &status, 0); } + char name[160]; + snprintf(name, sizeof(name), "a fork %s a pass: the child purges the arena that the pass had yet to come to", in_pass ? "in" : "outside of"); + check(name, pid > 0 && WIFEXITED(status) && WEXITSTATUS(status) == 0); + mi_heap_delete(heap); +} +#endif + +int main(void) { + if (!mi_option_is_enabled(mi_option_scavenger) || mi_option_get(mi_option_purge_delay) <= 0) { + fprintf(stderr, "test-purge-behind-pass: skipped (no scavenger, or no purge delay)\n"); + return 0; + } + // the scavenger starts when a second thread initializes or a thread first parks + void* warm = mi_malloc(64); mi_free(warm); + if (mi_on_thread_idle_start()) { mi_on_thread_idle_end(); } + + // 256 MiB first, so that the pass over it takes milliseconds (64 MiB where address space is scarce) + const int first_count = (sizeof(void*) >= 8 ? MAX_FIRST : MAX_FIRST / 4); + const int second_count = (sizeof(void*) >= 8 ? MAX_SECOND : MAX_SECOND / 4); + test_free_behind_pass(first_count, second_count); + test_fork_in_pass(first_count); + return failures; +}