diff --git a/CMakeLists.txt b/CMakeLists.txt index 9db485cd5..2b89d4968 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1147,7 +1147,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - set(mi_static_tests api api-fill stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot purge-zero park-handoff forced-purge purge-behind-pass free-before-init) + set(mi_static_tests api api-fill stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot purge-zero purge-committed park-handoff forced-purge purge-behind-pass free-before-init) if(NOT (MI_DEBUG_TSAN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_UBSAN)) list(APPEND mi_static_tests thp-optout) # counts madvise calls by interposing it, which a sanitizer runtime does first endif() @@ -1203,6 +1203,11 @@ if (MI_BUILD_TESTS) # and sweep every park, so the reclaim path is hit as hard as possible add_test(NAME test-park-handoff-eager COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES_MIN_INTERVAL=0 $) + # an allocation that does not ask for commit has to count its slices as the committing one does + add_test(NAME test-purge-committed-on-demand COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PAGE_COMMIT_ON_DEMAND=1 $) + # and an arena that is not committed up front is counted on its commit bits alone + add_test(NAME test-purge-committed-no-eager-commit COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 $) + # The allocator must never touch one of its own `__thread` variables from inside an allocation: # where `__thread` is emulated (Android before API 29) the first access on a thread mallocs, and # with the allocator being malloc that is a re-entrant allocation (oven-sh/bun#38051). Build the diff --git a/src/arena.c b/src/arena.c index 2b844cd9d..67ae4231d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -241,6 +241,22 @@ static size_t mi_page_full_size(mi_page_t* page) { Arena Allocation ----------------------------------------------------------- */ +// On an OS with overcommit, `mi_arena_reserve` does not count an eagerly committed arena as +// committed: the OS backs a slice the first time it is written. So a slice of such an arena counts +// as committed while its dirty bit is set: it is credited here when it is handed out for the first +// time, and debited in `mi_arena_purge` when a zero-claim purge clears the dirty bit again. +// An arena that was not committed up front is counted on its commit bits alone (`_mi_os_commit_ex`): +// a purge keeps those set, so a slice of it is counted once and stays counted. +static bool mi_arena_stat_on_dirty(const mi_arena_t* arena) { + return (arena->memid.initially_committed && _mi_os_has_overcommit() && !arena->memid.is_pinned /* huge pages, issue #1236 */); +} + +static void mi_arena_stat_touched(mi_arena_t* arena, size_t touched_slices) { + if (touched_slices > 0 && mi_arena_stat_on_dirty(arena)) { + mi_subproc_stat_increase(arena->subproc, committed, mi_size_of_slices(touched_slices)); + } +} + static mi_decl_noinline void* mi_arena_try_alloc_at( mi_arena_t* arena, size_t slice_count, bool commit, size_t tseq, mi_memid_t* memid) { @@ -255,12 +271,22 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( memid->is_pinned = arena->memid.is_pinned; // set the dirty bits and track which slices become accessible - size_t touched_slices = slice_count; + const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count); + size_t touched_slices = slice_count; // slices whose dirty bit was clear + size_t touched_committed = 0; // of those, the ones whose commit bit is set: counted by `mi_arena_stat_touched` if (arena->memid.initially_zero) { + if (already_committed < slice_count && mi_arena_stat_on_dirty(arena)) { + // committed in part (a purge decommitted some of it): the commit below counts the slices it commits, + // so count the clean slices among the committed ones here, before the dirty bits are set + for (size_t i = slice_index; i < slice_index + slice_count; i++) { + if (mi_bitmap_is_set(arena->slices_committed, i) && !mi_bitmap_is_set(arena->slices_dirty, i)) { touched_committed++; } + } + } size_t already_dirty = 0; memid->initially_zero = mi_bitmap_setN(arena->slices_dirty, slice_index, slice_count, &already_dirty); mi_assert_internal(already_dirty <= touched_slices); touched_slices -= already_dirty; + if (already_committed == slice_count) { touched_committed = touched_slices; } } else { // todo: properly count touched pages with a separate bitmap? @@ -270,7 +296,6 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( // set commit state if (commit) { // commit requested, but the range may not be committed as a whole: ensure it is committed now - const size_t already_committed = mi_bitmap_popcountN(arena->slices_committed, slice_index, slice_count); if (already_committed < slice_count) { // not all committed, try to commit now bool commit_zero = false; @@ -286,6 +311,8 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( // set the commit bits mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, NULL); + // the commit counted what it committed; the slices that were committed already count now + mi_arena_stat_touched(arena, touched_committed); // committed #if MI_DEBUG > 1 @@ -300,11 +327,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( else { // already fully committed. _mi_os_reuse(arena->subproc, p, mi_size_of_slices(slice_count)); - // if the OS has overcommit, and this is the first time we access these pages, then - // count the commit now (as at arena reserve we didn't count those commits as these are on-demand) - if (_mi_os_has_overcommit() && touched_slices > 0 && !arena->memid.is_pinned /* huge pages, issue #1236 */) { - mi_subproc_stat_increase( arena->subproc, committed, mi_size_of_slices(touched_slices)); - } + mi_arena_stat_touched(arena, touched_committed); } mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); @@ -321,12 +344,17 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( else { // no need to commit, but check if it is already fully committed memid->initially_committed = mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count); - if (!memid->initially_committed) { - // partly committed.. adjust stats - size_t already_committed_count = 0; - mi_bitmap_setN(arena->slices_committed, slice_index, slice_count, &already_committed_count); + if (memid->initially_committed) { + // nothing commits this range on demand later: count the slices it touches now + mi_arena_stat_touched(arena, touched_committed); + } + else { + // partly committed.. adjust stats: the range is committed on demand from here on, so the + // slices that were committed (and counted) are not counted any longer mi_bitmap_clearN(arena->slices_committed, slice_index, slice_count); - mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(already_committed_count)); + // a slice counted on its dirty bit was not counted while that bit was clear + const size_t counted = (mi_arena_stat_on_dirty(arena) ? already_committed - touched_committed : already_committed); + mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(counted)); } } @@ -2566,14 +2594,15 @@ static bool mi_arena_purge(mi_arena_t* arena, size_t slice_index, size_t slice_c // The OS guarantees this range reads back zero, so forget that it was ever dirty: the // next `mi_arenas_alloc` hands it out with `initially_zero`, `mi_zalloc` skips the // memset, and pages the caller never writes are never made resident again. + const size_t dirty_slices = mi_bitmap_popcountN(arena->slices_dirty, slice_index, slice_count); mi_bitmap_clearN(arena->slices_dirty, slice_index, slice_count); - // Do NOT touch the `committed` stat here. `committed` is keyed on the COMMIT bits, not the - // dirty bits: it is credited in `mi_arena_try_alloc_at` only for slices whose commit bit was - // clear (`slice_count - already_committed`, and the whole block is skipped when the range is - // already fully committed). A zero-claim purge deliberately KEEPS the commit bits set, so the - // next allocation credits nothing -- and debiting here would be an unmatched debit that walks - // `committed` down without bound (it is an int64; `mi_process_info` casts it to size_t, so it - // wraps). The real ratchet is the partial-commit branch below, which *clears* commit bits. + // A range that needs no recommit keeps its commit bits, so `_mi_os_purge_zero` debited + // nothing, and the next `mi_arena_try_alloc_at` credits `committed` again for every slice + // that was just un-dirtied (`mi_arena_stat_touched`). Debit those slices here, under the + // same condition, or `committed` grows by the size of every purge/reuse cycle, forever. + if (!needs_recommit && all_committed && dirty_slices > 0 && mi_arena_stat_on_dirty(arena)) { + mi_subproc_stat_decrease(arena->subproc, committed, mi_size_of_slices(dirty_slices)); + } } } diff --git a/test/test-purge-committed.c b/test/test-purge-committed.c new file mode 100644 index 000000000..1dc5899c9 --- /dev/null +++ b/test/test-purge-committed.c @@ -0,0 +1,134 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// The `committed` stat comes back down when a purge returns a range to the OS, and reads the same +// after every purge/reuse cycle. +// +// On an OS with overcommit (Linux, macOS) an arena is committed when it is reserved and the stat +// does not count that: a slice is counted the first time it is handed out. A purge on those +// systems (`MADV_DONTNEED`) leaves the range committed and the arena clears its dirty bits, so the +// next allocation of the range counts its slices as handed out for the first time, again. The purge +// has to debit what that allocation credits, or `committed` grows by the size of every cycle and +// never comes down: bun saw 28 GB "committed" against 0.6 GB resident after 8 days. +// +// `MIMALLOC_PAGE_COMMIT_ON_DEMAND=1` covers the other direction: an allocation that does not ask +// for commit, on a range that is committed already, has to credit the same slices, or the debit at +// the purge walks the stat below zero. `MIMALLOC_ARENA_EAGER_COMMIT=0` covers an arena that is +// not committed up front: its slices are counted when they are committed, and a purge must not +// debit what no allocation credited. + +#include "mimalloc.h" +#include "mimalloc-stats.h" +#include +#include +#include + +static int failures = 0; + +static void check(const char* name, bool ok) { + fprintf(stderr, "test: %s... %s\n", name, ok ? "ok." : "FAILED"); + if (!ok) failures++; +} + +#define MiB ((size_t)1024 * 1024) +#define ROUNDS (8) +#define BLOCKS (16) +#define SLACK (2 * MiB) // page meta data and the blocks the allocator keeps for itself + +// Whether a purge brings `committed` back down. Only a purge that zero-fills on the next access +// (Linux `MADV_DONTNEED`; `MADV_ZERO` on a recent macOS) un-dirties the range, and only an arena +// that was committed up front is counted on its dirty bits. Elsewhere the stat is a high-water mark +// by design: it does not come down after the purge, but it does not grow either. +static bool purge_counts_down(void) { + #if defined(__linux__) + return (mi_option_get(mi_option_arena_eager_commit) != 0); + #else + return false; + #endif +} + +static int64_t committed(void) { + mi_stats_t_decl(stats); + mi_subproc_stats_get_exclusive(mi_subproc_main(), &stats); + return stats.committed.current; +} + +// allocate `BLOCKS` blocks of `size`, write them, free them, and purge: `committed` has to be back +// where it started, and the same in every round +static void cycle(const char* name, size_t size) { + void* blocks[BLOCKS]; + // let the first round take its pages and meta data: the rounds after it reuse them + for (int i = 0; i < BLOCKS; i++) { blocks[i] = mi_malloc(size); memset(blocks[i], 1, size); } + for (int i = 0; i < BLOCKS; i++) { mi_free(blocks[i]); } + mi_collect(true); + const int64_t base = committed(); + + bool flat = true; + bool down = true; + bool nonneg = (base >= 0); + int64_t held0 = 0; + for (int round = 0; round < ROUNDS; round++) { + for (int i = 0; i < BLOCKS; i++) { blocks[i] = mi_malloc(size); memset(blocks[i], 1, size); } + const int64_t held = committed(); + if (round == 0) { held0 = held; } + for (int i = 0; i < BLOCKS; i++) { mi_free(blocks[i]); } + mi_collect(true); + const int64_t after = committed(); + fprintf(stderr, " %s round %d: held %lld KiB, after purge %lld KiB (base %lld KiB)\n", + name, round, (long long)(held / 1024), (long long)(after / 1024), (long long)(base / 1024)); + if (held > held0 + (int64_t)SLACK) { flat = false; } // grows with every cycle + if (after > base + (int64_t)SLACK) { down = false; } // the purge debited nothing + if (held < 0 || after < 0) { nonneg = false; } // debited more than was credited + } + if (!purge_counts_down()) { down = true; } + check(name, flat && down && nonneg); +} + +// Each round takes a block twice the size of the one before it, so every round's range covers slices +// the round before it took and purged, plus slices nothing took yet. In an arena that is not committed +// up front (`MIMALLOC_ARENA_EAGER_COMMIT=0`) the commit of such a range is partial. What the purge +// debits for it has to be what its allocation credited, or the stat drifts below zero, a little +// more with every round. +static void growing(const char* name) { + void* block = mi_malloc(64 * 1024); memset(block, 1, 64 * 1024); mi_free(block); + mi_collect(true); + const int64_t base = committed(); + bool nonneg = (base >= 0); + bool down = true; + int64_t after = base; + for (size_t size = 1 * MiB; size <= 32 * MiB; size *= 2) { + block = mi_malloc(size); memset(block, 1, size); + const int64_t held = committed(); + mi_free(block); + mi_collect(true); + after = committed(); + fprintf(stderr, " %s %zu MiB: held %lld KiB, after purge %lld KiB (base %lld KiB)\n", + name, size / MiB, (long long)(held / 1024), (long long)(after / 1024), (long long)(base / 1024)); + if (held < 0 || after < 0) { nonneg = false; } + } + if (after > base + (int64_t)SLACK && purge_counts_down()) { down = false; } + check(name, nonneg && down); +} + +int main(void) { + if (mi_option_get(mi_option_purge_delay) < 0) { + fprintf(stderr, "test-purge-committed: skipped (purging is off)\n"); + return 0; + } + mi_option_set(mi_option_purge_delay, 0); // a free purges at once + mi_option_set(mi_option_minimal_purge_size, 4 /* KiB */); // and purges the edges of a run too, not 2 MiB units only + + // first, while the arena is fresh: every block of the growing run has to reach slices nothing took yet + growing("committed stays at or above zero across blocks of growing size"); + cycle("committed is flat across 16 x 1 MiB purge/reuse cycles", 1 * MiB); + cycle("committed is flat across 16 x 64 KiB purge/reuse cycles", 64 * 1024); + cycle("committed is flat across 16 x 4 MiB purge/reuse cycles", 4 * MiB); + + if (failures == 0) { fprintf(stderr, "test-purge-committed: all ok.\n"); } + else { fprintf(stderr, "test-purge-committed: %d FAILED\n", failures); } + return (failures == 0 ? 0 : 1); +}