diff --git a/CMakeLists.txt b/CMakeLists.txt index d4b3587a4..67035b87e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1150,6 +1150,7 @@ if (MI_BUILD_TESTS) set(mi_static_tests api api-fill stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot purge-zero park-handoff forced-purge purge-behind-pass free-before-init tls-key-race) if(NOT (MI_DEBUG_TSAN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_UBSAN)) list(APPEND mi_static_tests thp-optout) # counts madvise calls by interposing it, which a sanitizer runtime does first + list(APPEND mi_static_tests huge-cycle) # blocks above 1 GiB, which a debug build reads back byte by byte to check that they are zero: too slow under a sanitizer endif() if(MI_BUILD_STATIC AND NOT MI_DEBUG_TSAN) # these reach into internals that the shared library (which the TSAN build links the tests against) does not export diff --git a/src/os.c b/src/os.c index ca37f1786..5bc5e50fc 100644 --- a/src/os.c +++ b/src/os.c @@ -142,6 +142,15 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) // todo: perhaps only do alignment hints if THP is enabled? if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL; + // No hint for huge blocks (> 1GiB, as mimalloc v2 did unconditionally). Blocks above + // `arena_max_object_size` are mapped from the OS directly and unmapped again on free. Since + // `aligned_base` only moves forward, each one would land in a fresh 512MiB region of the 2-level + // page map and leave a sub-map behind that is only freed at process exit: 4KiB of RSS per + // alloc/free cycle, for example per 4GiB WebAssembly memory reservation (oven-sh/bun#41459). + // Without a hint the OS hands back the range that was just unmapped, so the sub-map is reused. + // Upstream relaxed this cap in 82c1688674 (microsoft/mimalloc#1290) so that arena reservations + // above 1GiB still get a hint. Keep the cap until the page map can release empty sub-maps. + if (sze > 1*MI_GiB) return NULL; if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks. diff --git a/test/test-huge-cycle.c b/test/test-huge-cycle.c new file mode 100644 index 000000000..1a82054af --- /dev/null +++ b/test/test-huge-cycle.c @@ -0,0 +1,72 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. +-----------------------------------------------------------------------------*/ + +/* A block above `mi_arena_max_object_size()` does not live in an arena: it is mapped from the OS + and unmapped again when it is freed. `_mi_os_get_aligned_hint` (src/os.c) gave each of those + mappings an address from a counter that only moves forward, so every one landed in a part of + the address space that the page map had not seen before. The 2-level page map commits a + sub-map for each such part (64 KiB per 512 MiB of address space) and keeps it until the process + ends. A program that allocates and frees huge blocks one after the other (a runtime that + reserves 4 GiB of address space per WebAssembly memory, oven-sh/bun#41459) lost 64 KiB of + committed memory per block, 4 KiB of it resident. + + Without a hint the OS gives the range that was just unmapped to the next mapping of that size. + The test allocates and frees huge blocks, two at a time, and counts the 512 MiB parts of the + address space in which a block started: each one stands for a sub-map that stays behind. + + Expected with the bug present: one part per block. + Skipped where such a block cannot be allocated at all (a 32-bit address space). + Only a few blocks: a debug build reads every fresh block back to check that it is zero. */ + +#include +#include +#include +#include + +#define BLOCKS 6 +#define LIVE 2 +#define MAX_PARTS 4 // two blocks are live at a time: two parts when every range is used again +#define PART_SHIFT 29 // 512 MiB, what one sub-map of the page map covers + +static uintptr_t parts[BLOCKS]; +static int part_count; + +static void note_part(const void* p) { + const uintptr_t part = ((uintptr_t)p >> PART_SHIFT); + for (int i = 0; i < part_count; i++) { + if (parts[i] == part) return; + } + parts[part_count++] = part; +} + +int main(void) { + if (sizeof(void*) < 8) { + printf("test-huge-cycle: skipped, needs a 64-bit address space\n"); + return 0; + } + // above what an arena holds, and above 1 GiB also where an arena holds less than that + const size_t GiB = ((size_t)1 << 30); + const size_t size = (mi_arena_max_object_size() > GiB ? mi_arena_max_object_size() : GiB) + ((size_t)1 << 20); + + void* live[LIVE]; + for (int done = 0; done < BLOCKS; done += LIVE) { + for (int i = 0; i < LIVE; i++) { + live[i] = mi_malloc_aligned(size, 4096); + if (live[i] == NULL) { + printf("test-huge-cycle: skipped, cannot allocate a block of %zu MiB\n", size >> 20); + return 0; + } + memset(live[i], 1, 4096); + note_part(live[i]); + } + for (int i = 0; i < LIVE; i++) { mi_free(live[i]); } + } + + const int ok = (part_count <= MAX_PARTS); + printf("%s %d blocks of %zu MiB started in %d part(s) of the address space, expected at most %d\n", + (ok ? "ok " : "FAIL"), BLOCKS, size >> 20, part_count, MAX_PARTS); + return (ok ? 0 : 1); +}