os: no address hint for OS allocations above 1 GiB

CMakeLists.txt

@@ -1150,6 +1150,7 @@ if (MI_BUILD_TESTS)
   set(mi_static_tests api api-fill stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy abandoned-lazy fork-user-heap snapshot purge-zero park-handoff forced-purge purge-behind-pass free-before-init tls-key-race)
   if(NOT (MI_DEBUG_TSAN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_UBSAN))
     list(APPEND mi_static_tests thp-optout)  # counts madvise calls by interposing it, which a sanitizer runtime does first
+    list(APPEND mi_static_tests huge-cycle)  # blocks above 1 GiB, which a debug build reads back byte by byte to check that they are zero: too slow under a sanitizer
   endif()
   if(MI_BUILD_STATIC AND NOT MI_DEBUG_TSAN)
     # these reach into internals that the shared library (which the TSAN build links the tests against) does not export

src/os.c

@@ -142,6 +142,15 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze)
 
   // todo: perhaps only do alignment hints if THP is enabled?
   if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL;
+  // No hint for huge blocks (> 1GiB, as mimalloc v2 did unconditionally). Blocks above
+  // `arena_max_object_size` are mapped from the OS directly and unmapped again on free. Since
+  // `aligned_base` only moves forward, each one would land in a fresh 512MiB region of the 2-level
+  // page map and leave a sub-map behind that is only freed at process exit: 4KiB of RSS per
+  // alloc/free cycle, for example per 4GiB WebAssembly memory reservation (oven-sh/bun#41459).
+  // Without a hint the OS hands back the range that was just unmapped, so the sub-map is reused.
+  // Upstream relaxed this cap in 82c1688674 (microsoft/mimalloc#1290) so that arena reservations
+  // above 1GiB still get a hint. Keep the cap until the page map can release empty sub-maps.
+  if (sze > 1*MI_GiB) return NULL;
   if (mi_os_mem_config.virtual_address_bits < 46) return NULL;  // < 64TiB virtual address space
 
   size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks.

test/test-huge-cycle.c

@@ -0,0 +1,72 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license.
+-----------------------------------------------------------------------------*/
+
+/* A block above `mi_arena_max_object_size()` does not live in an arena: it is mapped from the OS
+   and unmapped again when it is freed. `_mi_os_get_aligned_hint` (src/os.c) gave each of those
+   mappings an address from a counter that only moves forward, so every one landed in a part of
+   the address space that the page map had not seen before. The 2-level page map commits a
+   sub-map for each such part (64 KiB per 512 MiB of address space) and keeps it until the process
+   ends. A program that allocates and frees huge blocks one after the other (a runtime that
+   reserves 4 GiB of address space per WebAssembly memory, oven-sh/bun#41459) lost 64 KiB of
+   committed memory per block, 4 KiB of it resident.
+
+   Without a hint the OS gives the range that was just unmapped to the next mapping of that size.
+   The test allocates and frees huge blocks, two at a time, and counts the 512 MiB parts of the
+   address space in which a block started: each one stands for a sub-map that stays behind.
+
+   Expected with the bug present: one part per block.
+   Skipped where such a block cannot be allocated at all (a 32-bit address space).
+   Only a few blocks: a debug build reads every fresh block back to check that it is zero. */
+
+#include <stdint.h>
+#include <stdio.h>
+#include <string.h>
+#include <mimalloc.h>
+
+#define BLOCKS       6
+#define LIVE         2
+#define MAX_PARTS    4             // two blocks are live at a time: two parts when every range is used again
+#define PART_SHIFT   29            // 512 MiB, what one sub-map of the page map covers
+
+static uintptr_t parts[BLOCKS];
+static int       part_count;
+
+static void note_part(const void* p) {
+  const uintptr_t part = ((uintptr_t)p >> PART_SHIFT);
+  for (int i = 0; i < part_count; i++) {
+    if (parts[i] == part) return;
+  }
+  parts[part_count++] = part;
+}
+
+int main(void) {
+  if (sizeof(void*) < 8) {
+    printf("test-huge-cycle: skipped, needs a 64-bit address space\n");
+    return 0;
+  }
+  // above what an arena holds, and above 1 GiB also where an arena holds less than that
+  const size_t GiB = ((size_t)1 << 30);
+  const size_t size = (mi_arena_max_object_size() > GiB ? mi_arena_max_object_size() : GiB) + ((size_t)1 << 20);
+
+  void* live[LIVE];
+  for (int done = 0; done < BLOCKS; done += LIVE) {
+    for (int i = 0; i < LIVE; i++) {
+      live[i] = mi_malloc_aligned(size, 4096);
+      if (live[i] == NULL) {
+        printf("test-huge-cycle: skipped, cannot allocate a block of %zu MiB\n", size >> 20);
+        return 0;
+      }
+      memset(live[i], 1, 4096);
+      note_part(live[i]);
+    }
+    for (int i = 0; i < LIVE; i++) { mi_free(live[i]); }
+  }
+
+  const int ok = (part_count <= MAX_PARTS);
+  printf("%s %d blocks of %zu MiB started in %d part(s) of the address space, expected at most %d\n",
+         (ok ? "ok  " : "FAIL"), BLOCKS, size >> 20, part_count, MAX_PARTS);
+  return (ok ? 0 : 1);
+}