Fold Bun's remaining patches into the fork, and take two upstream bitmap fixes

CMakeLists.txt

@@ -1165,7 +1165,7 @@ if (MI_BUILD_TESTS)
   enable_testing()
 
   # static link tests
-  set(mi_static_tests api api-fill stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy heap-new abandoned-lazy fork-user-heap snapshot purge-zero park-handoff forced-purge purge-behind-pass purge-age purge-resident park-again free-before-init tls-key-race thread-first-call)
+  set(mi_static_tests api api-fill exclusive-arena-bins stress-heaps stress-subprocs stress profile heap-mt heap-teardown heap-delete-race heap-churn heap-release-mt heap-aba heap-burst-destroy heap-new abandoned-lazy fork-user-heap snapshot purge-zero park-handoff forced-purge purge-behind-pass purge-age purge-resident park-again free-before-init tls-key-race thread-first-call)
   if(NOT (MI_DEBUG_TSAN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_UBSAN))
     list(APPEND mi_static_tests thp-optout)  # counts madvise calls by interposing it, which a sanitizer runtime does first
     list(APPEND mi_static_tests huge-cycle)  # blocks above 1 GiB, which a debug build reads back byte by byte to check that they are zero: too slow under a sanitizer
@@ -1190,7 +1190,7 @@ if (MI_BUILD_TESTS)
       message(STATUS "cannot build TSAN tests without MI_BUILD_SHARED being enabled")
     endif()
     set(mi_test_env "")
-    if(MI_GUARDED AND (TEST_NAME STREQUAL "purge-holes" OR TEST_NAME STREQUAL "purge-holes-large"))
+    if(MI_GUARDED AND (TEST_NAME STREQUAL "purge-holes" OR TEST_NAME STREQUAL "purge-holes-large" OR TEST_NAME STREQUAL "exclusive-arena-bins"))
       list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0)     # it knows the page layout, which guard pages change (set explicitly: the CI sets a rate for the whole job)
     elseif(MI_GUARDED AND TEST_NAME STREQUAL "profile")
       list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0)     # it checks what the samples add up to, which guarded samples in between change (set explicitly: the CI sets a rate for the whole job); the case with guarded blocks sets its own rate

src/alloc.c

@@ -759,6 +759,14 @@ it returns false, but we limit the total calls.
 -------------------------------------------------------*/
 #define MI_TRY_NEW_MAX (4)
 
+#if defined(MI_NEW_OOM) && !defined(_CPPUNWIND) && !defined(__cpp_exceptions)
+// Reports out of memory and does not return. Stands in for the
+// `throw std::bad_alloc()` that a build without exceptions cannot do, so a
+// throwing `new` never returns NULL to a caller the standard promised memory.
+mi_decl_externc mi_decl_noreturn void MI_NEW_OOM(void);
+#define MI_HAS_NEW_OOM 1
+#endif
+
 #ifdef __cplusplus
 #include <new>
 static bool mi_try_new_handler(bool nothrow) {
@@ -774,6 +782,10 @@ static bool mi_try_new_handler(bool nothrow) {
     if (!nothrow) {
       throw std::bad_alloc();
     }
+    #elif defined(MI_HAS_NEW_OOM)
+    if (!nothrow) {
+      MI_NEW_OOM();
+    }
     #else
     MI_UNUSED(nothrow);
     #endif

src/arena.c

@@ -2013,6 +2013,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start,
   // init bitmaps
   uint8_t* base = mi_arena_start(arena) + bitmap_base;
   arena->slices_free = mi_arena_bbitmap_init(subproc, slice_count, &base);
+  arena->slices_free->is_of_exclusive_arena = exclusive;
   arena->slices_committed = mi_arena_bitmap_init(slice_count, &base);
   arena->slices_dirty = mi_arena_bitmap_init(slice_count, &base);
   arena->slices_purge = mi_arena_bitmap_init(slice_count, &base);

src/bitmap.c

@@ -1814,7 +1814,8 @@ static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t
   mi_assert_internal(MI_BFIELD_BITS >= MI_BCHUNK_FIELDS);
   const mi_bfield_t cmap_mask  = mi_bfield_mask(cmap_max_count,0);
   const size_t cmap_cycle      = cmap_acc+1;
-  const mi_chunkbin_t bbin = mi_chunkbin_of(n);
+  mi_chunkbin_t bbin = mi_chunkbin_of(n);
+again: ;
   // visit each cmap entry
   size_t cmap_idx = 0;
   mi_bfield_cycle_iterate(cmap_mask, tseq, cmap_cycle, cmap_idx, X)
@@ -1857,7 +1858,12 @@ static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t
 
         size_t cidx;
         bool did_temp_clear_bits = false;
-        if ((*on_find)(chunk, n, &cidx, &did_temp_clear_bits)) {
+        const bool found = (*on_find)(chunk, n, &cidx, &did_temp_clear_bits);
+        if (did_temp_clear_bits) {
+          // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now
+          mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false);
+        }        
+        if (found) {
           if (cidx==0 && ibin == MI_CBIN_NONE) { // only the first block determines the size bin
             // this chunk is now reserved for the `bbin` size class
             mi_bbitmap_set_chunk_bin(bbitmap, chunk_idx, bbin);
@@ -1871,11 +1877,7 @@ static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t
           //       but that is ok as the chunkmap is a conservative approximation.
           // todo: should _on_find_ return a boolean if there is a chance all are clear to avoid calling `try_clear?`
           //       probably not as we already only call `try_clear` once we fail to clear any bits.
-          if (did_temp_clear_bits) {
-            // a concurrent find_and_claim may have cleared the chunkmap bit, restore it now
-            mi_bbitmap_chunkmap_set(bbitmap, chunk_idx, false);
-          }
-          else {
+          if (!did_temp_clear_bits) {
             mi_bbitmap_chunkmap_try_clear(bbitmap, chunk_idx);
           }
         }
@@ -1884,6 +1886,11 @@ static bool mi_bbitmap_try_find_and_clear_generic(mi_bbitmap_t* bbitmap, size_t
     }
   }
   mi_bfield_cycle_iterate_end(X);
+  // a request to an exclusive arena has no other arena to go to: it goes on to look in the chunks of every bin
+  if (bbin != MI_CBIN_HUGE && bbitmap->is_of_exclusive_arena) {
+    bbin = MI_CBIN_HUGE;
+    goto again;
+  }
   return false;
 }
 
@@ -1926,7 +1933,12 @@ static bool mi_bchunk_try_clearN_(mi_bbitmap_t* bbitmap, size_t chunk_idx, size_
   size_t count = 0;  // chunk count
   while (m > 0) {
     mi_bchunk_t* chunk = &bbitmap->chunks[chunk_idx + count];
-    if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, NULL)) {
+    bool did_temp_clear_bits = false;
+    if (!mi_bchunk_try_clearN(chunk, 0, (m > MI_BCHUNK_BITS ? MI_BCHUNK_BITS : m), NULL, &did_temp_clear_bits)) {
+      if (did_temp_clear_bits) {
+        // may have raced with a clearer (in on_find) so set the chunkmap bit conservatively
+        mi_bbitmap_chunkmap_set(bbitmap, chunk_idx + count, false);
+      }
       goto rollback;
     }
     m = (m <= MI_BCHUNK_BITS ? 0 : m - MI_BCHUNK_BITS);

src/bitmap.h

@@ -262,8 +262,9 @@ typedef mi_decl_bchunk_align struct mi_bbitmap_s {
   _Atomic(size_t)  chunk_count;         // total count of chunks (0 < N <= MI_BCHUNKMAP_BITS)
   _Atomic(size_t)  chunk_max_accessed;  // max chunk index that was once cleared or set
   mi_subproc_t*    subproc;             // constant, for stats
-  #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 3
-  size_t           _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 3];    // suppress warning on msvc by aligning manually
+  size_t           is_of_exclusive_arena;  // constant
+  #if (MI_BCHUNK_SIZE / MI_SIZE_SIZE) > 4
+  size_t           _padding[MI_BCHUNK_SIZE/MI_SIZE_SIZE - 4];    // suppress warning on msvc by aligning manually
   #endif
   mi_bchunkmap_t   chunkmap;
   mi_bchunkmap_t   chunkmap_bins[MI_CBIN_COUNT - 1];             // chunkmaps with bit set if the chunk is in that size class (excluding MI_CBIN_NONE)

src/heap.c

@@ -94,7 +94,7 @@ static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_he
     // allocate a fresh theap
     theap = _mi_theap_create(heap, mi_theap_get_default()->tld); // sets the theap thread local
     if (theap==NULL) {
-      _mi_error_message(EFAULT, "unable to allocate memory for a thread local heap\n");
+      _mi_error_message(ENOMEM, "unable to allocate memory for a thread local heap\n");
       return NULL;
     }
     _mi_heap_theap_set(heap, theap);
@@ -164,7 +164,7 @@ mi_heap_t* _mi_heap_new_for_subproc(mi_subproc_t* subproc, mi_arena_id_t exclusi
   // reserve a thread local slot for this heap (see also issue #1230)
   mi_thread_local_t theap_slot = (is_main_heap ? mi_thread_local_key_fast : _mi_thread_local_create());
   if (theap_slot == 0) {
-    _mi_error_message(EFAULT, "unable to dynamically create a thread local for a heap\n");
+    _mi_error_message(ENOMEM, "unable to dynamically create a thread local for a heap\n");
     mi_free(heap);
     return NULL;
   }

src/libc.c

@@ -96,8 +96,27 @@ int _mi_getenv(const char* name, char* result, size_t result_size) {
   return ENOENT;
 }
 #else
+#if defined(MI_GETENV_ALLOWED)
+// MI_GETENV_ALLOWED is a string of the names that are read from the environment, in lower case and separated
+// by commas ("mimalloc_purge_delay,mimalloc_arena_reserve"). No other name is read. An embedder can ask as well.
+mi_decl_externc bool mi_is_read_from_environment(const char* name);
+bool mi_is_read_from_environment(const char* name) {
+  const size_t len = _mi_strlen(name);
+  for (const char* s = MI_GETENV_ALLOWED; *s != 0; ) {
+    const char* end = s;
+    while (*end != 0 && *end != ',') { end++; }
+    if ((size_t)(end - s) == len && _mi_strnicmp(s, name, len) == 0) return true;
+    s = (*end == 0 ? end : end + 1);
+  }
+  return false;
+}
+#endif
+
 int _mi_getenv(const char* name, char* result, size_t result_size) {
   if (name==NULL || result == NULL || result_size < 64) return ENOENT;
+  #if defined(MI_GETENV_ALLOWED)
+  if (!mi_is_read_from_environment(name)) return ENOENT;
+  #endif
   // change the result of _mi_prim_getenv to an errno result
   const int res = _mi_prim_getenv(name,result,result_size);
   return (res > 0 ? 0 : (res == 0 ? ENOENT : EAGAIN));

src/os.c

@@ -211,6 +211,9 @@ bool _mi_os_secure_guard_page_set_at(mi_subproc_t* subproc, void* addr, mi_memid
     }
     else {
       ok = _mi_os_decommit(subproc, addr, _mi_os_secure_guard_page_size());
+      #if !defined(_WIN32)  // on Windows a decommitted page is not accessible, and `VirtualProtect` refuses it
+      ok = ok && _mi_os_protect(addr, _mi_os_secure_guard_page_size());
+      #endif
     }
   }
   if (!ok) {
@@ -238,6 +241,9 @@ bool _mi_os_secure_guard_page_reset_at(mi_subproc_t* subproc, void* addr, mi_mem
       return (*(arena->commit_fun))(true, addr, _mi_os_secure_guard_page_size(), NULL, arena->commit_fun_arg);
     }
     else {
+      #if !defined(_WIN32)
+      if (!_mi_os_unprotect(addr, _mi_os_secure_guard_page_size())) return false;
+      #endif
       return _mi_os_commit(subproc, addr, _mi_os_secure_guard_page_size(), NULL);
     }
   }

src/threadlocal.c

@@ -148,7 +148,7 @@ static mi_decl_noinline bool mi_thread_local_set_expand( mi_thread_local_t key,
   const size_t idx = mi_key_index(key);
   mi_thread_locals_t* tls = mi_thread_locals_expand(idx);
   if (tls==NULL) {
-    _mi_error_message(EFAULT,"unable to allocate thread local variables\n");
+    _mi_error_message(ENOMEM,"unable to allocate thread local variables\n");
     return false;
   }
   mi_assert_internal(tls == mi_thread_locals_get());

test/test-exclusive-arena-bins.c

@@ -0,0 +1,62 @@
+/* ----------------------------------------------------------------------------
+Copyright (c) 2026, Microsoft Research, Daan Leijen
+This is free software; you can redistribute it and/or modify it under the
+terms of the MIT license.
+-----------------------------------------------------------------------------*/
+
+// A heap with an exclusive arena has no other arena to go to. A chunk of an arena gets the size bin of the
+// first page that is placed in it, and takes no small pages once that is a page of several slices.
+// In an arena of two chunks, small blocks were refused with half of the arena free.
+
+#include <mimalloc.h>
+#include <stdint.h>
+#include <stdio.h>
+
+int main(void) {
+  const size_t KiB = 1024;
+  const size_t MiB = 1024*KiB;
+  // The sizes of a 64-bit build. With other sizes the state below is not reached, and all blocks are served anyway.
+  const size_t slice = 64*KiB;
+  const size_t chunk = 32*MiB;
+  const size_t size  = 2*chunk;
+  const size_t block = 1*KiB;
+
+  mi_arena_id_t arena;
+  if (mi_reserve_os_memory_ex(size, false /* commit */, false /* allow large */, true /* exclusive */, &arena) != 0) {
+    printf("test-exclusive-arena-bins: skipped (cannot reserve %zu MiB)\n", size/MiB);
+    return 0;
+  }
+  size_t area_size = 0;
+  const uint8_t* const start = (const uint8_t*)mi_arena_area(arena, &area_size);
+  mi_heap_t* const heap = mi_heap_new_in_arena(arena);
+  if (start == NULL || heap == NULL) {
+    fprintf(stderr, "FAILED: no heap in the arena\n");
+    return 1;
+  }
+
+  // small pages, until the first chunk has no room for a page of 8 slices
+  size_t served = 0;
+  for (;;) {
+    const uint8_t* const p = (const uint8_t*)mi_heap_malloc(heap, block);
+    if (p == NULL) break;
+    served += mi_usable_size(p);
+    if ((size_t)(p - start) >= chunk - 4*slice) break;
+  }
+  // which so becomes the first page of the second chunk
+  if (mi_heap_malloc(heap, 48*KiB) == NULL) {
+    fprintf(stderr, "FAILED: no room for a medium block\n");
+    return 1;
+  }
+  // small blocks again, until they are refused
+  for (void* p; (p = mi_heap_malloc(heap, block)) != NULL; ) {
+    served += mi_usable_size(p);
+  }
+
+  printf("test-exclusive-arena-bins: small blocks got %zu of %zu MiB\n", served/MiB, area_size/MiB);
+  // half of it at most when they are refused the second chunk; a debug build spends a fifth of each block on padding
+  if (served < (area_size/8)*5) {
+    fprintf(stderr, "FAILED: small blocks were refused with room left in the arena\n");
+    return 1;
+  }
+  return 0;
+}