From d38870faada8169d2becb3c09fd15ab72afed107 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 10 Aug 2026 18:43:31 -0700 Subject: [PATCH 001/280] use __builtin_thread_pointer on riscV, pr #1363 by @rui314 --- include/mimalloc/prim.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/prim.h b/include/mimalloc/prim.h index a55943f59..10e64b20d 100644 --- a/include/mimalloc/prim.h +++ b/include/mimalloc/prim.h @@ -275,7 +275,7 @@ static inline void mi_prim_tls_slot_set(size_t slot, void* value) mi_attr_noexce #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \ || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ - || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__))) + || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__) || defined(__riscv))) /* pr #1363 */ #define MI_USE_BUILTIN_THREAD_POINTER 1 #endif #endif From 680d2286fd2929f78f34d261fdade6a75ac698cf Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 10 Aug 2026 18:49:18 -0700 Subject: [PATCH 002/280] no parens on MI_SMALL_WSIZE_MAX --- include/mimalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 964d33e1d..19d6b8f1b 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -119,7 +119,7 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* // ------------------------------------------------------ // Extended functionality // ------------------------------------------------------ -#define MI_SMALL_WSIZE_MAX (128) +#define MI_SMALL_WSIZE_MAX 128 #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); From 932f6e5d539b60137cfd253e88eb0dc840e8e484 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 10 Aug 2026 18:54:11 -0700 Subject: [PATCH 003/280] add guard for thread_locals that are accessed after a the thread_locals are already free'd --- src/init.c | 16 ++++++++++++---- src/page-map.c | 4 ++-- src/page.c | 4 +--- src/prim/unix/prim.c | 2 +- src/threadlocal.c | 9 +++++++-- 5 files changed, 23 insertions(+), 12 deletions(-) diff --git a/src/init.c b/src/init.c index 9fcd2bf53..63f0a0e27 100644 --- a/src/init.c +++ b/src/init.c @@ -37,12 +37,20 @@ static const mi_page_t mi_page_empty = { #define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty) +#if MI_SMALL_WSIZE_MAX == 128 +#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p) +#elif MI_SMALL_WSIZE_MAX == 256 +#define MI_INIT_PAGES_DIRECT(p) MI_INIT128(p), MI_INIT128(p) +#else +#error define initializer for direct pages +#endif + #if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #elif (MI_PADDING>0) -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #else -#define MI_SMALL_PAGES_EMPTY { MI_INIT128(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } +#define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY() } #endif @@ -614,7 +622,7 @@ static void mi_process_done_once(void) { _mi_theap_merge_stats(subproc_main->theap_meta); _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free mi_heap_stats_merge_to_subproc(subproc_main->heap_main); - mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); + mi_subproc_stats_print_out(mi_subproc_main(), NULL, NULL); // note: can try to access (the now freed) thread_locals in mi_heap_theap_peek } } } diff --git a/src/page-map.c b/src/page-map.c index a7ca65ea6..35b0bbbb7 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -297,8 +297,8 @@ static bool mi_page_map_init_once(void) { #if MI_SECURE const bool commit = true; // the whole page map is valid and we can reliably check any pointer #else - const bool commit = false; // reserve_size <= 256*MI_KiB || // 40 virtual address bits - // mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); + const bool commit = reserve_size <= 256*MI_KiB || // 40 virtual address bits + mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); #endif mi_subproc_t* const subproc = _mi_subproc_main(); mi_memid_t memid; diff --git a/src/page.c b/src/page.c index 5ebd7402d..26d26751c 100644 --- a/src/page.c +++ b/src/page.c @@ -440,9 +440,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? if (pq->last==page && pq->first==page) { // the only page in the queue? mi_theap_t* theap = mi_page_theap(page); - #if MI_STAT>0 mi_theap_stat_counter_increase(theap, pages_retire, 1); - #endif page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); mi_assert_internal(pq >= theap->pages); const size_t index = pq - theap->pages; @@ -609,7 +607,7 @@ static mi_decl_noinline void mi_page_free_list_extend( mi_page_t* const page, co Page initialize and extend the capacity ----------------------------------------------------------- */ -#define MI_MAX_EXTEND_SIZE (4*1024) // heuristic, one OS page seems to work well. +#define MI_MAX_EXTEND_SIZE (8*1024) // heuristic, one or two OS pages seems to work well. #if (MI_SECURE>=2) #define MI_MIN_EXTEND (8*MI_SECURE) // extend at least by this many #else diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index a2f402428..a2c05e8a5 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -503,7 +503,7 @@ int _mi_prim_alloc(void* hint_addr, size_t size, size_t try_alignment, bool comm //--------------------------------------------- static void unix_mprotect_hint(int err) { - #if defined(__linux__) && (MI_SECURE>=5) // guard page around every mimalloc page + #if defined(__linux__) && (MI_SECURE>=5 || MI_GUARDED) // guard page around every mimalloc page if (err == ENOMEM) { _mi_warning_message("The next warning may be caused by a low memory map limit.\n" " On Linux this is controlled by the vm.max_map_count -- maybe increase it?\n" diff --git a/src/threadlocal.c b/src/threadlocal.c index 2b428f018..355498cee 100644 --- a/src/threadlocal.c +++ b/src/threadlocal.c @@ -175,8 +175,13 @@ bool _mi_thread_local_set( mi_thread_local_t key, void* val ) { // get a tls slot value static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key ) { mi_assert_internal(key!=0); - const mi_thread_locals_t* const tls = mi_thread_locals_get(); - mi_assert_internal(tls!=NULL); + const mi_thread_locals_t* const tls = mi_thread_locals_peek(); + if mi_unlikely(tls==NULL) { + // this can happen if a thread local is accessed after the thread local has been freed + // from mi_thread_done or mi_process_done. + // todo: can we remove this check? now we can still call this from process done when stats are printed (which calls mi_heap_theap_peek) + return NULL; + } const size_t idx = mi_key_index(key); if mi_likely(idx < tls->count && mi_key_version(key) == tls->slots[idx].version) { return tls->slots[idx].value; From 82fd55ff1253eca43d1dee3c7a22d2c18239a8ef Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 10 Aug 2026 23:56:03 -0700 Subject: [PATCH 004/280] wip: aligned page meta info for faster free --- include/mimalloc/internal.h | 34 +++++++- include/mimalloc/prim-tls.h | 14 ++-- include/mimalloc/types.h | 32 ++++++-- src/arena.c | 149 ++++++++++++++++++++++++++---------- src/free.c | 67 +++++++++------- src/init.c | 8 +- src/os.c | 2 +- test/test-stress.c | 2 +- 8 files changed, 222 insertions(+), 86 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index e42a574ac..1f711dd15 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -761,12 +761,38 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { #endif +#if MI_PAGE_META_ALIGNED +static inline mi_page_t* _mi_ptr_page_align0(const void* p) { + mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGN); + const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; + mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_COUNT); + return &page_metas[page_idx]; +} + +static inline mi_page_t* _mi_ptr_page_align(const void* p) { + mi_page_t* const page = _mi_ptr_page_align0(p); + return (page==NULL ? NULL : mi_atomic_load_relaxed(&page->self)); +} +#endif + static inline mi_page_t* _mi_ptr_page(const void* p) { mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); - #if MI_DEBUG || MI_SECURE || MI_FREE_IS_CHECKED - return _mi_checked_ptr_page(p); - #else - return _mi_unchecked_ptr_page(p); + #if MI_SECURE // || MI_FREE_IS_CHECKED + return _mi_checked_ptr_page(p); + #elif MI_PAGE_META_ALIGNED + mi_page_t* const page = _mi_ptr_page_align(p); + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if (cpage!=page) { + mi_assert(page == cpage); + } + #endif + // mi_assert_internal(page->block_size != 0); + return page; + #elif MI_DEBUG + return _mi_checked_ptr_page(p); + #else + return _mi_unchecked_ptr_page(p); #endif } diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 445529345..117bf2c23 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -89,16 +89,16 @@ static inline void** mi_prim_thread_pointer(void) { static inline void** mi_prim_thread_pointer(void) { void** tcb; #if defined(__APPLE__) // M1, issue rgb(62, 76, 62) - __asm__ volatile ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + __asm__ ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); #else - __asm__ volatile ("mrs %0, tpidr_el0" : "=r" (tcb)); + __asm__ ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif return tcb; } #elif defined(__riscv) static inline void** mi_prim_thread_pointer(void) { void** tcb; - __asm__ volatile ("mv %0, tp" : "=r" (tcb)); + __asm__ ("mv %0, tp" : "=r" (tcb)); return tcb; } #elif defined(__arm__) @@ -110,18 +110,18 @@ static inline void** mi_prim_thread_pointer(void) { #elif defined(__i386__) static inline void** mi_prim_thread_pointer(void) { void** tcb; - __asm__("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS + __asm__ ("movl %%gs:0, %0" : "=r" (tcb) : : ); // x86 32-bit always uses GS return tcb; } #elif defined(__x86_64__) static inline void** mi_prim_thread_pointer(void) { void** tcb; #if defined(__APPLE__) - __asm__("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS + __asm__ ("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS #elif (MI_INTPTR_SIZE==4) - __asm__("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI + __asm__ ("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI #else - __asm__("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS + __asm__ ("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS #endif return tcb; } diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ba4539bc7..ec4799b3b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -132,6 +132,13 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif +#define MI_OPT_META_ALIGNED 1 +#if MI_OPT_META_ALIGNED && MI_PAGE_META_IS_SEPARATED +#define MI_PAGE_META_ALIGNED_FREE_SMALL 0 +#define MI_PAGE_META_ALIGNED 1 +#define MI_PAGE_META_CHUNKS 8 +#endif + // We can choose to only put page info of small pages at the start of the page area. // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). @@ -188,6 +195,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_BCHUNK_BITS (1 << MI_BCHUNK_BITS_SHIFT) // sub-bitmaps in arena's are "bchunks" of 512 bits #define MI_ARENA_SLICE_SIZE (MI_ZU(1) << MI_ARENA_SLICE_SHIFT) // arena's allocate in slices of 64 KiB #define MI_ARENA_SLICE_ALIGN (MI_ARENA_SLICE_SIZE) +#define MI_ARENA_CHUNK_SIZE (MI_BCHUNK_BITS * MI_ARENA_SLICE_SIZE) #define MI_ARENA_MIN_OBJ_SLICES (1) #define MI_ARENA_MAX_CHUNK_OBJ_SLICES (MI_BCHUNK_BITS) // 32 MiB (or 8 MiB on 32-bit) @@ -212,11 +220,18 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_BIN_COUNT (MI_BIN_FULL+1) // We never allocate more than PTRDIFF_MAX (see also ) -#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX +#define MI_MAX_ALLOC_SIZE PTRDIFF_MAX // Minimal commit for a page on-demand commit -#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ +#define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ +#if MI_PAGE_META_ALIGNED +#define MI_PAGE_META_COUNT (MI_PAGE_META_CHUNKS * MI_BCHUNK_BITS) +#define MI_PAGE_META_ALIGN (MI_PAGE_META_COUNT * MI_ARENA_SLICE_SIZE) +#define MI_ARENA_ALIGN MI_PAGE_META_ALIGN +#else +#define MI_ARENA_ALIGN MI_ARENA_SLICE_ALIGN +#endif // ------------------------------------------------------ // Arena's are large reserved areas of memory allocated from @@ -388,7 +403,10 @@ typedef uintptr_t mi_thread_free_t; // - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` // - Using `uint16_t` does not seem to slow things down -typedef struct mi_page_s { +typedef struct mi_page_s { + #if (MI_PAGE_META_ALIGNED) + _Atomic(struct mi_page_s*) self; + #endif _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) @@ -411,10 +429,13 @@ typedef struct mi_page_s { struct mi_page_s* next; // next page owned by the theap with the same `block_size` struct mi_page_s* prev; // previous page owned by the theap with the same `block_size` mi_memid_t memid; // const: provenance of the page memory - + #if (MI_ENCODE_FREELIST || MI_PADDING) uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary #endif + #if (MI_PAGE_META_ALIGNED && MI_INTPTR_SIZE==8) + uintptr_t padding; + #endif } mi_page_t; @@ -422,7 +443,7 @@ typedef struct mi_page_s { // Object sizes // ------------------------------------------------------ -#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // pages must be aligned on this for the page map. +#define MI_PAGE_ALIGN MI_ARENA_SLICE_ALIGN // page area's must be aligned on this for the page map. #define MI_PAGE_MIN_START_BLOCK_ALIGN MI_MAX_ALIGN_SIZE // minimal block alignment for the first block in a page (16b) #define MI_PAGE_MAX_START_BLOCK_ALIGN2 (4*MI_KiB) // maximal block alignment for "power of 2"-sized blocks (such that we guarantee natural alignment) #define MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4*MI_KiB) // also aligns any multiple of this size to avoid TLB misses. @@ -692,6 +713,7 @@ typedef struct mi_arena_s { mi_memid_t memid; // provenance of the memory area mi_subproc_t* subproc; // subprocess this arena belongs to (`this 'element-of' this->subproc->arenas`) size_t arena_idx; // index in the arenas array + void* start; // actual start of the arena area (the arena_s info may come later due to guard pages etc.) size_t slice_count; // total size of the area in arena slices (of `MI_ARENA_SLICE_SIZE`) size_t info_slices; // initial slices reserved for the arena bitmaps diff --git a/src/arena.c b/src/arena.c index d14824e69..9127e721b 100644 --- a/src/arena.c +++ b/src/arena.c @@ -90,7 +90,7 @@ static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page #endif size_t mi_arena_min_alignment(void) { - return MI_ARENA_SLICE_ALIGN; + return MI_ARENA_ALIGN; } size_t mi_arena_min_size(void) { @@ -138,7 +138,7 @@ static size_t mi_arena_size(mi_arena_t* arena) { // Start of the arena memory area static uint8_t* mi_arena_start(mi_arena_t* arena) { - return ((uint8_t*)arena); + return ((uint8_t*)arena->start); } // Start of a slice @@ -149,6 +149,10 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) { mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { mi_assert_internal(slice_index < arena->slice_count); + #if MI_PAGE_META_ALIGNED + mi_page_t* const page = _mi_ptr_page_align(mi_arena_slice_start(arena,slice_index)); // todo: optimize? + return page; + #else if (arena->pages_meta != NULL) { mi_page_t* const page = &arena->pages_meta[slice_index]; #if MI_PAGE_META_ALIGNED_FREE_SMALL @@ -157,9 +161,10 @@ mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { #else return page; #endif - } + } // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL) return (mi_page_t*)mi_arena_slice_start(arena,slice_index); + #endif } // Arena area @@ -523,7 +528,7 @@ static mi_decl_noinline void* mi_arenas_try_alloc( if (req_arena != NULL) return NULL; // don't create arena's while preloading (todo: or should we?) - if (_mi_preloading()) return NULL; + // if (_mi_preloading()) return NULL; // don't create arena's if OS allocation is disallowed if (mi_option_is_enabled(mi_option_disallow_os_alloc)) return NULL; @@ -848,6 +853,24 @@ static size_t mi_page_block_start(size_t block_size, bool os_align) // Free a page without modifying page_bin stats static void mi_arenas_page_free_prim(mi_page_t* page); +static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) { + if (memid_slice.memkind == MI_MEM_ARENA) { + mi_arena_t* const arena = memid_slice.mem.arena.arena; + #if MI_PAGE_META_ALIGN + MI_UNUSED_RELEASE(arena); + mi_assert_internal(arena->pages_meta==NULL); + return _mi_ptr_page_align0(slice_start); + #else + MI_UNUSED(slice_start); + if (arena->pages_meta != NULL) { + mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); + return &arena->pages_meta[memid_slice.mem.arena.slice_index]; + } + #endif + } + return NULL; +} + // Allocate a fresh page static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit) { @@ -872,30 +895,26 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou size_t block_start = 0; // allocate page meta info at the arena start? - if (memid.memkind == MI_MEM_ARENA) { - mi_arena_t* const arena = memid.mem.arena.arena; - if (arena->pages_meta != NULL) { - mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); - mi_page_t* const page_meta = &arena->pages_meta[memid.mem.arena.slice_index]; - mi_assert_internal(page_meta->block_size == 0); - #if MI_PAGE_META_ALIGNED_FREE_SMALL - // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, - // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) - if (block_size > MI_SMALL_SIZE_MAX) + mi_page_t* const page_meta = mi_arena_page_meta(memid,slice_start); + if (page_meta!=NULL) { + mi_assert_internal(page_meta->block_size == 0); + #if MI_PAGE_META_ALIGNED_FREE_SMALL + // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, + // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) + if (block_size > MI_SMALL_SIZE_MAX) + #endif + { + page = page_meta; + page_meta_is_separate = true; + block_start = 0; + #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) + #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ #endif - { - page = page_meta; - page_meta_is_separate = true; - block_start = 0; - #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) - #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ - #endif - if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { - block_start += block_size; - } - mi_assert_internal(page->block_size == 0); - _mi_memzero_aligned(page, sizeof(*page)); + if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { + block_start += block_size; } + mi_assert_internal(page->block_size == 0); + _mi_memzero_aligned(page, sizeof(*page)); } } if (page == NULL) { @@ -967,12 +986,31 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou page->heap = _mi_theap_heap(theap); mi_page_set_theap(page,theap); - // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)); + // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)) + + #if MI_PAGE_META_ALIGNED + mi_atomic_store_ptr_release(mi_page_t,&page->self,page); + if (page_meta_is_separate) { + if (slice_count > 1 && reserved > 1) { + for(size_t i = 1; i < slice_count; i++) { + mi_assert_internal(page[i].block_size == 0); + mi_atomic_store_ptr_release(mi_page_t,&page[i].self,page); + } + } + } + mi_page_t* pstart = _mi_ptr_page_align0(slice_start); + mi_assert_internal(pstart->self==page); + if (reserved>1) { + mi_page_t* pend = _mi_ptr_page_align0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); + mi_assert_internal(pend->self==page); + } + #endif mi_assert_internal(page->free==NULL); mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page)); mi_assert_internal(mi_page_slice_start(page) == slice_start); mi_assert_internal(mi_page_size(page) <= page_noguard_size); + // now register in the arena_pages if (arena_pages!=NULL) { @@ -1508,13 +1546,25 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { return size; } +static size_t mi_arena_page_meta_slices(void) { + #if MI_PAGE_META_ALIGNED + return _mi_divide_up(MI_PAGE_META_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); + #else + return 0; + #endif +} + +static mi_arena_t* mi_arena_info(void* area) { + return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_slices())); +} + static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) { if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); - const size_t base_size = _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); + const size_t base_size = mi_size_of_slices(mi_arena_page_meta_slices()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free - #if MI_PAGE_META_IS_SEPARATED + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_ALIGNED const size_t pages_size = slice_count * sizeof(mi_page_t); #else const size_t pages_size = 0; @@ -1561,7 +1611,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, int numa_node, bool exclusive, mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) { - mi_assert_internal(_mi_is_aligned(start,MI_ARENA_SLICE_ALIGN)); + mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGN)); mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE); if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now @@ -1580,9 +1630,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, // _mi_warning_message("cannot use OS memory since it is too large with respect to the maximum object size (size %zu MiB, meta-info slices %zu, maximum object slices are %zu)", mi_size_of_slices(slice_count)/MI_MiB, info_slices, MI_ARENA_MAX_CHUNK_OBJ_SLICES); // return NULL; // } - - mi_arena_t* arena = (mi_arena_t*)start; - + // commit & zero if needed if (!memid.initially_committed) { size_t commit_size = mi_size_of_slices(info_slices); @@ -1590,10 +1638,10 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, if (!memid.is_pinned) { commit_size -= _mi_os_secure_guard_page_size(); } bool ok = false; if (commit_fun != NULL) { - ok = (*commit_fun)(true /* commit */, arena, commit_size, NULL, commit_fun_arg); + ok = (*commit_fun)(true /* commit */, start, commit_size, NULL, commit_fun_arg); } else { - ok = _mi_os_commit(subproc, arena, commit_size, NULL); + ok = _mi_os_commit(subproc, start, commit_size, NULL); } if (!ok) { _mi_warning_message("unable to commit meta-data for OS memory"); @@ -1603,13 +1651,15 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, else if (!memid.is_pinned) { // if MI_SECURE, set a guard page at the end of the arena info // todo: this does not respect the commit_fun as the memid is of external memory - _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)arena + mi_size_of_slices(info_slices), memid); + _mi_os_secure_guard_page_set_before(subproc, (uint8_t*)start + mi_size_of_slices(info_slices), memid); } if (!memid.initially_zero) { - _mi_memzero(arena, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size()); + _mi_memzero(start, mi_size_of_slices(info_slices) - _mi_os_secure_guard_page_size()); } // init + mi_arena_t* arena = mi_arena_info(start); + arena->start = start; arena->subproc = subproc; arena->memid = memid; arena->is_exclusive = exclusive; @@ -1637,7 +1687,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); } - #if MI_PAGE_META_IS_SEPARATED + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_ALIGNED arena->pages_meta = (mi_page_t*)base; base += (slice_count * sizeof(mi_page_t)); #else @@ -1646,7 +1696,23 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena))); // reserve our meta info (and reserve slices outside the memory area) + #if MI_PAGE_META_ALIGNED + for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_COUNT) { + // set all free slices (and skip the slices reserved for the page meta info) + const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_slices()); + const size_t start_idx = (i + meta_slices); + size_t count = MI_PAGE_META_COUNT - meta_slices; + if (start_idx < arena->slice_count) { + if (count + start_idx > arena->slice_count) { + count = arena->slice_count - start_idx; + mi_assert_internal(count > 0); + } + mi_bbitmap_unsafe_setN(arena->slices_free, start_idx, count); + } + } + #else mi_bbitmap_unsafe_setN(arena->slices_free, info_slices /* start */, arena->slice_count - info_slices); + #endif if (memid.initially_committed) { mi_bitmap_unsafe_setN(arena->slices_committed, 0, arena->slice_count); } @@ -1760,7 +1826,7 @@ static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool com return ENOMEM; } mi_memid_t memid; - void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_SLICE_ALIGN, commit, allow_large, &memid); + void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGN, commit, allow_large, &memid); if (start == NULL) return ENOMEM; if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { _mi_os_free_ex(subproc, start, size, commit, memid); @@ -1875,6 +1941,11 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, if (bit_of_page > 0) { c = '-'; } // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } + #if MI_PAGE_META_ALIGNED + else if ((slice_index % MI_PAGE_META_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_slices()) { + { c = 'i'; color = MI_GRAY; } + } + #endif // else if (mi_bitmap_is_setN(arena->pages_purge, slice_index + bit, NULL)) { c = '*'; } else if (mi_bbitmap_is_setN(arena->slices_free, slice_index+bit,1)) { if (mi_bitmap_is_set(arena->slices_purge, slice_index + bit)) { c = '~'; color = MI_ORANGE; } diff --git a/src/free.c b/src/free.c index 91b91797c..f4d8810c4 100644 --- a/src/free.c +++ b/src/free.c @@ -175,7 +175,7 @@ static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); return NULL; } - mi_page_t* page = _mi_safe_ptr_page(p); + mi_page_t* page = _mi_ptr_page(p); if (p != NULL && page == NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } @@ -187,16 +187,13 @@ static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) // Free a block // Fast path written carefully to prevent register spilling on the stack -static mi_decl_forceinline void mi_free_ex(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) +static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) { - if mi_unlikely(page==NULL) { // page will be NULL if p==NULL - if (pblock_size!=NULL) { *pblock_size = 0; } - return; - } mi_assert_internal(p!=NULL && page!=NULL); if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } - const mi_threadid_t xtid = (_mi_prim_thread_id() ^ mi_page_xthread_id(page)); + const mi_threadid_t ptid = mi_page_xthread_id(page); + const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid); if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page mi_block_t* const block = mi_validate_block_from_ptr(page,p); @@ -218,13 +215,30 @@ static mi_decl_forceinline void mi_free_ex(void* p, mi_page_t* page, size_t* pbl } } +static mi_decl_forceinline void mi_free_ex(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) +{ + if mi_unlikely(page==NULL) { // page will be NULL if p==NULL + if (pblock_size!=NULL) { *pblock_size = 0; } + return; + } + mi_free_nonnull(p,page,pblock_size,allow_collect); +} + void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept { mi_free_ex(p, page, NULL, true); } -void mi_free(void* p) mi_attr_noexcept { - mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); +void mi_free(void* p) mi_attr_noexcept { + #if MI_PAGE_META_ALIGNED + mi_page_t* page = _mi_ptr_page_align0(p); + if mi_unlikely(page==NULL) return; + page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); + mi_assert_internal(page == _mi_safe_ptr_page(p)); + mi_free_nonnull(p, page, NULL, true); + #else + mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); mi_free_ex(p, page, NULL, true); + #endif } void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { @@ -236,7 +250,7 @@ void mi_free_small(void* p) mi_attr_noexcept { // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. // If we keep page info in front of the page area for small objects, we can find the info // just by aligning down the pointer instead of looking it up in the page map. - #if MI_PAGE_META_ALIGNED_FREE_SMALL + #if !MI_PAGE_META_ALIGNED && MI_PAGE_META_ALIGNED_FREE_SMALL #if MI_GUARDED #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL) ignored as MI_GUARDED is defined" mi_free(p); @@ -462,22 +476,22 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { void mi_free_size(void* p, size_t size) mi_attr_noexcept { MI_UNUSED_RELEASE(size); #if MI_DEBUG - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); - if (page==NULL) return; - mi_assert(p!=NULL); - const size_t usable = _mi_page_usable_size(page,p); - if (size > usable) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); - mi_free(p); - return; - } - else if (size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); - mi_free(p); - return; - } + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); + if (page==NULL) return; + mi_assert(p!=NULL); + const size_t usable = _mi_page_usable_size(page,p); + if (size > usable) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); + mi_free(p); + return; + } + else if (size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); + mi_free(p); + return; + } #endif - #if MI_OPT_FREE_SMALL + #if MI_OPT_FREE_SMALL && !MI_PAGE_META_ALIGNED if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } @@ -506,9 +520,6 @@ void mi_cfree(void* p) mi_attr_noexcept { if mi_likely(page!=NULL) { _mi_free_in_page(p,page); } - // if (mi_is_in_heap_region(p)) { - // mi_free(p); - // } } diff --git a/src/init.c b/src/init.c index 63f0a0e27..9a8d4f790 100644 --- a/src/init.c +++ b/src/init.c @@ -14,6 +14,9 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array static const mi_page_t mi_page_empty = { + #if MI_PAGE_META_ALIGNED + MI_ATOMIC_VAR_INIT(NULL), // self + #endif MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free 0, // used @@ -31,7 +34,10 @@ static const mi_page_t mi_page_empty = { NULL, NULL, // next, prev MI_MEMID_STATIC, // memid #if (MI_PADDING || MI_ENCODE_FREELIST) - { 0, 0 } // keys + { 0, 0 }, // keys + #endif + #if (MI_PAGE_META_ALIGNED && MI_INTPTR_SIZE==8) + 0, #endif }; diff --git a/src/os.c b/src/os.c index f89eada96..50a403609 100644 --- a/src/os.c +++ b/src/os.c @@ -140,7 +140,7 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; - #if (MI_SECURE>=1 || defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode + #if (MI_SECURE>=1 || !defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point const uintptr_t r = _mi_theap_random_next(theap); diff --git a/test/test-stress.c b/test/test-stress.c index 79938e19b..006edb76d 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -309,7 +309,7 @@ static void test_stress(mi_subproc_id_t subproc) { if ((n + 1) % 10 == 0) { printf("- iterations left: %3d\n", ITER - (n + 1)); #ifndef USE_STD_MALLOC - // mi_debug_show_arenas(); + mi_debug_show_arenas(); #endif //mi_collect(true); //mi_debug_show_arenas(); From 67b324e77de5c0f0d007d1eedbe3e282c9b015ef Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 09:07:38 -0700 Subject: [PATCH 005/280] wip: aligned page meta info for OS allocations --- include/mimalloc/internal.h | 6 +++- include/mimalloc/types.h | 2 +- src/arena.c | 72 ++++++++++++++++++++++++++++++------- src/init.c | 8 +++-- test/test-api.c | 4 +-- test/test-stress.c | 2 ++ 6 files changed, 76 insertions(+), 18 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 1f711dd15..44cda9e91 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -843,7 +843,11 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) { } static inline bool mi_page_meta_is_separated(const mi_page_t* page) { - #if MI_PAGE_META_IS_SEPARATED + #if MI_PAGE_META_ALIGNED + MI_UNUSED_RELEASE(page); + mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + return true; + #elif MI_PAGE_META_IS_SEPARATED // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); #else diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ec4799b3b..fd62e88af 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -432,7 +432,7 @@ typedef struct mi_page_s { #if (MI_ENCODE_FREELIST || MI_PADDING) uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary - #endif + #endif #if (MI_PAGE_META_ALIGNED && MI_INTPTR_SIZE==8) uintptr_t padding; #endif diff --git a/src/arena.c b/src/arena.c index 9127e721b..d5f7150ba 100644 --- a/src/arena.c +++ b/src/arena.c @@ -764,7 +764,7 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl } static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { - MI_UNUSED_RELEASE(block_size); + MI_UNUSED(block_size); mi_assert_internal(parena_pages!=NULL); *parena_pages = NULL; @@ -803,19 +803,56 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ // otherwise fall back to the OS if (start == NULL) { + #if MI_PAGE_META_ALIGNED + size_t page_offset; // offset in the block for the page area + uint8_t* os_start; + if (block_alignment < MI_PAGE_META_ALIGN) { + page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); + os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGN, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + else { + // if we allow alignment >= MI_PAGE_META_ALIGN we need to substract 1 from a pointer + // in _mi_ptr_page_align (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. + _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB); + errno = EINVAL; + return NULL; + // page_offset = MI_PAGE_META_ALIGN; + // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGN /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + } + if (os_start==NULL) return NULL; + // commit page info and page + const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + 1; + const size_t min_page_meta = min_page_count * sizeof(mi_page_t); + bool is_zero; + bool ok = mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */); + if (ok && commit) { + ok = mi_arena_commit(heap->subproc,req_arena,os_start + page_offset,alloc_size,NULL,0); + } + if (!ok) { _mi_os_free(heap->subproc,os_start,alloc_size+page_offset,*memid); return NULL; } + if (!is_zero && !memid->initially_zero) { + _mi_memzero_aligned(os_start,min_page_meta); + } + start = os_start + page_offset; + mi_assert_internal(_mi_is_aligned(start,block_alignment)); + mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGN)); + mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGN) == os_start); + mi_assert_internal((uint8_t*)_mi_ptr_page_align0(start) < os_start + min_page_meta); + memid->initially_committed = true; // so we don't commit again + #else if (os_align) { // note: slice_count already includes the page mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment)); start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid); + mi_assert_internal(_mi_is_aligned(start + page_alignment, block_alignment)); } else { start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid); } + #endif } if (start == NULL) return NULL; mi_assert_internal(_mi_is_aligned(start, MI_PAGE_ALIGN)); - mi_assert_internal(!os_align || _mi_is_aligned(start + page_alignment, block_alignment)); return start; } @@ -854,20 +891,20 @@ static size_t mi_page_block_start(size_t block_size, bool os_align) static void mi_arenas_page_free_prim(mi_page_t* page); static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) { - if (memid_slice.memkind == MI_MEM_ARENA) { - mi_arena_t* const arena = memid_slice.mem.arena.arena; - #if MI_PAGE_META_ALIGN - MI_UNUSED_RELEASE(arena); - mi_assert_internal(arena->pages_meta==NULL); + #if MI_PAGE_META_ALIGNED + if (memid_slice.memkind == MI_MEM_ARENA || mi_memid_is_os(memid_slice)) { return _mi_ptr_page_align0(slice_start); - #else + } + #else + if (memid_slice.memkind == MI_MEM_ARENA) { MI_UNUSED(slice_start); + mi_arena_t* const arena = memid_slice.mem.arena.arena; if (arena->pages_meta != NULL) { mi_assert_internal(MI_PAGE_META_IS_SEPARATED!=0); return &arena->pages_meta[memid_slice.mem.arena.slice_index]; } - #endif } + #endif return NULL; } @@ -919,6 +956,9 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou } if (page == NULL) { // put page meta info in front of the slice + #if MI_PAGE_META_ALIGNED + mi_assert(false); + #endif page = (mi_page_t*)slice_start; block_start = mi_page_block_start(block_size, os_align); } @@ -991,13 +1031,16 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou #if MI_PAGE_META_ALIGNED mi_atomic_store_ptr_release(mi_page_t,&page->self,page); if (page_meta_is_separate) { - if (slice_count > 1 && reserved > 1) { - for(size_t i = 1; i < slice_count; i++) { + if (slice_count > 1) { + // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice + const size_t max_page_count = (reserved==1 && slice_count > 2 ? 2 : slice_count); + for(size_t i = 1; i < max_page_count; i++) { mi_assert_internal(page[i].block_size == 0); mi_atomic_store_ptr_release(mi_page_t,&page[i].self,page); } } } + #if MI_DEBUG>1 mi_page_t* pstart = _mi_ptr_page_align0(slice_start); mi_assert_internal(pstart->self==page); if (reserved>1) { @@ -1005,6 +1048,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou mi_assert_internal(pend->self==page); } #endif + #endif mi_assert_internal(page->free==NULL); mi_assert_internal(page_meta_is_separate == mi_page_meta_is_separated(page)); @@ -1069,8 +1113,12 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c // Allocate a page containing one block (very large, or with large alignment) static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) { + #if MI_PAGE_META_ALIGNED + const size_t info_size = 0; + #else const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); - const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + const size_t info_size = (os_align ? MI_PAGE_ALIGN : mi_page_info_size()); + #endif #if MI_SECURE < 2 const size_t slice_count = mi_slice_count_of_size(info_size + block_size); #else diff --git a/src/init.c b/src/init.c index 9a8d4f790..5a668835f 100644 --- a/src/init.c +++ b/src/init.c @@ -37,7 +37,7 @@ static const mi_page_t mi_page_empty = { { 0, 0 }, // keys #endif #if (MI_PAGE_META_ALIGNED && MI_INTPTR_SIZE==8) - 0, + 0, // padding #endif }; @@ -575,6 +575,10 @@ static void mi_process_init_once(void) { mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); } } + + #if MI_PAGE_META_ALIGNED + mi_assert_internal((sizeof(mi_page_t)%MI_MAX_ALIGN_SIZE) == 0); // or page->ma_offset might not work + #endif } // Initialize the process; called by thread_init or the process loader @@ -636,7 +640,7 @@ static void mi_process_done_once(void) { _mi_tls_slots_done(); _mi_subproc_main_done(); _mi_allocator_done(); - _mi_verbose_message("process done\n"); // : 0x%zx\n", mi_process_tld_main.thread_id); + _mi_verbose_message("process done %zu\n", sizeof(mi_page_t)); // : 0x%zx\n", mi_process_tld_main.thread_id); os_preloading = true; // don't call the C runtime anymore } diff --git a/test/test-api.c b/test/test-api.c index 2d2b988de..fb5caf6ff 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -110,7 +110,7 @@ int main(void) { }; #if MI_INTPTR_BITS > 32 CHECK_BODY("malloc-free-invalid-low") { - mi_free((void*)(MI_ZU(0x0000000003990080))); // issue #1087 + mi_cfree((void*)(MI_ZU(0x0000000003990080))); // issue #1087 }; #endif CHECK_BODY("calloc-overflow") { @@ -240,7 +240,7 @@ int main(void) { void* p[8]; const int max_align_shift = #if SIZE_MAX > UINT32_MAX - 28 + 28 /* up to 64 MiB alignment */ #else 20 #endif diff --git a/test/test-stress.c b/test/test-stress.c index 006edb76d..63a9bdc27 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -462,6 +462,8 @@ int main(int argc, char** argv) { // fputs(json,stderr); // mi_free(json); //} + #else + mi_debug_show_arenas(); #endif mi_collect(true); mi_stats_print(NULL); From 5fdf2e934df748c71fb0182794f1bc1af8b9355a Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 11:18:15 -0700 Subject: [PATCH 006/280] wip: aligned page meta data --- CMakeLists.txt | 22 ++++++++++--- include/mimalloc/bits.h | 2 +- include/mimalloc/internal.h | 6 ++-- include/mimalloc/types.h | 25 +++++++++----- src/arena.c | 66 +++++++++++++++++++++++++------------ src/free.c | 24 ++++++++------ src/init.c | 6 ++-- src/stats.c | 11 ++++--- test/test-stress.c | 2 -- 9 files changed, 106 insertions(+), 58 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 1d0c01d21..fcee31ace 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,7 +39,7 @@ option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen option(MI_LIBC_MUSL "Enable this when linking with musl libc" OFF) # advanced -option(MI_OPT_FREE_SMALL "Use optimized `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)" OFF) +option(MI_OPT_FREE "Use aligned `mi_free(_small)` without pagemap lookup (can only be used if SECURE mode is off)" ON) option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG or SECURE mode, or with Valgrind/ASAN)" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) @@ -67,6 +67,7 @@ option(MI_TLS_RECURSE_GUARD "Force a recursion check on thread local access (fo option(MI_WIN_USE_FLS "Use Fiber local storage on Windows to detect thread termination (deprecated)" OFF) option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) +option(MI_OPT_FREE_SMALL "Deprecated: Use aligned `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)" OFF) include(CheckLinkerFlag) # requires cmake 3.18 include(CheckIncludeFiles) @@ -397,14 +398,27 @@ elseif(MI_OVERRIDE AND NOT MI_NO_FREE_IS_CHECKED) list(APPEND mi_cflags_dynamic -DMI_FREE_IS_CHECKED=1) endif() -if(NOT MI_OPT_FREE_SMALL AND NOT MI_DEBUG AND NOT MI_GUARDED AND NOT MI_SECURE) +if(MI_OPT_FREE) + if("$CMAKE_SYSTEM_NAME" STREQUAL "WASI") # todo: also test for EMSCRIPTEN? or any 32-bit host? + message(STATUS "Disregarding MI_OPT_FREE as the host does not support large alignment well (WASI) (MI_OPT_FREE=OFF)") + elseif(MI_SECURE) + message(STATUS "Cannot enable MI_OPT_FREE in secure mode (MI_OPT_FREE=OFF)") + else() + message(STATUS "Use aligned `mi_free(_small)` to avoid a pagemap lookup (MI_OPT_FREE=ON)") + list(APPEND mi_defines MI_OPT_FREE=1) + endif() +endif() + +if(NOT MI_OPT_FREE_SMALL AND NOT MI_DEBUG AND NOT MI_GUARDED AND NOT MI_SECURE AND NOT MI_OPT_FREE) set(MI_OPT_FREE_SMALL ON) endif() if(MI_OPT_FREE_SMALL) - if(MI_GUARDED OR MI_SECURE) + if(MI_GUARDED OR MI_SECURE) message(STATUS "Cannot enable MI_OPT_FREE_SMALL in secure or guarded mode (MI_OPT_FREE_SMALL=OFF)") + elseif(MI_OPT_FREE) + message(STATUS "Cannot enable MI_OPT_FREE_SMALL with MI_OPT_FREE (MI_OPT_FREE_SMALL=OFF)") else() - message(STATUS "Optimize `mi_free_small` to not need a pagemap lookup (MI_OPT_FREE_SMALL=ON)") + message(STATUS "Use aligned `mi_free_small` to avoid a pagemap lookup (MI_OPT_FREE_SMALL=ON)") list(APPEND mi_defines MI_OPT_FREE_SMALL=1) endif() endif() diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index 8c3e4710f..55661bc40 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -138,7 +138,7 @@ typedef int32_t mi_ssize_t; // use a flat page-map or a 2-level one #ifndef MI_PAGE_MAP_FLAT -#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && MI_SECURE==0 // && !MI_PAGE_META_IS_SEPARATED +#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_OPT_FREE #define MI_PAGE_MAP_FLAT 1 #else #define MI_PAGE_MAP_FLAT 0 diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 44cda9e91..c11a2ba86 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -761,7 +761,7 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { #endif -#if MI_PAGE_META_ALIGNED +#if MI_PAGE_META_IS_ALIGNED static inline mi_page_t* _mi_ptr_page_align0(const void* p) { mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGN); const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; @@ -779,7 +779,7 @@ static inline mi_page_t* _mi_ptr_page(const void* p) { mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); #if MI_SECURE // || MI_FREE_IS_CHECKED return _mi_checked_ptr_page(p); - #elif MI_PAGE_META_ALIGNED + #elif MI_PAGE_META_IS_ALIGNED mi_page_t* const page = _mi_ptr_page_align(p); #if MI_DEBUG mi_page_t* const cpage = _mi_checked_ptr_page(p); @@ -843,7 +843,7 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) { } static inline bool mi_page_meta_is_separated(const mi_page_t* page) { - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED MI_UNUSED_RELEASE(page); mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); return true; diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index fd62e88af..f7feec2e0 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -123,7 +123,7 @@ terms of the MIT license. A copy of the license can be found in the file // Place page meta info at the start of the page area or keep it separate? // Separate keeps the page info at the arena start (default) which is more secure // and reduces wasted space due to alignment and block sizes. -// (but also reserves more memory up front (about 2MiB per GiB)) +// (but if MI_OPT_FREE is false, also reserves more memory up front (about 2MiB per GiB)) #if !defined(MI_PAGE_META_IS_SEPARATED) #if MI_PAGE_MAP_FLAT #define MI_PAGE_META_IS_SEPARATED 0 @@ -132,11 +132,18 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -#define MI_OPT_META_ALIGNED 1 -#if MI_OPT_META_ALIGNED && MI_PAGE_META_IS_SEPARATED +// We can choose to page meta info aligned at the start of every MI_PAGE_META_CHUNKS +// This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup. +// This only works if valid pointers are passed to `mi_free` though. However, checked +// free `mi_cfree` can still uses the page map to validate pointers. +#if MI_OPT_FREE +#if MI_PAGE_META_IS_SEPARATED #define MI_PAGE_META_ALIGNED_FREE_SMALL 0 -#define MI_PAGE_META_ALIGNED 1 -#define MI_PAGE_META_CHUNKS 8 +#define MI_PAGE_META_IS_ALIGNED 1 +#define MI_PAGE_META_CHUNKS MI_INTPTR_SIZE +#else +#warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" +#endif #endif // We can choose to only put page info of small pages at the start of the page area. @@ -225,9 +232,9 @@ terms of the MIT license. A copy of the license can be found in the file // Minimal commit for a page on-demand commit #define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ -#if MI_PAGE_META_ALIGNED +#if MI_PAGE_META_IS_ALIGNED #define MI_PAGE_META_COUNT (MI_PAGE_META_CHUNKS * MI_BCHUNK_BITS) -#define MI_PAGE_META_ALIGN (MI_PAGE_META_COUNT * MI_ARENA_SLICE_SIZE) +#define MI_PAGE_META_ALIGN (MI_PAGE_META_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit) #define MI_ARENA_ALIGN MI_PAGE_META_ALIGN #else #define MI_ARENA_ALIGN MI_ARENA_SLICE_ALIGN @@ -404,7 +411,7 @@ typedef uintptr_t mi_thread_free_t; // - Using `uint16_t` does not seem to slow things down typedef struct mi_page_s { - #if (MI_PAGE_META_ALIGNED) + #if (MI_PAGE_META_IS_ALIGNED) _Atomic(struct mi_page_s*) self; #endif _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) @@ -433,7 +440,7 @@ typedef struct mi_page_s { #if (MI_ENCODE_FREELIST || MI_PADDING) uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary #endif - #if (MI_PAGE_META_ALIGNED && MI_INTPTR_SIZE==8) + #if (MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8) uintptr_t padding; #endif } mi_page_t; diff --git a/src/arena.c b/src/arena.c index d5f7150ba..3a6af7723 100644 --- a/src/arena.c +++ b/src/arena.c @@ -149,7 +149,7 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) { mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { mi_assert_internal(slice_index < arena->slice_count); - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED mi_page_t* const page = _mi_ptr_page_align(mi_arena_slice_start(arena,slice_index)); // todo: optimize? return page; #else @@ -217,6 +217,13 @@ static size_t mi_page_full_size(mi_page_t* page) { } } +static size_t mi_arena_page_meta_slices(void) { + #if MI_PAGE_META_IS_ALIGNED + return _mi_divide_up(MI_PAGE_META_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); + #else + return 0; + #endif +} /* ----------------------------------------------------------- Arena Allocation @@ -803,7 +810,7 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ // otherwise fall back to the OS if (start == NULL) { - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED size_t page_offset; // offset in the block for the page area uint8_t* os_start; if (block_alignment < MI_PAGE_META_ALIGN) { @@ -856,6 +863,8 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ return start; } +#if !MI_PAGE_META_IS_ALIGNED +// Only used for non-separate pages static size_t mi_page_block_start(size_t block_size, bool os_align) { size_t offset; @@ -886,14 +895,34 @@ static size_t mi_page_block_start(size_t block_size, bool os_align) } return _mi_align_up(offset,MI_MAX_ALIGN_SIZE); } +#endif // Free a page without modifying page_bin stats static void mi_arenas_page_free_prim(mi_page_t* page); static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_start) { - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED if (memid_slice.memkind == MI_MEM_ARENA || mi_memid_is_os(memid_slice)) { - return _mi_ptr_page_align0(slice_start); + // ensure the meta data is committed + if (memid_slice.memkind == MI_MEM_ARENA) { + mi_arena_t* const arena = memid_slice.mem.arena.arena; + uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGN); + mi_assert_internal(meta_slices >= mi_arena_start(arena)); + const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; + if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { + // try to commit now + const size_t meta_slice_count = mi_arena_page_meta_slices(); + const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; + if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, commit_size /* dont count? */)) { + // if the commit fails return NULL + return NULL; + } + // set the commit bits + mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL); + } + } + mi_page_t* const page_meta = _mi_ptr_page_align0(slice_start); + return page_meta; } #else if (memid_slice.memkind == MI_MEM_ARENA) { @@ -955,12 +984,15 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou } } if (page == NULL) { + #if MI_PAGE_META_IS_ALIGNED + // can only happen on failing to commit the page meta info + _mi_arenas_free(_mi_theap_subproc(theap),slice_start,alloc_size,memid); + return NULL; + #else // put page meta info in front of the slice - #if MI_PAGE_META_ALIGNED - mi_assert(false); - #endif page = (mi_page_t*)slice_start; block_start = mi_page_block_start(block_size, os_align); + #endif } mi_assert_internal(block_start % MI_MAX_ALIGN_SIZE == 0); @@ -1028,7 +1060,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou mi_page_set_theap(page,theap); // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)) - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED mi_atomic_store_ptr_release(mi_page_t,&page->self,page); if (page_meta_is_separate) { if (slice_count > 1) { @@ -1113,7 +1145,7 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c // Allocate a page containing one block (very large, or with large alignment) static mi_page_t* mi_arenas_page_singleton_alloc(mi_theap_t* theap, size_t block_size, size_t block_alignment) { - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED const size_t info_size = 0; #else const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); @@ -1594,14 +1626,6 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { return size; } -static size_t mi_arena_page_meta_slices(void) { - #if MI_PAGE_META_ALIGNED - return _mi_divide_up(MI_PAGE_META_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); - #else - return 0; - #endif -} - static mi_arena_t* mi_arena_info(void* area) { return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_slices())); } @@ -1612,7 +1636,7 @@ static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_bas const size_t base_size = mi_size_of_slices(mi_arena_page_meta_slices()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free - #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED const size_t pages_size = slice_count * sizeof(mi_page_t); #else const size_t pages_size = 0; @@ -1735,7 +1759,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, for (size_t i = 0; i < MI_ARENA_BIN_COUNT; i++) { arena->pages_main.pages_abandoned[i] = mi_arena_bitmap_init(slice_count, &base); } - #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED arena->pages_meta = (mi_page_t*)base; base += (slice_count * sizeof(mi_page_t)); #else @@ -1744,7 +1768,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, mi_assert_internal(mi_size_of_slices(info_slices) >= (size_t)(base - mi_arena_start(arena))); // reserve our meta info (and reserve slices outside the memory area) - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_COUNT) { // set all free slices (and skip the slices reserved for the page meta info) const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_slices()); @@ -1989,7 +2013,7 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, if (bit_of_page > 0) { c = '-'; } // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED else if ((slice_index % MI_PAGE_META_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_slices()) { { c = 'i'; color = MI_GRAY; } } diff --git a/src/free.c b/src/free.c index f4d8810c4..6ca0a468d 100644 --- a/src/free.c +++ b/src/free.c @@ -229,15 +229,19 @@ void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept { } void mi_free(void* p) mi_attr_noexcept { - #if MI_PAGE_META_ALIGNED - mi_page_t* page = _mi_ptr_page_align0(p); - if mi_unlikely(page==NULL) return; - page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); - mi_assert_internal(page == _mi_safe_ptr_page(p)); - mi_free_nonnull(p, page, NULL, true); + #if MI_PAGE_META_IS_ALIGNED + mi_page_t* page = _mi_ptr_page_align0(p); + if mi_unlikely(page==NULL) return; + #if MI_DEBUG + mi_page_t* const cpage = _mi_safe_ptr_page(p); + mi_assert_internal(cpage!=NULL); + #endif + page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); + mi_assert_internal(page == cpage); + mi_free_nonnull(p, page, NULL, true); #else - mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); - mi_free_ex(p, page, NULL, true); + mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); + mi_free_ex(p, page, NULL, true); #endif } @@ -250,7 +254,7 @@ void mi_free_small(void* p) mi_attr_noexcept { // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. // If we keep page info in front of the page area for small objects, we can find the info // just by aligning down the pointer instead of looking it up in the page map. - #if !MI_PAGE_META_ALIGNED && MI_PAGE_META_ALIGNED_FREE_SMALL + #if !MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_ALIGNED_FREE_SMALL #if MI_GUARDED #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL) ignored as MI_GUARDED is defined" mi_free(p); @@ -491,7 +495,7 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { return; } #endif - #if MI_OPT_FREE_SMALL && !MI_PAGE_META_ALIGNED + #if MI_OPT_FREE_SMALL && !MI_OPT_FREE if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } diff --git a/src/init.c b/src/init.c index 5a668835f..068ff7412 100644 --- a/src/init.c +++ b/src/init.c @@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file // Empty page used to initialize the small free pages array static const mi_page_t mi_page_empty = { - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED MI_ATOMIC_VAR_INIT(NULL), // self #endif MI_ATOMIC_VAR_INIT(0), // xthread_id @@ -36,7 +36,7 @@ static const mi_page_t mi_page_empty = { #if (MI_PADDING || MI_ENCODE_FREELIST) { 0, 0 }, // keys #endif - #if (MI_PAGE_META_ALIGNED && MI_INTPTR_SIZE==8) + #if (MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8) 0, // padding #endif }; @@ -576,7 +576,7 @@ static void mi_process_init_once(void) { } } - #if MI_PAGE_META_ALIGNED + #if MI_PAGE_META_IS_ALIGNED mi_assert_internal((sizeof(mi_page_t)%MI_MAX_ALIGN_SIZE) == 0); // or page->ma_offset might not work #endif } diff --git a/src/stats.c b/src/stats.c index de1c24393..6aaba3ddf 100644 --- a/src/stats.c +++ b/src/stats.c @@ -416,12 +416,13 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stat_print_ex(&stats->heaps, "heaps", 0, out, arg, ""); mi_stat_counter_print(&stats->heaps_delete_wait, "heap waits", out, arg); _mi_fprintf(out, arg, "\n"); - - mi_print_header("process", out, arg); - mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); - _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); - mi_process_info_print_out(out, arg); } + + mi_print_header("process", out, arg); + mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); + _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); + mi_process_info_print_out(out, arg); + _mi_fprintf(out, arg, "\n"); } diff --git a/test/test-stress.c b/test/test-stress.c index 63a9bdc27..006edb76d 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -462,8 +462,6 @@ int main(int argc, char** argv) { // fputs(json,stderr); // mi_free(json); //} - #else - mi_debug_show_arenas(); #endif mi_collect(true); mi_stats_print(NULL); From 70c4250b7797f72368c6f425ecf00a9e0f971491 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 12:09:20 -0700 Subject: [PATCH 007/280] fix padding for 32-bit --- ide/vs2022/mimalloc-lib.vcxproj | 16 ++++++++-------- ide/vs2022/mimalloc-override-dll.vcxproj | 16 ++++++++-------- include/mimalloc/internal.h | 2 +- include/mimalloc/types.h | 8 ++++++-- src/init.c | 6 ++++-- 5 files changed, 27 insertions(+), 21 deletions(-) diff --git a/ide/vs2022/mimalloc-lib.vcxproj b/ide/vs2022/mimalloc-lib.vcxproj index 216f418c0..cba6f88f0 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj +++ b/ide/vs2022/mimalloc-lib.vcxproj @@ -174,7 +174,7 @@ true Default ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -194,7 +194,7 @@ true Default ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -222,7 +222,7 @@ true Default ../../include - MI_DEBUG=3;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=3;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -250,7 +250,7 @@ true Default ../../include - MI_DEBUG=3;MI_GUARDED=0;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=3;MI_GUARDED=0;%(PreprocessorDefinitions); CompileAsCpp false stdcpp20 @@ -278,7 +278,7 @@ true Default ../../include - %(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -307,7 +307,7 @@ true Default ../../include - %(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -343,7 +343,7 @@ true Default ../../include - %(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -380,7 +380,7 @@ true Default ../../include - %(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj b/ide/vs2022/mimalloc-override-dll.vcxproj index 18fce6938..25269e3d8 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj +++ b/ide/vs2022/mimalloc-override-dll.vcxproj @@ -170,7 +170,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false CompileAsCpp @@ -201,7 +201,7 @@ true true ../../include - MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false CompileAsCpp @@ -232,7 +232,7 @@ true true ../../include - MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false CompileAsCpp @@ -263,7 +263,7 @@ true true ../../include - MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); + MI_OPT_FREE=1;MI_DEBUG=4;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions); MultiThreadedDebugDLL false CompileAsCpp @@ -295,7 +295,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -330,7 +330,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -365,7 +365,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false @@ -401,7 +401,7 @@ true true ../../include - MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG + MI_OPT_FREE=1;MI_SHARED_LIB;MI_SHARED_LIB_EXPORT;MI_MALLOC_OVERRIDE;%(PreprocessorDefinitions);NDEBUG AssemblyAndSourceCode $(IntDir) false diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index c11a2ba86..e717e9676 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -722,7 +722,7 @@ typedef struct mi_page_map_s { size_t reserved_size; // full reserved size (mi_page_map_t + submaps) mi_memid_t memid; // provenance mi_lock_t lock; // used when allocating new submaps - mi_decl_cache_align _Atomic(mi_submap_t) submaps[1]; + _Atomic(mi_submap_t) submaps[1]; } mi_page_map_t; extern mi_decl_hidden _Atomic(mi_page_map_t*) __mi_page_map; diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index f7feec2e0..06ed57d28 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -440,8 +440,12 @@ typedef struct mi_page_s { #if (MI_ENCODE_FREELIST || MI_PADDING) uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary #endif - #if (MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8) - uintptr_t padding; + #if MI_PAGE_META_IS_ALIGNED + #if MI_INTPTR_SIZE==8 || (MI_INTPTR_SIZE==4 && !MI_ENCODE_FREELIST && !MI_PADDING) + uintptr_t padding[1]; + #elif MI_INTPTR_SIZE==4 + uintptr_t padding[3]; + #endif #endif } mi_page_t; diff --git a/src/init.c b/src/init.c index 068ff7412..53262e968 100644 --- a/src/init.c +++ b/src/init.c @@ -36,8 +36,10 @@ static const mi_page_t mi_page_empty = { #if (MI_PADDING || MI_ENCODE_FREELIST) { 0, 0 }, // keys #endif - #if (MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8) - 0, // padding + #if (MI_INTPTR_SIZE==4 && !MI_ENCODE_FREELIST && !MI_PADDING) + { 0 }, // padding + #elif (MI_INTPTR_SIZE==4) + { 0, 0, 0 }, #endif }; From 1e748098bfa1ca1b8358d9163ece7416c3b01cfa Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 12:25:09 -0700 Subject: [PATCH 008/280] fix 32-bit compilation --- include/mimalloc/types.h | 1 + src/free.c | 5 ++++- src/init.c | 6 ++++-- 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 06ed57d28..ea33c0b21 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -441,6 +441,7 @@ typedef struct mi_page_s { uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary #endif #if MI_PAGE_META_IS_ALIGNED + // with alignment (MI_OPT_FREE) we need sizeof(mi_page_t) to be a multiple of MI_MAX_ALIGN_SIZE #if MI_INTPTR_SIZE==8 || (MI_INTPTR_SIZE==4 && !MI_ENCODE_FREELIST && !MI_PADDING) uintptr_t padding[1]; #elif MI_INTPTR_SIZE==4 diff --git a/src/free.c b/src/free.c index 6ca0a468d..68a9783fc 100644 --- a/src/free.c +++ b/src/free.c @@ -107,7 +107,10 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { const size_t diff = (uint8_t*)p - mi_page_start(page); const size_t block_size = mi_page_block_size(page); - const size_t adjust = (_mi_is_power_of_two(block_size) ? diff & (block_size - 1) : diff % block_size); + size_t adjust = diff & (block_size - 1); + if mi_unlikely(!_mi_is_power_of_two(block_size)) { + adjust = diff % block_size; + } return (mi_block_t*)((uintptr_t)p - adjust); } diff --git a/src/init.c b/src/init.c index 53262e968..c4ed58698 100644 --- a/src/init.c +++ b/src/init.c @@ -36,11 +36,13 @@ static const mi_page_t mi_page_empty = { #if (MI_PADDING || MI_ENCODE_FREELIST) { 0, 0 }, // keys #endif - #if (MI_INTPTR_SIZE==4 && !MI_ENCODE_FREELIST && !MI_PADDING) + #if MI_PAGE_META_IS_ALIGNED + #if MI_INTPTR_SIZE==8 || (MI_INTPTR_SIZE==4 && !MI_ENCODE_FREELIST && !MI_PADDING) { 0 }, // padding - #elif (MI_INTPTR_SIZE==4) + #elif MI_INTPTR_SIZE==4 { 0, 0, 0 }, #endif + #endif }; #define MI_PAGE_EMPTY() ((mi_page_t*)&mi_page_empty) From ad6c7e6bb6976ba5539111fea448e8328fc9c246 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 12:58:31 -0700 Subject: [PATCH 009/280] enable Zacas extension with -DMI_OPT_ARCH=ON on riscV --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fcee31ace..3356f48c6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -399,8 +399,8 @@ elseif(MI_OVERRIDE AND NOT MI_NO_FREE_IS_CHECKED) endif() if(MI_OPT_FREE) - if("$CMAKE_SYSTEM_NAME" STREQUAL "WASI") # todo: also test for EMSCRIPTEN? or any 32-bit host? - message(STATUS "Disregarding MI_OPT_FREE as the host does not support large alignment well (WASI) (MI_OPT_FREE=OFF)") + if(CMAKE_SYSTEM_NAME STREQUAL "WASI") # todo: also test for EMSCRIPTEN? or any 32-bit host? + message(STATUS "Disregarding MI_OPT_FREE as the target does not support large alignment well (WASI) (MI_OPT_FREE=OFF)") elseif(MI_SECURE) message(STATUS "Cannot enable MI_OPT_FREE in secure mode (MI_OPT_FREE=OFF)") else() @@ -591,6 +591,8 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM set(MI_OPT_ARCH_FLAGS "-march=haswell;-mavx2") # fast bit scan (since 2013) elseif(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH_FLAGS "-march=armv8.1-a") # fast atomics (since 2016) + elseif(MI_ARCH STREQUAL "riscv64") + set(MI_OPT_ARCH_FLAGS "-march=rv64gc_zacas") # fast atomics (since 2023) endif() endif() endif() From c21535b2371ced7c99adb4919529d39a82c3f81f Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 15:33:05 -0700 Subject: [PATCH 010/280] use multi value option for MI_FREE_IS_CHECKED --- CMakeLists.txt | 30 +++++++++++++++++++++--------- include/mimalloc/internal.h | 12 ++++++------ 2 files changed, 27 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3356f48c6..3bee0c2bd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -13,7 +13,9 @@ option(MI_GUARDED "Build with guard pages behind certain object alloca option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016))" OFF) option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) -option(MI_FREE_IS_CHECKED "Always check for invalid pointers in 'free' (always enabled in debug and secure builds, or when MI_OVERRIDE is ON)" OFF) + +set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': ALWAYS, SHARED, NEVER, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") +set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "DEFAULT" "NEVER" "SHARED" "ALWAYS") option(MI_DEBUG "Enable assertion checks (enabled by default in a debug build)" OFF) option(MI_DEBUG_INTERNAL "Enable assertion and internal invariant checks (enabled by default in a debug build)" OFF) @@ -52,7 +54,6 @@ option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead option(MI_NO_USE_CXX "Use plain C compilation (has priority over MI_USE_CXX)" OFF) option(MI_NO_OPT_ARCH "Do not use architecture specific optimizations (like '-march=armv8.1-a' for example) (has priority over MI_OPT_ARCH)" OFF) option(MI_NO_DEBUG "Disable all assertion checks even in debug builds" OFF) -option(MI_NO_FREE_IS_CHECKED "Always disable checking for invalid pointers in release builds" OFF) # experimental option(MI_WIN_INIT_USE_RAW_DLLMAIN "Use the raw DLL main entry point for mimalloc initialization; can be more robust but can also lead to link errors with other libraries" OFF) @@ -387,22 +388,33 @@ else() endif() endif() -if(MI_DEBUG OR MI_SECURE) - set(MI_FREE_IS_CHECKED ON) +if(MI_FREE_IS_CHECKED STREQUAL "DEFAULT") + if (MI_DEBUG OR MI_SECURE) + set(MI_FREE_IS_CHECKED ALWAYS) + else() + set(MI_FREE_IS_CHECKED NEVER) + endif() endif() -if(MI_FREE_IS_CHECKED) - message(STATUS "Check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=ON)") + +if(MI_FREE_IS_CHECKED MATCHES "ALWAYS|ON") + message(STATUS "Check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=ALWAYS)") list(APPEND mi_defines MI_FREE_IS_CHECKED=1) -elseif(MI_OVERRIDE AND NOT MI_NO_FREE_IS_CHECKED) - message(STATUS "Check validity of pointers passed to 'mi_free' only in the dynamic link library (MI_OVERRIDE=ON, MI_FREE_IS_CHECKED=OFF)") +elseif(MI_FREE_IS_CHECKED STREQUAL "SHARED") + message(STATUS "Check validity of pointers passed to 'mi_free' only in the dynamic link library (MI_FREE_IS_CHECKED=SHARED)") list(APPEND mi_cflags_dynamic -DMI_FREE_IS_CHECKED=1) +elseif(MI_FREE_IS_CHECKED MATCHES "NEVER|OFF") + message(STATUS "Do not check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=NEVER)") +else() + message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, NEVER, SHARED, or ALWAYS)") endif() if(MI_OPT_FREE) if(CMAKE_SYSTEM_NAME STREQUAL "WASI") # todo: also test for EMSCRIPTEN? or any 32-bit host? message(STATUS "Disregarding MI_OPT_FREE as the target does not support large alignment well (WASI) (MI_OPT_FREE=OFF)") elseif(MI_SECURE) - message(STATUS "Cannot enable MI_OPT_FREE in secure mode (MI_OPT_FREE=OFF)") + message(STATUS "Disregarding MI_OPT_FREE in secure mode (MI_SECURE=ON, MI_OPT_FREE=OFF)") + elseif(MI_FREE_IS_CHECKED MATCHES "ALWAYS|ON") + message(STATUS "Disregarding MI_OPT_FREE as pointers are always checked for validity (MI_FREE_IS_CHECKED=ALWAYS, MI_OPT_FREE=OFF)") else() message(STATUS "Use aligned `mi_free(_small)` to avoid a pagemap lookup (MI_OPT_FREE=ON)") list(APPEND mi_defines MI_OPT_FREE=1) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index e717e9676..c7c852066 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -762,6 +762,8 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { #endif #if MI_PAGE_META_IS_ALIGNED +// if the page meta data is aligned in front of pages we can find it efficiently +// without needing to go through the page map (for valid pointers). static inline mi_page_t* _mi_ptr_page_align0(const void* p) { mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGN); const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; @@ -771,23 +773,21 @@ static inline mi_page_t* _mi_ptr_page_align0(const void* p) { static inline mi_page_t* _mi_ptr_page_align(const void* p) { mi_page_t* const page = _mi_ptr_page_align0(p); - return (page==NULL ? NULL : mi_atomic_load_relaxed(&page->self)); + if mi_unlikely(page==NULL) return NULL; + return mi_atomic_load_relaxed(&page->self); } #endif static inline mi_page_t* _mi_ptr_page(const void* p) { mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); - #if MI_SECURE // || MI_FREE_IS_CHECKED + #if MI_SECURE || MI_FREE_IS_CHECKED return _mi_checked_ptr_page(p); #elif MI_PAGE_META_IS_ALIGNED mi_page_t* const page = _mi_ptr_page_align(p); #if MI_DEBUG mi_page_t* const cpage = _mi_checked_ptr_page(p); - if (cpage!=page) { - mi_assert(page == cpage); - } + if (cpage!=page) { mi_assert(page == cpage); } #endif - // mi_assert_internal(page->block_size != 0); return page; #elif MI_DEBUG return _mi_checked_ptr_page(p); From b0ed24efd9a64dfa1a10dd9ea26d2be21282f819 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 16:19:09 -0700 Subject: [PATCH 011/280] reduce default random keys per page to 1 (still use 2 for MI_SECURE>4) --- include/mimalloc/internal.h | 24 +++++++++++++++++++----- include/mimalloc/types.h | 24 ++++++++++++++---------- src/arena.c | 14 +++++++------- src/free.c | 6 +++++- src/init.c | 13 ++++++------- src/options.c | 8 ++++++++ src/page.c | 4 ++++ 7 files changed, 63 insertions(+), 30 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index c7c852066..7067f41d3 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -805,8 +805,8 @@ static inline size_t mi_page_block_size(const mi_page_t* page) { // Page start static inline uint8_t* mi_page_start(const mi_page_t* page) { - // multiplication must be done in `size_t`; in a 32-bit multiplication the offset wraps for pages whose blocks start 4 GiB or more after the page meta info - return (uint8_t*)page + (((size_t)page->page_ma_offset) * MI_MAX_ALIGN_SIZE); + // multiplication must be done in `size_t` + return (uint8_t*)page + (((size_t)page->page_zoffset) * MI_SIZE_SIZE); } static inline size_t mi_page_size(const mi_page_t* page) { @@ -1192,13 +1192,27 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) { } static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { - void* p = (void*)(mi_rotr(x - keys[0], keys[0]) ^ keys[1]); + #if MI_PAGE_KEY_COUNT==2 + uintptr_t k1 = keys[0]; + uintptr_t k2 = keys[1]; + #else + uintptr_t k1 = keys[0]; + uintptr_t k2 = mi_rotr(k1,13); + #endif + void* p = (void*)(mi_rotr(x - k1, k1) ^ k2); return (p==null ? NULL : p); } static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { - uintptr_t x = (uintptr_t)(p==NULL ? null : p); - return mi_rotl(x ^ keys[1], keys[0]) + keys[0]; + #if MI_PAGE_KEY_COUNT==2 + uintptr_t k1 = keys[0]; + uintptr_t k2 = keys[1]; + #else + uintptr_t k1 = keys[0]; + uintptr_t k2 = mi_rotr(k1,13); + #endif + uintptr_t x = (uintptr_t)(p==NULL ? null : p); + return mi_rotl(x ^ k2, k1) + k1; } static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) { diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ea33c0b21..5830ac93b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -113,6 +113,12 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_CHECK_DOUBLE_FREE 1 #endif +#if MI_SECURE>=5 +#define MI_PAGE_KEY_COUNT 2 +#else +#define MI_PAGE_KEY_COUNT 1 +#endif + // Enable large pages for objects between 64KiB and 512KiB. // This should perhaps be disabled by default as for many workloads the block sizes above 64 KiB // are quite random which can lead to too many partially used large pages (but see issue #1104). @@ -141,6 +147,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_PAGE_META_ALIGNED_FREE_SMALL 0 #define MI_PAGE_META_IS_ALIGNED 1 #define MI_PAGE_META_CHUNKS MI_INTPTR_SIZE +#define MI_PAGE_KEY2 0 #else #warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" #endif @@ -169,6 +176,9 @@ terms of the MIT license. A copy of the license can be found in the file #warning "mimalloc assertions enabled in a release build" #endif +#ifndef MI_PAGE_KEY2 +#define MI_PAGE_KEY2 1 +#endif // -------------------------------------------------------------- // Sizes of internal data-structures @@ -426,7 +436,7 @@ typedef struct mi_page_s { _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) size_t block_size; // const: size available in each block (always `>0`) - uint32_t page_ma_offset; // const: offset relative to the page (in MI_MAX_ALIGN_SIZE parts) to the start of the blocks + uint32_t page_zoffset; // const: offset relative to the page (in size_t parts) to the start of the blocks uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) uint16_t reserved; // number of blocks reserved in memory @@ -438,15 +448,9 @@ typedef struct mi_page_s { mi_memid_t memid; // const: provenance of the page memory #if (MI_ENCODE_FREELIST || MI_PADDING) - uintptr_t keys[2]; // const: two random keys to encode the free lists (see `_mi_block_next`) or padding canary - #endif - #if MI_PAGE_META_IS_ALIGNED - // with alignment (MI_OPT_FREE) we need sizeof(mi_page_t) to be a multiple of MI_MAX_ALIGN_SIZE - #if MI_INTPTR_SIZE==8 || (MI_INTPTR_SIZE==4 && !MI_ENCODE_FREELIST && !MI_PADDING) - uintptr_t padding[1]; - #elif MI_INTPTR_SIZE==4 - uintptr_t padding[3]; - #endif + uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary + #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 + uintptr_t padding[1]; // make it 128 bytes for best codegen in mi_ptr_page_align #endif } mi_page_t; diff --git a/src/arena.c b/src/arena.c index 3a6af7723..b6a5d942f 100644 --- a/src/arena.c +++ b/src/arena.c @@ -24,8 +24,8 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #include "mimalloc/prim-tls.h" #include "bitmap.h" -#if (MI_ARENA_MAX_SIZE > MI_MAX_ALIGN_SIZE*UINT32_MAX) -#error "The page_t.page_ma_offset field is not large enough to cover a full arena" +#if (MI_ARENA_MAX_SIZE > MI_SIZE_SIZE*UINT32_MAX) +#error "The page_t.page_zoffset field is not large enough to cover a full arena" #endif /* ----------------------------------------------------------- @@ -1044,8 +1044,8 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou uint8_t* const start = slice_start + block_start; mi_assert_internal(start > (uint8_t*)page); const size_t offset = start - (uint8_t*)page; - mi_assert_internal((offset % MI_MAX_ALIGN_SIZE) == 0 && (offset / MI_MAX_ALIGN_SIZE) <= UINT32_MAX); - page->page_ma_offset = (uint32_t)(offset / MI_MAX_ALIGN_SIZE); + mi_assert_internal((offset % MI_SIZE_SIZE) == 0 && (offset / MI_SIZE_SIZE) <= UINT32_MAX); + page->page_zoffset = (uint32_t)(offset / MI_SIZE_SIZE); // initialize page meta-data page->reserved = (uint16_t)reserved; @@ -1803,11 +1803,11 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t s mi_assert(start!=NULL); if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } if (start==NULL) return false; - if (!_mi_is_aligned(start, MI_ARENA_SLICE_SIZE)) { + if (!_mi_is_aligned(start, MI_ARENA_ALIGN)) { // we can align the start since the memid tracks the real base of the memory. - void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_SLICE_SIZE); + void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGN); const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start; - if (diff >= size || (size - diff) < MI_ARENA_SLICE_SIZE) { + if (diff >= size || (size - diff) < MI_ARENA_ALIGN) { _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size); return false; } diff --git a/src/free.c b/src/free.c index 68a9783fc..e05e769b0 100644 --- a/src/free.c +++ b/src/free.c @@ -235,7 +235,7 @@ void mi_free(void* p) mi_attr_noexcept { #if MI_PAGE_META_IS_ALIGNED mi_page_t* page = _mi_ptr_page_align0(p); if mi_unlikely(page==NULL) return; - #if MI_DEBUG + #if MI_DEBUG>1 mi_page_t* const cpage = _mi_safe_ptr_page(p); mi_assert_internal(cpage!=NULL); #endif @@ -603,7 +603,11 @@ static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* bloc uint32_t canary = padding->canary; uintptr_t keys[2]; keys[0] = page->keys[0]; + #if MI_PAGE_KEY_COUNT==2 keys[1] = page->keys[1]; + #else + keys[1] = keys[0]; + #endif bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize); mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); return ok; diff --git a/src/init.c b/src/init.c index c4ed58698..0c87b2f3d 100644 --- a/src/init.c +++ b/src/init.c @@ -26,7 +26,7 @@ static const mi_page_t mi_page_empty = { NULL, // local_free MI_ATOMIC_VAR_INIT(0), // xthread_free 0, // block_size - 0, // page_ma_offset + 0, // page_zoffset 0, // slice_pcommitted 0, // reserved capacity NULL, // theap @@ -34,14 +34,13 @@ static const mi_page_t mi_page_empty = { NULL, NULL, // next, prev MI_MEMID_STATIC, // memid #if (MI_PADDING || MI_ENCODE_FREELIST) + #if MI_PAGE_KEY_COUNT==2 { 0, 0 }, // keys + #else + { 0 }, // key #endif - #if MI_PAGE_META_IS_ALIGNED - #if MI_INTPTR_SIZE==8 || (MI_INTPTR_SIZE==4 && !MI_ENCODE_FREELIST && !MI_PADDING) + #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 { 0 }, // padding - #elif MI_INTPTR_SIZE==4 - { 0, 0, 0 }, - #endif #endif }; @@ -581,7 +580,7 @@ static void mi_process_init_once(void) { } #if MI_PAGE_META_IS_ALIGNED - mi_assert_internal((sizeof(mi_page_t)%MI_MAX_ALIGN_SIZE) == 0); // or page->ma_offset might not work + mi_assert_internal((sizeof(mi_page_t)%MI_SIZE_SIZE) == 0); // or page->zoffset might not work #endif } diff --git a/src/options.c b/src/options.c index 8bf13dcd4..a2faad4b9 100644 --- a/src/options.c +++ b/src/options.c @@ -249,6 +249,14 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ #if MI_TSAN _mi_fprintf(out, arg, "thread santizer enabled\n"); #endif + #if MI_PAGE_META_IS_ALIGNED + _mi_fprintf(out, arg, "free: aligned, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_FREE_IS_CHECKED + _mi_fprintf(out, arg, "free: checked, page size: %zu\n", sizeof(mi_page_t)); + #endif + #if MI_ENCODE_FREELIST + _mi_fprintf(out, arg, "free lists: encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); + #endif } mi_decl_export void mi_options_print(void) mi_attr_noexcept { diff --git a/src/page.c b/src/page.c index 26d26751c..417e1008b 100644 --- a/src/page.c +++ b/src/page.c @@ -711,8 +711,10 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(page->reserved > 0); #if (MI_PADDING || MI_ENCODE_FREELIST) page->keys[0] = _mi_theap_random_next(theap); + #if MI_PAGE_KEY_COUNT==2 page->keys[1] = _mi_theap_random_next(theap); #endif + #endif #if MI_DEBUG>2 if (page->memid.initially_zero) { mi_track_mem_defined(mi_page_start(page), mi_page_committed(page)); @@ -735,8 +737,10 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(!mi_page_has_interior_pointers(page)); #if (MI_PADDING || MI_ENCODE_FREELIST) mi_assert_internal(page->keys[0] != 0); + #if MI_PAGE_KEY_COUNT==2 mi_assert_internal(page->keys[1] != 0); #endif + #endif mi_assert_expensive(mi_page_is_valid_init(page)); // initialize an initial free list From 6c139ccbb0d5cc5586aa5e13f0f57805a0cdc82d Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 16:38:10 -0700 Subject: [PATCH 012/280] remove recurse guard in options, see also issue #1362 --- CMakeLists.txt | 10 ++++++++++ src/options.c | 15 +-------------- test/test-api.c | 2 +- 3 files changed, 12 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3bee0c2bd..50014e238 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -914,6 +914,16 @@ if (MI_BUILD_TESTS) endif() endforeach() + # static override test + if(MI_BUILD_OBJECT AND NOT (MI_CYGWIN OR MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) + add_executable(mimalloc-test-stress-static test/test-stress.c) + target_compile_definitions(mimalloc-test-stress-static PRIVATE ${mi_defines} "USE_STD_MALLOC=1") + target_compile_options(mimalloc-test-stress-static PRIVATE ${mi_cflags}) + target_include_directories(mimalloc-test-stress-static PRIVATE include) + target_link_libraries(mimalloc-test-stress-static PRIVATE mimalloc-obj) # link statically + add_test(NAME test-stress-static COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_VERBOSE=1 MIMALLOC_SHOW_STATS=1 $) + endif() + # dynamic override test if(MI_BUILD_SHARED AND NOT (MI_CYGWIN OR MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) add_executable(mimalloc-test-stress-dynamic test/test-stress.c) diff --git a/src/options.c b/src/options.c index a2faad4b9..0ab6efd3a 100644 --- a/src/options.c +++ b/src/options.c @@ -441,25 +441,12 @@ static void mi_add_stderr_output(void) { static _Atomic(size_t) error_count; // = 0; // when >= max_error_count stop emitting errors static _Atomic(size_t) warning_count; // = 0; // when >= max_warning_count stop emitting warnings -// When overriding malloc, we may recurse into mi_vfprintf if an allocation -// inside the C runtime causes another message. -// In some cases (like on macOS) the loader already allocates which -// calls into mimalloc; if we then access thread locals (like `recurse`) -// this may crash as the access may call _tlv_bootstrap that tries to -// (recursively) invoke malloc again to allocate space for the thread local -// variables on demand. This is why we use a _mi_preloading test on such -// platforms. However, C code generator may move the initial thread local address -// load before the `if` and we therefore split it out in a separate function. -static mi_decl_thread bool recurse = false; - static mi_decl_noinline bool mi_recurse_enter_prim(void) { - if (recurse) return false; - recurse = true; return true; } static mi_decl_noinline void mi_recurse_exit_prim(void) { - recurse = false; + /* nothing */ } static bool mi_recurse_enter(void) { diff --git a/test/test-api.c b/test/test-api.c index fb5caf6ff..1e6825d89 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -442,7 +442,7 @@ int main(void) { #if (MI_INTPTR_SIZE > 4) CHECK_BODY("arena_reserve") { - result = (0==mi_reserve_os_memory(16*MI_GiB,false,true)); + result = (0==mi_reserve_os_memory(34*MI_GiB,false,true)); } #endif From 0ba7c98e164e400f2adaff4b5720ab47190e4575 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 17:13:33 -0700 Subject: [PATCH 013/280] fix secure check --- CMakeLists.txt | 8 +++++--- include/mimalloc/types.h | 5 ++--- src/arena.c | 2 +- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 50014e238..7e80d7a16 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -390,20 +390,22 @@ endif() if(MI_FREE_IS_CHECKED STREQUAL "DEFAULT") if (MI_DEBUG OR MI_SECURE) - set(MI_FREE_IS_CHECKED ALWAYS) + set(MI_FREE_IS_CHECKED "ALWAYS") else() - set(MI_FREE_IS_CHECKED NEVER) + set(MI_FREE_IS_CHECKED "NEVER") endif() endif() if(MI_FREE_IS_CHECKED MATCHES "ALWAYS|ON") message(STATUS "Check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=ALWAYS)") list(APPEND mi_defines MI_FREE_IS_CHECKED=1) +elseif(MI_SECURE) + message(STATUS "Check validity of pointers passed to 'mi_free' (MI_SECURE=ON, MI_FREE_IS_CHECKED=ALWAYS)") elseif(MI_FREE_IS_CHECKED STREQUAL "SHARED") message(STATUS "Check validity of pointers passed to 'mi_free' only in the dynamic link library (MI_FREE_IS_CHECKED=SHARED)") list(APPEND mi_cflags_dynamic -DMI_FREE_IS_CHECKED=1) elseif(MI_FREE_IS_CHECKED MATCHES "NEVER|OFF") - message(STATUS "Do not check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=NEVER)") + message(STATUS "Do not check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=${MI_FREE_IS_CHECKED})") else() message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, NEVER, SHARED, or ALWAYS)") endif() diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 5830ac93b..08bf78c53 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -113,7 +113,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_CHECK_DOUBLE_FREE 1 #endif -#if MI_SECURE>=5 +#if MI_SECURE>=4 || (MI_FREE_IS_CHECKED && !MI_OPT_FREE) #define MI_PAGE_KEY_COUNT 2 #else #define MI_PAGE_KEY_COUNT 1 @@ -142,12 +142,11 @@ terms of the MIT license. A copy of the license can be found in the file // This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup. // This only works if valid pointers are passed to `mi_free` though. However, checked // free `mi_cfree` can still uses the page map to validate pointers. -#if MI_OPT_FREE +#if MI_OPT_FREE && !MI_FREE_IS_CHECKED #if MI_PAGE_META_IS_SEPARATED #define MI_PAGE_META_ALIGNED_FREE_SMALL 0 #define MI_PAGE_META_IS_ALIGNED 1 #define MI_PAGE_META_CHUNKS MI_INTPTR_SIZE -#define MI_PAGE_KEY2 0 #else #warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" #endif diff --git a/src/arena.c b/src/arena.c index b6a5d942f..df9715384 100644 --- a/src/arena.c +++ b/src/arena.c @@ -24,7 +24,7 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #include "mimalloc/prim-tls.h" #include "bitmap.h" -#if (MI_ARENA_MAX_SIZE > MI_SIZE_SIZE*UINT32_MAX) +#if (MI_ARENA_MAX_SIZE - MI_SIZE_SIZE > MI_SIZE_SIZE*UINT32_MAX) #error "The page_t.page_zoffset field is not large enough to cover a full arena" #endif From df80a9d08997fb89ce6d575f5918f8409fa7c968 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 17:17:32 -0700 Subject: [PATCH 014/280] fix linking of static override test --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7e80d7a16..fa05594a0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -922,7 +922,7 @@ if (MI_BUILD_TESTS) target_compile_definitions(mimalloc-test-stress-static PRIVATE ${mi_defines} "USE_STD_MALLOC=1") target_compile_options(mimalloc-test-stress-static PRIVATE ${mi_cflags}) target_include_directories(mimalloc-test-stress-static PRIVATE include) - target_link_libraries(mimalloc-test-stress-static PRIVATE mimalloc-obj) # link statically + target_link_libraries(mimalloc-test-stress-static PRIVATE mimalloc-obj ${mi_libraries}) # override statically with an object file add_test(NAME test-stress-static COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_VERBOSE=1 MIMALLOC_SHOW_STATS=1 $) endif() From 4073f72f9df145b2ae06e245bdf8655f7e5d8ddc Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 17:32:44 -0700 Subject: [PATCH 015/280] add android github action --- .github/workflows/test.yaml | 19 +++++++++++++++++++ src/bitmap.h | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 734cc8fc1..f2557fa70 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,6 +33,8 @@ jobs: tests: alpine-arm32 # qemu - os: windows-latest tests: mingw-ucrt64 + - os: ubuntu-22.04-arm + tests: android-arm64 # - os: ubuntu-latest # slow, so disable by default # tests: freebsd-arm64 # qemu @@ -444,3 +446,20 @@ jobs: cmake --build out/secure-arm32 --parallel 4 --config Release ctest --test-dir out/secure-arm32 --verbose --timeout 240 -C Release + - name: Android, Setup + if: matrix.tests == 'android-arm64' + id: setup-ndk + uses: nttld/setup-ndk@v1 + with: + link-to-sdk: true + + - name: Android, Debug + if: matrix.tests == 'android-arm64' + run: | + cmake . -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_TOOLCHAIN_FILE=${{ steps:setup-ndk.outputs.ndk-path }}/build/cmake/android.toolchain.cmake \ + -DANDROID_ABI=arm64-v8a \ + -DANDROID_PLATFORM=android-21 + cmake --build out/android-arm64 --parallel 4 --config Debug + ctest --test-dir out/android-arm64 --verbose --timeout 240 -C Debug + diff --git a/src/bitmap.h b/src/bitmap.h index 8522d346d..1465c8634 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -96,7 +96,7 @@ typedef mi_bchunk_t mi_bchunkmap_t; #define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS) #define MI_BITMAP_MIN_CHUNK_COUNT (1) #if MI_SIZE_BITS > 32 -#define MI_BITMAP_DEFAULT_CHUNK_COUNT (64) // 2 GiB on 64-bit -- this is for the page map +#define MI_BITMAP_DEFAULT_CHUNK_COUNT (128) // 4 GiB on 64-bit -- this is for the page map #else #define MI_BITMAP_DEFAULT_CHUNK_COUNT (1) #endif From 4032adb77cd4a40859277f228c6386352f992d3a Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 17:42:46 -0700 Subject: [PATCH 016/280] fix android action --- .github/workflows/test.yaml | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f2557fa70..27b3b32d7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -451,15 +451,19 @@ jobs: id: setup-ndk uses: nttld/setup-ndk@v1 with: + ndk-version: r27d link-to-sdk: true + local-cache: true - name: Android, Debug if: matrix.tests == 'android-arm64' + env: + ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | cmake . -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=${{ steps:setup-ndk.outputs.ndk-path }}/build/cmake/android.toolchain.cmake \ + -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ -DANDROID_ABI=arm64-v8a \ - -DANDROID_PLATFORM=android-21 + -DANDROID_PLATFORM=android-27 cmake --build out/android-arm64 --parallel 4 --config Debug ctest --test-dir out/android-arm64 --verbose --timeout 240 -C Debug From 9b06bcd48b15e7021cfb9c6fb4d48a7ede0eeae3 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 17:47:44 -0700 Subject: [PATCH 017/280] try to fix android action --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 27b3b32d7..d6d296dc7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,7 +33,7 @@ jobs: tests: alpine-arm32 # qemu - os: windows-latest tests: mingw-ucrt64 - - os: ubuntu-22.04-arm + - os: ubuntu-latest tests: android-arm64 # - os: ubuntu-latest # slow, so disable by default # tests: freebsd-arm64 # qemu From ffc9770bb0d81bbd28b71b7579520f30cf9318db Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 17:51:47 -0700 Subject: [PATCH 018/280] try to fix android action --- .github/workflows/test.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d6d296dc7..efa397ab7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -461,9 +461,9 @@ jobs: ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | cmake . -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ - -DANDROID_ABI=arm64-v8a \ - -DANDROID_PLATFORM=android-27 + -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake cmake --build out/android-arm64 --parallel 4 --config Debug ctest --test-dir out/android-arm64 --verbose --timeout 240 -C Debug + # -DANDROID_ABI=arm64-v8a \ + # -DANDROID_PLATFORM=android-27 From 97577b73b87696e698d8e50ff6c23e876a43c9e7 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 17:54:05 -0700 Subject: [PATCH 019/280] try to fix android action --- .github/workflows/test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index efa397ab7..d9b0d26b7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -461,7 +461,8 @@ jobs: ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | cmake . -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake + -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ + -DANDROID_PLATFORM=android-27 cmake --build out/android-arm64 --parallel 4 --config Debug ctest --test-dir out/android-arm64 --verbose --timeout 240 -C Debug From b76294d302307b975b14d78fc184ad5d6d913b13 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:00:32 -0700 Subject: [PATCH 020/280] try to fix android action --- .github/workflows/test.yaml | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d9b0d26b7..99102b478 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -449,22 +449,32 @@ jobs: - name: Android, Setup if: matrix.tests == 'android-arm64' id: setup-ndk - uses: nttld/setup-ndk@v1 + uses: android-actions/setup-android@v2 with: ndk-version: r27d link-to-sdk: true local-cache: true - - name: Android, Debug + - name: Android, Debug Build if: matrix.tests == 'android-arm64' env: ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | cmake . -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ - -DANDROID_PLATFORM=android-27 + -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-27 cmake --build out/android-arm64 --parallel 4 --config Debug - ctest --test-dir out/android-arm64 --verbose --timeout 240 -C Debug + + - name: Android, Debug Run + uses: reactivecircus/android-emulator-runner@v2 + with: + api-level: 27 + target: default + arch: arm64-v8a + script: | + adb push out/android-arm64/mimalloc-test-api /data/local/tmp/ + adb shell "/data/local/tmp/mimalloc-test-api" + # -DANDROID_ABI=arm64-v8a \ # -DANDROID_PLATFORM=android-27 From 6f890ff03428274cbe7f9b26783aa2a858b9d474 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:00:53 -0700 Subject: [PATCH 021/280] try to fix android action --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 99102b478..40438b826 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -466,6 +466,7 @@ jobs: cmake --build out/android-arm64 --parallel 4 --config Debug - name: Android, Debug Run + if: matrix.tests == 'android-arm64' uses: reactivecircus/android-emulator-runner@v2 with: api-level: 27 From 6d21779fe6f9dfde6b7664b6ec66a338528a276b Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:03:20 -0700 Subject: [PATCH 022/280] try to fix android action --- .github/workflows/test.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 40438b826..d30da6ae3 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -460,7 +460,8 @@ jobs: env: ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | - cmake . -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ + env + cmake . -G Ninja -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-27 cmake --build out/android-arm64 --parallel 4 --config Debug From 9cea819865045bccaf8862619c628646d3ac74e8 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:05:23 -0700 Subject: [PATCH 023/280] try to fix android action --- .github/workflows/test.yaml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d30da6ae3..1183693d5 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -457,12 +457,10 @@ jobs: - name: Android, Debug Build if: matrix.tests == 'android-arm64' - env: - ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | env cmake . -G Ninja -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ + -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-27 cmake --build out/android-arm64 --parallel 4 --config Debug From d83813e82ec715003119309d3f32be2f88f548b0 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:09:49 -0700 Subject: [PATCH 024/280] try to fix android action --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1183693d5..b5b4603e0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,7 +33,7 @@ jobs: tests: alpine-arm32 # qemu - os: windows-latest tests: mingw-ucrt64 - - os: ubuntu-latest + - os: ubuntu-22.04-arm tests: android-arm64 # - os: ubuntu-latest # slow, so disable by default # tests: freebsd-arm64 # qemu From d9cbb98d9f16fdb3d4c3793b3ad1902cb1bcbd2e Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:16:46 -0700 Subject: [PATCH 025/280] try to fix android action --- .github/workflows/test.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b5b4603e0..58b08f376 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,7 +33,7 @@ jobs: tests: alpine-arm32 # qemu - os: windows-latest tests: mingw-ucrt64 - - os: ubuntu-22.04-arm + - os: ubuntu-latest tests: android-arm64 # - os: ubuntu-latest # slow, so disable by default # tests: freebsd-arm64 # qemu @@ -449,7 +449,7 @@ jobs: - name: Android, Setup if: matrix.tests == 'android-arm64' id: setup-ndk - uses: android-actions/setup-android@v2 + uses: nttld/setup-ndk@v1 # android-actions/setup-android@v2 with: ndk-version: r27d link-to-sdk: true @@ -457,11 +457,13 @@ jobs: - name: Android, Debug Build if: matrix.tests == 'android-arm64' + env: + ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | env cmake . -G Ninja -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ - -DANDROID_ABI=arm64-v8a -DANDROID_PLATFORM=android-27 + -DANDROID_ABI=x86_64 -DANDROID_PLATFORM=android-27 cmake --build out/android-arm64 --parallel 4 --config Debug - name: Android, Debug Run @@ -470,7 +472,7 @@ jobs: with: api-level: 27 target: default - arch: arm64-v8a + arch: x86_64 script: | adb push out/android-arm64/mimalloc-test-api /data/local/tmp/ adb shell "/data/local/tmp/mimalloc-test-api" From 6c653a0fd07bbfbf46c6ff27ecdc37bf4147789a Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:17:58 -0700 Subject: [PATCH 026/280] try to fix android action --- .github/workflows/test.yaml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 58b08f376..5a6bfe5a4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -34,7 +34,7 @@ jobs: - os: windows-latest tests: mingw-ucrt64 - os: ubuntu-latest - tests: android-arm64 + tests: android-x64 # - os: ubuntu-latest # slow, so disable by default # tests: freebsd-arm64 # qemu @@ -447,7 +447,7 @@ jobs: ctest --test-dir out/secure-arm32 --verbose --timeout 240 -C Release - name: Android, Setup - if: matrix.tests == 'android-arm64' + if: matrix.tests == 'android-x64' id: setup-ndk uses: nttld/setup-ndk@v1 # android-actions/setup-android@v2 with: @@ -456,25 +456,25 @@ jobs: local-cache: true - name: Android, Debug Build - if: matrix.tests == 'android-arm64' + if: matrix.tests == 'android-x64' env: ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | env - cmake . -G Ninja -B out/android-arm64 -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ + cmake . -G Ninja -B out/android-x64 -DCMAKE_BUILD_TYPE=Debug \ + -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ -DANDROID_ABI=x86_64 -DANDROID_PLATFORM=android-27 - cmake --build out/android-arm64 --parallel 4 --config Debug + cmake --build out/android-x64 --parallel 4 --config Debug - name: Android, Debug Run - if: matrix.tests == 'android-arm64' + if: matrix.tests == 'android-x64' uses: reactivecircus/android-emulator-runner@v2 with: api-level: 27 target: default arch: x86_64 script: | - adb push out/android-arm64/mimalloc-test-api /data/local/tmp/ + adb push out/android-x64/mimalloc-test-api /data/local/tmp/ adb shell "/data/local/tmp/mimalloc-test-api" From 288a1c0357940b93906c5903caf9b8d035bf723c Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:19:42 -0700 Subject: [PATCH 027/280] try to fix android action --- .github/workflows/test.yaml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5a6bfe5a4..77802f5b7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -449,7 +449,7 @@ jobs: - name: Android, Setup if: matrix.tests == 'android-x64' id: setup-ndk - uses: nttld/setup-ndk@v1 # android-actions/setup-android@v2 + uses: android-actions/setup-android@v2 with: ndk-version: r27d link-to-sdk: true @@ -457,12 +457,12 @@ jobs: - name: Android, Debug Build if: matrix.tests == 'android-x64' - env: - ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} + # env: + # ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} run: | env cmake . -G Ninja -B out/android-x64 -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK_HOME/build/cmake/android.toolchain.cmake \ + -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ -DANDROID_ABI=x86_64 -DANDROID_PLATFORM=android-27 cmake --build out/android-x64 --parallel 4 --config Debug From 8266f70436f32441df128989e81ddea43e792d26 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:28:46 -0700 Subject: [PATCH 028/280] disable android ci for now --- .github/workflows/test.yaml | 64 +++++++++++++++++-------------------- 1 file changed, 30 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 77802f5b7..f88ddb82b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -446,37 +446,33 @@ jobs: cmake --build out/secure-arm32 --parallel 4 --config Release ctest --test-dir out/secure-arm32 --verbose --timeout 240 -C Release - - name: Android, Setup - if: matrix.tests == 'android-x64' - id: setup-ndk - uses: android-actions/setup-android@v2 - with: - ndk-version: r27d - link-to-sdk: true - local-cache: true - - - name: Android, Debug Build - if: matrix.tests == 'android-x64' - # env: - # ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} - run: | - env - cmake . -G Ninja -B out/android-x64 -DCMAKE_BUILD_TYPE=Debug \ - -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ - -DANDROID_ABI=x86_64 -DANDROID_PLATFORM=android-27 - cmake --build out/android-x64 --parallel 4 --config Debug - - - name: Android, Debug Run - if: matrix.tests == 'android-x64' - uses: reactivecircus/android-emulator-runner@v2 - with: - api-level: 27 - target: default - arch: x86_64 - script: | - adb push out/android-x64/mimalloc-test-api /data/local/tmp/ - adb shell "/data/local/tmp/mimalloc-test-api" - - - # -DANDROID_ABI=arm64-v8a \ - # -DANDROID_PLATFORM=android-27 + # - name: Android, Setup + # if: matrix.tests == 'android-x64' + # uses: android-actions/setup-android@v4 + # #id: setup-ndk + # # with: + # # ndk-version: r27d + # # link-to-sdk: true + # # local-cache: true + + # - name: Android, Debug Build + # if: matrix.tests == 'android-x64' + # # env: + # # ANDROID_NDK_HOME: ${{ steps.setup-ndk.outputs.ndk-path }} + # run: | + # env + # cmake . -G Ninja -B out/android-x64 -DCMAKE_BUILD_TYPE=Debug \ + # -DCMAKE_TOOLCHAIN_FILE=$ANDROID_NDK/build/cmake/android.toolchain.cmake \ + # -DANDROID_ABI=x86_64 -DANDROID_PLATFORM=android-27 + # cmake --build out/android-x64 --parallel 4 --config Debug + + # - name: Android, Debug Run + # if: matrix.tests == 'android-x64' + # uses: reactivecircus/android-emulator-runner@v2 + # with: + # api-level: 27 + # target: default + # arch: x86_64 + # script: | + # adb push out/android-x64/mimalloc-test-api /data/local/tmp/ + # adb shell "/data/local/tmp/mimalloc-test-api" From 797eed31641fb9b5b74aa7c885db775809c66163 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:30:46 -0700 Subject: [PATCH 029/280] disable android ci for now --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f88ddb82b..a8c35ffa7 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -33,8 +33,8 @@ jobs: tests: alpine-arm32 # qemu - os: windows-latest tests: mingw-ucrt64 - - os: ubuntu-latest - tests: android-x64 + # - os: ubuntu-latest + # tests: android-x64 # - os: ubuntu-latest # slow, so disable by default # tests: freebsd-arm64 # qemu From 486f2afe3c6c701b701cdeadfdc10633ef9663a9 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 18:51:23 -0700 Subject: [PATCH 030/280] try to fix spurious ubsan error --- CMakeLists.txt | 2 +- src/bitmap.h | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index fa05594a0..8e314acf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -389,7 +389,7 @@ else() endif() if(MI_FREE_IS_CHECKED STREQUAL "DEFAULT") - if (MI_DEBUG OR MI_SECURE) + if (MI_SECURE) set(MI_FREE_IS_CHECKED "ALWAYS") else() set(MI_FREE_IS_CHECKED "NEVER") diff --git a/src/bitmap.h b/src/bitmap.h index 1465c8634..ba8224b8a 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -95,11 +95,11 @@ typedef mi_bchunk_t mi_bchunkmap_t; #define MI_BITMAP_MAX_CHUNK_COUNT (MI_BCHUNKMAP_BITS) #define MI_BITMAP_MIN_CHUNK_COUNT (1) -#if MI_SIZE_BITS > 32 -#define MI_BITMAP_DEFAULT_CHUNK_COUNT (128) // 4 GiB on 64-bit -- this is for the page map -#else +// #if MI_SIZE_BITS > 32 +// #define MI_BITMAP_DEFAULT_CHUNK_COUNT (128) // 4 GiB on 64-bit -- this is for the page map +// #else #define MI_BITMAP_DEFAULT_CHUNK_COUNT (1) -#endif +// #endif #define MI_BITMAP_MAX_BIT_COUNT (MI_BITMAP_MAX_CHUNK_COUNT * MI_BCHUNK_BITS) // 16 GiB arena #define MI_BITMAP_MIN_BIT_COUNT (MI_BITMAP_MIN_CHUNK_COUNT * MI_BCHUNK_BITS) // 32 MiB arena #define MI_BITMAP_DEFAULT_BIT_COUNT (MI_BITMAP_DEFAULT_CHUNK_COUNT * MI_BCHUNK_BITS) // 2 GiB arena From 755c559ac6ac5e3368377630e9fcb5f996dd0f69 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 11 Aug 2026 19:00:45 -0700 Subject: [PATCH 031/280] reduce arena reserve test --- test/test-api.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-api.c b/test/test-api.c index 1e6825d89..fb5caf6ff 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -442,7 +442,7 @@ int main(void) { #if (MI_INTPTR_SIZE > 4) CHECK_BODY("arena_reserve") { - result = (0==mi_reserve_os_memory(34*MI_GiB,false,true)); + result = (0==mi_reserve_os_memory(16*MI_GiB,false,true)); } #endif From 0d178264f906ebe2e43d90ce63745c5222026b64 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 11:35:39 -0700 Subject: [PATCH 032/280] clean up and rename options --- CMakeLists.txt | 46 +++++------- include/mimalloc/internal.h | 42 +++++------ include/mimalloc/types.h | 41 +++++----- src/arena.c | 54 +++++++------- src/free.c | 145 +++++++++++++++++++----------------- src/options.c | 5 ++ 6 files changed, 166 insertions(+), 167 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8e314acf1..750edf3b2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -14,8 +14,8 @@ option(MI_USE_CXX "Use the C++ compiler to compile the library (instea option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016))" OFF) option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) -set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': ALWAYS, SHARED, NEVER, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") -set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "DEFAULT" "NEVER" "SHARED" "ALWAYS") +set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': ON, SHARED, OFF, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") +set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "DEFAULT" "OFF" "SHARED" "ON") option(MI_DEBUG "Enable assertion checks (enabled by default in a debug build)" OFF) option(MI_DEBUG_INTERNAL "Enable assertion and internal invariant checks (enabled by default in a debug build)" OFF) @@ -41,7 +41,7 @@ option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen option(MI_LIBC_MUSL "Enable this when linking with musl libc" OFF) # advanced -option(MI_OPT_FREE "Use aligned `mi_free(_small)` without pagemap lookup (can only be used if SECURE mode is off)" ON) +option(MI_FREE_USE_PAGEMAP "Always use the page map when free-ing a pointer" OFF) option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG or SECURE mode, or with Valgrind/ASAN)" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) @@ -390,49 +390,41 @@ endif() if(MI_FREE_IS_CHECKED STREQUAL "DEFAULT") if (MI_SECURE) - set(MI_FREE_IS_CHECKED "ALWAYS") + set(MI_FREE_IS_CHECKED ON) else() - set(MI_FREE_IS_CHECKED "NEVER") + set(MI_FREE_IS_CHECKED OFF) endif() endif() -if(MI_FREE_IS_CHECKED MATCHES "ALWAYS|ON") - message(STATUS "Check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=ALWAYS)") +if(MI_FREE_IS_CHECKED STREQUAL "ON") + message(STATUS "Check validity eof pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=ON)") list(APPEND mi_defines MI_FREE_IS_CHECKED=1) elseif(MI_SECURE) - message(STATUS "Check validity of pointers passed to 'mi_free' (MI_SECURE=ON, MI_FREE_IS_CHECKED=ALWAYS)") + message(STATUS "Check validity of pointers passed to 'mi_free' (MI_SECURE=ON, MI_FREE_IS_CHECKED=ON)") elseif(MI_FREE_IS_CHECKED STREQUAL "SHARED") - message(STATUS "Check validity of pointers passed to 'mi_free' only in the dynamic link library (MI_FREE_IS_CHECKED=SHARED)") + message(STATUS "Check validity of pointers passed to 'mi_free' only in the dynamic/shared link library (MI_FREE_IS_CHECKED=SHARED)") list(APPEND mi_cflags_dynamic -DMI_FREE_IS_CHECKED=1) -elseif(MI_FREE_IS_CHECKED MATCHES "NEVER|OFF") - message(STATUS "Do not check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=${MI_FREE_IS_CHECKED})") +elseif(MI_FREE_IS_CHECKED STREQUAL "OFF") + # message(STATUS "Do not check validity of pointers passed to 'mi_free' (MI_FREE_IS_CHECKED=OFF)") else() - message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, NEVER, SHARED, or ALWAYS)") + message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, OFF, SHARED, or ON)") endif() -if(MI_OPT_FREE) - if(CMAKE_SYSTEM_NAME STREQUAL "WASI") # todo: also test for EMSCRIPTEN? or any 32-bit host? - message(STATUS "Disregarding MI_OPT_FREE as the target does not support large alignment well (WASI) (MI_OPT_FREE=OFF)") - elseif(MI_SECURE) - message(STATUS "Disregarding MI_OPT_FREE in secure mode (MI_SECURE=ON, MI_OPT_FREE=OFF)") - elseif(MI_FREE_IS_CHECKED MATCHES "ALWAYS|ON") - message(STATUS "Disregarding MI_OPT_FREE as pointers are always checked for validity (MI_FREE_IS_CHECKED=ALWAYS, MI_OPT_FREE=OFF)") - else() - message(STATUS "Use aligned `mi_free(_small)` to avoid a pagemap lookup (MI_OPT_FREE=ON)") - list(APPEND mi_defines MI_OPT_FREE=1) - endif() +if(MI_FREE_USE_PAGEMAP) + message(STATUS "Always use the pagemap in mi_free (MI_FREE_USE_PAGEMAP=ON)") +elseif(CMAKE_SYSTEM_NAME STREQUAL "WASI") + message(STATUS "Always use the pagemap in mi_free since the target (WASI) does not support large OS aligned allocations well (MI_FREE_USE_PAGEMAP=ON)") + list(APPEND mi_defines MI_FREE_USE_PAGEMAP=1) endif() -if(NOT MI_OPT_FREE_SMALL AND NOT MI_DEBUG AND NOT MI_GUARDED AND NOT MI_SECURE AND NOT MI_OPT_FREE) -set(MI_OPT_FREE_SMALL ON) -endif() +# deprecated if(MI_OPT_FREE_SMALL) if(MI_GUARDED OR MI_SECURE) message(STATUS "Cannot enable MI_OPT_FREE_SMALL in secure or guarded mode (MI_OPT_FREE_SMALL=OFF)") elseif(MI_OPT_FREE) message(STATUS "Cannot enable MI_OPT_FREE_SMALL with MI_OPT_FREE (MI_OPT_FREE_SMALL=OFF)") else() - message(STATUS "Use aligned `mi_free_small` to avoid a pagemap lookup (MI_OPT_FREE_SMALL=ON)") + message(STATUS "Use aligned `mi_free_small` (MI_OPT_FREE_SMALL=ON)") list(APPEND mi_defines MI_OPT_FREE_SMALL=1) endif() endif() diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 7067f41d3..c569b508c 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -764,15 +764,15 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { #if MI_PAGE_META_IS_ALIGNED // if the page meta data is aligned in front of pages we can find it efficiently // without needing to go through the page map (for valid pointers). -static inline mi_page_t* _mi_ptr_page_align0(const void* p) { - mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGN); +static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { + mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; - mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_COUNT); + mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_ALIGNED_COUNT); return &page_metas[page_idx]; } -static inline mi_page_t* _mi_ptr_page_align(const void* p) { - mi_page_t* const page = _mi_ptr_page_align0(p); +static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { + mi_page_t* const page = _mi_aligned_ptr_page0(p); if mi_unlikely(page==NULL) return NULL; return mi_atomic_load_relaxed(&page->self); } @@ -782,13 +782,8 @@ static inline mi_page_t* _mi_ptr_page(const void* p) { mi_assert_internal(p==NULL || mi_is_in_heap_region(p)); #if MI_SECURE || MI_FREE_IS_CHECKED return _mi_checked_ptr_page(p); - #elif MI_PAGE_META_IS_ALIGNED - mi_page_t* const page = _mi_ptr_page_align(p); - #if MI_DEBUG - mi_page_t* const cpage = _mi_checked_ptr_page(p); - if (cpage!=page) { mi_assert(page == cpage); } - #endif - return page; + #elif MI_PAGE_META_IS_ALIGNED + return _mi_aligned_ptr_page(p); #elif MI_DEBUG return _mi_checked_ptr_page(p); #else @@ -848,7 +843,7 @@ static inline bool mi_page_meta_is_separated(const mi_page_t* page) { mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); return true; #elif MI_PAGE_META_IS_SEPARATED - // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_ALIGNED_FREE_SMALL + // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); #else MI_UNUSED(page); @@ -1192,26 +1187,24 @@ static inline bool mi_is_in_same_page(const void* p, const void* q) { } static inline void* mi_ptr_decode(const void* null, const mi_encoded_t x, const uintptr_t* keys) { + const uintptr_t k1 = keys[0]; #if MI_PAGE_KEY_COUNT==2 - uintptr_t k1 = keys[0]; - uintptr_t k2 = keys[1]; + const uintptr_t k2 = keys[1]; #else - uintptr_t k1 = keys[0]; - uintptr_t k2 = mi_rotr(k1,13); + const uintptr_t k2 = mi_rotr(k1,13); #endif void* p = (void*)(mi_rotr(x - k1, k1) ^ k2); return (p==null ? NULL : p); } static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const uintptr_t* keys) { + const uintptr_t k1 = keys[0]; #if MI_PAGE_KEY_COUNT==2 - uintptr_t k1 = keys[0]; - uintptr_t k2 = keys[1]; + const uintptr_t k2 = keys[1]; #else - uintptr_t k1 = keys[0]; - uintptr_t k2 = mi_rotr(k1,13); + const uintptr_t k2 = mi_rotr(k1,13); #endif - uintptr_t x = (uintptr_t)(p==NULL ? null : p); + const uintptr_t x = (uintptr_t)(p==NULL ? null : p); return mi_rotl(x ^ k2, k1) + k1; } @@ -1249,14 +1242,15 @@ static inline void mi_block_set_nextx(const void* null, mi_block_t* block, const mi_track_mem_noaccess(block,sizeof(mi_block_t)); } +mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next); // in options.c + static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* block) { #if MI_ENCODE_FREELIST mi_block_t* next = mi_block_nextx(page,block,page->keys); // check for free list corruption: is `next` at least in the same page? // todo: check if `next` is `page->block_size` aligned? if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) { - _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); - next = NULL; + return _mi_block_next_is_corrupted(page,block,next); // returns NULL } return next; #else diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 08bf78c53..b8b51164b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -113,7 +113,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_CHECK_DOUBLE_FREE 1 #endif -#if MI_SECURE>=4 || (MI_FREE_IS_CHECKED && !MI_OPT_FREE) +#if MI_SECURE>=4 || (MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP) #define MI_PAGE_KEY_COUNT 2 #else #define MI_PAGE_KEY_COUNT 1 @@ -129,7 +129,6 @@ terms of the MIT license. A copy of the license can be found in the file // Place page meta info at the start of the page area or keep it separate? // Separate keeps the page info at the arena start (default) which is more secure // and reduces wasted space due to alignment and block sizes. -// (but if MI_OPT_FREE is false, also reserves more memory up front (about 2MiB per GiB)) #if !defined(MI_PAGE_META_IS_SEPARATED) #if MI_PAGE_MAP_FLAT #define MI_PAGE_META_IS_SEPARATED 0 @@ -138,35 +137,39 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// We can choose to page meta info aligned at the start of every MI_PAGE_META_CHUNKS +// We can choose to page meta info aligned at the start of every MI_PAGE_META_ALIGNED_CHUNKS // This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup. // This only works if valid pointers are passed to `mi_free` though. However, checked // free `mi_cfree` can still uses the page map to validate pointers. -#if MI_OPT_FREE && !MI_FREE_IS_CHECKED +#if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP #if MI_PAGE_META_IS_SEPARATED -#define MI_PAGE_META_ALIGNED_FREE_SMALL 0 #define MI_PAGE_META_IS_ALIGNED 1 -#define MI_PAGE_META_CHUNKS MI_INTPTR_SIZE +#define MI_PAGE_META_ALIGNED_CHUNKS MI_INTPTR_SIZE +#ifdef MI_PAGE_META_SMALL_IS_ALIGNED +#undef MI_PAGE_META_SMALL_IS_ALIGNED +#endif #else #warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" #endif #endif -// We can choose to only put page info of small pages at the start of the page area. +// Deprecated: We can choose to only put page info of small pages at the start of the page area. // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). -#if MI_OPT_FREE_SMALL && !defined(MI_PAGE_META_ALIGNED_FREE_SMALL) -#define MI_PAGE_META_ALIGNED_FREE_SMALL 1 -#elif !defined(MI_PAGE_META_ALIGNED_FREE_SMALL) -#define MI_PAGE_META_ALIGNED_FREE_SMALL 0 +#if !MI_PAGE_META_IS_ALIGNED +#if MI_OPT_FREE_SMALL && !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#define MI_PAGE_META_SMALL_IS_ALIGNED 1 +#elif !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#define MI_PAGE_META_SMALL_IS_ALIGNED 0 +#endif #endif // Configuration checks #if !MI_PAGE_META_IS_SEPARATED && MI_SECURE #error "secure mode should use separated page infos" #endif -#if MI_PAGE_META_ALIGNED_FREE_SMALL && MI_SECURE -#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL)" +#if MI_PAGE_META_SMALL_IS_ALIGNED && MI_SECURE +#error "secure mode cannot use MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED)" #endif #if MI_PAGE_META_IS_SEPARATED && MI_PAGE_MAP_FLAT #error "cannot have a flat page map with separated page infos" @@ -175,10 +178,6 @@ terms of the MIT license. A copy of the license can be found in the file #warning "mimalloc assertions enabled in a release build" #endif -#ifndef MI_PAGE_KEY2 -#define MI_PAGE_KEY2 1 -#endif - // -------------------------------------------------------------- // Sizes of internal data-structures // (comments specify sizes on 64-bit, usually 32-bit is halved) @@ -242,11 +241,11 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_PAGE_MIN_COMMIT_SIZE (16*MI_KiB) /* MI_ARENA_SLICE_SIZE */ #if MI_PAGE_META_IS_ALIGNED -#define MI_PAGE_META_COUNT (MI_PAGE_META_CHUNKS * MI_BCHUNK_BITS) -#define MI_PAGE_META_ALIGN (MI_PAGE_META_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit) -#define MI_ARENA_ALIGN MI_PAGE_META_ALIGN +#define MI_PAGE_META_ALIGNED_COUNT (MI_PAGE_META_ALIGNED_CHUNKS * MI_BCHUNK_BITS) +#define MI_PAGE_META_ALIGNMENT (MI_PAGE_META_ALIGNED_COUNT * MI_ARENA_SLICE_SIZE) // 256 MiB (32 MiB on 32-bit) +#define MI_ARENA_ALIGNMENT MI_PAGE_META_ALIGNMENT #else -#define MI_ARENA_ALIGN MI_ARENA_SLICE_ALIGN +#define MI_ARENA_ALIGNMENT MI_ARENA_SLICE_ALIGN #endif // ------------------------------------------------------ diff --git a/src/arena.c b/src/arena.c index df9715384..e7209954d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -90,7 +90,7 @@ static bool mi_heap_has_page(mi_heap_t* heap, mi_arena_t* arena, mi_page_t* page #endif size_t mi_arena_min_alignment(void) { - return MI_ARENA_ALIGN; + return MI_ARENA_ALIGNMENT; } size_t mi_arena_min_size(void) { @@ -150,19 +150,19 @@ uint8_t* mi_arena_slice_start(mi_arena_t* arena, size_t slice_index) { mi_page_t* mi_arena_page_at_slice(mi_arena_t* arena, size_t slice_index) { mi_assert_internal(slice_index < arena->slice_count); #if MI_PAGE_META_IS_ALIGNED - mi_page_t* const page = _mi_ptr_page_align(mi_arena_slice_start(arena,slice_index)); // todo: optimize? + mi_page_t* const page = _mi_aligned_ptr_page(mi_arena_slice_start(arena,slice_index)); // todo: optimize? return page; #else if (arena->pages_meta != NULL) { mi_page_t* const page = &arena->pages_meta[slice_index]; - #if MI_PAGE_META_ALIGNED_FREE_SMALL + #if MI_PAGE_META_SMALL_IS_ALIGNED // pages with small blocks still have the page at the start of the slice (and set the `block_size` in pages_meta to 0) if (page->block_size>0) return page; #else return page; #endif } - // fall through (for MI_PAGE_META_ALIGNED_FREE_SMALL) + // fall through (for MI_PAGE_META_SMALL_IS_ALIGNED) return (mi_page_t*)mi_arena_slice_start(arena,slice_index); #endif } @@ -219,7 +219,7 @@ static size_t mi_page_full_size(mi_page_t* page) { static size_t mi_arena_page_meta_slices(void) { #if MI_PAGE_META_IS_ALIGNED - return _mi_divide_up(MI_PAGE_META_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); + return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); #else return 0; #endif @@ -813,18 +813,18 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ #if MI_PAGE_META_IS_ALIGNED size_t page_offset; // offset in the block for the page area uint8_t* os_start; - if (block_alignment < MI_PAGE_META_ALIGN) { + if (block_alignment < MI_PAGE_META_ALIGNMENT) { page_offset = (block_alignment < MI_PAGE_ALIGN ? MI_PAGE_ALIGN : block_alignment); - os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGN, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, MI_PAGE_META_ALIGNMENT, 0 /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); } else { - // if we allow alignment >= MI_PAGE_META_ALIGN we need to substract 1 from a pointer - // in _mi_ptr_page_align (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. + // if we allow alignment >= MI_PAGE_META_ALIGNMENT we need to substract 1 from a pointer + // in _mi_aligned_ptr_page (and test for (intptr_t)p < 0 instead of NULL). We avoid this by limiting the max alignment. _mi_warning_message("requested alignment is too large (%zu KiB)\n", block_alignment / MI_KiB); errno = EINVAL; return NULL; - // page_offset = MI_PAGE_META_ALIGN; - // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGN /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); + // page_offset = MI_PAGE_META_ALIGNMENT; + // os_start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size + page_offset, block_alignment, MI_PAGE_META_ALIGNMENT /* align offset */, false /* commit */, false /* allow large */, req_arena, memid); } if (os_start==NULL) return NULL; // commit page info and page @@ -841,9 +841,9 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ } start = os_start + page_offset; mi_assert_internal(_mi_is_aligned(start,block_alignment)); - mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGN)); - mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGN) == os_start); - mi_assert_internal((uint8_t*)_mi_ptr_page_align0(start) < os_start + min_page_meta); + mi_assert_internal(_mi_is_aligned(os_start,MI_PAGE_META_ALIGNMENT)); + mi_assert_internal(_mi_align_down_ptr(start-1,MI_PAGE_META_ALIGNMENT) == os_start); + mi_assert_internal((uint8_t*)_mi_aligned_ptr_page0(start) < os_start + min_page_meta); memid->initially_committed = true; // so we don't commit again #else if (os_align) { @@ -906,7 +906,7 @@ static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_s // ensure the meta data is committed if (memid_slice.memkind == MI_MEM_ARENA) { mi_arena_t* const arena = memid_slice.mem.arena.arena; - uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGN); + uint8_t* const meta_slices = (uint8_t*)_mi_align_down_ptr(slice_start,MI_PAGE_META_ALIGNMENT); mi_assert_internal(meta_slices >= mi_arena_start(arena)); const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { @@ -921,7 +921,7 @@ static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_s mi_bitmap_setN(arena->slices_committed, meta_slice_index, meta_slice_count, NULL); } } - mi_page_t* const page_meta = _mi_ptr_page_align0(slice_start); + mi_page_t* const page_meta = _mi_aligned_ptr_page0(slice_start); return page_meta; } #else @@ -964,7 +964,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou mi_page_t* const page_meta = mi_arena_page_meta(memid,slice_start); if (page_meta!=NULL) { mi_assert_internal(page_meta->block_size == 0); - #if MI_PAGE_META_ALIGNED_FREE_SMALL + #if MI_PAGE_META_SMALL_IS_ALIGNED // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) if (block_size > MI_SMALL_SIZE_MAX) @@ -1073,10 +1073,10 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou } } #if MI_DEBUG>1 - mi_page_t* pstart = _mi_ptr_page_align0(slice_start); + mi_page_t* pstart = _mi_aligned_ptr_page0(slice_start); mi_assert_internal(pstart->self==page); if (reserved>1) { - mi_page_t* pend = _mi_ptr_page_align0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); + mi_page_t* pend = _mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); mi_assert_internal(pend->self==page); } #endif @@ -1683,7 +1683,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, int numa_node, bool exclusive, mi_memid_t memid, mi_commit_fun_t* commit_fun, void* commit_fun_arg, mi_arena_id_t* arena_id) { - mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGN)); + mi_assert_internal(_mi_is_aligned(start,MI_ARENA_ALIGNMENT)); mi_assert_internal(mi_size_of_slices(slice_count)>=MI_ARENA_MIN_SIZE); if (slice_count > MI_BITMAP_MAX_BIT_COUNT) { // 16 GiB for now @@ -1769,11 +1769,11 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, // reserve our meta info (and reserve slices outside the memory area) #if MI_PAGE_META_IS_ALIGNED - for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_COUNT) { + for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) { // set all free slices (and skip the slices reserved for the page meta info) const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_slices()); const size_t start_idx = (i + meta_slices); - size_t count = MI_PAGE_META_COUNT - meta_slices; + size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices; if (start_idx < arena->slice_count) { if (count + start_idx > arena->slice_count) { count = arena->slice_count - start_idx; @@ -1803,11 +1803,11 @@ static bool mi_manage_os_memory_ex2(mi_subproc_t* subproc, void* start, size_t s mi_assert(start!=NULL); if (arena_id != NULL) { *arena_id = _mi_arena_id_none(); } if (start==NULL) return false; - if (!_mi_is_aligned(start, MI_ARENA_ALIGN)) { + if (!_mi_is_aligned(start, MI_ARENA_ALIGNMENT)) { // we can align the start since the memid tracks the real base of the memory. - void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGN); + void* const aligned_start = _mi_align_up_ptr(start, MI_ARENA_ALIGNMENT); const size_t diff = (uint8_t*)aligned_start - (uint8_t*)start; - if (diff >= size || (size - diff) < MI_ARENA_ALIGN) { + if (diff >= size || (size - diff) < MI_ARENA_ALIGNMENT) { _mi_warning_message("after alignment, the size of the arena becomes too small (memory at %p with size %zu)\n", start, size); return false; } @@ -1898,7 +1898,7 @@ static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool com return ENOMEM; } mi_memid_t memid; - void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGN, commit, allow_large, &memid); + void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); if (start == NULL) return ENOMEM; if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { _mi_os_free_ex(subproc, start, size, commit, memid); @@ -2014,7 +2014,7 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } #if MI_PAGE_META_IS_ALIGNED - else if ((slice_index % MI_PAGE_META_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_slices()) { + else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_slices()) { { c = 'i'; color = MI_GRAY; } } #endif diff --git a/src/free.c b/src/free.c index e05e769b0..5321bdaae 100644 --- a/src/free.c +++ b/src/free.c @@ -233,15 +233,15 @@ void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept { void mi_free(void* p) mi_attr_noexcept { #if MI_PAGE_META_IS_ALIGNED - mi_page_t* page = _mi_ptr_page_align0(p); - if mi_unlikely(page==NULL) return; - #if MI_DEBUG>1 - mi_page_t* const cpage = _mi_safe_ptr_page(p); - mi_assert_internal(cpage!=NULL); + mi_page_t* const page0 = _mi_aligned_ptr_page0(p); + if mi_unlikely(page0==NULL) return; + #if MI_DEBUG + mi_page_t* const cpage = mi_validate_ptr_page(p,"mi_free"); #endif - page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); - mi_assert_internal(page == cpage); - mi_free_nonnull(p, page, NULL, true); + mi_page_t* const page = mi_atomic_load_ptr_relaxed(mi_page_t,&page0->self); + mi_assert(cpage==page); + mi_assert_internal(page != NULL); + mi_free_nonnull(p,page,NULL,true); #else mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); mi_free_ex(p, page, NULL, true); @@ -254,15 +254,25 @@ void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { } void mi_free_small(void* p) mi_attr_noexcept { - // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. - // If we keep page info in front of the page area for small objects, we can find the info - // just by aligning down the pointer instead of looking it up in the page map. - #if !MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_ALIGNED_FREE_SMALL + #if MI_PAGE_META_IS_ALIGNED && MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE + mi_page_t* const page0 = _mi_aligned_ptr_page0(p); + if mi_unlikely(page0==NULL) return; + #if MI_DEBUG + mi_assert(page0 == _mi_checked_ptr_page(p)); + mi_page_t* const page = mi_atomic_load_ptr_relaxed(mi_page_t,&page0->self); + mi_assert(page==page0); // for small pages, page0 is the right one + mi_assert_internal(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX)); + #endif + mi_free_nonnull(p,page0,NULL,true); + #elif MI_PAGE_META_SMALL_IS_ALIGNED + // deprecated: We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. + // If we keep page info in front of the page area for small objects, we can find the info + // just by aligning down the pointer instead of looking it up in the page map. #if MI_GUARDED - #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL) ignored as MI_GUARDED is defined" + #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED) ignored as MI_GUARDED is defined" mi_free(p); #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE - #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_ALIGNED_FREE_SMALL) ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size" + #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED) ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size" mi_free(p); #else mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); @@ -286,6 +296,59 @@ void _mi_free_subproc_safe(void* p) mi_attr_noexcept { mi_free_ex(p, page, NULL, false); } +// ------------------------------------------------------ +// Free variants +// ------------------------------------------------------ + +void mi_free_size(void* p, size_t size) mi_attr_noexcept { + MI_UNUSED_RELEASE(size); + #if MI_DEBUG + const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); + if (page==NULL) return; + mi_assert(p!=NULL); + const size_t usable = _mi_page_usable_size(page,p); + if (size > usable) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); + mi_free(p); + return; + } + else if (size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); + mi_free(p); + return; + } + #endif + #if MI_PAGE_META_SMALL_IS_ALIGNED + if mi_likely(size <= MI_SMALL_SIZE_MAX) { + mi_free_small(p); + } + else + #endif + { + mi_free(p); + } +} + +void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free_size(p,size); +} + +void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { + MI_UNUSED_RELEASE(alignment); + mi_assert(((uintptr_t)p % alignment) == 0); + mi_free(p); +} + +// checked free +void mi_cfree(void* p) mi_attr_noexcept { + mi_page_t* const page = _mi_checked_ptr_page(p); + if mi_likely(page!=NULL) { + _mi_free_in_page(p,page); + } +} + // -------------------------------------------------------------------------------------------- // `mi_free_try_collect_mt`: Potentially collect a page in a free in an abandoned page. @@ -476,60 +539,6 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { } -// ------------------------------------------------------ -// Free variants -// ------------------------------------------------------ - -void mi_free_size(void* p, size_t size) mi_attr_noexcept { - MI_UNUSED_RELEASE(size); - #if MI_DEBUG - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); - if (page==NULL) return; - mi_assert(p!=NULL); - const size_t usable = _mi_page_usable_size(page,p); - if (size > usable) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); - mi_free(p); - return; - } - else if (size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); - mi_free(p); - return; - } - #endif - #if MI_OPT_FREE_SMALL && !MI_OPT_FREE - if mi_likely(size <= MI_SMALL_SIZE_MAX) { - mi_free_small(p); - } - else - #endif - { - mi_free(p); - } -} - -void mi_free_size_aligned(void* p, size_t size, size_t alignment) mi_attr_noexcept { - MI_UNUSED_RELEASE(alignment); - mi_assert(((uintptr_t)p % alignment) == 0); - mi_free_size(p,size); -} - -void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { - MI_UNUSED_RELEASE(alignment); - mi_assert(((uintptr_t)p % alignment) == 0); - mi_free(p); -} - -// checked free -void mi_cfree(void* p) mi_attr_noexcept { - mi_page_t* const page = _mi_checked_ptr_page(p); - if mi_likely(page!=NULL) { - _mi_free_in_page(p,page); - } -} - - // ------------------------------------------------------ // Check for double free in secure and debug mode // This is somewhat expensive so only enabled for secure mode 4 diff --git a/src/options.c b/src/options.c index 0ab6efd3a..64a13f709 100644 --- a/src/options.c +++ b/src/options.c @@ -608,6 +608,11 @@ void _mi_error_message(int err, const char* fmt, ...) { } } +mi_decl_noinline mi_block_t* _mi_block_next_is_corrupted(const mi_page_t* page, const mi_block_t* block, const mi_block_t* next) { + _mi_error_message(EFAULT, "corrupted free list entry of size %zub at %p: value 0x%zx\n", mi_page_block_size(page), block, (uintptr_t)next); + return NULL; +} + // -------------------------------------------------------- // Initialize options by checking the environment // -------------------------------------------------------- From b9c167959e5cf8c9c780985227761b973c3bb916 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 12:02:09 -0700 Subject: [PATCH 033/280] fix win32 32-bit compilation --- include/mimalloc/bits.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index 55661bc40..d3c1fa39c 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -138,7 +138,7 @@ typedef int32_t mi_ssize_t; // use a flat page-map or a 2-level one #ifndef MI_PAGE_MAP_FLAT -#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_OPT_FREE +#if MI_MAX_VABITS <= 40 && !defined(__APPLE__) && !MI_SECURE && !MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP #define MI_PAGE_MAP_FLAT 1 #else #define MI_PAGE_MAP_FLAT 0 From 9ad0fa48885dd525ffcae267f3ff6624a011e999 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 16:23:21 -0700 Subject: [PATCH 034/280] optimize checked_ptr_page --- CMakeLists.txt | 2 +- include/mimalloc/bits.h | 4 ++-- include/mimalloc/internal.h | 9 +++++++-- src/page-map.c | 21 +++++++++++---------- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 750edf3b2..f3a15237a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -41,7 +41,7 @@ option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen option(MI_LIBC_MUSL "Enable this when linking with musl libc" OFF) # advanced -option(MI_FREE_USE_PAGEMAP "Always use the page map when free-ing a pointer" OFF) +option(MI_FREE_USE_PAGEMAP "Always use the page map when free-ing a pointer -- enable only for systems that don't support large OS aligned allocations well." OFF) option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG or SECURE mode, or with Valgrind/ASAN)" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index d3c1fa39c..c7ce294f9 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -127,10 +127,10 @@ typedef int32_t mi_ssize_t; #define MI_MAX_VABITS (32) #endif -// the MI_MIN_VABITS determine how many bits of the address are always mapped in the page_map +// the MI_MIN_VABITS determine how many bits of the address space are always committed in the page_map #if MI_MAX_VABITS <= 32 #define MI_MIN_VABITS (32) -#elif MI_MAX_VABITS <= 43 +#elif MI_MAX_VABITS <= 42 #define MI_MIN_VABITS MI_MAX_VABITS #else #define MI_MIN_VABITS (43) /* 8 TiB */ diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index c569b508c..9b8bd1cfc 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -750,10 +750,15 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { static inline mi_page_t* _mi_checked_ptr_page(const void* p) { const mi_page_map_t* pmap = _mi_page_map(); - const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); - if mi_unlikely(idx > committed_count) return NULL; + #if MI_MIN_VABITS < MI_INTPTR_BITS + if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + if mi_unlikely(idx >= committed_count) return NULL; + } + #endif + mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); mi_submap_t const sub = _mi_page_map_at(pmap,idx); if mi_unlikely(sub == NULL) return NULL; return sub[sub_idx]; diff --git a/src/page-map.c b/src/page-map.c index 35b0bbbb7..14872883c 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -289,17 +289,14 @@ static bool mi_page_map_init_once(void) { // Allocate the page map and commit bits mi_assert(MI_MAX_VABITS >= vbits); - const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + mi_assert(MI_MIN_VABITS <= vbits); + const size_t reserve_count = (MI_ZU(1) << (vbits - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); const size_t os_page_size = _mi_os_page_size(); const size_t reserve_size = _mi_align_up( sizeof(mi_page_map_t) + ((reserve_count - 1) * sizeof(mi_submap_t)), os_page_size); const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; const size_t extra_reserve_size = reserve_size + submap_size; - #if MI_SECURE - const bool commit = true; // the whole page map is valid and we can reliably check any pointer - #else - const bool commit = reserve_size <= 256*MI_KiB || // 40 virtual address bits - mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); - #endif + const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || // 42 virtual address bits + mi_option_is_enabled(mi_option_pagemap_commit); // || _mi_os_has_overcommit(); mi_subproc_t* const subproc = _mi_subproc_main(); mi_memid_t memid; mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid); @@ -319,15 +316,19 @@ static bool mi_page_map_init_once(void) { commit_count = mi_page_map_count_of_size(reserve_size); } else { - // commit first page + // commit first entries up to MI_MIN_VABITS entries + const size_t min_commit_count = (MI_ZU(1) << (MI_MIN_VABITS - MI_PAGE_MAP_SUB_SHIFT - MI_ARENA_SLICE_SHIFT)); + const size_t min_commit_size = _mi_align_up( sizeof(mi_page_map_t) + ((min_commit_count-1) * sizeof(mi_submap_t)), os_page_size); + mi_assert_internal(min_commit_size <= reserve_size); bool is_zero; - if (!_mi_os_commit(subproc,pmap,os_page_size,&is_zero)) { + if (!_mi_os_commit(subproc,pmap,min_commit_size,&is_zero)) { mi_page_map_cannot_commit(); _mi_os_free(subproc,pmap,extra_reserve_size,memid); return false; }; mi_assert_internal(is_zero || memid.initially_zero); - commit_count = mi_page_map_count_of_size(os_page_size); + commit_count = mi_page_map_count_of_size(min_commit_size); + mi_assert_internal(commit_count >= min_commit_count); } // ensure there is a submap for the NULL address From 458290d0afc078b7b2f5ac45dbeab6c9921fda0d Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 21:02:42 -0700 Subject: [PATCH 035/280] add more debug assertion to find test bug on macOS 14 --- src/page-map.c | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/src/page-map.c b/src/page-map.c index 14872883c..561a20981 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -329,6 +329,11 @@ static bool mi_page_map_init_once(void) { mi_assert_internal(is_zero || memid.initially_zero); commit_count = mi_page_map_count_of_size(min_commit_size); mi_assert_internal(commit_count >= min_commit_count); + #if MI_DEBUG>1 + size_t subidx; + const size_t idx = _mi_page_map_index((void*)(MI_ZU(1)< Date: Wed, 12 Aug 2026 21:05:10 -0700 Subject: [PATCH 036/280] find test bug on macOS 14 --- include/mimalloc/internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 9b8bd1cfc..6cb3c54ea 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -753,10 +753,10 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); #if MI_MIN_VABITS < MI_INTPTR_BITS - if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + //if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); if mi_unlikely(idx >= committed_count) return NULL; - } + //} #endif mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); mi_submap_t const sub = _mi_page_map_at(pmap,idx); From 7595dee2da4c0ac9f2ae2a24378becafda9c6814 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 21:20:49 -0700 Subject: [PATCH 037/280] find test bug on macOS 14 --- include/mimalloc/internal.h | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 6cb3c54ea..d4c7b6f97 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -755,7 +755,10 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { #if MI_MIN_VABITS < MI_INTPTR_BITS //if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); - if mi_unlikely(idx >= committed_count) return NULL; + if mi_unlikely(idx >= committed_count) { + _mi_warning_message("invalid pointer %p, idx: %zu, committed count: %zu, min_vabits: %zu\n", p, idx, committed_count, MI_MIN_VABITS); + return NULL; + } //} #endif mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); From f9c2ae794c6e894769cb790b1afbdbe6ba28d115 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 21:24:02 -0700 Subject: [PATCH 038/280] find test bug on macOS 14 --- include/mimalloc/internal.h | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index d4c7b6f97..dff0aa5fd 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -748,6 +748,7 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { return _mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL } +#include static inline mi_page_t* _mi_checked_ptr_page(const void* p) { const mi_page_map_t* pmap = _mi_page_map(); size_t sub_idx; @@ -756,7 +757,7 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { //if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); if mi_unlikely(idx >= committed_count) { - _mi_warning_message("invalid pointer %p, idx: %zu, committed count: %zu, min_vabits: %zu\n", p, idx, committed_count, MI_MIN_VABITS); + fprintf(stderr,"invalid pointer %p, idx: %zu, committed count: %zu, min_vabits: %zu\n", p, idx, committed_count, MI_MIN_VABITS); return NULL; } //} From ed3620bcbe7848ffa09a04531f857980037624fb Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 21:29:09 -0700 Subject: [PATCH 039/280] find test bug on macOS 14 --- include/mimalloc/internal.h | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index dff0aa5fd..fbe6e7928 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -754,13 +754,15 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); #if MI_MIN_VABITS < MI_INTPTR_BITS - //if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { - const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { if mi_unlikely(idx >= committed_count) { - fprintf(stderr,"invalid pointer %p, idx: %zu, committed count: %zu, min_vabits: %zu\n", p, idx, committed_count, MI_MIN_VABITS); return NULL; } - //} + } + if mi_unlikely(idx >= committed_count) { + fprintf(stderr,"invalid pointer %p, idx: %zu, committed count: %zu, min_vabits: %d\n", p, idx, committed_count, MI_MIN_VABITS); + } #endif mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); mi_submap_t const sub = _mi_page_map_at(pmap,idx); From 21696da013dc286f45264ca6ef532f15fc351642 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 21:42:07 -0700 Subject: [PATCH 040/280] fix test bug on macOS 14 --- include/mimalloc/internal.h | 19 +++++++------------ 1 file changed, 7 insertions(+), 12 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index fbe6e7928..c009e3e5a 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -748,23 +748,18 @@ static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { return _mi_page_map_at(pmap,idx)[sub_idx]; // NULL if p==NULL } -#include static inline mi_page_t* _mi_checked_ptr_page(const void* p) { const mi_page_map_t* pmap = _mi_page_map(); size_t sub_idx; const size_t idx = _mi_page_map_index(p, &sub_idx); - #if MI_MIN_VABITS < MI_INTPTR_BITS const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); - if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { - if mi_unlikely(idx >= committed_count) { - return NULL; - } - } - if mi_unlikely(idx >= committed_count) { - fprintf(stderr,"invalid pointer %p, idx: %zu, committed count: %zu, min_vabits: %d\n", p, idx, committed_count, MI_MIN_VABITS); - } - #endif - mi_assert_internal(idx < mi_atomic_load_relaxed(&pmap->committed_count)); + if mi_unlikely(idx >= committed_count) return NULL; + // #if MI_MIN_VABITS < MI_INTPTR_BITS // is still invalid if free is called before the pagemap is initialized + // if mi_unlikely(((uintptr_t)p >> MI_MIN_VABITS) != 0) { + // const size_t committed_count = mi_atomic_load_relaxed(&pmap->committed_count); + // if mi_unlikely(idx >= committed_count) return NULL; + // } + // #endif mi_submap_t const sub = _mi_page_map_at(pmap,idx); if mi_unlikely(sub == NULL) return NULL; return sub[sub_idx]; From f69cd7bf66a4aba0e1e1db4fe563d9ebf792957b Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 12 Aug 2026 21:43:04 -0700 Subject: [PATCH 041/280] always fully commit the page map on overcommit systems --- src/page-map.c | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/src/page-map.c b/src/page-map.c index 561a20981..949e37ac3 100644 --- a/src/page-map.c +++ b/src/page-map.c @@ -296,7 +296,7 @@ static bool mi_page_map_init_once(void) { const size_t submap_size = MI_PAGE_MAP_SUB_SIZE; const size_t extra_reserve_size = reserve_size + submap_size; const bool commit = (vbits == MI_MIN_VABITS) || (reserve_size <= 64*MI_KiB) || // 42 virtual address bits - mi_option_is_enabled(mi_option_pagemap_commit); // || _mi_os_has_overcommit(); + mi_option_is_enabled(mi_option_pagemap_commit) || _mi_os_has_overcommit(); mi_subproc_t* const subproc = _mi_subproc_main(); mi_memid_t memid; mi_page_map_t* const pmap = (mi_page_map_t*)_mi_os_alloc_aligned(subproc, extra_reserve_size, 1, commit, true /* allow large */, &memid); @@ -329,11 +329,6 @@ static bool mi_page_map_init_once(void) { mi_assert_internal(is_zero || memid.initially_zero); commit_count = mi_page_map_count_of_size(min_commit_size); mi_assert_internal(commit_count >= min_commit_count); - #if MI_DEBUG>1 - size_t subidx; - const size_t idx = _mi_page_map_index((void*)(MI_ZU(1)< Date: Thu, 13 Aug 2026 08:05:13 -0700 Subject: [PATCH 042/280] fix max arena object size for aligned page meta --- include/mimalloc.h | 1 + src/arena.c | 43 +++++++++++++++++++++++++++---------------- 2 files changed, 28 insertions(+), 16 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index fe5ee1878..581b2bf5b 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -333,6 +333,7 @@ mi_decl_export void mi_debug_show_arenas(void) mi_attr_noexcept; mi_decl_export void mi_arenas_print(void) mi_attr_noexcept; mi_decl_export size_t mi_arena_min_alignment(void); mi_decl_export size_t mi_arena_min_size(void); +mi_decl_export size_t mi_arena_max_object_size(void); typedef void* mi_arena_id_t; mi_decl_export void* mi_arena_area(mi_arena_id_t arena_id, size_t* size); diff --git a/src/arena.c b/src/arena.c index e7209954d..ff6979a28 100644 --- a/src/arena.c +++ b/src/arena.c @@ -97,14 +97,33 @@ size_t mi_arena_min_size(void) { return MI_ARENA_MIN_SIZE; } -static size_t mi_arena_max_object_size(void) { +// slices reserved for page meta info at the start of aligned chunks +static size_t mi_arena_page_meta_aligned_slice_count(void) { + #if MI_PAGE_META_IS_ALIGNED + return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); + #else + return 0; + #endif +} + +// fixed limit for the maximum object size in an arena +static size_t mi_arena_max_fixed_object_size(void) { + #if MI_PAGE_META_IS_ALIGNED + return (MI_PAGE_META_ALIGNMENT - mi_arena_page_meta_aligned_slice_count()); + #else + return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info + #endif +} + +// Maximum object size allowed to be allocated in an arena +size_t mi_arena_max_object_size(void) { size_t max_size = mi_option_get_size(mi_option_arena_max_object_size); max_size = _mi_align_up(max_size, MI_ARENA_SLICE_SIZE); if (max_size <= MI_ARENA_MIN_OBJ_SIZE) { return MI_ARENA_MIN_OBJ_SIZE; } - else if (max_size >= MI_ARENA_MAX_SIZE - (MI_BCHUNK_BITS*MI_ARENA_SLICE_SIZE)) { // minus an initial chunk to accommodate meta info - return (MI_ARENA_MAX_SIZE - (MI_BCHUNK_BITS*MI_ARENA_SLICE_SIZE)); + else if (max_size >= mi_arena_max_fixed_object_size()) { + return mi_arena_max_fixed_object_size(); } else { return max_size; @@ -217,14 +236,6 @@ static size_t mi_page_full_size(mi_page_t* page) { } } -static size_t mi_arena_page_meta_slices(void) { - #if MI_PAGE_META_IS_ALIGNED - return _mi_divide_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE); - #else - return 0; - #endif -} - /* ----------------------------------------------------------- Arena Allocation ----------------------------------------------------------- */ @@ -911,7 +922,7 @@ static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_s const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { // try to commit now - const size_t meta_slice_count = mi_arena_page_meta_slices(); + const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, commit_size /* dont count? */)) { // if the commit fails return NULL @@ -1627,13 +1638,13 @@ static size_t mi_arena_pages_size(size_t slice_count, size_t* bitmap_base) { } static mi_arena_t* mi_arena_info(void* area) { - return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_slices())); + return (mi_arena_t*)((uint8_t*)area + mi_size_of_slices(mi_arena_page_meta_aligned_slice_count())); } static size_t mi_arena_info_slices_needed(size_t slice_count, size_t* bitmap_base) { if (slice_count == 0) slice_count = MI_BCHUNK_BITS; mi_assert_internal((slice_count % MI_BCHUNK_BITS) == 0); - const size_t base_size = mi_size_of_slices(mi_arena_page_meta_slices()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); + const size_t base_size = mi_size_of_slices(mi_arena_page_meta_aligned_slice_count()) + _mi_align_up(sizeof(mi_arena_t), MI_BCHUNK_SIZE); const size_t bitmaps_count = 4 + MI_ARENA_BIN_COUNT; // commit, dirty, purge, pages, and abandoned const size_t bitmaps_size = bitmaps_count * mi_bitmap_size(slice_count, NULL) + mi_bbitmap_size(slice_count, NULL); // + free #if MI_PAGE_META_IS_SEPARATED && !MI_PAGE_META_IS_ALIGNED @@ -1771,7 +1782,7 @@ static mi_arena_t* mi_arena_initialize(mi_subproc_t* subproc, void* start, #if MI_PAGE_META_IS_ALIGNED for(size_t i = 0; i < arena->slice_count; i += MI_PAGE_META_ALIGNED_COUNT) { // set all free slices (and skip the slices reserved for the page meta info) - const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_slices()); + const size_t meta_slices = (i==0 ? info_slices : mi_arena_page_meta_aligned_slice_count()); const size_t start_idx = (i + meta_slices); size_t count = MI_PAGE_META_ALIGNED_COUNT - meta_slices; if (start_idx < arena->slice_count) { @@ -2014,7 +2025,7 @@ static size_t mi_debug_show_page_bfield(char* buf, size_t* k, mi_arena_t* arena, // else if (_mi_meta_is_meta_page(arena->subproc,start)) { c = 'm'; color = MI_GRAY; } else if (slice_index + bit < arena->info_slices) { c = 'i'; color = MI_GRAY; } #if MI_PAGE_META_IS_ALIGNED - else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_slices()) { + else if ((slice_index % MI_PAGE_META_ALIGNED_COUNT) == 0 && (size_t)bit <= mi_arena_page_meta_aligned_slice_count()) { { c = 'i'; color = MI_GRAY; } } #endif From 7720a475136a10fc90c8e8b3d5507814bf4bc962 Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 13 Aug 2026 09:56:11 -0700 Subject: [PATCH 043/280] detect double free through padding --- include/mimalloc/internal.h | 15 ++++-- include/mimalloc/types.h | 27 +++++----- src/alloc.c | 2 +- src/free.c | 100 +++++++++++++++++++++--------------- test/test-wrong.c | 3 ++ 5 files changed, 88 insertions(+), 59 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index c009e3e5a..873c6d9ac 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1217,13 +1217,22 @@ static inline mi_encoded_t mi_ptr_encode(const void* null, const void* p, const static inline uint32_t mi_ptr_encode_canary(const void* null, const void* p, const uintptr_t* keys) { const uint32_t x = (uint32_t)(mi_ptr_encode(null,p,keys)); // make the lowest byte 0 to prevent spurious read overflows which could be a security issue (issue #951) + // also clear bit 9 which we set only when a block is freed. #if MI_BIG_ENDIAN - return (x & 0x00FFFFFF); + return (x & 0x00FFFEFF); #else - return (x & 0xFFFFFF00); + return (x & 0xFFFFFE00); #endif } +static inline uint32_t mi_ptr_encode_canary_freed(void) { + return (0x00DEAD00); // set bit 9 so it is different from any valid canary +} + +static inline bool mi_ptr_decode_canary_is_freed(uint32_t canary) { + return (canary == mi_ptr_encode_canary_freed()); +} + static inline mi_block_t* mi_block_nextx( const void* null, const mi_block_t* block, const uintptr_t* keys ) { mi_track_mem_defined(block,sizeof(mi_block_t)); mi_block_t* next; @@ -1255,7 +1264,7 @@ static inline mi_block_t* mi_block_next(const mi_page_t* page, const mi_block_t* mi_block_t* next = mi_block_nextx(page,block,page->keys); // check for free list corruption: is `next` at least in the same page? // todo: check if `next` is `page->block_size` aligned? - if mi_unlikely(next!=NULL && !mi_is_in_same_page(block, next)) { + if mi_unlikely(next!=NULL && !mi_page_contains_address(page,next)) { return _mi_block_next_is_corrupted(page,block,next); // returns NULL } return next; diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index b8b51164b..62b58e48e 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -55,11 +55,11 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_STAT 1 // Define MI_SECURE to enable security mitigations -// #define MI_SECURE 1 // guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption -// #define MI_SECURE 2 // randomize relative allocation addresses (within mimalloc pages) -// #define MI_SECURE 3 // encode free lists (detect corrupted free list (buffer overflow), and invalid pointer free) -// #define MI_SECURE 4 // checks for double free (may be more expensive) (`-DMI_SECURE=ON`) -// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`) +// #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption +// #define MI_SECURE 2 // randomize relative allocation addresses within mimalloc pages +// #define MI_SECURE 3 // check buffer overflow, check double free, encode free lists (and detect corrupted free lists) +// #define MI_SECURE 4 // same as level 3 for now (`-DMI_SECURE=ON`) +// #define MI_SECURE 5 // guard page at the end of each mimalloc page (expensive!) (`-DMI_SECURE_FULL=ON`), and byte-precise buffer overflow checks. #if !defined(MI_SECURE) #define MI_SECURE 0 @@ -92,14 +92,14 @@ terms of the MIT license. A copy of the license can be found in the file #endif // Reserve extra padding at the end of each block to be more resilient against theap block overflows. -// The padding can detect buffer overflow on free. +// The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. #if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) #define MI_PADDING 1 #endif -// Check padding bytes; allows byte-precise buffer overflow detection -#if !defined(MI_PADDING_CHECK) && MI_PADDING && (MI_SECURE>=3 || MI_DEBUG>=1) -#define MI_PADDING_CHECK 1 +// Check for byte-precise buffer overflow? +#if !defined(MI_PADDING_CHECK_BYTES) && MI_PADDING && (MI_SECURE>=5 || MI_DEBUG>=1) +#define MI_PADDING_CHECK_BYTES 1 #endif @@ -109,11 +109,12 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ENCODE_FREELIST 1 #endif -#if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) -#define MI_CHECK_DOUBLE_FREE 1 -#endif +// Deprecated (checked with padding) +// #if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) +// #define MI_CHECK_DOUBLE_FREE 1 +// #endif -#if MI_SECURE>=4 || (MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP) +#if MI_SECURE>=4 || MI_PADDING || (MI_FREE_IS_CHECKED && MI_FREE_USE_PAGEMAP) #define MI_PAGE_KEY_COUNT 2 #else #define MI_PAGE_KEY_COUNT 1 diff --git a/src/alloc.c b/src/alloc.c index d57309fc9..30a365ef7 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -103,7 +103,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess padding->canary = mi_ptr_encode_canary(page,block,page->keys); padding->delta = (uint32_t)(delta); - #if MI_PADDING_CHECK + #if MI_PADDING_CHECK_BYTES if (!mi_page_is_huge(page)) { uint8_t* fill = (uint8_t*)padding - delta; const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes diff --git a/src/free.c b/src/free.c index 5321bdaae..135a75773 100644 --- a/src/free.c +++ b/src/free.c @@ -13,10 +13,10 @@ terms of the MIT license. A copy of the license can be found in the file #endif // forward declarations -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block); -static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); +// static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block); // ------------------------------------------------------ @@ -27,18 +27,20 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); // fast path written carefully to prevent spilling on the stack static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full) { - MI_UNUSED(was_guarded); // checks - if mi_unlikely(mi_check_is_double_free(page, block)) return; - if (!was_guarded) { mi_check_padding(page, block); } - if (track_stats) { mi_stat_free(page, block); } + size_t usable_size; + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; + // if mi_unlikely(!mi_check_is_double_free(page,block)) return; // checked with padding + + if (track_stats) { + mi_stat_free(page, block); + mi_track_free_size(block, usable_size); + } #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - size_t dbgsize = mi_page_block_size(page); - if (dbgsize > 1*MI_MiB) { dbgsize = 1*MI_MiB; } + const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); #endif - if (track_stats) { mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); } // faster then mi_usable_size as we already know the page and that p is unaligned - + // actual free: push on the local free list mi_block_set_next(page, block, page->local_free); page->local_free = block; @@ -63,17 +65,17 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // Free a block multi-threaded static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_collect) mi_attr_noexcept { - // todo: we cannot safely check for double free in _mt -- should check when collecting the thread_free list - if (!was_guarded) { mi_check_padding(page, block); } // checking padding is safe for mt + size_t usable_size; + if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt + // adjust stats (after padding check ) mi_stat_free(page, block); // stat_free may access the padding - mi_track_free_size(block, mi_page_usable_size_of(page, block, was_guarded)); + mi_track_free_size(block, usable_size); // _mi_padding_shrink(page, block, sizeof(mi_block_t)); #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN // note: when tracking, cannot use mi_usable_size with multi-threading if (!was_guarded) { - size_t dbgsize = mi_usable_size(block); - if (dbgsize > 1*MI_MiB) { dbgsize = 1*MI_MiB; } + const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); } #endif @@ -540,11 +542,12 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { // ------------------------------------------------------ +// Deprecated: double free is checked with padding now. // Check for double free in secure and debug mode // This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ -#if MI_CHECK_DOUBLE_FREE +#if MI_CHECK_DOUBLE_FREE // linear check if the free list contains a specific element static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) { const size_t max_count = page->capacity; // can never hold more blocks than the capacity @@ -576,8 +579,8 @@ static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, con // Used for double free checking to avoid checking free lists too frequently static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { mi_block_t* n = mi_block_nextx(page,block,page->keys); - return (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? - (n==NULL || mi_is_in_same_page(block,n))); // quick check: in the same page or NULL? + return (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? + (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? } // check if `block` was free'd before @@ -604,20 +607,17 @@ static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block // --------------------------------------------------------------------------- #if MI_PADDING // && !MI_TRACK_ENABLED -static bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize) { +static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_t* block, size_t* delta, size_t* bsize, bool* double_free) { *bsize = mi_page_usable_block_size(page); - const mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + *bsize); mi_track_mem_defined(padding,sizeof(mi_padding_t)); *delta = padding->delta; - uint32_t canary = padding->canary; - uintptr_t keys[2]; - keys[0] = page->keys[0]; - #if MI_PAGE_KEY_COUNT==2 - keys[1] = page->keys[1]; - #else - keys[1] = keys[0]; - #endif - bool ok = (mi_ptr_encode_canary(page,block,keys) == canary && *delta <= *bsize); + const uint32_t canary = padding->canary; + const bool ok = (mi_ptr_encode_canary(page,block,page->keys) == canary && *delta <= *bsize); + if (double_free!=NULL) { + if mi_unlikely(!ok) { *double_free = mi_ptr_decode_canary_is_freed(canary); } // double free? + else { padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed + } mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); return ok; } @@ -631,7 +631,7 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl else { size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); mi_assert_internal(ok); mi_assert_internal(delta <= bsize); return (ok ? bsize - delta : 0); } @@ -644,7 +644,7 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); mi_assert_internal(ok); if (!ok || (bsize - delta) >= min_size) return; // usually already enough space mi_assert_internal(bsize >= min_size); @@ -673,16 +673,17 @@ void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const si } #endif -#if MI_PADDING && MI_PADDING_CHECK +#if MI_PADDING -static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong) { +static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, size_t* size, size_t* wrong, bool* is_double_free) { size_t bsize; size_t delta; - bool ok = mi_page_decode_padding(page, block, &delta, &bsize); + bool ok = mi_page_decode_padding(page, block, &delta, &bsize, is_double_free ); *size = *wrong = bsize; if (!ok) return false; mi_assert_internal(bsize >= delta); *size = bsize - delta; + #if MI_PADDING_CHECK_BYTES if (!mi_page_is_huge(page)) { uint8_t* fill = (uint8_t*)block + bsize - delta; const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // check at most the first N padding bytes @@ -696,22 +697,37 @@ static bool mi_verify_padding(const mi_page_t* page, const mi_block_t* block, si } mi_track_mem_noaccess(fill, maxpad); } + #endif return ok; } -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { - size_t size; - size_t wrong; - if (!mi_verify_padding(page,block,&size,&wrong)) { - _mi_error_message(EFAULT, "buffer overflow in theap block %p of size %zu: write after %zu bytes\n", block, size, wrong ); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) { + if mi_unlikely(is_guarded) { + const size_t bsize = mi_page_block_size(page); + *usable_size = (bsize - _mi_os_page_size()); + return true; + } + else { + size_t wrong; + bool is_double_free; + if mi_unlikely(!mi_verify_padding(page,block,usable_size,&wrong,&is_double_free)) { + if (is_double_free) { + _mi_error_message(EAGAIN, "double free detected of heap block %p with size %zu\n", block, *usable_size); + } + else { + _mi_error_message(EFAULT, "buffer overflow in heap block %p of size %zu: write after %zu bytes\n", block, *usable_size, wrong ); + } + return false; + } + return true; } } #else -static void mi_check_padding(const mi_page_t* page, const mi_block_t* block) { - MI_UNUSED(page); - MI_UNUSED(block); +mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size) { + *usable_size = mi_page_usable_size_of(page,block,is_guarded); + return true; } #endif diff --git a/test/test-wrong.c b/test/test-wrong.c index 666d99f59..4e0590839 100644 --- a/test/test-wrong.c +++ b/test/test-wrong.c @@ -65,6 +65,9 @@ int main(int argc, char** argv) { char* c = (char*)mi(malloc)(3); printf("invalid byte: over: %d, under: %d\n", c[4], c[-1]); mi(free)(c); + + // double free + mi(free)(c); // undefined access long* q = (long*)mi(malloc)(sizeof(long)); From 9fb59460ed91215a4501ec27bc151ce8ca466d7f Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 13 Aug 2026 09:57:35 -0700 Subject: [PATCH 044/280] always show errors in CI --- .github/workflows/test.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index a8c35ffa7..fe97d2f53 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -15,6 +15,8 @@ jobs: build: name: Test runs-on: ${{ matrix.os }} + env: + MIMALLOC_SHOW_ERRORS: 1 strategy: matrix: tests: [basic,extra] From 490dc85f9b1d56563464aad891479fee12871c2d Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 13 Aug 2026 12:05:42 -0700 Subject: [PATCH 045/280] clean up cmake options --- CMakeLists.txt | 181 ++++++++++++++++++++++++++++++++----------------- 1 file changed, 118 insertions(+), 63 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f3a15237a..7305ce6a9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,8 +4,9 @@ project(libmimalloc C) set(CMAKE_C_STANDARD 11) set(CMAKE_CXX_STANDARD 17) -option(MI_SECURE "Use security mitigations (like meta data guard pages, allocation randomization, double-free mitigation, and free-list corruption detection)" OFF) -option(MI_SECURE_FULL "Use full security mitigations including guard pages at the end of each mimalloc page (may be expensive)" OFF) +set(MI_SECURE "OFF" CACHE STRING "Use security mitigations (like meta data guard pages, allocation randomization, double-free mitigation, and free-list corruption detection): OFF, ON, FULL. The FULL setting adds guard pages at the end of each mimalloc page (which may be expensive)") +set_property(CACHE MI_SECURE PROPERTY STRINGS "OFF" "ON" "FULL") + option(MI_OVERRIDE "Override the standard malloc interface (i.e. define entry points for 'malloc', 'free', etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) @@ -14,12 +15,11 @@ option(MI_USE_CXX "Use the C++ compiler to compile the library (instea option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016))" OFF) option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) -set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': ON, SHARED, OFF, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") -set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "DEFAULT" "OFF" "SHARED" "ON") +set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") +set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "OFF" "SHARED" "ON" "DEFAULT") -option(MI_DEBUG "Enable assertion checks (enabled by default in a debug build)" OFF) -option(MI_DEBUG_INTERNAL "Enable assertion and internal invariant checks (enabled by default in a debug build)" OFF) -option(MI_DEBUG_FULL "Enable assertion checks and expensive internal heap invariant checking" OFF) +set(MI_DEBUG "DEFAULT" CACHE STRING "Enable assertion checks: OFF, ON, INTERNAL, FULL, DEFAULT. INTERNAL adds extra internal invariant checks, while FULL adds expensive heap invariant checks") +set_property(CACHE MI_DEBUG PROPERTY STRINGS "OFF" "ON" "INTERNAL" "FULL" "DEFAULT") option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) @@ -36,19 +36,17 @@ option(MI_BUILD_TESTS "Build test executables" ON) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) option(MI_WIN_REDIRECT "Use redirection module ('mimalloc-redirect') on Windows if compiling mimalloc as a DLL" ON) -option(MI_WIN_DIRECT_TLS "Use only direct TLS slots on Windows to avoid extra tests in the malloc fast path (only works if the program uses less than 64 TlsAlloc'd slots in total)" OFF) -option(MI_LOCAL_DYNAMIC_TLS "Use local-dynamic-tls, a slightly slower but dlopen-compatible thread local storage mechanism (Unix)" OFF) option(MI_LIBC_MUSL "Enable this when linking with musl libc" OFF) # advanced +option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) +option(MI_NO_THP "Disable transparent huge pages support on Linux/Android for the mimalloc process only" OFF) option(MI_FREE_USE_PAGEMAP "Always use the page map when free-ing a pointer -- enable only for systems that don't support large OS aligned allocations well." OFF) option(MI_PADDING "Enable padding to detect heap block overflow (always on in DEBUG or SECURE mode, or with Valgrind/ASAN)" OFF) option(MI_NO_PADDING "Force no use of padding even in DEBUG mode etc." OFF) option(MI_SKIP_COLLECT_ON_EXIT "Skip collecting memory on program exit" OFF) -option(MI_NO_THP "Disable transparent huge pages support on Linux/Android for the mimalloc process only" OFF) option(MI_EXTRA_CPPDEFS "Extra pre-processor definitions (use as `-DMI_EXTRA_CPPDEFS=\"opt1=val1;opt2=val2\"`)" "") option(MI_SEE_ASM "Generate assembly files" OFF) -option(MI_INSTALL_TOPLEVEL "Install directly into $CMAKE_INSTALL_PREFIX instead of PREFIX/lib/mimalloc-version" OFF) # negated options for vcpkg features option(MI_NO_USE_CXX "Use plain C compilation (has priority over MI_USE_CXX)" OFF) @@ -56,19 +54,28 @@ option(MI_NO_OPT_ARCH "Do not use architecture specific optimizations (lik option(MI_NO_DEBUG "Disable all assertion checks even in debug builds" OFF) # experimental -option(MI_WIN_INIT_USE_RAW_DLLMAIN "Use the raw DLL main entry point for mimalloc initialization; can be more robust but can also lead to link errors with other libraries" OFF) -option(MI_WIN_INIT_USE_TLS_DLLMAIN "Use legacy TLS entries with DllMain for mimalloc initialization" OFF) - -option(MI_TLS_MODEL_LOCAL "Force TLS model based on thread local variables" OFF) -option(MI_TLS_MODEL_PTHREADS "Force TLS model based on pthreads" OFF) -option(MI_TLS_MODEL_FIXED "Force TLS model based on fixed thread local slots" OFF) -option(MI_TLS_RECURSE_GUARD "Force a recursion check on thread local access (for MI_TLS_MODEL_LOCAL only)" OFF) +option(MI_TLS_RECURSE_GUARD "Force a recursion check on thread local access (for MI_TLS_MODEL_LOCAL only)" OFF) +set(MI_WIN_INIT "DEFAULT" CACHE STRING "Force Windows intialization strategy: CRT_TLS (default), RAW_DLLMAIN (use raw DLL main entry point), TLS_DLLMAIN (TLS entries with DllMain), FLS (legacy fiber API), DEFAULT") +set_property(CACHE MI_WIN_INIT PROPERTY STRINGS "CRT_TLS" "RAW_DLLMAIN" "TLS_DLLMAIN" "FLS" "DEFAULT") +set(MI_TLS_MODEL "DEFAULT" CACHE STRING "Force TLS model: LOCAL (thread local variables), PTHREADS (pthread locals), LOCAL_DYNAMIC (dlopen-compatible thread local storage mechanism (Unix)), WIN32 (direct Windows TLS slots), FIXED (fixed thread local slots), DEFAULT.") +set_property(CACHE MI_TLS_MODEL PROPERTY STRINGS "LOCAL" "PTHREADS" "LOCAL_DYNAMIC" "WIN32" "FIXED" "DEFAULT") # deprecated options -option(MI_WIN_USE_FLS "Use Fiber local storage on Windows to detect thread termination (deprecated)" OFF) -option(MI_CHECK_FULL "Use full internal invariant checking in DEBUG mode (deprecated, use MI_DEBUG_FULL instead)" OFF) -option(MI_USE_LIBATOMIC "Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) -option(MI_OPT_FREE_SMALL "Deprecated: Use aligned `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)" OFF) +option(MI_USE_LIBATOMIC "Deprecated: Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) +option(MI_OPT_FREE_SMALL "Deprecated: Use aligned `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)" OFF) +option(MI_DEBUG_INTERNAL "Deprecated: Enable assertion and internal invariant checks (use MI_DEBUG=INTERNAL instead)" OFF) +option(MI_DEBUG_FULL "Deprecated: Enable assertion checks and expensive internal heap invariant checking (use MI_DEBUG=FULL instead)" OFF) +option(MI_CHECK_FULL "Deprecated: Use full internal invariant checking in DEBUG mode (use MI_DEBUG=FULL instead)" OFF) +option(MI_SECURE_FULL "Deprecated: Use full security mitigations including guard pages at the end of each mimalloc page (may be expensive) (use MI_SECURE=FULL instead)" OFF) +option(MI_TLS_MODEL_LOCAL "Deprecated: Force TLS model based on thread local variables (use MI_TLS_MODEL=LOCAL instead)" OFF) +option(MI_TLS_MODEL_PTHREADS "Deprecated: Force TLS model based on pthreads (use MI_TLS_MODEL=PTHREADS instead)" OFF) +option(MI_TLS_MODEL_FIXED "Deprecated: Force TLS model based on fixed thread local slots (use MI_TLS_MODEL=FIXED instead)" OFF) +option(MI_LOCAL_DYNAMIC_TLS "Deprecated: Use local-dynamic-tls, a slightly slower but dlopen-compatible thread local storage mechanism (Unix) (use MI_TLS_MODEL=LOCAL_DYNAMIC instead)" OFF) +option(MI_WIN_USE_FLS "Deprecated: Use Fiber local storage on Windows to detect thread termination (use MI_WIN_INIT=FLS instead)" OFF) +option(MI_WIN_INIT_USE_RAW_DLLMAIN "Deprecated: Use the raw DLL main entry point for mimalloc initialization; can be more robust but can also lead to link errors with other libraries (use MI_WIN_INIT=RAW_DLLMAIN instead)" OFF) +option(MI_WIN_INIT_USE_TLS_DLLMAIN "Deprecated: Use legacy TLS entries with DllMain for mimalloc initialization (use MI_WIN_INIT=TLS_DLLMAIN instead)" OFF) +option(MI_WIN_DIRECT_TLS "Deprecated: Use only direct TLS slots on Windows to avoid extra tests in the malloc fast path (only works if the program uses less than 64 TlsAlloc'd slots in total)" OFF) + include(CheckLinkerFlag) # requires cmake 3.18 include(CheckIncludeFiles) @@ -266,11 +273,14 @@ if(WIN32) endif() if(MI_SECURE_FULL) - set(MI_SECURE ON) - message(STATUS "Set full secure build (MI_SECURE_FULL=ON)") + message(STATUS "MI_SECURE_FULL is deprecated, use MI_SECURE=FULL instead") + set(MI_SECURE "FULL") +endif() +if(MI_SECURE STREQUAL "FULL") + message(STATUS "Use full secure build (MI_SECURE=FULL, may be expensive)") list(APPEND mi_defines MI_SECURE=5) elseif(MI_SECURE) - message(STATUS "Set secure build (MI_SECURE=ON)") + message(STATUS "Use secure build (MI_SECURE=${MI_SECURE})") list(APPEND mi_defines MI_SECURE=4) endif() @@ -338,39 +348,49 @@ if(MI_SEE_ASM) endif() endif() -if(MI_CHECK_FULL) - message(STATUS "The MI_CHECK_FULL option is deprecated, use MI_DEBUG_FULL instead") - set(MI_DEBUG_FULL "ON") -endif() if (MI_SKIP_COLLECT_ON_EXIT) message(STATUS "Skip collecting memory on program exit (MI_SKIP_COLLECT_ON_EXIT=ON)") list(APPEND mi_defines MI_SKIP_COLLECT_ON_EXIT=1) endif() -if(MI_DEBUG_FULL) - message(STATUS "Set debug level to full assertion and internal invariant checking (MI_DEBUG_FULL=ON, expensive)") - set(MI_DEBUG ON) - list(APPEND mi_defines MI_DEBUG=3) # full invariant checking (mi_assert, mi_assert_internal, and mi_assert_expensive) +if(MI_CHECK_FULL OR MI_DEBUG_FULL) + message(STATUS "Use of MI_CHECK_FULL/MI_DEBUG_FULL is deprecated, use MI_DEBUG=FULL instead") + if(MI_DEBUG STREQUAL "DEFAULT") + set(MI_DEBUG "FULL") + endif() elseif(MI_DEBUG_INTERNAL) - message(STATUS "Set debug level to internal assertion and invariant checking (MI_DEBUG_INTERNAL=ON)") - set(MI_DEBUG ON) - list(APPEND mi_defines MI_DEBUG=2) # invariant checking (mi_assert and mi_assert_internal) -elseif(MI_DEBUG) - message(STATUS "Set debug level to assertion checking (MI_DEBUG=ON)") - list(APPEND mi_defines MI_DEBUG=1) # assertion checking (mi_assert) -elseif(MI_NO_DEBUG) + message(STATUS "Use of MI_DEBUG_INTERNAL is deprecated, use MI_DEBUG=INTERNAL instead") + if(MI_DEBUG STREQUAL "DEFAULT") + set(MI_DEBUG "INTERNAL") + endif() +endif() + +if(MI_DEBUG STREQUAL "DEFAULT") + if(CMAKE_BUILD_TYPE MATCHES "Debug") + set(MI_DEBUG "INTERNAL") + else() + set(MI_DEBUG "OFF") + endif() +endif() + +if(MI_NO_DEBUG) message(STATUS "Disable assertion checks (MI_NO_DEBUG=ON)") list(APPEND mi_defines NDEBUG=1) -elseif(CMAKE_BUILD_TYPE MATCHES "Debug") - message(STATUS "Set debug level to internal assertion and invariant checking (CMAKE_BUILD_TYPE=Debug)") - set(MI_DEBUG_INTERNAL ON) - set(MI_DEBUG ON) + set(MI_DEBUG OFF) +elseif(MI_DEBUG STREQUAL "FULL") + message(STATUS "Set debug level to full assertion and internal invariant checking (MI_DEBUG=FULL, expensive)") + list(APPEND mi_defines MI_DEBUG=3) # full invariant checking (mi_assert, mi_assert_internal, and mi_assert_expensive) +elseif(MI_DEBUG STREQUAL "INTERNAL") + message(STATUS "Set debug level to internal assertion and invariant checking (MI_DEBUG=INTERNAL)") list(APPEND mi_defines MI_DEBUG=2) # invariant checking (mi_assert and mi_assert_internal) +elseif(MI_DEBUG) + message(STATUS "Set debug level to assertion checking (MI_DEBUG=${MI_DEBUG})") + list(APPEND mi_defines MI_DEBUG=1) # assertion checking (mi_assert) endif() if(MI_DEBUG AND !MI_GUARDED) - message(STATUS "Enable MI_GUARDED (since MI_DEBUG=ON)") + message(STATUS "Enable MI_GUARDED (since MI_DEBUG is enabled)") set(MI_GUARDED=ON) endif() if(MI_GUARDED) @@ -481,19 +501,42 @@ if(MI_LIBC_MUSL) list(APPEND mi_defines MI_LIBC_MUSL=1) endif() -if(MI_TLS_MODEL_PTHREADS) - message(STATUS "Use TLS model based on pthreads (MI_TLS_MODEL_PTHREADS=ON)") +if(MI_TLS_MODEL STREQUAL "DEFAULT") + if(MI_TLS_MODEL_PTHREADS) + message(STATUS "Use of MI_TLS_MODEL_PTHREADS is deprecated, use MI_TLS_MODEL=PTHREADS instead") + set(MI_TLS_MODEL "PTHREADS") + elseif(MI_TLS_MODEL_FIXED) + message(STATUS "Use of MI_TLS_MODEL_FIXED is deprecated, use MI_TLS_MODEL=FIXED instead") + set(MI_TLS_MODEL "FIXED") + elseif(MI_TLS_MODEL_LOCAL) + message(STATUS "Use of MI_TLS_MODEL_LOCAL is deprecated, use MI_TLS_MODEL=LOCAL instead") + set(MI_TLS_MODEL "LOCAL") + elseif(MI_LOCAL_DYNAMIC_TLS) + message(STATUS "Use of MI_LOCAL_DYNAMIC_TLS is deprecated, use MI_TLS_MODEL=LOCAL_DYNAMIC instead") + set(MI_TLS_MODEL "LOCAL_DYNAMIC") + endif() +endif() +if(MI_TLS_MODEL STREQUAL "PTHREADS") + message(STATUS "Use TLS model based on pthreads (MI_TLS_MODEL=PTHREADS)") list(APPEND mi_defines MI_TLS_MODEL_PTHREADS=1) -elseif(MI_TLS_MODEL_FIXED) - message(STATUS "Use TLS model based on fixed thread local slots (MI_TLS_MODEL_FIXED=ON)") +elseif(MI_TLS_MODEL STREQUAL "FIXED") + message(STATUS "Use TLS model based on fixed thread local slots (MI_TLS_MODEL=FIXED)") message(STATUS "Note: fixed TLS slots do not allow multiple mimalloc instances in a single process.") - list(APPEND mi_defines MI_TLS_MODEL_FIXED=1) -elseif(MI_TLS_MODEL_LOCAL) - message(STATUS "Use TLS model based on thread local (MI_TLS_MODEL_LOCAL=ON)") + list(APPEND mi_defines MI_TLS_MODEL=FIXED=1) +elseif(MI_TLS_MODEL STREQUAL "LOCAL") + message(STATUS "Use TLS model based on thread local (MI_TLS_MODEL=LOCAL)") if(APPLE) - message(WARNING "On macOS the MI_TLS_MODEL_LOCAL does not work reliably when dynamically overriding mimalloc. MI_TLS_MODEL_PTHREADS is recommended instead.") + message(WARNING "On macOS the MI_TLS_MODEL=LOCAL does not work reliably when dynamically overriding mimalloc. MI_TLS_MODEL=PTHREADS is recommended instead.") endif() list(APPEND mi_defines MI_TLS_MODEL_LOCAL=1) +elseif(MI_TLS_MODEL STREQUAL "LOCAL_DYNAMIC") + message(STATUS "Use TLS model based on dlopen compatible dynamic thread locals (MI_TLS_MODEL=LOCAL_DYNAMIC)") + list(APPEND mi_defines MI_TLS_MODEL_LOCAL=1) +elseif(MI_TLS_MODEL STREQUAL "WIN32") + message(STATUS "Use TLS model based on direct Win32 TlsAlloc'd slots (MI_TLS_MODEL=WIN32)") + list(APPEND mi_defines MI_TLS_MODEL_WIN32=1) +elseif(NOT (MI_TLS_MODEL STREQUAL "DEFAULT")) + message(STATUS "Illegal option value: MI_TLS_MODEL=${MI_TLS_MODEL} (ignored)") endif() if(MI_TLS_RECURSE_GUARD) @@ -515,19 +558,31 @@ if(MI_WIN_DIRECT_TLS) list(APPEND mi_defines MI_WIN_DIRECT_TLS=1) endif() -if(MI_WIN_INIT_USE_RAW_DLLMAIN) - message(STATUS "Use raw Dll main entry on Windows to initialize and finalize mimalloc (MI_WIN_INIT_USE_RAW_DLLMAIN=ON)") +if(MI_WIN_INIT STREQUAL "DEFAULT") + if(MI_WIN_INIT_USE_RAW_DLLMAIN) + message(STATUS "Use of MI_WIN_INIT_USE_RAW_DLLMAIN is deprecated, use MI_WIN_INIT=RAW_DLLMAIN instead") + set(MI_WIN_INIT "RAW_DLLMAIN") + elseif(MI_WIN_INIT_USE_TLS_DLLMAIN) + message(STATUS "Use of MI_WIN_INIT_USE_TLS_DLLMAIN is deprecated, use MI_WIN_INIT=TLS_DLLMAIN instead") + set(MI_WIN_INIT "TLS_DLLMAIN") + elseif(MI_WIN_USE_FLS) + message(STATUS "Use of MI_WIN_USE_FLS is deprecated, use MI_WIN_INIT=FLS instead") + set(MI_WIN_INIT "FLS") + endif() +endif() + +if(MI_WIN_INIT STREQUAL "RAW_DLLMAIN") + message(STATUS "Use raw Dll main entry on Windows to initialize and finalize mimalloc (MI_WIN_INIT=RAW_DLLMAIN)") message(STATUS "Note: this can lead to link errors if other libraries also try to define a raw Dll main entry point") list(APPEND mi_defines MI_WIN_INIT_USE_RAW_DLLMAIN=1) -elseif(MI_WIN_INIT_USE_TLS_DLLMAIN) - message(STATUS "Use Dll main entry with TLS entries on Windows to initialize and finalize mimalloc (MI_WIN_INIT_USE_TLS_DLLMAIN=ON)") +elseif(MI_WIN_INIT STREQUAL "TLS_DLLMAIN") + message(STATUS "Use Dll main entry with TLS entries on Windows to initialize and finalize mimalloc (MI_WIN_INIT=TLS_DLLMAIN)") list(APPEND mi_defines MI_WIN_INIT_USE_TLS_DLLMAIN=1) -elseif(MI_WIN_USE_FLS) - message(STATUS "Deprecated: Use the Fiber API to detect thread termination (deprecated) (MI_WIN_USE_FLS=ON)") +elseif(MI_WIN_INIT STREQUAL "FLS") + message(STATUS "Use the Fiber API to detect thread termination (deprecated) (MI_WIN_INIT=FLS)") list(APPEND mi_defines MI_WIN_INIT_USE_FLS=1) -else() - # default uses CRT with TLS entries for initialization and finalization - # list(APPEND mi_defines MI_WIN_INIT_USE_CRT_TLS=1) +elseif(NOT (MI_WIN_INIT STREQUAL "DEFAULT")) + message(STATUS "Illegal option value: MI_WIN_INIT=${MI_WIN_INIT} (ignored)") endif() if(MI_ARCH MATCHES "riscv") @@ -563,7 +618,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel") endif() if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku" AND NOT MI_CLANG_CL) - if(MI_LOCAL_DYNAMIC_TLS) + if(MI_TLS_MODEL STREQUAL "LOCAL_DYNAMIC") list(APPEND mi_cflags -ftls-model=local-dynamic) else() if(MI_LIBC_MUSL) From 4ea234bba51d52da90142a28063c442a66eaf834 Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 13 Aug 2026 15:24:55 -0700 Subject: [PATCH 046/280] make mi_track a multi-choice option --- CMakeLists.txt | 80 ++++++++++++++++++++++++++------------------------ 1 file changed, 42 insertions(+), 38 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7305ce6a9..13e95c80a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -21,11 +21,11 @@ set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "OFF" "SHARED" "ON" "DEFA set(MI_DEBUG "DEFAULT" CACHE STRING "Enable assertion checks: OFF, ON, INTERNAL, FULL, DEFAULT. INTERNAL adds extra internal invariant checks, while FULL adds expensive heap invariant checks") set_property(CACHE MI_DEBUG PROPERTY STRINGS "OFF" "ON" "INTERNAL" "FULL" "DEFAULT") +set(MI_TRACK "OFF" CACHE STRING "Enable tracking: OFF, ASAN (address sanitizer), VALGRIND, ETW (Windows event tracing). Adds a small overhead.") +set_property(CACHE MI_TRACK PROPERTY STRINGS "OFF" "ASAN" "VALGRIND" "ETW") + option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) -option(MI_TRACK_VALGRIND "Compile with Valgrind support (adds a small overhead)" OFF) -option(MI_TRACK_ASAN "Compile with address sanitizer support (adds a small overhead)" OFF) -option(MI_TRACK_ETW "Compile with Windows event tracing (ETW) support (adds a small overhead)" OFF) option(MI_BUILD_SHARED "Build shared library" ON) option(MI_BUILD_STATIC "Build static library" ON) @@ -55,8 +55,10 @@ option(MI_NO_DEBUG "Disable all assertion checks even in debug builds" # experimental option(MI_TLS_RECURSE_GUARD "Force a recursion check on thread local access (for MI_TLS_MODEL_LOCAL only)" OFF) + set(MI_WIN_INIT "DEFAULT" CACHE STRING "Force Windows intialization strategy: CRT_TLS (default), RAW_DLLMAIN (use raw DLL main entry point), TLS_DLLMAIN (TLS entries with DllMain), FLS (legacy fiber API), DEFAULT") set_property(CACHE MI_WIN_INIT PROPERTY STRINGS "CRT_TLS" "RAW_DLLMAIN" "TLS_DLLMAIN" "FLS" "DEFAULT") + set(MI_TLS_MODEL "DEFAULT" CACHE STRING "Force TLS model: LOCAL (thread local variables), PTHREADS (pthread locals), LOCAL_DYNAMIC (dlopen-compatible thread local storage mechanism (Unix)), WIN32 (direct Windows TLS slots), FIXED (fixed thread local slots), DEFAULT.") set_property(CACHE MI_TLS_MODEL PROPERTY STRINGS "LOCAL" "PTHREADS" "LOCAL_DYNAMIC" "WIN32" "FIXED" "DEFAULT") @@ -75,6 +77,9 @@ option(MI_WIN_USE_FLS "Deprecated: Use Fiber local storage on Windows to option(MI_WIN_INIT_USE_RAW_DLLMAIN "Deprecated: Use the raw DLL main entry point for mimalloc initialization; can be more robust but can also lead to link errors with other libraries (use MI_WIN_INIT=RAW_DLLMAIN instead)" OFF) option(MI_WIN_INIT_USE_TLS_DLLMAIN "Deprecated: Use legacy TLS entries with DllMain for mimalloc initialization (use MI_WIN_INIT=TLS_DLLMAIN instead)" OFF) option(MI_WIN_DIRECT_TLS "Deprecated: Use only direct TLS slots on Windows to avoid extra tests in the malloc fast path (only works if the program uses less than 64 TlsAlloc'd slots in total)" OFF) +option(MI_TRACK_VALGRIND "Deprecated: Compile with Valgrind support (adds a small overhead) (use MI_TRACK=VALGRIND instead)" OFF) +option(MI_TRACK_ASAN "Deprecated: Compile with address sanitizer support (adds a small overhead) (use MI_TRACK=ASAN instead)" OFF) +option(MI_TRACK_ETW "Deprecated: Compile with Windows event tracing (ETW) support (adds a small overhead) (use MI_TRACK=ETW instead)" OFF) include(CheckLinkerFlag) # requires cmake 3.18 @@ -285,54 +290,53 @@ elseif(MI_SECURE) endif() if(MI_TRACK_VALGRIND) + message(STATUS "Use of MI_TRACK_VALGRIND is deprecated, use MI_TRACK=VALGRIND instead") + set(MI_TRACK "VALGRIND") +elseif(MI_TRACK_ASAN) + message(STATUS "Use of MI_TRACK_ASAN is deprecated, use MI_TRACK=ASAN instead") + set(MI_TRACK "ASAN") +elseif(MI_TRACK_ETW) + message(STATUS "Use of MI_TRACK_ETW is deprecated, use MI_TRACK=ETW instead") + set(MI_TRACK "ETW") +endif() + +if(MI_TRACK STREQUAL "VALGRIND") CHECK_INCLUDE_FILES("valgrind/valgrind.h;valgrind/memcheck.h" MI_HAS_VALGRINDH) if (NOT MI_HAS_VALGRINDH) - set(MI_TRACK_VALGRIND OFF) + set(MI_TRACK OFF) message(WARNING "Cannot find the 'valgrind/valgrind.h' and 'valgrind/memcheck.h' -- install valgrind first?") - message(STATUS "Disabling Valgrind support (MI_TRACK_VALGRIND=OFF)") + message(STATUS "Disabling Valgrind support (MI_TRACK=OFF)") else() - message(STATUS "Compile with Valgrind support (MI_TRACK_VALGRIND=ON)") + message(STATUS "Compile with Valgrind support (MI_TRACK=VALGRIND)") list(APPEND mi_defines MI_TRACK_VALGRIND=1) endif() -endif() - -if(MI_TRACK_ASAN) +elseif(MI_TRACK STREQUAL "ASAN") if (APPLE AND MI_OVERRIDE) - set(MI_TRACK_ASAN OFF) - message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_TRACK_ASAN=OFF)") - endif() - if (MI_TRACK_VALGRIND) - set(MI_TRACK_ASAN OFF) - message(WARNING "Cannot enable address sanitizer support with also Valgrind support enabled (MI_TRACK_ASAN=OFF)") - endif() - if(MI_TRACK_ASAN) + set(MI_TRACK OFF) + message(WARNING "Cannot enable address sanitizer support on macOS if MI_OVERRIDE is ON (MI_TRACK=OFF)") + else() CHECK_INCLUDE_FILES("sanitizer/asan_interface.h" MI_HAS_ASANH) if (NOT MI_HAS_ASANH) - set(MI_TRACK_ASAN OFF) + set(MI_TRACK OFF) message(WARNING "Cannot find the 'sanitizer/asan_interface.h' -- install address sanitizer support first") - message(STATUS "Compile **without** address sanitizer support (MI_TRACK_ASAN=OFF)") + message(STATUS "Compile **without** address sanitizer support (MI_TRACK=OFF)") else() - message(STATUS "Compile with address sanitizer support (MI_TRACK_ASAN=ON)") + message(STATUS "Compile with address sanitizer support (MI_TRACK=ASAN)") list(APPEND mi_defines MI_TRACK_ASAN=1) list(APPEND mi_cflags -fsanitize=address) list(APPEND mi_libraries -fsanitize=address) endif() endif() -endif() - -if(MI_TRACK_ETW) +elseif(MI_TRACK STREQUAL "ETW") if(NOT WIN32) - set(MI_TRACK_ETW OFF) - message(WARNING "Can only enable ETW support on Windows (MI_TRACK_ETW=OFF)") - endif() - if (MI_TRACK_VALGRIND OR MI_TRACK_ASAN) - set(MI_TRACK_ETW OFF) - message(WARNING "Cannot enable ETW support with also Valgrind or ASAN support enabled (MI_TRACK_ETW=OFF)") - endif() - if(MI_TRACK_ETW) - message(STATUS "Compile with Windows event tracing support (MI_TRACK_ETW=ON)") + set(MI_TRACK OFF) + message(WARNING "Can only enable ETW support on Windows (MI_TRACK=OFF)") + else() + message(STATUS "Compile with Windows event tracing support (MI_TRACK=ETW)") list(APPEND mi_defines MI_TRACK_ETW=1) endif() +elseif(MI_TRACK) + message(STATUS "Invalid option value: MI_TRACK=${MI_TRACK} (ignored)") endif() if(MI_SEE_ASM) @@ -536,7 +540,7 @@ elseif(MI_TLS_MODEL STREQUAL "WIN32") message(STATUS "Use TLS model based on direct Win32 TlsAlloc'd slots (MI_TLS_MODEL=WIN32)") list(APPEND mi_defines MI_TLS_MODEL_WIN32=1) elseif(NOT (MI_TLS_MODEL STREQUAL "DEFAULT")) - message(STATUS "Illegal option value: MI_TLS_MODEL=${MI_TLS_MODEL} (ignored)") + message(STATUS "Invalid option value: MI_TLS_MODEL=${MI_TLS_MODEL} (ignored)") endif() if(MI_TLS_RECURSE_GUARD) @@ -582,7 +586,7 @@ elseif(MI_WIN_INIT STREQUAL "FLS") message(STATUS "Use the Fiber API to detect thread termination (deprecated) (MI_WIN_INIT=FLS)") list(APPEND mi_defines MI_WIN_INIT_USE_FLS=1) elseif(NOT (MI_WIN_INIT STREQUAL "DEFAULT")) - message(STATUS "Illegal option value: MI_WIN_INIT=${MI_WIN_INIT} (ignored)") + message(STATUS "Invalid option value: MI_WIN_INIT=${MI_WIN_INIT} (ignored)") endif() if(MI_ARCH MATCHES "riscv") @@ -752,10 +756,10 @@ set(mi_libname "mimalloc") if(MI_SECURE) set(mi_libname "${mi_libname}-secure") endif() -if(MI_TRACK_VALGRIND) +if(MI_TRACK STREQUAL "VALGRIND") set(mi_libname "${mi_libname}-valgrind") endif() -if(MI_TRACK_ASAN) +if(MI_TRACK STREQUAL "ASAN") set(mi_libname "${mi_libname}-asan") endif() string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) @@ -964,7 +968,7 @@ if (MI_BUILD_TESTS) endforeach() # static override test - if(MI_BUILD_OBJECT AND NOT (MI_CYGWIN OR MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) + if(MI_BUILD_OBJECT AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) add_executable(mimalloc-test-stress-static test/test-stress.c) target_compile_definitions(mimalloc-test-stress-static PRIVATE ${mi_defines} "USE_STD_MALLOC=1") target_compile_options(mimalloc-test-stress-static PRIVATE ${mi_cflags}) @@ -974,7 +978,7 @@ if (MI_BUILD_TESTS) endif() # dynamic override test - if(MI_BUILD_SHARED AND NOT (MI_CYGWIN OR MI_TRACK_ASAN OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) + if(MI_BUILD_SHARED AND NOT (MI_CYGWIN OR (MI_TRACK STREQUAL "ASAN") OR MI_DEBUG_TSAN OR MI_DEBUG_UBSAN)) # AND NOT (APPLE AND MI_USE_CXX)) add_executable(mimalloc-test-stress-dynamic test/test-stress.c) target_compile_definitions(mimalloc-test-stress-dynamic PRIVATE ${mi_defines} "USE_STD_MALLOC=1") target_compile_options(mimalloc-test-stress-dynamic PRIVATE ${mi_cflags}) From 389d5dfe07267e8c99e9dd94e0a683820daf8e64 Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 13 Aug 2026 17:24:10 -0700 Subject: [PATCH 047/280] clean up mi_validate_ptr --- include/mimalloc/internal.h | 4 +- include/mimalloc/prim-tls.h | 4 +- src/free.c | 116 ++++++++++++++++------------- src/prim/osx/alloc-override-zone.c | 6 +- 4 files changed, 70 insertions(+), 60 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 873c6d9ac..d00d6f253 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -331,8 +331,8 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); // "free.c" -void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept; -void _mi_free_subproc_safe_in_page(void* p, mi_page_t* page) mi_attr_noexcept; +void _mi_free_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept; +void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept; void _mi_free_subproc_safe(void* p) mi_attr_noexcept; void _mi_page_unguard_all(mi_page_t* page); size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept; diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 117bf2c23..b0b5c137a 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -89,7 +89,7 @@ static inline void** mi_prim_thread_pointer(void) { static inline void** mi_prim_thread_pointer(void) { void** tcb; #if defined(__APPLE__) // M1, issue rgb(62, 76, 62) - __asm__ ("mrs %0, tpidrro_el0\nbic %0, %0, #7" : "=r" (tcb)); + __asm__ ("mrs %0, tpidrro_el0\n\tbic %0, %0, #7" : "=r" (tcb)); #else __asm__ ("mrs %0, tpidr_el0" : "=r" (tcb)); #endif @@ -104,7 +104,7 @@ static inline void** mi_prim_thread_pointer(void) { #elif defined(__arm__) static inline void** mi_prim_thread_pointer(void) { void** tcb; - __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\nbic %0, %0, #3" : "=r" (tcb)); + __asm__ volatile ("mrc p15, 0, %0, c13, c0, 3\n\tbic %0, %0, #3" : "=r" (tcb)); return tcb; } #elif defined(__i386__) diff --git a/src/free.c b/src/free.c index 135a75773..d4dba16fb 100644 --- a/src/free.c +++ b/src/free.c @@ -172,22 +172,50 @@ void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) // Get the page belonging to a pointer // Does further checks in debug mode to see if this was a valid pointer. -static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) +static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, bool free_small, mi_page_t** ppage) { MI_UNUSED_RELEASE(msg); #if MI_DEBUG if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); - return NULL; - } - mi_page_t* page = _mi_ptr_page(p); - if (p != NULL && page == NULL) { - _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); + return false; } - return page; + #endif + + mi_page_t* page; + #if MI_PAGE_META_IS_ALIGNED + page = _mi_aligned_ptr_page0(p); + if mi_unlikely(page==NULL) return false; // p==NULL => page==NULL + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + #endif + + #if MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE + if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_relaxed(mi_page_t,&page->self)); } + else { page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); } + #else + MI_UNUSED(free_small); + page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); + #endif + + mi_assert_internal(page!=NULL); + mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); #else - return _mi_ptr_page(p); + page = _mi_ptr_page(p); + if mi_unlikely(page==NULL) { + #if MI_DEBUG + if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + return false; + } #endif + *ppage = page; + return true; +} + +static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) { + mi_page_t* page; + return (mi_validate_ptr_page_nonnull(p,msg,false,&page) ? page : NULL); } // Free a block @@ -220,53 +248,30 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t } } -static mi_decl_forceinline void mi_free_ex(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) -{ - if mi_unlikely(page==NULL) { // page will be NULL if p==NULL - if (pblock_size!=NULL) { *pblock_size = 0; } - return; - } - mi_free_nonnull(p,page,pblock_size,allow_collect); -} - -void _mi_free_in_page(void* p, mi_page_t* page) mi_attr_noexcept { - mi_free_ex(p, page, NULL, true); +void _mi_free_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { + mi_assert_internal(p!=NULL && page!=NULL); + mi_free_nonnull(p, page, NULL, true /* allow collect? */); } void mi_free(void* p) mi_attr_noexcept { - #if MI_PAGE_META_IS_ALIGNED - mi_page_t* const page0 = _mi_aligned_ptr_page0(p); - if mi_unlikely(page0==NULL) return; - #if MI_DEBUG - mi_page_t* const cpage = mi_validate_ptr_page(p,"mi_free"); - #endif - mi_page_t* const page = mi_atomic_load_ptr_relaxed(mi_page_t,&page0->self); - mi_assert(cpage==page); - mi_assert_internal(page != NULL); - mi_free_nonnull(p,page,NULL,true); - #else - mi_page_t* const page = mi_validate_ptr_page(p,"mi_free"); - mi_free_ex(p, page, NULL, true); - #endif + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free",false,&page)) { + _mi_free_in_page_nonnull(p, page); + } } void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { - mi_page_t* const page = mi_validate_ptr_page(p,"mi_ufree"); - mi_free_ex(p, page, pblock_size, true); + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_ufree",false,&page)) { + mi_free_nonnull(p, page, pblock_size, true /* allow collect? */); + } + else { + if (pblock_size!=NULL) { *pblock_size = 0; } + } } void mi_free_small(void* p) mi_attr_noexcept { - #if MI_PAGE_META_IS_ALIGNED && MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE - mi_page_t* const page0 = _mi_aligned_ptr_page0(p); - if mi_unlikely(page0==NULL) return; - #if MI_DEBUG - mi_assert(page0 == _mi_checked_ptr_page(p)); - mi_page_t* const page = mi_atomic_load_ptr_relaxed(mi_page_t,&page0->self); - mi_assert(page==page0); // for small pages, page0 is the right one - mi_assert_internal(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX)); - #endif - mi_free_nonnull(p,page0,NULL,true); - #elif MI_PAGE_META_SMALL_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED // deprecated: We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. // If we keep page info in front of the page area for small objects, we can find the info // just by aligning down the pointer instead of looking it up in the page map. @@ -281,21 +286,26 @@ void mi_free_small(void* p) mi_attr_noexcept { mi_assert(page == mi_validate_ptr_page(p,"mi_free_small")); mi_assert((void*)page == _mi_align_down_ptr(mi_page_start(page),MI_SMALL_PAGE_SIZE)); mi_assert(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX)); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small` - mi_free_ex(p, page, NULL, true); + mi_free_nonnull(p, page, NULL, true); #endif #else - mi_free(p); + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free_small",true,&page)) { + _mi_free_in_page_nonnull(p, page); + } #endif } -void _mi_free_subproc_safe_in_page(void* p, mi_page_t* page) mi_attr_noexcept { - mi_free_ex(p, page, NULL, false); +void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { + mi_free_nonnull(p, page, NULL, false /* allow collect? */); } // Free a pointer that is potentially allocated in a different sub-process void _mi_free_subproc_safe(void* p) mi_attr_noexcept { - mi_page_t* const page = mi_validate_ptr_page(p,"_mi_free_subproc_safe"); - mi_free_ex(p, page, NULL, false); + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"_mi_free_subproc_safe",false,&page)) { + _mi_free_subproc_safe_in_page_nonnull(p, page); + } } // ------------------------------------------------------ @@ -347,7 +357,7 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { void mi_cfree(void* p) mi_attr_noexcept { mi_page_t* const page = _mi_checked_ptr_page(p); if mi_likely(page!=NULL) { - _mi_free_in_page(p,page); + _mi_free_in_page_nonnull(p,page); } } diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index c859ae991..db45aca63 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -74,13 +74,13 @@ static void zone_free(malloc_zone_t* zone, void* p) { mi_page_t* const page = _mi_checked_ptr_page(p); if mi_likely(page!=NULL) { mi_assert_internal(_mi_thread_is_initialized()); - _mi_free_in_page(p,page); + _mi_free_in_page_nonnull(p,page); // if mi_likely(_mi_thread_is_initialized()) { - // _mi_free_in_page(p,page); + // _mi_free_in_page_nonnull(p,page); // } // else { // // during thread shutdown `_pthread_tsd_cleanup` may call `zone_free` on a pointer that was allocated in another subproc. - // _mi_free_subproc_safe_in_page(p,page); + // _mi_free_subproc_safe_in_page_nonnull(p,page); // } } else if (!is_mimalloc_zone(zone)) { // can happen due to interpose From 2dd48f1c9fc06b2d275bb567d85a8c41b454f292 Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 13 Aug 2026 17:57:52 -0700 Subject: [PATCH 048/280] keep max 3 pages as retired --- src/page.c | 45 ++++++++++++++++++++++++++------------------- 1 file changed, 26 insertions(+), 19 deletions(-) diff --git a/src/page.c b/src/page.c index 417e1008b..d315e85e2 100644 --- a/src/page.c +++ b/src/page.c @@ -411,7 +411,8 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { // _mi_arenas_collect(false, false, theap->tld); // allow purging } -#define MI_RETIRE_CYCLES (16) +#define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "cycles" before free'ing it */ +#define MI_RETIRE_MAX_PAGES (3) /* keep at most N pages per size bin as retired */ // Retire a page with no more used blocks // Important to not retire too quickly though as new @@ -438,7 +439,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { #if MI_RETIRE_CYCLES > 0 const size_t bsize = mi_page_block_size(page); if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? - if (pq->last==page && pq->first==page) { // the only page in the queue? + if (pq->count <= MI_RETIRE_MAX_PAGES) { // at most N pages left? // pq->last==page && pq->first==page) { // the only page in the queue? mi_theap_t* theap = mi_page_theap(page); mi_theap_stat_counter_increase(theap, pages_retire, 1); page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); @@ -477,29 +478,35 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) { } } +static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) { + if (mi_page_all_free(page)) { + page->retire_expire--; + if (page->retire_expire == 0 || force) { + _mi_page_free(page, pq); + } + else { + // keep retired, update min/max + if (bin < *min) *min = bin; + if (bin > *max) *max = bin; + } + } + else { + page->retire_expire = 0; + } +} + // free retired pages: we don't need to look at the entire queues // since we only retire pages that are at the head position in a queue. void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { size_t min = MI_BIN_FULL; size_t max = 0; for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) { - mi_page_queue_t* pq = &theap->pages[bin]; - mi_page_t* page = pq->first; - if (page != NULL && page->retire_expire != 0) { - if (mi_page_all_free(page)) { - page->retire_expire--; - if (page->retire_expire == 0 || force) { - _mi_page_free(page, pq); - } - else { - // keep retired, update min/max - if (bin < min) min = bin; - if (bin > max) max = bin; - } - } - else { - page->retire_expire = 0; - } + mi_page_queue_t* pq = &theap->pages[bin]; + mi_page_t* page = pq->first; + for (int i = 0; iretire_expire==0; i++) { + mi_page_t* next = page-> next; + mi_page_try_retire(pq,page,bin,force,&min,&max); + page = next; } } theap->page_retired_min = min; From 2dc67efbddfb7844bd40533d76721e3893a16023 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 08:43:59 -0700 Subject: [PATCH 049/280] use load_acquire for ptr_page; use armv8.3 for MI_OPT_ARCH and rva22_zacas for riscV --- .github/workflows/test.yaml | 18 +++++++++--------- CMakeLists.txt | 14 +++++++------- include/mimalloc/internal.h | 4 ++-- src/arena.c | 4 ++-- src/free.c | 14 +++++++------- 5 files changed, 27 insertions(+), 27 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index fe97d2f53..020311e9e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -60,12 +60,12 @@ jobs: cmake --build out/release --parallel 4 --config Release ctest --test-dir out/release --verbose --timeout 240 -C Release - - name: Release, asm + - name: Release, Asm if: matrix.tests == 'basic' && runner.os == 'Linux' uses: actions/upload-artifact@v7 with: - name: ${{ matrix.os }}-alloc.c.s - path: out/release/CMakeFiles/mimalloc-static.dir/src/alloc.c.s + name: ${{ matrix.os }}-asm + path: out/release/CMakeFiles/mimalloc-static.dir/src/*.c.s - name: Secure @@ -178,12 +178,12 @@ jobs: cmake --build out/release-clang-cxx --parallel 8 --config Release ctest --test-dir out/release-clang-cxx --verbose --timeout 240 -C Release - - name: Release, C++, clang++, asm + - name: Release, C++, clang++, Asm if: matrix.tests == 'extra' && runner.os == 'Linux' uses: actions/upload-artifact@v7 with: - name: ${{ matrix.os }}-clang-cxx-alloc.s - path: out/release-clang-cxx/alloc.s + name: ${{ matrix.os }}-clang-cxx-asm + path: out/release-clang-cxx/*.s - name: Debug, ASAN if: matrix.tests == 'extra' && runner.os == 'Linux' @@ -361,12 +361,12 @@ jobs: cmake --build out/release-riscv64 --parallel 4 --config Release ctest --test-dir out/release-riscv64 --verbose --timeout 240 -C Release - - name: Alpine risc64, asm + - name: Alpine risc64, Asm if: matrix.tests == 'alpine-riscv64' uses: actions/upload-artifact@v7 with: - name: alpine-riscv64-alloc.c.s - path: out/release-riscv64/CMakeFiles/mimalloc.dir/src/alloc.c.s + name: alpine-riscv64-asm + path: out/release-riscv64/CMakeFiles/mimalloc.dir/src/*.c.s - name: Alpine riscv64, Secure if: matrix.tests == 'alpine-riscv64' diff --git a/CMakeLists.txt b/CMakeLists.txt index 13e95c80a..47472099c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_GUARDED "Build with guard pages behind certain object allocations (enabled by default in a debug build)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) -option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016))" OFF) +option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.3-a' (2016), for riscV: '-march=rva22_zacas' (2024))" OFF) option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") @@ -50,7 +50,7 @@ option(MI_SEE_ASM "Generate assembly files" OFF) # negated options for vcpkg features option(MI_NO_USE_CXX "Use plain C compilation (has priority over MI_USE_CXX)" OFF) -option(MI_NO_OPT_ARCH "Do not use architecture specific optimizations (like '-march=armv8.1-a' for example) (has priority over MI_OPT_ARCH)" OFF) +option(MI_NO_OPT_ARCH "Do not use architecture specific optimizations (like '-march=armv8.3-a' for example) (has priority over MI_OPT_ARCH)" OFF) option(MI_NO_DEBUG "Disable all assertion checks even in debug builds" OFF) # experimental @@ -188,7 +188,7 @@ endif() if(MI_NO_OPT_ARCH) set(MI_OPT_ARCH "OFF") elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH "ON") # enable armv8.1-a by default on arm64 unless MI_NO_OPT_ARCH is set + set(MI_OPT_ARCH "ON") # enable armv8.3-a by default (has LDAPR) on arm64 unless MI_NO_OPT_ARCH is set endif() if(DEFINED ENV{MI_TEST_LIGHT}) @@ -647,7 +647,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM if(MI_OPT_ARCH) if(APPLE AND CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND CMAKE_OSX_ARCHITECTURES) # to support multi-arch binaries (#999) if("arm64" IN_LIST CMAKE_OSX_ARCHITECTURES) - list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_arm64;-march=armv8.1-a") + list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_arm64;-march=armv8.3-a") endif() if("x86_64" IN_LIST CMAKE_OSX_ARCHITECTURES) list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_x86_64;-march=haswell;-Xarch_x86_64;-mavx2") @@ -655,9 +655,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM elseif(MI_ARCH STREQUAL "x64") set(MI_OPT_ARCH_FLAGS "-march=haswell;-mavx2") # fast bit scan (since 2013) elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH_FLAGS "-march=armv8.1-a") # fast atomics (since 2016) + set(MI_OPT_ARCH_FLAGS "-march=armv8.3-a") # fast atomics (since 2016) elseif(MI_ARCH STREQUAL "riscv64") - set(MI_OPT_ARCH_FLAGS "-march=rv64gc_zacas") # fast atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rva22_zacas") # fast bit scan and atomics (since 2023) endif() endif() endif() @@ -668,7 +668,7 @@ if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) # vs2017+ if(MI_ARCH STREQUAL "x64") set(MI_OPT_ARCH_FLAGS "/arch:AVX2") elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH_FLAGS "/arch:armv8.1") # fast atomics (LSE) + set(MI_OPT_ARCH_FLAGS "/arch:armv8.3") # fast atomics (LDAPR) endif() endif() endif() diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index d00d6f253..de9c77b5b 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -738,7 +738,7 @@ static inline mi_page_map_t* _mi_page_map(void) { } static inline mi_submap_t _mi_page_map_at(const mi_page_map_t* pmap, size_t idx) { - return mi_atomic_load_ptr_relaxed(mi_page_t*, &pmap->submaps[idx]); + return mi_atomic_load_ptr_acquire(mi_page_t*, &pmap->submaps[idx]); } static inline mi_page_t* _mi_unchecked_ptr_page(const void* p) { @@ -780,7 +780,7 @@ static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { mi_page_t* const page = _mi_aligned_ptr_page0(p); if mi_unlikely(page==NULL) return NULL; - return mi_atomic_load_relaxed(&page->self); + return mi_atomic_load_acquire(&page->self); } #endif diff --git a/src/arena.c b/src/arena.c index ff6979a28..c90c8e265 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1085,10 +1085,10 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou } #if MI_DEBUG>1 mi_page_t* pstart = _mi_aligned_ptr_page0(slice_start); - mi_assert_internal(pstart->self==page); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pstart->self)==page); if (reserved>1) { mi_page_t* pend = _mi_aligned_ptr_page0(slice_start + (slice_count*MI_ARENA_SLICE_SIZE) - 1); - mi_assert_internal(pend->self==page); + mi_assert_internal(mi_atomic_load_ptr_acquire(mi_page_t,&pend->self)==page); } #endif #endif diff --git a/src/free.c b/src/free.c index d4dba16fb..d4e5a83bd 100644 --- a/src/free.c +++ b/src/free.c @@ -191,11 +191,11 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, #endif #if MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE - if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_relaxed(mi_page_t,&page->self)); } - else { page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); } + if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + else { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); } #else MI_UNUSED(free_small); - page = mi_atomic_load_ptr_relaxed(mi_page_t,&page->self); + page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); #endif mi_assert_internal(page!=NULL); @@ -256,7 +256,7 @@ void _mi_free_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { void mi_free(void* p) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free",false,&page)) { - _mi_free_in_page_nonnull(p, page); + mi_free_nonnull(p, page, NULL, true /* allow collect? */); } } @@ -291,7 +291,7 @@ void mi_free_small(void* p) mi_attr_noexcept { #else mi_page_t* page; if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free_small",true,&page)) { - _mi_free_in_page_nonnull(p, page); + mi_free_nonnull(p, page, NULL, true /* allow collect? */); } #endif } @@ -304,7 +304,7 @@ void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noe void _mi_free_subproc_safe(void* p) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_validate_ptr_page_nonnull(p,"_mi_free_subproc_safe",false,&page)) { - _mi_free_subproc_safe_in_page_nonnull(p, page); + mi_free_nonnull(p, page, NULL, false /* allow collect? */); } } @@ -357,7 +357,7 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { void mi_cfree(void* p) mi_attr_noexcept { mi_page_t* const page = _mi_checked_ptr_page(p); if mi_likely(page!=NULL) { - _mi_free_in_page_nonnull(p,page); + mi_free_nonnull(p, page, NULL, true /* allow collect? */); } } From a32be20f81714fbcf8ea2e1c628b94e8ccf3903e Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 08:48:41 -0700 Subject: [PATCH 050/280] fix collect_retire condition --- src/page.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/page.c b/src/page.c index d315e85e2..130d08d50 100644 --- a/src/page.c +++ b/src/page.c @@ -503,8 +503,8 @@ void _mi_theap_collect_retired(mi_theap_t* theap, bool force) { for(size_t bin = theap->page_retired_min; bin <= theap->page_retired_max; bin++) { mi_page_queue_t* pq = &theap->pages[bin]; mi_page_t* page = pq->first; - for (int i = 0; iretire_expire==0; i++) { - mi_page_t* next = page-> next; + for (int i = 0; iretire_expire!=0; i++) { + mi_page_t* next = page->next; mi_page_try_retire(pq,page,bin,force,&min,&max); page = next; } From 79c50117b461d9331b8cccd32af16cf1189e051c Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 08:54:29 -0700 Subject: [PATCH 051/280] show env in Debug CI --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 020311e9e..9c5cc59dc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -49,6 +49,7 @@ jobs: - name: Debug if: matrix.tests == 'basic' run: | + env cmake . -B out/debug -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON cmake --build out/debug --parallel 4 --config Debug ctest --test-dir out/debug --verbose --timeout 240 -C Debug From 81154310e781ca4f54f99b04a04bbc72a53c16fc Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 08:55:28 -0700 Subject: [PATCH 052/280] show env in Debug CI --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9c5cc59dc..aadce5a9e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -139,6 +139,7 @@ jobs: - name: Debug, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | + env cmake . -B out/debug-tls-pthreads -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_PTHREADS=ON cmake --build out/debug-tls-pthreads --parallel 4 --config Debug ctest --test-dir out/debug-tls-pthreads --verbose --timeout 240 -C Debug From 7ecdf7e4600b7a75987064cbb99df98aa4971c21 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 09:00:32 -0700 Subject: [PATCH 053/280] use rv64gcb_zacas on riscv --- .github/workflows/test.yaml | 1 - CMakeLists.txt | 6 ++++-- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index aadce5a9e..9c5cc59dc 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -139,7 +139,6 @@ jobs: - name: Debug, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | - env cmake . -B out/debug-tls-pthreads -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_PTHREADS=ON cmake --build out/debug-tls-pthreads --parallel 4 --config Debug ctest --test-dir out/debug-tls-pthreads --verbose --timeout 240 -C Debug diff --git a/CMakeLists.txt b/CMakeLists.txt index 47472099c..471a885fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_GUARDED "Build with guard pages behind certain object allocations (enabled by default in a debug build)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) -option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.3-a' (2016), for riscV: '-march=rva22_zacas' (2024))" OFF) +option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.3-a' (2016), for riscV: '-march=rv64gcb_zacas' (2024))" OFF) option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") @@ -657,7 +657,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM elseif(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH_FLAGS "-march=armv8.3-a") # fast atomics (since 2016) elseif(MI_ARCH STREQUAL "riscv64") - set(MI_OPT_ARCH_FLAGS "-march=rva22_zacas") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas") # fast bit scan and atomics (since 2023) + elseif(MI_ARCH STREQUAL "riscv32") + set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas") # fast bit scan and atomics (since 2023) endif() endif() endif() From 3e78d8f0b450848b6176211453bd974c804dd644 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 09:15:05 -0700 Subject: [PATCH 054/280] add zalars for riscv --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 471a885fb..e13a5d4e6 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_GUARDED "Build with guard pages behind certain object allocations (enabled by default in a debug build)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) -option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.3-a' (2016), for riscV: '-march=rv64gcb_zacas' (2024))" OFF) +option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.3-a' (2016), for riscV: '-march=rv64gcb_zacas_zalasr' (2024))" OFF) option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") @@ -657,9 +657,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM elseif(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH_FLAGS "-march=armv8.3-a") # fast atomics (since 2016) elseif(MI_ARCH STREQUAL "riscv64") - set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas_zalasr") # fast bit scan and atomics (since 2023) elseif(MI_ARCH STREQUAL "riscv32") - set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas_zalars") # fast bit scan and atomics (since 2023) endif() endif() endif() From 150fd517e7cf28749d892060d29b09ed385a7dfd Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 09:20:21 -0700 Subject: [PATCH 055/280] add zalars for riscv --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e13a5d4e6..046687d32 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -657,9 +657,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM elseif(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH_FLAGS "-march=armv8.3-a") # fast atomics (since 2016) elseif(MI_ARCH STREQUAL "riscv64") - set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas_zalasr") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas_zalasr -menable-experimental-extensions") # fast bit scan and atomics (since 2023) elseif(MI_ARCH STREQUAL "riscv32") - set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas_zalars") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas_zalars -menable-experimental-extensions") # fast bit scan and atomics (since 2023) endif() endif() endif() From 4fda8a342d80c6ee3940ee130410f8a88783312c Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 09:20:54 -0700 Subject: [PATCH 056/280] add zalars for riscv --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 046687d32..f405b9b1a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -659,7 +659,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM elseif(MI_ARCH STREQUAL "riscv64") set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas_zalasr -menable-experimental-extensions") # fast bit scan and atomics (since 2023) elseif(MI_ARCH STREQUAL "riscv32") - set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas_zalars -menable-experimental-extensions") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas_zalasr -menable-experimental-extensions") # fast bit scan and atomics (since 2023) endif() endif() endif() From 5e1a950a10423bbc91c831581c6462111ed7af43 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 09:26:14 -0700 Subject: [PATCH 057/280] add zalasr for riscv --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f405b9b1a..9b05f1a63 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -657,9 +657,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM elseif(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH_FLAGS "-march=armv8.3-a") # fast atomics (since 2016) elseif(MI_ARCH STREQUAL "riscv64") - set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas_zalasr -menable-experimental-extensions") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-menable-experimental-extensions;-march=rv64gcb_zacas_zalasr") # fast bit scan and atomics (since 2023) elseif(MI_ARCH STREQUAL "riscv32") - set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas_zalasr -menable-experimental-extensions") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-menable-experimental-extensions;-march=rv32gcb_zacas_zalasr") # fast bit scan and atomics (since 2023) endif() endif() endif() From 5a429ab29e4ecc966ac63e9c6075c1181864b538 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 09:32:56 -0700 Subject: [PATCH 058/280] remove zalasr for riscv until wider support --- CMakeLists.txt | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b05f1a63..25cc2b86b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_GUARDED "Build with guard pages behind certain object allocations (enabled by default in a debug build)" OFF) option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) -option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.3-a' (2016), for riscV: '-march=rv64gcb_zacas_zalasr' (2024))" OFF) +option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.3-a' (2016), for riscV: '-march=rv64gcb_zacas' (2023))" OFF) option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") @@ -657,9 +657,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM elseif(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH_FLAGS "-march=armv8.3-a") # fast atomics (since 2016) elseif(MI_ARCH STREQUAL "riscv64") - set(MI_OPT_ARCH_FLAGS "-menable-experimental-extensions;-march=rv64gcb_zacas_zalasr") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas") # fast bit scan and atomics (since 2023) TODO: add zalasr (2025) when possible. elseif(MI_ARCH STREQUAL "riscv32") - set(MI_OPT_ARCH_FLAGS "-menable-experimental-extensions;-march=rv32gcb_zacas_zalasr") # fast bit scan and atomics (since 2023) + set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas") # fast bit scan and atomics (since 2023) endif() endif() endif() From 66383f06385c3c513c291bb882763fe1ab4e18b8 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Fri, 14 Aug 2026 18:48:43 +0100 Subject: [PATCH 059/280] fix numa node count on linux: return the count, not the highest node index The scan starts at node1, so on a machine with nodes 0..N-1 `last_found` ends at N-1 and is returned as the count: 2 nodes reported as 1, 4 as 3. Callers use it as a bound on node ids (`numa_node % numa_count`), so the highest node gets folded onto node 0. Return `last_found + 1`, as the Windows primitive does with `GetNumaHighestNodeNumber() + 1`. Follow-up to 870ac856. --- src/prim/unix/prim.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index bde6183b9..34dd2c2c8 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -609,7 +609,7 @@ size_t _mi_prim_numa_node(void) { size_t _mi_prim_numa_node_count(void) { char buf[128]; - unsigned last_found = 1; + unsigned last_found = 0; for(unsigned node = 1; node < 256; node++) { // enumerate node entries -- todo: it there a more efficient way to do this? (but ensure there is no allocation) _mi_snprintf(buf, 127, "/sys/devices/system/node/node%u", node); @@ -618,7 +618,7 @@ size_t _mi_prim_numa_node_count(void) { } else { last_found = node; } // highest found node } - return last_found; + return last_found + 1; } #elif defined(__FreeBSD__) && __FreeBSD_version >= 1200000 From 66ee6ff30a847931e7ac860f116ba96b9deb600b Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 11:07:26 -0700 Subject: [PATCH 060/280] fix MI_FREE_USE_PAGEMAP option --- CMakeLists.txt | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 25cc2b86b..bd5916576 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -434,12 +434,14 @@ else() message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, OFF, SHARED, or ON)") endif() +if(CMAKE_SYSTEM_NAME STREQUAL "WASI") + message(STATUS "Enabling MI_FREE_USE_PAGEMAP since the target (WASI) does not support large OS aligned allocations well") + set(MI_FREE_USE_PAGEMAP ON) +endif() if(MI_FREE_USE_PAGEMAP) message(STATUS "Always use the pagemap in mi_free (MI_FREE_USE_PAGEMAP=ON)") -elseif(CMAKE_SYSTEM_NAME STREQUAL "WASI") - message(STATUS "Always use the pagemap in mi_free since the target (WASI) does not support large OS aligned allocations well (MI_FREE_USE_PAGEMAP=ON)") list(APPEND mi_defines MI_FREE_USE_PAGEMAP=1) -endif() +endif() # deprecated if(MI_OPT_FREE_SMALL) From 6e5aa7f49db4386511a466adf61a4be7673eea88 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 11:07:42 -0700 Subject: [PATCH 061/280] fix max arena object size --- src/arena.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index c90c8e265..b61095819 100644 --- a/src/arena.c +++ b/src/arena.c @@ -109,7 +109,7 @@ static size_t mi_arena_page_meta_aligned_slice_count(void) { // fixed limit for the maximum object size in an arena static size_t mi_arena_max_fixed_object_size(void) { #if MI_PAGE_META_IS_ALIGNED - return (MI_PAGE_META_ALIGNMENT - mi_arena_page_meta_aligned_slice_count()); + return (MI_PAGE_META_ALIGNMENT - _mi_align_up(MI_PAGE_META_ALIGNED_COUNT * sizeof(mi_page_t), MI_ARENA_SLICE_SIZE)); #else return (MI_ARENA_MAX_SIZE - MI_ARENA_CHUNK_SIZE); // minus an initial chunk to accommodate meta info #endif From 8d947ca3d3ba6d2edd7276d363319286cbdd7188 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 11:08:11 -0700 Subject: [PATCH 062/280] adjust stats better when needing overallocation for alignment --- src/free.c | 1 + src/os.c | 29 +++++++++++++++++------------ src/page.c | 9 +++++---- src/stats.c | 9 ++++++--- 4 files changed, 29 insertions(+), 19 deletions(-) diff --git a/src/free.c b/src/free.c index d4e5a83bd..29091d9b0 100644 --- a/src/free.c +++ b/src/free.c @@ -201,6 +201,7 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, mi_assert_internal(page!=NULL); mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); #else + MI_UNUSED(free_small); page = _mi_ptr_page(p); if mi_unlikely(page==NULL) { #if MI_DEBUG diff --git a/src/os.c b/src/os.c index 50a403609..d1255d7e0 100644 --- a/src/os.c +++ b/src/os.c @@ -235,7 +235,7 @@ bool _mi_os_secure_guard_page_reset_before(mi_subproc_t* subproc, void* addr, mi static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size); -static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size) { +static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size_t commit_size, bool adjust) { mi_assert_internal(subproc!=NULL); mi_assert_internal((size % _mi_os_page_size()) == 0); if (addr == NULL) return; // || _mi_os_is_huge_reserved(addr) @@ -243,10 +243,14 @@ static void mi_os_prim_free(mi_subproc_t* subproc, void* addr, size_t size, size if (err != 0) { _mi_warning_message("unable to free OS memory (error: %d (0x%x), size: 0x%zx bytes, address: %p)\n", err, err, size, addr); } - if (commit_size > 0) { - mi_subproc_stat_decrease(subproc, committed, commit_size); + if (adjust) { + if (commit_size>0) { mi_subproc_stat_adjust_decrease(subproc, committed, commit_size); } + mi_subproc_stat_adjust_decrease(subproc, reserved, size); + } + else { + if (commit_size>0) { mi_subproc_stat_decrease(subproc, committed, commit_size); } + mi_subproc_stat_decrease(subproc, reserved, size); } - mi_subproc_stat_decrease(subproc, reserved, size); } void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_committed, mi_memid_t memid) { @@ -274,7 +278,7 @@ void _mi_os_free_ex(mi_subproc_t* subproc, void* addr, size_t size, bool still_c mi_os_free_huge_os_pages(subproc, base, csize); } else { - mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0)); + mi_os_prim_free(subproc, base, csize, (still_committed ? commit_size : 0), false); } } else { @@ -369,7 +373,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t _mi_warning_message("unable to allocate aligned OS memory directly, fall back to over-allocation (size: 0x%zx bytes, address: %p, alignment: 0x%zx, commit: %d)\n", size, p, alignment, commit); } #endif - if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0)); } + if (p != NULL) { mi_os_prim_free(subproc, p, size, (commit ? size : 0), true /* adjust so we "forget" the previous reservation */); } if (size >= (SIZE_MAX - alignment)) return NULL; // overflow const size_t over_size = size + alignment; @@ -388,14 +392,15 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t // explicitly commit only the aligned part if (commit) { if (!_mi_os_commit(subproc, p, size, NULL)) { - mi_os_prim_free(subproc, os_base, over_size, 0); + mi_os_prim_free(subproc, os_base, over_size, 0, true); return NULL; } } } else { // mmap can free inside an allocation // overallocate... - p = mi_os_prim_alloc(subproc, over_size, 1, commit, false, &os_is_large, &os_is_zero); + // note: we keep `allow_large` but this only works if we can partially free any large OS pages later on. This seems to be the case though. + p = mi_os_prim_alloc(subproc, over_size, 1, commit, allow_large /* or false? */, &os_is_large, &os_is_zero); if (p == NULL) return NULL; // and selectively unmap parts around the over-allocated area. @@ -404,8 +409,8 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t const size_t mid_size = _mi_align_up(size, _mi_os_page_size()); const size_t post_size = over_size - pre_size - mid_size; mi_assert_internal(pre_size < over_size&& post_size < over_size&& mid_size >= size); - if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0)); } - if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0)); } + if (pre_size > 0) { mi_os_prim_free(subproc, p, pre_size, (commit ? pre_size : 0), true /* adjust */); } + if (post_size > 0) { mi_os_prim_free(subproc, (uint8_t*)aligned_p + mid_size, post_size, (commit ? post_size : 0), true /* adjust */); } // we can return the aligned pointer on `mmap` systems p = aligned_p; os_base = aligned_p; // since we freed the pre part, `*base == p`. @@ -787,7 +792,7 @@ void* _mi_os_alloc_huge_os_pages(mi_subproc_t* subproc, size_t pages, int numa_n // no success, issue a warning and break if (p != NULL) { _mi_warning_message("could not allocate contiguous huge OS page %zu at %p\n", page, addr); - mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); + mi_os_prim_free(subproc, p, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, true /* adjust */); } break; } @@ -834,7 +839,7 @@ static void mi_os_free_huge_os_pages(mi_subproc_t* subproc, void* p, size_t size if (p==NULL || size==0) return; uint8_t* base = (uint8_t*)p; while (size >= MI_HUGE_OS_PAGE_SIZE) { - mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE); + mi_os_prim_free(subproc, base, MI_HUGE_OS_PAGE_SIZE, MI_HUGE_OS_PAGE_SIZE, false /* adjust? */); size -= MI_HUGE_OS_PAGE_SIZE; base += MI_HUGE_OS_PAGE_SIZE; } diff --git a/src/page.c b/src/page.c index 130d08d50..02ca72044 100644 --- a/src/page.c +++ b/src/page.c @@ -411,7 +411,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq) { // _mi_arenas_collect(false, false, theap->tld); // allow purging } -#define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "cycles" before free'ing it */ +#define MI_RETIRE_CYCLES (16) /* keep a retired page around for about 16 "admin cycles" before free'ing it */ #define MI_RETIRE_MAX_PAGES (3) /* keep at most N pages per size bin as retired */ // Retire a page with no more used blocks @@ -438,8 +438,8 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { mi_page_queue_t* pq = mi_page_queue_of(page); #if MI_RETIRE_CYCLES > 0 const size_t bsize = mi_page_block_size(page); - if mi_likely( /* bsize < MI_MAX_RETIRE_SIZE && */ !mi_page_queue_is_special(pq)) { // not full or huge queue? - if (pq->count <= MI_RETIRE_MAX_PAGES) { // at most N pages left? // pq->last==page && pq->first==page) { // the only page in the queue? + if mi_likely( pq->count <= MI_RETIRE_MAX_PAGES && !mi_page_queue_is_special(pq)) { // not full or huge queue? + if (pq->count==1 || bsize < MI_SMALL_SIZE_MAX) { mi_theap_t* theap = mi_page_theap(page); mi_theap_stat_counter_increase(theap, pages_retire, 1); page->retire_expire = (bsize <= MI_SMALL_MAX_OBJ_SIZE ? MI_RETIRE_CYCLES : MI_RETIRE_CYCLES/4); @@ -450,7 +450,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index > theap->page_retired_max) theap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); return; // don't free after all - } + } } #endif _mi_page_free(page, pq); @@ -479,6 +479,7 @@ static void mi_theap_collect_full_pages(mi_theap_t* theap) { } static void mi_page_try_retire(mi_page_queue_t* pq, mi_page_t* page, size_t bin, bool force, size_t* min, size_t* max) { + mi_assert_internal(page!=NULL && page->retire_expire!=0); if (mi_page_all_free(page)) { page->retire_expire--; if (page->retire_expire == 0 || force) { diff --git a/src/stats.c b/src/stats.c index 6aaba3ddf..2a07f0198 100644 --- a/src/stats.c +++ b/src/stats.c @@ -69,14 +69,17 @@ void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; // adjust atomically + const size_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); mi_atomic_addi64_relaxed(&stat->current, amount); - mi_atomic_addi64_relaxed(&stat->total, amount); + const size_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); + if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); } } static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; stat->current += amount; - stat->total += amount; + if (stat->total==stat->peak) { stat->peak += amount; } + stat->total += amount; } void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) { @@ -102,7 +105,7 @@ static void mi_stat_count_add_mt(mi_stat_count_t* stat, const mi_stat_count_t* s const int64_t prev_current = mi_atomic_addi64_relaxed(&stat->current, src_current); // Global current plus thread peak approximates new global peak - // note: peak scores do really not work across threads. + // note: peak scores do not really work across threads. // we used to just add them together but that often overestimates in practice. // similarly, max does not seem to work well. The current approach // by Artem Kharytoniuk (@artem-lunarg) seems to work better, see PR#1112 From a89f12bf5acdcd73489b814e5c9e566362d71208 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 11:42:58 -0700 Subject: [PATCH 063/280] don't purge on thread abandon --- src/arena.c | 6 ++---- src/options.c | 2 +- src/theap.c | 4 +++- test/test-stress.c | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/src/arena.c b/src/arena.c index b61095819..18b951383 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2382,10 +2382,8 @@ static int mi_arena_try_purge(mi_arena_t* arena, mi_msecs_t now, bool force) // expired yet? mi_msecs_t expire = mi_atomic_loadi64_relaxed(&arena->purge_expire); - if (!force) { - if (expire==0) return -1; - if (expire > now) return 0; - } + if (expire==0) return -1; + if (!force && expire > now) return 0; // reset expire mi_atomic_storei64_release(&arena->purge_expire, (mi_msecs_t)0); diff --git a/src/options.c b/src/options.c index 64a13f709..a35661ece 100644 --- a/src/options.c +++ b/src/options.c @@ -146,7 +146,7 @@ static mi_option_desc_t mi_options[_mi_option_last] = { 10, MI_OPTION_UNINIT, MI_OPTION(deprecated_max_segment_reclaim)}, // max. percentage of the abandoned segments to be reclaimed per try. { 0, MI_OPTION_UNINIT, MI_OPTION(destroy_on_exit)}, // release all OS memory on process exit; careful with dangling pointer or after-exit frees! { MI_DEFAULT_ARENA_RESERVE, MI_OPTION_UNINIT, MI_OPTION(arena_reserve) }, // reserve memory N KiB at a time (=1GiB) (use `option_get_size`) - { 1, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's + { 4, MI_OPTION_UNINIT, MI_OPTION(arena_purge_mult) }, // purge delay multiplier for arena's { 1, MI_OPTION_UNINIT, MI_OPTION_LEGACY(deprecated_purge_extend_delay, decommit_extend_delay) }, { MI_DEFAULT_DISALLOW_ARENA_ALLOC, MI_OPTION_UNINIT, MI_OPTION(disallow_arena_alloc) }, // 1 = do not use arena's for allocation (except if using specific arena id's) { 400, MI_OPTION_UNINIT, MI_OPTION(retry_on_oom) }, // windows only: retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. diff --git a/src/theap.c b/src/theap.c index 06feaf1d1..5d37893dd 100644 --- a/src/theap.c +++ b/src/theap.c @@ -139,7 +139,9 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) // collect arenas (this is program wide so don't force purges on abandonment of threads) //mi_atomic_storei64_release(&theap->tld->subproc->purge_expire, 1); - _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld); + if (collect != MI_ABANDON) { + _mi_arenas_collect(collect == MI_FORCE /* force purge? */, collect >= MI_FORCE /* visit all? */, theap->tld); + } // merge statistics _mi_theap_merge_stats(theap); diff --git a/test/test-stress.c b/test/test-stress.c index 006edb76d..350fb6139 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -399,7 +399,7 @@ int main(int argc, char** argv) { #endif #if !defined(NDEBUG) && !defined(USE_STD_MALLOC) mi_option_set(mi_option_arena_reserve, (long)(mi_arena_min_size()/1024) /* in KiB ! */); - mi_option_set(mi_option_purge_delay,1); + // mi_option_set(mi_option_purge_delay,1); #endif #if defined(NDEBUG) && !defined(USE_STD_MALLOC) // mi_option_set(mi_option_purge_delay,-1); @@ -463,7 +463,7 @@ int main(int argc, char** argv) { // mi_free(json); //} #endif - mi_collect(true); + // mi_collect(true); mi_stats_print(NULL); #endif //bench_end_program(); From 601096559bcbb60e6210d4c78933ca0a08effd8d Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 12:15:51 -0700 Subject: [PATCH 064/280] update comments --- include/mimalloc/types.h | 7 ++++--- src/free.c | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 62b58e48e..ba99ce07f 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -109,7 +109,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_ENCODE_FREELIST 1 #endif -// Deprecated (checked with padding) +// Deprecated (double-free is now checked with padding) // #if (MI_ENCODE_FREELIST && (MI_SECURE>=4 || MI_DEBUG!=0)) // #define MI_CHECK_DOUBLE_FREE 1 // #endif @@ -141,7 +141,7 @@ terms of the MIT license. A copy of the license can be found in the file // We can choose to page meta info aligned at the start of every MI_PAGE_META_ALIGNED_CHUNKS // This can be used to have a faster `mi_free(_small)` as we can avoid a page_map lookup. // This only works if valid pointers are passed to `mi_free` though. However, checked -// free `mi_cfree` can still uses the page map to validate pointers. +// free `mi_cfree` can still use the page map to validate pointers. #if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP #if MI_PAGE_META_IS_SEPARATED #define MI_PAGE_META_IS_ALIGNED 1 @@ -154,7 +154,8 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Deprecated: We can choose to only put page info of small pages at the start of the page area. +// Deprecated (in favor of MI_PAGE_META_IS_ALIGNED): +// We can choose to only put page info of small pages at the start of the page area. // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). #if !MI_PAGE_META_IS_ALIGNED diff --git a/src/free.c b/src/free.c index 29091d9b0..faffc9d18 100644 --- a/src/free.c +++ b/src/free.c @@ -191,6 +191,7 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, #endif #if MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE + // for mi_free_small we can avoid a load-acquire if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } else { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); } #else @@ -291,7 +292,7 @@ void mi_free_small(void* p) mi_attr_noexcept { #endif #else mi_page_t* page; - if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free_small",true,&page)) { + if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free_small",true /* is_small? */,&page)) { mi_free_nonnull(p, page, NULL, true /* allow collect? */); } #endif From 6afcf9d825b07a95c339aeef76dd47d85e9e0c5b Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 12:34:52 -0700 Subject: [PATCH 065/280] move pages_direct in front for a zero offset in malloc --- .github/workflows/test.yaml | 10 +++++++++- include/mimalloc/types.h | 4 +++- src/init.c | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 9c5cc59dc..91ffb03b0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -146,10 +146,18 @@ jobs: - name: Release, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | - cmake . -B out/release-tls-pthreads -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON + cmake . -B out/release-tls-pthreads -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON -DMI_SEE_ASM=ON cmake --build out/release-tls-pthreads --parallel 4 --config Release ctest --test-dir out/release-tls-pthreads --verbose --timeout 240 -C Release + - name: Release, TLS pthreads, Asm + if: matrix.tests == 'extra' && runner.os == 'linux' + uses: actions/upload-artifact@v7 + with: + name: ${{ matrix.os }}-pthreads-asm + path: out/release-tls-pthreads/CMakeFiles/mimalloc-static.dir/src/*.c.s + + # Extra - name: Release, SIMD if: matrix.tests == 'extra' diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ba99ce07f..e110d5c7a 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -558,6 +558,9 @@ typedef struct mi_padding_s { // A thread-local heap ("theap") owns a set of thread-local pages. struct mi_theap_s { + // put in front for fast small allocations + mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + mi_tld_t* tld; // thread-local data _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) @@ -588,7 +591,6 @@ struct mi_theap_s { size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) size_t guarded_sample_count; // current sample count (counting down to 0) #endif - mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin") mi_memid_t memid; // provenance of the theap struct itself (meta or os) mi_stats_t stats; // thread-local statistics diff --git a/src/init.c b/src/init.c index 0c87b2f3d..e1a544746 100644 --- a/src/init.c +++ b/src/init.c @@ -118,6 +118,7 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = { }; mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { + MI_SMALL_PAGES_EMPTY, // direct small pages &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc @@ -138,7 +139,6 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { #if MI_GUARDED 0, 0, 0, 1, // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) #endif - MI_SMALL_PAGES_EMPTY, MI_PAGE_QUEUES_EMPTY, MI_MEMID_STATIC, MI_STATS_NULL, // stats From 40518eb0980f8cedca58dfa56ca0c4a774647715 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 12:41:48 -0700 Subject: [PATCH 066/280] add full secure to CI --- .github/workflows/test.yaml | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 91ffb03b0..837a9300d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -50,7 +50,7 @@ jobs: if: matrix.tests == 'basic' run: | env - cmake . -B out/debug -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug --parallel 4 --config Debug ctest --test-dir out/debug --verbose --timeout 240 -C Debug @@ -80,7 +80,7 @@ jobs: - name: Debug, C++ if: matrix.tests == 'basic' && runner.os != 'Windows' # windows already uses C++ by default run: | - cmake . -B out/debug-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + cmake . -B out/debug-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON cmake --build out/debug-cxx --parallel 8 --config Debug ctest --test-dir out/debug-cxx --verbose --timeout 240 -C Debug @@ -176,7 +176,7 @@ jobs: - name: Debug, C++, clang++ if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-clang-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + CC=clang CXX=clang++ cmake . -B out/debug-clang-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON cmake --build out/debug-clang-cxx --parallel 8 --config Debug ctest --test-dir out/debug-clang-cxx --verbose --timeout 240 -C Debug @@ -197,32 +197,32 @@ jobs: - name: Debug, ASAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-asan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON + CC=clang CXX=clang++ cmake . -B out/debug-asan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_TRACK=ASAN cmake --build out/debug-asan --parallel 8 --config Debug ctest --test-dir out/debug-asan --verbose --timeout 240 -C Debug - name: Debug, UBSAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-ubsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON + CC=clang CXX=clang++ cmake . -B out/debug-ubsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_UBSAN=ON cmake --build out/debug-ubsan --parallel 8 --config Debug ctest --test-dir out/debug-ubsan --verbose --timeout 240 -C Debug - name: Debug, TSAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-tsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_TSAN=ON + CC=clang CXX=clang++ cmake . -B out/debug-tsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_TSAN=ON cmake --build out/debug-tsan --parallel 8 --config Debug ctest --test-dir out/debug-tsan --verbose --timeout 240 -C Debug - name: Debug, Guarded if: matrix.tests == 'extra' && runner.os != 'macOS' run: | - cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_GUARDED=ON + cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=ON cmake --build out/debug-guarded --parallel 8 --config Debug ctest --test-dir out/debug-guarded --verbose --timeout 240 -C Debug env: - MIMALLOC_GUARDED_SAMPLE_RATE: 100 + MIMALLOC_GUARDED_SAMPLE_RATE: 10 - name: Release, Guarded if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') @@ -231,7 +231,14 @@ jobs: cmake --build out/release-guarded --parallel 8 --config Release ctest --test-dir out/release-guarded --verbose --timeout 240 -C Release env: - MIMALLOC_GUARDED_SAMPLE_RATE: 100 + MIMALLOC_GUARDED_SAMPLE_RATE: 10 + + - name: Debug, Secure Full + if: matrix.tests == 'extra' && runner.os != 'macOS' + run: | + cmake . -B out/debug-secure-full -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_SECURE=FULL + cmake --build out/debug-secure-full --parallel 8 --config Debug + ctest --test-dir out/debug-secure-full --verbose --timeout 240 -C Debug # MSYS2 mingw64 - name: mingw-ucrt64 x64, Setup @@ -250,7 +257,7 @@ jobs: shell: msys2 {0} run: | env - cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-mingw-ucrt64-x64 --parallel 4 --config Debug ctest --test-dir out/debug-mingw-ucrt64-x64 --verbose --timeout 240 -C Debug @@ -280,7 +287,7 @@ jobs: shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/debug-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-arm64 --parallel 4 --config Debug ctest --test-dir out/debug-arm64 --verbose --timeout 240 -C Debug @@ -318,7 +325,7 @@ jobs: if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-x64 --parallel 4 --config Debug ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug @@ -358,7 +365,7 @@ jobs: MI_TEST_LIGHT: 1 run: | uname -m - cmake . -B out/debug-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-riscv64 --parallel 4 --config Debug ctest --test-dir out/debug-riscv64 --verbose --timeout 240 -C Debug @@ -400,7 +407,7 @@ jobs: if: matrix.tests == 'alpine-x64' shell: alpine-x64.sh {0} run: | - cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-x64 --parallel 4 --config Debug ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug @@ -437,7 +444,7 @@ jobs: MI_TEST_LIGHT: 1 run: | uname -m - cmake . -B out/debug-arm32 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + cmake . -B out/debug-arm32 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-arm32 --parallel 4 --config Debug ctest --test-dir out/debug-arm32 --verbose --timeout 240 -C Debug From a6fd08b53dbc9660b14a09546410036abd5f9057 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 12:48:22 -0700 Subject: [PATCH 067/280] increase TSAN iterations to 500 in the CI --- test/test-stress.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-stress.c b/test/test-stress.c index 350fb6139..76f4c1e41 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -45,7 +45,7 @@ terms of the MIT license. #if defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits static int THREADS = NTHREADS/4; static int SCALE = 10; -static int ITER = 300; +static int ITER = 500; #elif defined(MI_UBSAN) // with undefined behavious sanitizer reduce parameters to stay within the azure pipeline limits static int THREADS = NTHREADS/4; static int SCALE = 25; From 5c625a9bf87c224911d0c112f3b0d876f2291c4e Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 12:49:09 -0700 Subject: [PATCH 068/280] increase TSAN iterations to 500 in the CI --- test/test-stress.c | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/test/test-stress.c b/test/test-stress.c index 76f4c1e41..1cf876e5b 100644 --- a/test/test-stress.c +++ b/test/test-stress.c @@ -42,9 +42,13 @@ terms of the MIT license. // > mimalloc-test-stress [THREADS] [SCALE] [ITER] // // argument defaults -#if defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits +#if defined(MI_TSAN) && MI_TEST_LIGHT // with thread-sanitizer reduce the threads to test within the azure pipeline limits static int THREADS = NTHREADS/4; static int SCALE = 10; +static int ITER = 100; +#elif defined(MI_TSAN) // with thread-sanitizer reduce the threads to test within the azure pipeline limits +static int THREADS = NTHREADS/4; +static int SCALE = 25; static int ITER = 500; #elif defined(MI_UBSAN) // with undefined behavious sanitizer reduce parameters to stay within the azure pipeline limits static int THREADS = NTHREADS/4; From 28162adfb8ead6c44dbcc6ca8bc72d64f353f2e3 Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Fri, 14 Aug 2026 22:52:29 +0200 Subject: [PATCH 069/280] Add theap stats retrieval --- include/mimalloc-stats.h | 3 +++ src/stats.c | 7 +++++++ 2 files changed, 10 insertions(+) diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index b4e19c09a..f8be41aa6 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -140,6 +140,9 @@ mi_decl_export bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_ mi_decl_export char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL mi_decl_export void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept; +// stats from a theap +mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept; + // stats from a subprocess and its heaps aggregated mi_decl_export bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept; mi_decl_export char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL diff --git a/src/stats.c b/src/stats.c index 2a07f0198..9220bfec2 100644 --- a/src/stats.c +++ b/src/stats.c @@ -464,6 +464,10 @@ static const mi_stats_t* mi_heap_get_stats(mi_heap_t* heap) { else return mi_stats_merge_theap_to_heap(theap); } +static const mi_stats_t* mi_theap_get_stats(mi_theap_t* theap) { + return &theap->stats; +} + // deprecated void mi_stats_reset(void) mi_attr_noexcept { if (!mi_theap_is_initialized(_mi_theap_default())) return; @@ -623,6 +627,9 @@ bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_attr_noexcept { return mi_stats_copy(stats, mi_heap_get_stats(heap)); } +bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept { + return mi_stats_copy(stats, mi_theap_get_stats(theap)); +} static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) { mi_stats_t* stats = (mi_stats_t*)arg; From 388b99a6351a43268d372b4722be4b6a30317926 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 14 Aug 2026 14:11:09 -0700 Subject: [PATCH 070/280] add debug check for invalid pointers in aligned mode --- src/free.c | 1 + 1 file changed, 1 insertion(+) diff --git a/src/free.c b/src/free.c index faffc9d18..f4744514a 100644 --- a/src/free.c +++ b/src/free.c @@ -188,6 +188,7 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, if mi_unlikely(page==NULL) return false; // p==NULL => page==NULL #if MI_DEBUG mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } #endif #if MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE From 6762739af1533ac15b2b47cf00639e78b1aa7b90 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 15 Aug 2026 08:49:13 -0700 Subject: [PATCH 071/280] add extra debug checks to track spurious CI failures --- include/mimalloc/internal.h | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index de9c77b5b..0bc394519 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -780,6 +780,13 @@ static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { mi_page_t* const page = _mi_aligned_ptr_page0(p); if mi_unlikely(page==NULL) return NULL; + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { + _mi_error_message(EINVAL, "_mi_aligned_ptr_page: invalid pointer: %p\n", p); + return NULL; + } + #endif return mi_atomic_load_acquire(&page->self); } #endif From af7bfeccf8593bfe4902dc413ebc06891bb364a4 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 10 Aug 2026 18:49:18 -0700 Subject: [PATCH 072/280] no parens on MI_SMALL_WSIZE_MAX --- include/mimalloc.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 964d33e1d..19d6b8f1b 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -119,7 +119,7 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* // ------------------------------------------------------ // Extended functionality // ------------------------------------------------------ -#define MI_SMALL_WSIZE_MAX (128) +#define MI_SMALL_WSIZE_MAX 128 #define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); From 2aa191b4ca8d5f1d0899072ce961e5c99e5cface Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 15 Aug 2026 09:30:30 -0700 Subject: [PATCH 073/280] fix aligned address hints for large alignments --- src/os.c | 38 ++++++++++++++++++++++---------------- 1 file changed, 22 insertions(+), 16 deletions(-) diff --git a/src/os.c b/src/os.c index d1255d7e0..65c5657b7 100644 --- a/src/os.c +++ b/src/os.c @@ -112,47 +112,49 @@ bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero // space (64TiB) we use this technique. (but see issue #939) #if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) -// Return a MI_HINT_ALIGN (4MiB) aligned address that is probably available. +// Return a `try_alignment` aligned address that is probably available. // If this returns NULL, the OS will determine the address but on some OS's that may not be // properly aligned which can be more costly as it needs to be adjusted afterwards. // In secure mode, for a size > 16GiB this always returns NULL in order to guarantee good ASLR randomization; // (otherwise an initial large allocation of say 2TiB has a 50% chance to include (known) addresses // in the middle of the 2TiB - 6TiB address range (see issue #372)) -#define MI_HINT_ALIGN ((uintptr_t)4 << 20) // 4MiB alignment #define MI_HINT_BASE ((uintptr_t)2 << 40) // 2TiB start #define MI_HINT_AREA ((uintptr_t)4 << 40) // upto (2+4) 6TiB (since before win8 there is "only" 8TiB available to processes) #define MI_HINT_MAX ((uintptr_t)30 << 40) // wrap after 30TiB (area after 32TiB is used for huge OS pages) -void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) +void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) { static mi_decl_cache_align _Atomic(uintptr_t) aligned_base; // = 0 // todo: perhaps only do alignment hints if THP is enabled? - if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > MI_HINT_ALIGN) return NULL; + if (try_alignment <= mi_os_mem_config.alloc_granularity || try_alignment > 16*MI_GiB) return NULL; if (mi_os_mem_config.virtual_address_bits < 46) return NULL; // < 64TiB virtual address space - size = _mi_align_up(size, MI_HINT_ALIGN); + + size_t req_size = sze + _mi_os_page_size(); // always reserve a bit more to create virtual gaps between hinted blocks. + req_size += (try_alignment - 1); // ensure we can align in the requested size + req_size = _mi_align_up(req_size, _mi_os_large_page_size()); #if (MI_SECURE>=1) - if (size > 16*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 + if (req_size > 32*MI_GiB) return NULL; // guarantee the chance of fixed valid address is at most 1/(MI_HINT_AREA / 1<<34) = 1/256 #endif - size += MI_HINT_ALIGN; // put in virtual gaps between hinted blocks; this splits VLA's but increases guarded areas. - - uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, size); + + uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; - #if (MI_SECURE>=1 || !defined(NDEBUG)) // security: randomize start of aligned allocations unless in debug mode + #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point const uintptr_t r = _mi_theap_random_next(theap); - init = init + ((MI_HINT_ALIGN * ((r>>17) & 0xFFFFF)) % MI_HINT_AREA); // (randomly 20 bits)*4MiB == 0 to 4TiB + init = init + ((MI_MiB * ((r>>17) & 0x3FFFFF)) % MI_HINT_AREA); // (randomly 22 bits)* 1MiB == 0 to 4TiB #endif - uintptr_t expected = hint + size; + uintptr_t expected = hint + req_size; mi_atomic_cas_strong_acq_rel(&aligned_base, &expected, init); - hint = mi_atomic_add_acq_rel(&aligned_base, size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + hint = mi_atomic_add_acq_rel(&aligned_base, req_size); // this may still give 0 or > MI_HINT_MAX but that is ok, it is a hint after all + if (hint==0) return NULL; } - mi_assert_internal(hint%MI_HINT_ALIGN == 0); - if (hint%try_alignment != 0) return NULL; - return (void*)hint; + const uintptr_t hint_align = _mi_align_up(hint,try_alignment); + mi_assert_internal(hint_align + sze < hint + req_size); + return (void*)hint_align; } #else void* _mi_os_get_aligned_hint(size_t try_alignment, size_t size) { @@ -350,8 +352,12 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = _mi_align_up(size, _mi_os_page_size()); + #if MI_INTPTR_SIZE >= 8 + const bool try_direct_alloc = true; + #else // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size. const bool try_direct_alloc = (alignment <= mi_os_mem_config.alloc_granularity || alignment <= size/4); + #endif bool os_is_large = false; bool os_is_zero = false; From 8b81a1f6eb8d95a9960edf2bbc065aefc6ddcd35 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 15 Aug 2026 12:12:13 -0700 Subject: [PATCH 074/280] ensure enough meta pages are committed for OS allocated pages --- src/arena.c | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/src/arena.c b/src/arena.c index 18b951383..614b02157 100644 --- a/src/arena.c +++ b/src/arena.c @@ -781,8 +781,8 @@ static mi_page_t* mi_arenas_page_try_find_abandoned(mi_theap_t* theap, size_t sl return NULL; } -static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { - MI_UNUSED(block_size); +static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_count, size_t max_page_meta_count, size_t block_alignment, bool os_align, bool commit, mi_memid_t* memid, mi_arena_pages_t** parena_pages ) { + MI_UNUSED(max_page_meta_count); mi_assert_internal(parena_pages!=NULL); *parena_pages = NULL; @@ -839,8 +839,9 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ } if (os_start==NULL) return NULL; // commit page info and page - const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + 1; + const size_t min_page_count = _mi_divide_up(page_offset + MI_ARENA_SLICE_SIZE,MI_ARENA_SLICE_SIZE) + max_page_meta_count; const size_t min_page_meta = min_page_count * sizeof(mi_page_t); + mi_assert_internal(min_page_meta < page_offset); bool is_zero; bool ok = mi_arena_commit(heap->subproc,req_arena,os_start,min_page_meta,&is_zero,min_page_meta /* don't count in stats? */); if (ok && commit) { @@ -921,7 +922,7 @@ static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_s mi_assert_internal(meta_slices >= mi_arena_start(arena)); const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { - // try to commit now + // try to commit all page meta slices now const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, commit_size /* dont count? */)) { @@ -952,10 +953,13 @@ static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_s static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_count, size_t block_size, size_t block_alignment, bool commit) { const bool os_align = (block_alignment > MI_PAGE_MAX_OVERALLOC_ALIGN); + const bool singleton = (os_align || block_size > MI_LARGE_MAX_OBJ_SIZE); + const size_t max_page_meta_count = (singleton && slice_count > 2 ? 2 : slice_count); + const size_t alloc_size = mi_size_of_slices(slice_count); mi_memid_t memid = _mi_memid_none(); mi_arena_pages_t* arena_pages = NULL; - uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,block_size,block_alignment,os_align,commit,&memid,&arena_pages); + uint8_t* const slice_start = mi_arenas_page_alloc_fresh_area(theap,slice_count,max_page_meta_count,block_alignment,os_align,commit,&memid,&arena_pages); if (!slice_start) return NULL; // guard page at the end of mimalloc page? @@ -1076,8 +1080,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou if (page_meta_is_separate) { if (slice_count > 1) { // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice - const size_t max_page_count = (reserved==1 && slice_count > 2 ? 2 : slice_count); - for(size_t i = 1; i < max_page_count; i++) { + for(size_t i = 1; i < max_page_meta_count; i++) { mi_assert_internal(page[i].block_size == 0); mi_atomic_store_ptr_release(mi_page_t,&page[i].self,page); } From 34ec0c118820e75e25a9feb6fce864cb6701724a Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 15 Aug 2026 16:52:35 -0700 Subject: [PATCH 075/280] fix compilation error --- src/arena.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index 614b02157..201d188ee 100644 --- a/src/arena.c +++ b/src/arena.c @@ -860,7 +860,6 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ #else if (os_align) { // note: slice_count already includes the page - mi_assert_internal(slice_count >= mi_slice_count_of_size(block_size) + mi_slice_count_of_size(page_alignment)); start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, block_alignment, page_alignment /* align offset */, commit, allow_large, req_arena, memid); mi_assert_internal(_mi_is_aligned(start + page_alignment, block_alignment)); } From 342cdf1ef0964b4a6e6bd848185baee3891ba4f5 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 15 Aug 2026 17:01:57 -0700 Subject: [PATCH 076/280] fix commit stats for partial commits --- include/mimalloc/internal.h | 2 +- src/arena.c | 14 ++++++++------ src/os.c | 7 ++++--- 3 files changed, 13 insertions(+), 10 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 0bc394519..83c34f6a5 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -213,7 +213,7 @@ bool _mi_os_reset(mi_subproc_t* subproc, void* addr, size_t size); bool _mi_os_decommit(mi_subproc_t* subproc, void* addr, size_t size); void _mi_os_reuse(mi_subproc_t* subproc, void* p, size_t size); mi_decl_nodiscard bool _mi_os_commit(mi_subproc_t* subproc, void* p, size_t size, bool* is_zero); -mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_size); +mi_decl_nodiscard bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed); mi_decl_nodiscard bool _mi_os_protect(void* addr, size_t size); bool _mi_os_unprotect(void* addr, size_t size); bool _mi_os_purge(mi_subproc_t* subproc, void* p, size_t size); diff --git a/src/arena.c b/src/arena.c index 201d188ee..b7bd7eb79 100644 --- a/src/arena.c +++ b/src/arena.c @@ -130,13 +130,13 @@ size_t mi_arena_max_object_size(void) { } } -mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t already_committed) { +mi_decl_nodiscard static bool mi_arena_commit(mi_subproc_t* subproc, mi_arena_t* arena, void* start, size_t size, bool* is_zero, size_t stat_already_committed) { mi_assert_internal(subproc!=NULL); if (arena != NULL && arena->commit_fun != NULL) { return (*arena->commit_fun)(true, start, size, is_zero, arena->commit_fun_arg); } - else if (already_committed > 0) { - return _mi_os_commit_ex(subproc, start, size, is_zero, already_committed); + else if (stat_already_committed > 0) { + return _mi_os_commit_ex(subproc, start, size, is_zero, stat_already_committed); } else { return _mi_os_commit(subproc, start, size, is_zero); @@ -273,7 +273,7 @@ static mi_decl_noinline void* mi_arena_try_alloc_at( if (already_committed < slice_count) { // not all committed, try to commit now bool commit_zero = false; - if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(slice_count - already_committed))) { + if (!mi_arena_commit(arena->subproc, arena, p, mi_size_of_slices(slice_count), &commit_zero, mi_size_of_slices(already_committed))) { // if the commit fails, release ownership, and return NULL; // note: this does not roll back dirty bits but that is ok. mi_bbitmap_setN(arena->slices_free, slice_index, slice_count); @@ -921,10 +921,12 @@ static mi_page_t* mi_arena_page_meta(mi_memid_t memid_slice, const void* slice_s mi_assert_internal(meta_slices >= mi_arena_start(arena)); const size_t meta_slice_index = (meta_slices - mi_arena_start(arena)) / MI_ARENA_SLICE_SIZE; if mi_unlikely(mi_bitmap_is_clear(arena->slices_committed, meta_slice_index)) { - // try to commit all page meta slices now + // try to commit all page meta slices now const size_t meta_slice_count = mi_arena_page_meta_aligned_slice_count(); + // the following assertion does not hold in a concurrent setting.. + // mi_assert_internal(mi_bitmap_is_clearN(arena->slices_committed, meta_slice_index, meta_slice_count)); const size_t commit_size = meta_slice_count * MI_ARENA_SLICE_SIZE; - if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, commit_size /* dont count? */)) { + if (!mi_arena_commit(arena->subproc, arena, meta_slices, commit_size, NULL, 0)) { // if the commit fails return NULL return NULL; } diff --git a/src/os.c b/src/os.c index 65c5657b7..c74bc736a 100644 --- a/src/os.c +++ b/src/os.c @@ -555,7 +555,8 @@ static void* mi_os_page_align_area_conservative(void* addr, size_t size, size_t* return mi_os_page_align_areax(true, addr, size, newsize); } -bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_size) { +bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero, size_t stat_already_committed) { + mi_assert_internal(size >= stat_already_committed); if (is_zero != NULL) { *is_zero = false; } mi_subproc_stat_counter_increase(subproc, commit_calls, 1); @@ -580,12 +581,12 @@ bool _mi_os_commit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* is_z if (os_is_zero) { mi_track_mem_defined(start,csize); } else { mi_track_mem_undefined(start,csize); } #endif - mi_subproc_stat_increase(subproc, committed, stat_size); // use size for precise commit vs. decommit + mi_subproc_stat_increase(subproc, committed, size - stat_already_committed); // for precise commit vs. decommit return true; } bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero) { - return _mi_os_commit_ex(subproc, addr, size, is_zero, size); + return _mi_os_commit_ex(subproc, addr, size, is_zero, 0); } static bool mi_os_decommit_ex(mi_subproc_t* subproc, void* addr, size_t size, bool* needs_recommit, size_t stat_size) { From 90b13d70852f97716e6ee46f93590fe8ba474fcd Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 15 Aug 2026 17:14:49 -0700 Subject: [PATCH 077/280] set heaps test to eager arena commit to find rare Debug CI failure --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bd5916576..41343ddac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -965,7 +965,7 @@ if (MI_BUILD_TESTS) if(MI_GUARDED) add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) elseif(TEST_NAME STREQUAL "stress-heaps") - add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=1 $) + add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=1 MIMALLOC_PAGE_COMMIT_ON_DEMAND=1 $) else() add_test(NAME test-${TEST_NAME} COMMAND mimalloc-test-${TEST_NAME}) endif() From e927d7b0d2bc730f32903d5842e008c7d3a0c43b Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 17 Aug 2026 22:50:24 -0700 Subject: [PATCH 078/280] optimize page layout for malloc/free --- include/mimalloc/types.h | 31 ++++++++++++++++--------------- src/alloc.c | 14 ++++++++++---- src/free.c | 9 +++------ src/init.c | 14 +++++++------- 4 files changed, 36 insertions(+), 32 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index e110d5c7a..aa2cac2a2 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -392,6 +392,9 @@ typedef size_t mi_page_flags_t; // This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`. typedef uintptr_t mi_thread_free_t; +// Convenience +typedef size_t mi_used_t; + // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: // `free` for blocks that can be allocated, @@ -417,29 +420,27 @@ typedef uintptr_t mi_thread_free_t; // that case the `xthreadid` is 0 or 4 (4 is for abandoned pages that // are in the `pages_abandoned` lists of an arena, these are called "mapped" abandoned pages). // - page flags are in the bottom 3 bits of `xthread_id` for the fast path in `mi_free`. -// - The layout is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` -// - Using `uint16_t` does not seem to slow things down +// - The layout below is optimized for `free.c:mi_free` and `alloc.c:mi_page_alloc` typedef struct mi_page_s { #if (MI_PAGE_META_IS_ALIGNED) - _Atomic(struct mi_page_s*) self; + _Atomic(struct mi_page_s*) self; // points to the actual page info (for pages that span multiple slices) #endif _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) - mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) - uint32_t used; // number of blocks in use (including blocks in `thread_free`) + mi_used_t used; // number of blocks in use (including blocks in `thread_free`) + mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + size_t block_size; // const: size available in each block (always `>0`) + size_t page_zoffset; // const: relative offset from the page (in size_t parts) to the start of the blocks uint16_t capacity; // number of blocks committed + uint16_t reserved; // number of blocks reserved in memory + uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) uint8_t retire_expire; // expiration count for retired blocks bool free_is_zero; // `true` if the blocks in the free list are zero initialized - - mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) + + // next cache line _Atomic(mi_thread_free_t) xthread_free; // list of deferred free blocks freed by other threads (= `mi_block_t* | (1 if owned)`) - - size_t block_size; // const: size available in each block (always `>0`) - uint32_t page_zoffset; // const: offset relative to the page (in size_t parts) to the start of the blocks - uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) - uint16_t reserved; // number of blocks reserved in memory - mi_theap_t* theap; // the theap owning this page (may not be valid or NULL for abandoned pages) mi_heap_t* heap; // const: the heap owning this page @@ -449,8 +450,8 @@ typedef struct mi_page_s { #if (MI_ENCODE_FREELIST || MI_PADDING) uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary - #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 - uintptr_t padding[1]; // make it 128 bytes for best codegen in mi_ptr_page_align + // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 + // uintptr_t padding[1]; // make it 128 bytes for best codegen in mi_ptr_page_align #endif } mi_page_t; diff --git a/src/alloc.c b/src/alloc.c index 30a365ef7..2b6809cba 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -39,15 +39,21 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // check the free list mi_block_t* const block = page->free; - if mi_unlikely(block == NULL) { + const mi_used_t used = page->used; + #if defined(__GNUC__) && MI_ARCH_ARM64 + __asm("" : : : "memory"); // induce ldp instruction on aarch64 + #endif + if (block == NULL) { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); if (ppage != NULL) { *ppage = page; }; // pop from the free list - page->free = mi_block_next(page, block); - page->used++; + mi_block_t* next = mi_block_next(page,block); + block->next = 0; // don't leak internal data + page->free = next; + page->used = used+1; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); @@ -78,7 +84,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // zero the block? note: we need to zero the full block size (issue #63) if mi_likely(!zero) { // #if MI_SECURE - block->next = 0; // don't leak internal data + // block->next = 0; // don't leak internal data // #endif #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } diff --git a/src/free.c b/src/free.c index f4744514a..daaf0549a 100644 --- a/src/free.c +++ b/src/free.c @@ -42,14 +42,11 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool #endif // actual free: push on the local free list + const mi_used_t used = page->used - 1; mi_block_set_next(page, block, page->local_free); + page->used = used; page->local_free = block; - #if defined(__clang__) && defined(__aarch64__) - if mi_unlikely(page->used-- == 1) // better code on arm64 than using `--page->used == 0` - #else - if mi_unlikely(--page->used == 0) - #endif - { + if mi_unlikely(used==0) { if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page) _mi_page_retire(page); } diff --git a/src/init.c b/src/init.c index e1a544746..04b4b666f 100644 --- a/src/init.c +++ b/src/init.c @@ -20,15 +20,15 @@ static const mi_page_t mi_page_empty = { MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free 0, // used - 0, // capacity - 0, // retire_expire - false, // is_zero NULL, // local_free - MI_ATOMIC_VAR_INIT(0), // xthread_free 0, // block_size 0, // page_zoffset - 0, // slice_pcommitted + 0, // capacity 0, // reserved capacity + 0, // slice_pcommitted + 0, // retire_expire + false, // is_zero + MI_ATOMIC_VAR_INIT(0), // xthread_free NULL, // theap NULL, // heap NULL, NULL, // next, prev @@ -39,8 +39,8 @@ static const mi_page_t mi_page_empty = { #else { 0 }, // key #endif - #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 - { 0 }, // padding + // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 + // { 0 }, // padding #endif }; From 15ea2514966789a5c288a0c17f9bb4ec8772a5bb Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 17 Aug 2026 22:59:41 -0700 Subject: [PATCH 079/280] fix asan check --- CMakeLists.txt | 2 +- src/alloc.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 41343ddac..cd915e3a3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -965,7 +965,7 @@ if (MI_BUILD_TESTS) if(MI_GUARDED) add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) elseif(TEST_NAME STREQUAL "stress-heaps") - add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=1 MIMALLOC_PAGE_COMMIT_ON_DEMAND=1 $) + add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=0 $) else() add_test(NAME test-${TEST_NAME} COMMAND mimalloc-test-${TEST_NAME}) endif() diff --git a/src/alloc.c b/src/alloc.c index 2b6809cba..624bd60d8 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -51,6 +51,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // pop from the free list mi_block_t* next = mi_block_next(page,block); + mi_track_mem_undefined(block,sizeof(*block)); block->next = 0; // don't leak internal data page->free = next; page->used = used+1; From 9cc85fc0d3faf7c87f79840f2fd1a17f7a12b31d Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 18 Aug 2026 09:20:21 -0700 Subject: [PATCH 080/280] make page commit on demand more fine grained --- src/arena.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/arena.c b/src/arena.c index b7bd7eb79..8e8cbb926 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1142,7 +1142,7 @@ static mi_page_t* mi_arenas_page_regular_alloc(mi_theap_t* theap, size_t slice_c // 2. find a free block, potentially allocating a new arena const long commit_on_demand = mi_option_get(mi_option_page_commit_on_demand); - const bool commit = (slice_count <= mi_slice_count_of_size(mi_page_min_commit_size()) || // always commit small pages + const bool commit = (mi_page_min_commit_size() >= slice_count * MI_ARENA_SLICE_SIZE || // always commit small pages (slice_count >= mi_slice_count_of_size(UINT16_MAX * _mi_os_page_size())) || // always commit pages too large to hold a 32-bit slice_committed (commit_on_demand == 2 && _mi_os_has_overcommit()) || (commit_on_demand == 0)); page = mi_arenas_page_alloc_fresh(theap, slice_count, block_size, 1, commit); From fd8f92e128424d590a5f5f114ca33acd3b7c8090 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 18 Aug 2026 09:33:12 -0700 Subject: [PATCH 081/280] use page_offset instread of zoffset --- include/mimalloc/internal.h | 2 +- include/mimalloc/types.h | 2 +- src/arena.c | 9 ++------- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 83c34f6a5..95c92b8fa 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -814,7 +814,7 @@ static inline size_t mi_page_block_size(const mi_page_t* page) { // Page start static inline uint8_t* mi_page_start(const mi_page_t* page) { // multiplication must be done in `size_t` - return (uint8_t*)page + (((size_t)page->page_zoffset) * MI_SIZE_SIZE); + return (uint8_t*)page + page->page_offset; } static inline size_t mi_page_size(const mi_page_t* page) { diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index aa2cac2a2..bb00cdbdd 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -432,7 +432,7 @@ typedef struct mi_page_s { mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) size_t block_size; // const: size available in each block (always `>0`) - size_t page_zoffset; // const: relative offset from the page (in size_t parts) to the start of the blocks + size_t page_offset; // const: relative offset from the page to the start of the blocks uint16_t capacity; // number of blocks committed uint16_t reserved; // number of blocks reserved in memory uint16_t slice_pcommitted; // committed size in OS page sizes relative to the first arena slice of the page data (or 0 if the page is fully committed already) diff --git a/src/arena.c b/src/arena.c index 8e8cbb926..ca28d593c 100644 --- a/src/arena.c +++ b/src/arena.c @@ -24,9 +24,6 @@ The arena allocation needs to be thread safe and we use an atomic bitmap to allo #include "mimalloc/prim-tls.h" #include "bitmap.h" -#if (MI_ARENA_MAX_SIZE - MI_SIZE_SIZE > MI_SIZE_SIZE*UINT32_MAX) -#error "The page_t.page_zoffset field is not large enough to cover a full arena" -#endif /* ----------------------------------------------------------- Arena id's @@ -1059,10 +1056,8 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou // initialize the page start uint8_t* const start = slice_start + block_start; mi_assert_internal(start > (uint8_t*)page); - const size_t offset = start - (uint8_t*)page; - mi_assert_internal((offset % MI_SIZE_SIZE) == 0 && (offset / MI_SIZE_SIZE) <= UINT32_MAX); - page->page_zoffset = (uint32_t)(offset / MI_SIZE_SIZE); - + page->page_offset = start - (uint8_t*)page; + // initialize page meta-data page->reserved = (uint16_t)reserved; page->block_size = block_size; From 7c66c3f4c3ead23705a3ad66797dc996bc775a6a Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 18 Aug 2026 09:39:09 -0700 Subject: [PATCH 082/280] minor optimization --- src/alloc.c | 4 ++-- src/init.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 624bd60d8..bb98facbb 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -40,8 +40,8 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // check the free list mi_block_t* const block = page->free; const mi_used_t used = page->used; - #if defined(__GNUC__) && MI_ARCH_ARM64 - __asm("" : : : "memory"); // induce ldp instruction on aarch64 + #if defined(__GNUC__) + __asm("" : : : "memory"); // alway load used before the test #endif if (block == NULL) { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); diff --git a/src/init.c b/src/init.c index 04b4b666f..4d85bc23a 100644 --- a/src/init.c +++ b/src/init.c @@ -22,7 +22,7 @@ static const mi_page_t mi_page_empty = { 0, // used NULL, // local_free 0, // block_size - 0, // page_zoffset + 0, // page_offset 0, // capacity 0, // reserved capacity 0, // slice_pcommitted From b2ef7421e4a038002a98f2d255e1d8fc240eef06 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 18 Aug 2026 11:14:24 -0700 Subject: [PATCH 083/280] remove incorrect assertion --- src/prim/osx/alloc-override-zone.c | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index db45aca63..1d21741bc 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -73,7 +73,9 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { static void zone_free(malloc_zone_t* zone, void* p) { mi_page_t* const page = _mi_checked_ptr_page(p); if mi_likely(page!=NULL) { - mi_assert_internal(_mi_thread_is_initialized()); + // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free` + // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc. + // mi_assert_internal(_mi_thread_is_initialized()); _mi_free_in_page_nonnull(p,page); // if mi_likely(_mi_thread_is_initialized()) { // _mi_free_in_page_nonnull(p,page); From 53b364dafcc6bf32f2fecdd503532dd8c33fbb48 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 18 Aug 2026 11:15:09 -0700 Subject: [PATCH 084/280] all free to reabandon pages even if the thread is uninitialized, issue #1364 --- include/mimalloc/internal.h | 4 +++ src/arena.c | 59 ++++++++++++++----------------------- 2 files changed, 26 insertions(+), 37 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 95c92b8fa..ca46e3222 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -405,6 +405,10 @@ void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount); #define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt) #define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt) +#define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); } +#define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); } +#define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); } +#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } /* ----------------------------------------------------------- pthread thread locals diff --git a/src/arena.c b/src/arena.c index ca28d593c..a09a309fe 100644 --- a/src/arena.c +++ b/src/arena.c @@ -628,10 +628,10 @@ void* _mi_arenas_alloc(mi_heap_t* heap, size_t size, bool commit, bool allow_lar // release ownership of a page. This may free the page if all blocks were concurrently // freed in the meantime. Returns true if the page was freed. -static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) { +static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); - mi_assert_internal(mi_theap_matches_thread(current_theap)); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { @@ -639,8 +639,8 @@ static bool mi_abandoned_page_unown(mi_page_t* page, mi_theap_t* current_theap) while mi_unlikely(mi_tf_block(tf_old) != NULL) { _mi_page_free_collect(page, false); // update used if (mi_page_all_free(page)) { // it may become free just before unowning it - _mi_arenas_page_unabandon(page, current_theap); - _mi_arenas_page_free(page, current_theap); + _mi_arenas_page_unabandon(page, current_theapx); + _mi_arenas_page_free(page, current_theapx); return true; } tf_old = mi_atomic_load_relaxed(&page->xthread_free); @@ -1291,15 +1291,9 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(page->next==NULL && page->prev==NULL); mi_assert_internal(mi_theap_matches_thread(current_theapx)); - if (current_theapx != NULL) { - mi_theap_stat_decrease(current_theapx, page_bins[_mi_page_stats_bin(page)], 1); - mi_theap_stat_decrease(current_theapx, pages, 1); - } - else { - mi_heap_t* const heap = mi_page_heap(page); - mi_heap_stat_decrease(heap, page_bins[_mi_page_stats_bin(page)], 1); - mi_heap_stat_decrease(heap, pages, 1); - } + mi_heap_t* const heap = mi_page_heap(page); + mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); + mi_theapx_stat_decrease(heap, current_theapx, pages, 1); mi_arenas_page_free_prim(page); } @@ -1307,19 +1301,18 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { Arena abandon ----------------------------------------------------------- */ -void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { +void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(!mi_page_all_free(page)); mi_assert_internal(page->next==NULL && page->prev == NULL); - mi_assert_internal(mi_theap_matches_thread(current_theap)); + mi_assert_internal(mi_theap_matches_thread(current_theapx)); // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); // add to abandoned? - mi_heap_t* heap = mi_page_heap(page); - mi_assert_internal(heap==_mi_theap_heap(current_theap)); + mi_heap_t* heap = mi_page_heap(page); if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { // make available for allocations size_t bin = _mi_bin(mi_page_block_size(page)); @@ -1339,8 +1332,8 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { const bool was_clear = mi_bitmap_set(arena_pages->pages_abandoned[bin], slice_index); MI_UNUSED(was_clear); mi_assert_internal(was_clear); mi_atomic_increment_relaxed(&heap->abandoned_count[bin]); - mi_theap_stat_increase(current_theap, pages_abandoned, 1); - mi_abandoned_page_unown(page, current_theap); + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); return; } } @@ -1357,8 +1350,8 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theap) { heap->os_abandoned_pages = page; } } - mi_theap_stat_increase(current_theap, pages_abandoned, 1); - mi_abandoned_page_unown(page, current_theap); + mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); + mi_abandoned_page_unown(page, current_theapx); } @@ -1377,16 +1370,13 @@ bool _mi_arenas_page_try_reabandon_to_mapped(mi_page_t* page) { } else { // do not use _mi_heap_theap as we may call this during shutdown of threads and don't want to reinitialize the theap - mi_theap_t* const theap = _mi_page_associated_theap_peek(page); - if (theap == NULL) { - return false; - } - else { - mi_theap_stat_counter_increase(theap, pages_reabandon_full, 1); - mi_theap_stat_adjust_decrease(theap, pages_abandoned, 1); // adjust as we are not abandoning fresh - _mi_arenas_page_abandon(page, theap); - return true; - } + mi_theap_t* const theapx = _mi_page_associated_theap_peek(page); // can be NULL + mi_heap_t* const heap = mi_page_heap(page); + // if (theapx==NULL) return false; + mi_theapx_stat_counter_increase(heap, theapx, pages_reabandon_full, 1); + mi_theapx_stat_adjust_decrease(heap, theapx, pages_abandoned, 1); // adjust as we are not abandoning fresh + _mi_arenas_page_abandon(page, theapx); + return true; } } @@ -1430,12 +1420,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { } } } - if (current_theapx!=NULL) { - mi_theap_stat_decrease(current_theapx, pages_abandoned, 1); - } - else { - mi_heap_stat_decrease(heap, pages_abandoned, 1); - } + mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1); } From 2c22450537521bff0645616b872e3a08718d7c04 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 18 Aug 2026 23:23:01 -0700 Subject: [PATCH 085/280] cfree returns a boolean (true on succes) --- include/mimalloc.h | 2 +- include/mimalloc/internal.h | 2 -- src/free.c | 23 +++++++++++------------ src/prim/osx/alloc-override-zone.c | 22 ++++++---------------- 4 files changed, 18 insertions(+), 31 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 581b2bf5b..229e39484 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -539,7 +539,7 @@ mi_decl_export void mi_option_set_default(mi_option_t option, long value); // note: we use `mi_cfree` as "checked free" and it checks if the pointer is in our theap before free-ing. // ------------------------------------------------------------------------------------------------------- -mi_decl_export void mi_cfree(void* p) mi_attr_noexcept; +mi_decl_export bool mi_cfree(void* p) mi_attr_noexcept; mi_decl_export void* mi__expand(void* p, size_t newsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_malloc_good_size(size_t size) mi_attr_noexcept; diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index ca46e3222..67a08258b 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -331,8 +331,6 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); // "free.c" -void _mi_free_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept; -void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept; void _mi_free_subproc_safe(void* p) mi_attr_noexcept; void _mi_page_unguard_all(mi_page_t* page); size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept; diff --git a/src/free.c b/src/free.c index daaf0549a..82ecf2601 100644 --- a/src/free.c +++ b/src/free.c @@ -183,7 +183,7 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, #if MI_PAGE_META_IS_ALIGNED page = _mi_aligned_ptr_page0(p); if mi_unlikely(page==NULL) return false; // p==NULL => page==NULL - #if MI_DEBUG + #if MI_DEBUG mi_page_t* const cpage = _mi_checked_ptr_page(p); if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } #endif @@ -248,11 +248,6 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t } } -void _mi_free_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { - mi_assert_internal(p!=NULL && page!=NULL); - mi_free_nonnull(p, page, NULL, true /* allow collect? */); -} - void mi_free(void* p) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free",false,&page)) { @@ -296,9 +291,9 @@ void mi_free_small(void* p) mi_attr_noexcept { #endif } -void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { - mi_free_nonnull(p, page, NULL, false /* allow collect? */); -} +// void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { +// mi_free_nonnull(p, page, NULL, false /* allow collect? */); +// } // Free a pointer that is potentially allocated in a different sub-process void _mi_free_subproc_safe(void* p) mi_attr_noexcept { @@ -319,12 +314,12 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { if (page==NULL) return; mi_assert(p!=NULL); const size_t usable = _mi_page_usable_size(page,p); - if (size > usable) { + if mi_unlikely(size > usable) { _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); mi_free(p); return; } - else if (size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { + else if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); mi_free(p); return; @@ -354,10 +349,14 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { } // checked free -void mi_cfree(void* p) mi_attr_noexcept { +bool mi_cfree(void* p) mi_attr_noexcept { mi_page_t* const page = _mi_checked_ptr_page(p); if mi_likely(page!=NULL) { mi_free_nonnull(p, page, NULL, true /* allow collect? */); + return true; + } + else { + return false; } } diff --git a/src/prim/osx/alloc-override-zone.c b/src/prim/osx/alloc-override-zone.c index 1d21741bc..48f979582 100644 --- a/src/prim/osx/alloc-override-zone.c +++ b/src/prim/osx/alloc-override-zone.c @@ -71,22 +71,12 @@ static void* zone_valloc(malloc_zone_t* zone, size_t size) { } static void zone_free(malloc_zone_t* zone, void* p) { - mi_page_t* const page = _mi_checked_ptr_page(p); - if mi_likely(page!=NULL) { - // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free` - // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc. - // mi_assert_internal(_mi_thread_is_initialized()); - _mi_free_in_page_nonnull(p,page); - // if mi_likely(_mi_thread_is_initialized()) { - // _mi_free_in_page_nonnull(p,page); - // } - // else { - // // during thread shutdown `_pthread_tsd_cleanup` may call `zone_free` on a pointer that was allocated in another subproc. - // _mi_free_subproc_safe_in_page_nonnull(p,page); - // } - } - else if (!is_mimalloc_zone(zone)) { // can happen due to interpose - zone->free(zone,p); + // during C++ thread shutdown `_pthread_tsd_cleanup` may call `zone_free` + // after mimalloc mi_thread_done, and also on a pointer that was allocated in another subproc. + if mi_unlikely(!mi_cfree(p)) { + if (!is_mimalloc_zone(zone)) { // can happen due to interpose + zone->free(zone,p); + } } } From 559ffde8bd8dbf7a8e4ac14b4cb3e45cfc436ea2 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 18 Aug 2026 23:30:29 -0700 Subject: [PATCH 086/280] bump version to v1.15.0 --- .github/workflows/release.yaml | 4 ++-- .github/workflows/stale.yaml | 2 +- cmake/mimalloc-config-version.cmake | 4 ++-- include/mimalloc.h | 2 +- readme.md | 16 ++++++++++++---- 5 files changed, 18 insertions(+), 10 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index dd60b12e3..b82b8199c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ permissions: contents: write env: - RELEASE: Release v3.4.5 + RELEASE: Release v3.5.0 FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true name: Release @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - branch: [v3.4.5, v2.4.5, v1.9.15] # [dev,dev2,dev3] + branch: [v3.5.0, v2.5.0, v1.15.0] # [dev,dev2,dev3] # we build on the oldest ubuntu version for better binary compatibility. os: [windows-latest, macOS-latest, macos-15-intel, ubuntu-22.04, ubuntu-22.04-arm] diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index dff9cb656..fda392a10 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -1,7 +1,7 @@ on: workflow_dispatch: # allow running the workflow manually schedule: - - cron: "15 21 * * *" # minute, hour, day (1-31), month (1-12), day of the week (0 - 6 or SUN-SAT) + - cron: "01 21 * * *" # minute, hour, day (1-31), month (1-12), day of the week (0 - 6 or SUN-SAT) name: Close inactive issues jobs: diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index e153b575a..aebbc1d6d 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 1) -set(mi_version_minor 9) -set(mi_version_patch 15) +set(mi_version_minor 15) +set(mi_version_patch 0) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/include/mimalloc.h b/include/mimalloc.h index 19d6b8f1b..39d8e8a05 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 10915 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 11500 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes diff --git a/readme.md b/readme.md index 2395b2c4e..e0b9eb8f8 100644 --- a/readme.md +++ b/readme.md @@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.4.5` (2026-08-05) recommended. -Latest v2 release: `v2.4.5` (2026-08-05) stable. -Latest v1 release: `v1.9.15` (2026-08-05) legacy. +Latest release : `v3.5.0` (2026-08-18) recommended. +Latest v2 release: `v2.5.0` (2026-08-18) stable. +Latest v1 release: `v1.15.0` (2026-08-18) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -85,11 +85,19 @@ New development is mostly on v3, while v1 and v2 are maintained with security an and has more efficient heap-walking (for the CPython GC for example). (release tags: `v3.x`, development branch `dev3`). - __v2__: stable mimalloc version. Uses thread-local segments to reduce fragmentation. (release tags: `v2.x`, development branch `dev2` and `main`) -- __v1__: legacy version: initial design of mimalloc (release tags: `v1.9.x`, development branch `dev`). +- __v1__: legacy version: initial design of mimalloc (release tags: `v1.x`, development branch `dev`). __Send PR's against this version if possible.__ ### Releases +* 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) Increased performance for `free` + using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for + faster load-acquire, increase retired page count from 1 to 3, faster double free + detection in secure mode, use faster atomics for the arena bitmaps (thank you Alan + Andrade, #1346), faster pagemap lookup. + Other: fix numa sparse node count detection (#1365), add theap stats retrieval, + use __builtin_thread_pointer on riscV (#1363), fix C mode compilation on x86 with + msvc (#1361), improved mingw-ucrt64 support. * 2026-08-05, `v1.9.15`, `v2.4.5`, `v3.4.5`: Fix compilation with xmalloc (#1353), make the build deterministic (#1355), mi_zalloc_aligned fix (#763), improve support for mingw (ucrt64), fix fputs fallback on windows (#1354), (v3): use proper lock backoff for first-class heap deletion, improved riscv64 codegen. From dac1f4b339849ca0356fda251b1cd2f6d0322081 Mon Sep 17 00:00:00 2001 From: daanx Date: Tue, 18 Aug 2026 23:46:30 -0700 Subject: [PATCH 087/280] update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index e0b9eb8f8..a92903b27 100644 --- a/readme.md +++ b/readme.md @@ -90,7 +90,7 @@ New development is mostly on v3, while v1 and v2 are maintained with security an ### Releases -* 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) Increased performance for `free` +* 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) slightly better performance for `free` using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for faster load-acquire, increase retired page count from 1 to 3, faster double free detection in secure mode, use faster atomics for the arena bitmaps (thank you Alan From 604c252a0ab7415893803e37e83c52943c706f45 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 19 Aug 2026 14:44:01 -0700 Subject: [PATCH 088/280] fix compilation error with clang on windows, issue #1370 --- include/mimalloc/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 67a08258b..754b60075 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -789,7 +789,7 @@ static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { return NULL; } #endif - return mi_atomic_load_acquire(&page->self); + return mi_atomic_load_ptr_acquire(mi_page_t, &page->self); } #endif From 0a4ea88d8c7fff9d6001449311b179582db6dcab Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 19 Aug 2026 22:43:29 -0700 Subject: [PATCH 089/280] possible fix linking error with clang on windows, issue #1370 --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index bb98facbb..237d20f32 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -242,7 +242,7 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz } } -inline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { +extern inline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { return _mi_theap_malloc_zero_ex(theap, size, zero, 0, ppage); } From 0424ab27f88121b0170e891169ebde21c1f595e9 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 19 Aug 2026 22:49:52 -0700 Subject: [PATCH 090/280] Pin GitHub Actions to full-length commit SHAs, see PR #1368 --- .github/workflows/release.yaml | 4 ++-- .github/workflows/stale.yaml | 2 +- .github/workflows/test.yaml | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index b82b8199c..e495a23dc 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,7 +26,7 @@ jobs: steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 with: ref: ${{ matrix.branch }} fetch-tags: true @@ -45,7 +45,7 @@ jobs: # Create a release: https://github.com/softprops/action-gh-release (MIT license) - name: Release - uses: softprops/action-gh-release@v3 + uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 if: ${{ env.RELEASE != '' && env.RELEASE != 'no' }} with: name: ${{ env.RELEASE }} diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index fda392a10..71a91a887 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -11,7 +11,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@v10 + - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 with: days-before-issue-stale: 360 days-before-issue-close: 14 diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 24e2afbc7..9e3d5b494 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -29,7 +29,7 @@ jobs: steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository - uses: actions/checkout@v6 + uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 # Basic - name: Debug @@ -168,7 +168,7 @@ jobs: # MSYS2 mingw64 UCRT64 - name: mingw-ucrt64 x64, Setup if: matrix.tests == 'mingw-ucrt64' - uses: msys2/setup-msys2@v2 + uses: msys2/setup-msys2@66cd2cce69caa17b53920067426061ca1de3a884 # v2.32.0 with: msystem: UCRT64 install: | From 0ea1bb2267cbe2878d3cfdcd704d9f13d9f8b1b2 Mon Sep 17 00:00:00 2001 From: daanx Date: Wed, 19 Aug 2026 23:03:59 -0700 Subject: [PATCH 091/280] add plain clang to win32 tests, #1370 --- .github/workflows/test.yaml | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2660be20f..f18aeecba 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -91,7 +91,7 @@ jobs: cmake --build out/release-cxx --parallel 8 --config Release ctest --test-dir out/release-cxx --verbose --timeout 240 -C Release - # Extra, Windows x86 + # Extra Windows: x86, clang-cl, and clang - name: Debug, Win32 if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') run: | @@ -109,16 +109,30 @@ jobs: - name: Debug, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') run: | - cmake . -B out/debug-clang -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl + cmake . -B out/debug-clang-cl -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl + cmake --build out/debug-clang-cl --parallel 4 --config Debug + ctest --test-dir out/debug-clang-cl --verbose --timeout 240 -C Debug + + - name: Release, clang-cl + if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + run: | + cmake . -B out/release-clang-cl -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl + cmake --build out/release-clang-cl --parallel 4 --config Release + ctest --test-dir out/release-clang-cl --verbose --timeout 240 -C Release + + - name: Debug, Win32, clang + if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + run: | + cmake . -B out/debug-clang -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang cmake --build out/debug-clang --parallel 4 --config Debug ctest --test-dir out/debug-clang --verbose --timeout 240 -C Debug - - name: Release, clang-cl + - name: Release, Win32, clang if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') run: | - cmake . -B out/release-clang -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl + cmake . -B out/release-clang -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang cmake --build out/release-clang --parallel 4 --config Release - ctest --test-dir out/release-clang --verbose --timeout 240 -C Release + ctest --test-dir out/release-clang --verbose --timeout 240 -C Release # Extra, macOS fixed TLS - name: Debug, TLS fixed From 870c30dc00e02ea277b9037f1da89ad86f29884a Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 20 Aug 2026 21:20:27 -0700 Subject: [PATCH 092/280] nice names for some CI steps --- .github/workflows/test.yaml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index f18aeecba..423b3f340 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -106,28 +106,28 @@ jobs: cmake --build out/release-x86 --parallel 4 --config Release ctest --test-dir out/release-x86 --verbose --timeout 240 -C Release - - name: Debug, clang-cl + - name: Debug, Windows, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') run: | cmake . -B out/debug-clang-cl -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl cmake --build out/debug-clang-cl --parallel 4 --config Debug ctest --test-dir out/debug-clang-cl --verbose --timeout 240 -C Debug - - name: Release, clang-cl + - name: Release, Windows, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') run: | cmake . -B out/release-clang-cl -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl cmake --build out/release-clang-cl --parallel 4 --config Release ctest --test-dir out/release-clang-cl --verbose --timeout 240 -C Release - - name: Debug, Win32, clang + - name: Debug, Windows, clang if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') run: | cmake . -B out/debug-clang -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang cmake --build out/debug-clang --parallel 4 --config Debug ctest --test-dir out/debug-clang --verbose --timeout 240 -C Debug - - name: Release, Win32, clang + - name: Release, Windows, clang if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') run: | cmake . -B out/release-clang -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang @@ -135,14 +135,14 @@ jobs: ctest --test-dir out/release-clang --verbose --timeout 240 -C Release # Extra, macOS fixed TLS - - name: Debug, TLS fixed + - name: Debug, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' run: | cmake . -B out/debug-tls-fixed -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_FIXED=ON cmake --build out/debug-tls-fixed --parallel 4 --config Debug ctest --test-dir out/debug-tls-fixed --verbose --timeout 240 -C Debug - - name: Release, TLS fixed + - name: Release, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' run: | cmake . -B out/release-tls-fixed -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_FIXED=ON @@ -150,21 +150,21 @@ jobs: ctest --test-dir out/release-tls-fixed --verbose --timeout 240 -C Release # Extra, Linux pthread TLS - - name: Debug, TLS pthreads + - name: Debug, Linux, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | cmake . -B out/debug-tls-pthreads -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_PTHREADS=ON cmake --build out/debug-tls-pthreads --parallel 4 --config Debug ctest --test-dir out/debug-tls-pthreads --verbose --timeout 240 -C Debug - - name: Release, TLS pthreads + - name: Release, Linux, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | cmake . -B out/release-tls-pthreads -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON -DMI_SEE_ASM=ON cmake --build out/release-tls-pthreads --parallel 4 --config Release ctest --test-dir out/release-tls-pthreads --verbose --timeout 240 -C Release - - name: Release, TLS pthreads, Asm + - name: Release, Linux, TLS pthreads, Asm if: matrix.tests == 'extra' && runner.os == 'linux' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: From a15a51dd281d4fada484d933f7b6428a45a48cf2 Mon Sep 17 00:00:00 2001 From: daanx Date: Thu, 20 Aug 2026 21:21:01 -0700 Subject: [PATCH 093/280] fix assertion check in realloc, issue #1371 --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 237d20f32..9e86e5eae 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -404,7 +404,7 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi if (theap!=NULL) #endif { - if (mi_page_heap(page)==_mi_theap_heap(theap)) { // and within the same heap + if (mi_page_heap(page)==_mi_theap_heap_peek(theap)) { // and within the same heap mi_assert_internal(p!=NULL); // todo: do not track as the usable size is still the same in the free; adjust potential padding? // mi_track_resize(p,size,newsize) From 0a0c2962dd08a47eeadb4b24d69873f5d1f47c50 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 22 Aug 2026 03:14:28 -0700 Subject: [PATCH 094/280] improve cache associativity --- src/arena.c | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index a09a309fe..1b3b94c03 100644 --- a/src/arena.c +++ b/src/arena.c @@ -989,8 +989,15 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ #endif - if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { - block_start += block_size; + if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && + _mi_is_power_of_two(block_size)) + { + if (block_size < 64) { + block_start += mi_page_info_size() + 3*block_size; + } + else { + block_start += _mi_align_up(mi_page_info_size(), block_size); + } } mi_assert_internal(page->block_size == 0); _mi_memzero_aligned(page, sizeof(*page)); From dc71e905fa448ede55c44eac8db3e23dd1f064be Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 22 Aug 2026 03:14:38 -0700 Subject: [PATCH 095/280] remove cookie field --- .github/workflows/test.yaml | 10 +++++----- CMakeLists.txt | 2 +- include/mimalloc/types.h | 1 - src/init.c | 5 ----- src/theap.c | 2 +- 5 files changed, 7 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 423b3f340..29066bfa0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -66,7 +66,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-asm - path: out/release/CMakeFiles/mimalloc-static.dir/src/*.c.s + path: out/release/CMakeFiles/mimalloc-static.dir/src/*.s - name: Secure @@ -168,8 +168,8 @@ jobs: if: matrix.tests == 'extra' && runner.os == 'linux' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ matrix.os }}-pthreads-asm - path: out/release-tls-pthreads/CMakeFiles/mimalloc-static.dir/src/*.c.s + name: ${{ matrix.os }}-pthreads-asm + path: out/release-tls-pthreads/CMakeFiles/mimalloc-static.dir/src/*.s # Extra @@ -206,7 +206,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-clang-cxx-asm - path: out/release-clang-cxx/*.s + path: out/release-clang-cxx/CMakeFiles/mimalloc-static.dir/src/*.s - name: Debug, ASAN if: matrix.tests == 'extra' && runner.os == 'Linux' @@ -396,7 +396,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: alpine-riscv64-asm - path: out/release-riscv64/CMakeFiles/mimalloc.dir/src/*.c.s + path: out/release-riscv64/CMakeFiles/mimalloc-static.dir/src/*.s - name: Alpine riscv64, Secure if: matrix.tests == 'alpine-riscv64' diff --git a/CMakeLists.txt b/CMakeLists.txt index cd915e3a3..bc86fd5cd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -344,7 +344,7 @@ if(MI_SEE_ASM) if(CMAKE_C_COMPILER_ID MATCHES "MSVC" OR MI_CLANG_CL) list(APPEND mi_cflags -FA) else() - list(APPEND mi_cflags -save-temps) + list(APPEND mi_cflags -save-temps=obj) if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND CMAKE_C_COMPILER_VERSION VERSION_GREATER 14) message(STATUS "No GNU Line marker") list(APPEND mi_cflags -Wno-gnu-line-marker) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index bb00cdbdd..dd0d861cc 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -568,7 +568,6 @@ struct mi_theap_s { _Atomic(size_t) refcount; // reference count unsigned long long heartbeat; // monotonic heartbeat count - uintptr_t cookie; // random cookie to verify pointers (see `_mi_ptr_cookie`) mi_random_ctx_t random; // random number context used for secure allocation size_t page_count; // total number of pages in the `pages` queues. size_t page_retired_min; // smallest retired index (retired pages are fully free, but still in the page queues) diff --git a/src/init.c b/src/init.c index 4d85bc23a..68e94aaa2 100644 --- a/src/init.c +++ b/src/init.c @@ -124,7 +124,6 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount 0, // heartbeat - 0, // cookie { {0}, {0}, 0, true }, // random 0, // page count MI_BIN_FULL, 0, // page retired min/max @@ -578,10 +577,6 @@ static void mi_process_init_once(void) { mi_reserve_os_memory((size_t)ksize*MI_KiB, true, true); } } - - #if MI_PAGE_META_IS_ALIGNED - mi_assert_internal((sizeof(mi_page_t)%MI_SIZE_SIZE) == 0); // or page->zoffset might not work - #endif } // Initialize the process; called by thread_init or the process loader diff --git a/src/theap.c b/src/theap.c index 5d37893dd..d9fb4309d 100644 --- a/src/theap.c +++ b/src/theap.c @@ -286,7 +286,7 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) else { _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random } - theap->cookie = _mi_theap_random_next(theap) | 1; + // theap->cookie = _mi_theap_random_next(theap) | 1; mi_theap_guarded_init(theap); // needs theap->random if (!theap->is_detached) { mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem From cda82f99b762f8074e9850857ebb68c13da44664 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 22 Aug 2026 03:27:58 -0700 Subject: [PATCH 096/280] check double free's even without padding --- include/mimalloc/types.h | 2 ++ src/free.c | 25 +++++++++++++------------ 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index dd0d861cc..701f9a4be 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -65,6 +65,8 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_SECURE 0 #endif +#define MI_PADDING 0 + // Define MI_DEBUG for assertion and invariant checking // #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON) // #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON) diff --git a/src/free.c b/src/free.c index 82ecf2601..cf21b6ad1 100644 --- a/src/free.c +++ b/src/free.c @@ -14,9 +14,9 @@ terms of the MIT license. A copy of the license can be found in the file // forward declarations mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); +mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block); static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); -// static bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block); // ------------------------------------------------------ @@ -30,7 +30,7 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool // checks size_t usable_size; if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; - // if mi_unlikely(!mi_check_is_double_free(page,block)) return; // checked with padding + if mi_unlikely(!mi_check_double_free(page,block)) return; // usually checked with padding if (track_stats) { mi_stat_free(page, block); @@ -551,12 +551,13 @@ mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { // ------------------------------------------------------ -// Deprecated: double free is checked with padding now. +// Deprecated: double free is usually checked with padding now +// as that is faster and works better for cross-thread free'ing. // Check for double free in secure and debug mode // This is somewhat expensive so only enabled for secure mode 4 // ------------------------------------------------------ -#if MI_CHECK_DOUBLE_FREE +#if MI_SECURE>=3 && !MI_PADDING // linear check if the free list contains a specific element static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, const mi_block_t* elem, const char* list_kind) { const size_t max_count = page->capacity; // can never hold more blocks than the capacity @@ -572,7 +573,7 @@ static bool mi_list_contains(const mi_page_t* page, const mi_block_t* list, cons return false; } -static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, const mi_block_t* block) { +static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const mi_block_t* block) { // The decoded value is in the same page (or NULL). // Walk the free lists to verify positively if it is already freed if (mi_list_contains(page, page->free, block, "free") || @@ -580,9 +581,9 @@ static mi_decl_noinline bool mi_check_is_double_freex(const mi_page_t* page, con mi_list_contains(page, mi_page_thread_free(page), block, "thread free")) { _mi_error_message(EAGAIN, "double free detected of block %p with size %zu\n", block, mi_page_block_size(page)); - return true; + return false; } - return false; + return true; } // Used for double free checking to avoid checking free lists too frequently @@ -593,20 +594,20 @@ static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi } // check if `block` was free'd before -static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { +static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) { if mi_unlikely(mi_block_could_be_double_free(page,block)) // quick check: next field is aligned in the same page or NULL? { // Suspicious: decoded value a in block is in the same page (or NULL) -- maybe a double free? // (continue in separate function to improve code generation) - return mi_check_is_double_freex(page, block); + return mi_check_double_freex(page, block); } - else return false; + else return true; } #else -static inline bool mi_check_is_double_free(const mi_page_t* page, const mi_block_t* block) { +static inline bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(page); MI_UNUSED(block); - return false; + return true; } #endif From 622d421f647fc41a2dfe90bc45174770129d78b4 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 22 Aug 2026 03:49:29 -0700 Subject: [PATCH 097/280] fix natural alignment --- src/arena.c | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index 1b3b94c03..b32a52216 100644 --- a/src/arena.c +++ b/src/arena.c @@ -993,11 +993,13 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou _mi_is_power_of_two(block_size)) { if (block_size < 64) { - block_start += mi_page_info_size() + 3*block_size; + block_start += sizeof(mi_page_t) + 3*block_size; } else { - block_start += _mi_align_up(mi_page_info_size(), block_size); + block_start += sizeof(mi_page_t); } + block_start = _mi_align_up(block_start, MI_MAX_ALIGN_SIZE); + block_start = _mi_align_up(block_start, block_size); // to maintain natural alignment } mi_assert_internal(page->block_size == 0); _mi_memzero_aligned(page, sizeof(*page)); From d962692647aeba222b809d7d39c1154f9a7d3c80 Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Fri, 28 Aug 2026 13:49:48 +0200 Subject: [PATCH 098/280] workflows: Enable clang-cl tests for ARM --- .github/workflows/test.yaml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 29066bfa0..00c94550e 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -93,42 +93,42 @@ jobs: # Extra Windows: x86, clang-cl, and clang - name: Debug, Win32 - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + if: matrix.tests == 'extra' && runner.os == 'Windows' run: | cmake . -B out/debug-x86 -DCMAKE_BUILD_TYPE=Debug -A Win32 cmake --build out/debug-x86 --parallel 4 --config Debug ctest --test-dir out/debug-x86 --verbose --timeout 240 -C Debug - name: Release, Win32 - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + if: matrix.tests == 'extra' && runner.os == 'Windows' run: | cmake . -B out/release-x86 -DCMAKE_BUILD_TYPE=Release -A Win32 cmake --build out/release-x86 --parallel 4 --config Release ctest --test-dir out/release-x86 --verbose --timeout 240 -C Release - name: Debug, Windows, clang-cl - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + if: matrix.tests == 'extra' && runner.os == 'Windows' run: | cmake . -B out/debug-clang-cl -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl cmake --build out/debug-clang-cl --parallel 4 --config Debug ctest --test-dir out/debug-clang-cl --verbose --timeout 240 -C Debug - name: Release, Windows, clang-cl - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + if: matrix.tests == 'extra' && runner.os == 'Windows' run: | cmake . -B out/release-clang-cl -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl cmake --build out/release-clang-cl --parallel 4 --config Release ctest --test-dir out/release-clang-cl --verbose --timeout 240 -C Release - name: Debug, Windows, clang - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + if: matrix.tests == 'extra' && runner.os == 'Windows' run: | cmake . -B out/debug-clang -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang cmake --build out/debug-clang --parallel 4 --config Debug ctest --test-dir out/debug-clang --verbose --timeout 240 -C Debug - name: Release, Windows, clang - if: matrix.tests == 'extra' && runner.os == 'Windows' && !endsWith(matrix.os,'arm') + if: matrix.tests == 'extra' && runner.os == 'Windows' run: | cmake . -B out/release-clang -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang cmake --build out/release-clang --parallel 4 --config Release From 0e9c24f8632d323f6081cb0d4acf32814122d12e Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Fri, 28 Aug 2026 14:07:28 +0200 Subject: [PATCH 099/280] workflows: Add MI_NO_USE_CXX variants for Windows/clang tests --- .github/workflows/test.yaml | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 00c94550e..007744cfb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -113,6 +113,13 @@ jobs: cmake --build out/debug-clang-cl --parallel 4 --config Debug ctest --test-dir out/debug-clang-cl --verbose --timeout 240 -C Debug + - name: Debug, Windows, clang-cl, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/debug-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -A Win32 -T ClangCl + cmake --build out/debug-clang-cl-nocxx --parallel 4 --config Debug + ctest --test-dir out/debug-clang-cl-nocxx --verbose --timeout 240 -C Debug + - name: Release, Windows, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' run: | @@ -120,6 +127,13 @@ jobs: cmake --build out/release-clang-cl --parallel 4 --config Release ctest --test-dir out/release-clang-cl --verbose --timeout 240 -C Release + - name: Release, Windows, clang-cl, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/release-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -A Win32 -T ClangCl + cmake --build out/release-clang-cl-nocxx --parallel 4 --config Release + ctest --test-dir out/release-clang-cl-nocxx --verbose --timeout 240 -C Release + - name: Debug, Windows, clang if: matrix.tests == 'extra' && runner.os == 'Windows' run: | @@ -127,6 +141,13 @@ jobs: cmake --build out/debug-clang --parallel 4 --config Debug ctest --test-dir out/debug-clang --verbose --timeout 240 -C Debug + - name: Debug, Windows, clang, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/debug-clang-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -DCMAKE_C_COMPILER=clang + cmake --build out/debug-clang-nocxx --parallel 4 --config Debug + ctest --test-dir out/debug-clang-nocxx --verbose --timeout 240 -C Debug + - name: Release, Windows, clang if: matrix.tests == 'extra' && runner.os == 'Windows' run: | @@ -134,6 +155,13 @@ jobs: cmake --build out/release-clang --parallel 4 --config Release ctest --test-dir out/release-clang --verbose --timeout 240 -C Release + - name: Release, Windows, clang, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/release-clang-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -DCMAKE_C_COMPILER=clang + cmake --build out/release-clang-nocxx --parallel 4 --config Release + ctest --test-dir out/release-clang-nocxx --verbose --timeout 240 -C Release + # Extra, macOS fixed TLS - name: Debug, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' From a032d354427cd24587b1c1ab0e90e20eec3022e5 Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Fri, 28 Aug 2026 14:48:56 +0200 Subject: [PATCH 100/280] workflows: Add MI_NO_USE_CXX variants for MSVC tests --- .github/workflows/test.yaml | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 007744cfb..d8fa3493c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -91,7 +91,21 @@ jobs: cmake --build out/release-cxx --parallel 8 --config Release ctest --test-dir out/release-cxx --verbose --timeout 240 -C Release - # Extra Windows: x86, clang-cl, and clang + # Extra Windows: MI_NO_USE_CXX, x86, clang-cl, and clang + - name: Debug, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/debug-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_NO_USE_CXX=ON + cmake --build out/debug-nocxx --parallel 8 --config Debug + ctest --test-dir out/debug-nocxx --verbose --timeout 240 -C Debug + + - name: Release, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/release-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_NO_USE_CXX=ON + cmake --build out/release-nocxx --parallel 8 --config Release + ctest --test-dir out/release-nocxx --verbose --timeout 240 -C Release + - name: Debug, Win32 if: matrix.tests == 'extra' && runner.os == 'Windows' run: | @@ -99,6 +113,13 @@ jobs: cmake --build out/debug-x86 --parallel 4 --config Debug ctest --test-dir out/debug-x86 --verbose --timeout 240 -C Debug + - name: Debug, Win32, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/debug-x86-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -A Win32 + cmake --build out/debug-x86-nocxx --parallel 4 --config Debug + ctest --test-dir out/debug-x86-nocxx --verbose --timeout 240 -C Debug + - name: Release, Win32 if: matrix.tests == 'extra' && runner.os == 'Windows' run: | @@ -106,6 +127,13 @@ jobs: cmake --build out/release-x86 --parallel 4 --config Release ctest --test-dir out/release-x86 --verbose --timeout 240 -C Release + - name: Release, Win32, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/release-x86-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -A Win32 + cmake --build out/release-x86-nocxx --parallel 4 --config Release + ctest --test-dir out/release-x86-nocxx --verbose --timeout 240 -C Release + - name: Debug, Windows, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' run: | From 80dbcb456c353a73c34c68998176ae68e9ebe47a Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Fri, 28 Aug 2026 14:59:38 +0200 Subject: [PATCH 101/280] workflows: Add clang-cl tests targetting the native architecture --- .github/workflows/test.yaml | 36 ++++++++++++++++++++++++++++++++---- 1 file changed, 32 insertions(+), 4 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d8fa3493c..596ced3e0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -134,31 +134,59 @@ jobs: cmake --build out/release-x86-nocxx --parallel 4 --config Release ctest --test-dir out/release-x86-nocxx --verbose --timeout 240 -C Release + - name: Debug, Win32, clang-cl + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/debug-x86-clang-cl -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl + cmake --build out/debug-x86-clang-cl --parallel 4 --config Debug + ctest --test-dir out/debug-x86-clang-cl --verbose --timeout 240 -C Debug + + - name: Debug, Win32, clang-cl, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/debug-x86-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -A Win32 -T ClangCl + cmake --build out/debug-x86-clang-cl-nocxx --parallel 4 --config Debug + ctest --test-dir out/debug-x86-clang-cl-nocxx --verbose --timeout 240 -C Debug + + - name: Release, Win32, clang-cl + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/release-x86-clang-cl -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl + cmake --build out/release-x86-clang-cl --parallel 4 --config Release + ctest --test-dir out/release-x86-clang-cl --verbose --timeout 240 -C Release + + - name: Release, Win32, clang-cl, MI_NO_USE_CXX + if: matrix.tests == 'extra' && runner.os == 'Windows' + run: | + cmake . -B out/release-x86-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -A Win32 -T ClangCl + cmake --build out/release-x86-clang-cl-nocxx --parallel 4 --config Release + ctest --test-dir out/release-x86-clang-cl-nocxx --verbose --timeout 240 -C Release + - name: Debug, Windows, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/debug-clang-cl -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl + cmake . -B out/debug-clang-cl -DCMAKE_BUILD_TYPE=Debug -T ClangCl cmake --build out/debug-clang-cl --parallel 4 --config Debug ctest --test-dir out/debug-clang-cl --verbose --timeout 240 -C Debug - name: Debug, Windows, clang-cl, MI_NO_USE_CXX if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/debug-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -A Win32 -T ClangCl + cmake . -B out/debug-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -T ClangCl cmake --build out/debug-clang-cl-nocxx --parallel 4 --config Debug ctest --test-dir out/debug-clang-cl-nocxx --verbose --timeout 240 -C Debug - name: Release, Windows, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/release-clang-cl -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl + cmake . -B out/release-clang-cl -DCMAKE_BUILD_TYPE=Release -T ClangCl cmake --build out/release-clang-cl --parallel 4 --config Release ctest --test-dir out/release-clang-cl --verbose --timeout 240 -C Release - name: Release, Windows, clang-cl, MI_NO_USE_CXX if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/release-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -A Win32 -T ClangCl + cmake . -B out/release-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -T ClangCl cmake --build out/release-clang-cl-nocxx --parallel 4 --config Release ctest --test-dir out/release-clang-cl-nocxx --verbose --timeout 240 -C Release From 819be2c09a6994860312352ab70ea0dff5c832ca Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Fri, 28 Aug 2026 17:13:39 +0200 Subject: [PATCH 102/280] workflows: Add matrix entry for C vs C++ --- .github/workflows/test.yaml | 373 ++++++++++++++++-------------------- 1 file changed, 160 insertions(+), 213 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 596ced3e0..ed721fc6c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -21,6 +21,7 @@ jobs: matrix: tests: [basic,extra] os: [windows-latest, macos-latest, ubuntu-latest, macos-14, macos-15-intel, ubuntu-22.04-arm, windows-11-arm] + c_or_cxx: [c, cxx] exclude: - os: macos-14 tests: extra @@ -45,21 +46,45 @@ jobs: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + # Choose MI_USE_CXX/MI_NO_USE_CXX values + - name: Compute CMake args + id: cmake_args + shell: bash + run: | + CMAKE_CXX_OPT_C="-DMI_USE_CXX=OFF -DMI_NO_USE_CXX=ON" + CMAKE_CXX_OPT_CXX="-DMI_USE_CXX=ON -DMI_NO_USE_CXX=OFF" + # On MSVC, C++ mode is default, so there C mode explicitly disables C++. + # Everywhere else, C mode is default, so explicitly enable C + if [ "${{ matrix.c_or_cxx }}" = "cxx" ]; then + CMAKE_CXX_OPT_GCC=$CMAKE_CXX_OPT_CXX + CMAKE_CXX_OPT_MSVC= + else # Default to C mode + CMAKE_CXX_OPT_GCC= + CMAKE_CXX_OPT_MSVC=$CMAKE_CXX_OPT_C + fi + echo "CMAKE_CXX_OPT_GCC=$CMAKE_CXX_OPT_GCC" >> $GITHUB_OUTPUT + echo "CMAKE_CXX_OPT_MSVC=$CMAKE_CXX_OPT_MSVC" >> $GITHUB_OUTPUT + if [ "${{ runner.os }}" = "Windows" ]; then + echo "CMAKE_CXX_OPT_DEFAULT=$CMAKE_CXX_OPT_MSVC" >> $GITHUB_OUTPUT + else + echo "CMAKE_CXX_OPT_DEFAULT=$CMAKE_CXX_OPT_GCC" >> $GITHUB_OUTPUT + fi + # Basic - name: Debug if: matrix.tests == 'basic' run: | env - cmake . -B out/debug -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL - cmake --build out/debug --parallel 4 --config Debug - ctest --test-dir out/debug --verbose --timeout 240 -C Debug + cmake . -B out/debug-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release if: matrix.tests == 'basic' run: | - cmake . -B out/release -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON - cmake --build out/release --parallel 4 --config Release - ctest --test-dir out/release --verbose --timeout 240 -C Release + cmake . -B out/release-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Release, Asm if: matrix.tests == 'basic' && runner.os == 'Linux' @@ -72,181 +97,110 @@ jobs: - name: Secure if: matrix.tests == 'basic' run: | - cmake . -B out/secure -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - cmake --build out/secure --parallel 4 --config Release - ctest --test-dir out/secure --verbose --timeout 240 -C Release - - # Basic, C++ - - name: Debug, C++ - if: matrix.tests == 'basic' && runner.os != 'Windows' # windows already uses C++ by default - run: | - cmake . -B out/debug-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON - cmake --build out/debug-cxx --parallel 8 --config Debug - ctest --test-dir out/debug-cxx --verbose --timeout 240 -C Debug - - - name: Release, C++ - if: matrix.tests == 'basic' && runner.os != 'Windows' # windows already uses C++ by default - run: | - cmake . -B out/release-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON - cmake --build out/release-cxx --parallel 8 --config Release - ctest --test-dir out/release-cxx --verbose --timeout 240 -C Release + cmake . -B out/secure-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/secure-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/secure-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - # Extra Windows: MI_NO_USE_CXX, x86, clang-cl, and clang - - name: Debug, MI_NO_USE_CXX + # Extra Windows: x86, clang-cl, and clang + - name: Debug if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/debug-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_NO_USE_CXX=ON - cmake --build out/debug-nocxx --parallel 8 --config Debug - ctest --test-dir out/debug-nocxx --verbose --timeout 240 -C Debug + cmake . -B out/debug-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-${{ matrix.c_or_cxx }} --parallel 8 --config Debug + ctest --test-dir out/debug-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - - name: Release, MI_NO_USE_CXX + - name: Release if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/release-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_NO_USE_CXX=ON - cmake --build out/release-nocxx --parallel 8 --config Release - ctest --test-dir out/release-nocxx --verbose --timeout 240 -C Release + cmake . -B out/release-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-${{ matrix.c_or_cxx }} --parallel 8 --config Release + ctest --test-dir out/release-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Win32 if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/debug-x86 -DCMAKE_BUILD_TYPE=Debug -A Win32 - cmake --build out/debug-x86 --parallel 4 --config Debug - ctest --test-dir out/debug-x86 --verbose --timeout 240 -C Debug - - - name: Debug, Win32, MI_NO_USE_CXX - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/debug-x86-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -A Win32 - cmake --build out/debug-x86-nocxx --parallel 4 --config Debug - ctest --test-dir out/debug-x86-nocxx --verbose --timeout 240 -C Debug + cmake . -B out/debug-x86-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -A Win32 ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-x86-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-x86-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Win32 if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/release-x86 -DCMAKE_BUILD_TYPE=Release -A Win32 - cmake --build out/release-x86 --parallel 4 --config Release - ctest --test-dir out/release-x86 --verbose --timeout 240 -C Release - - - name: Release, Win32, MI_NO_USE_CXX - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/release-x86-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -A Win32 - cmake --build out/release-x86-nocxx --parallel 4 --config Release - ctest --test-dir out/release-x86-nocxx --verbose --timeout 240 -C Release + cmake . -B out/release-x86-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -A Win32 ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-x86-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-x86-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Win32, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/debug-x86-clang-cl -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl - cmake --build out/debug-x86-clang-cl --parallel 4 --config Debug - ctest --test-dir out/debug-x86-clang-cl --verbose --timeout 240 -C Debug - - - name: Debug, Win32, clang-cl, MI_NO_USE_CXX - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/debug-x86-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -A Win32 -T ClangCl - cmake --build out/debug-x86-clang-cl-nocxx --parallel 4 --config Debug - ctest --test-dir out/debug-x86-clang-cl-nocxx --verbose --timeout 240 -C Debug + cmake . -B out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Win32, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/release-x86-clang-cl -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl - cmake --build out/release-x86-clang-cl --parallel 4 --config Release - ctest --test-dir out/release-x86-clang-cl --verbose --timeout 240 -C Release - - - name: Release, Win32, clang-cl, MI_NO_USE_CXX - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/release-x86-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -A Win32 -T ClangCl - cmake --build out/release-x86-clang-cl-nocxx --parallel 4 --config Release - ctest --test-dir out/release-x86-clang-cl-nocxx --verbose --timeout 240 -C Release + cmake . -B out/release-x86-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-x86-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-x86-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Windows, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/debug-clang-cl -DCMAKE_BUILD_TYPE=Debug -T ClangCl - cmake --build out/debug-clang-cl --parallel 4 --config Debug - ctest --test-dir out/debug-clang-cl --verbose --timeout 240 -C Debug - - - name: Debug, Windows, clang-cl, MI_NO_USE_CXX - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/debug-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -T ClangCl - cmake --build out/debug-clang-cl-nocxx --parallel 4 --config Debug - ctest --test-dir out/debug-clang-cl-nocxx --verbose --timeout 240 -C Debug + cmake . -B out/debug-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Windows, clang-cl if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/release-clang-cl -DCMAKE_BUILD_TYPE=Release -T ClangCl - cmake --build out/release-clang-cl --parallel 4 --config Release - ctest --test-dir out/release-clang-cl --verbose --timeout 240 -C Release - - - name: Release, Windows, clang-cl, MI_NO_USE_CXX - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/release-clang-cl-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -T ClangCl - cmake --build out/release-clang-cl-nocxx --parallel 4 --config Release - ctest --test-dir out/release-clang-cl-nocxx --verbose --timeout 240 -C Release + cmake . -B out/release-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Windows, clang if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/debug-clang -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang - cmake --build out/debug-clang --parallel 4 --config Debug - ctest --test-dir out/debug-clang --verbose --timeout 240 -C Debug - - - name: Debug, Windows, clang, MI_NO_USE_CXX - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/debug-clang-nocxx -DCMAKE_BUILD_TYPE=Debug -DMI_NO_USE_CXX=ON -DCMAKE_C_COMPILER=clang - cmake --build out/debug-clang-nocxx --parallel 4 --config Debug - ctest --test-dir out/debug-clang-nocxx --verbose --timeout 240 -C Debug + cmake . -B out/debug-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/debug-clang-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Windows, clang if: matrix.tests == 'extra' && runner.os == 'Windows' run: | - cmake . -B out/release-clang -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang - cmake --build out/release-clang --parallel 4 --config Release - ctest --test-dir out/release-clang --verbose --timeout 240 -C Release - - - name: Release, Windows, clang, MI_NO_USE_CXX - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/release-clang-nocxx -DCMAKE_BUILD_TYPE=Release -DMI_NO_USE_CXX=ON -DCMAKE_C_COMPILER=clang - cmake --build out/release-clang-nocxx --parallel 4 --config Release - ctest --test-dir out/release-clang-nocxx --verbose --timeout 240 -C Release + cmake . -B out/release-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} + cmake --build out/release-clang-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release # Extra, macOS fixed TLS - name: Debug, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' run: | - cmake . -B out/debug-tls-fixed -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_FIXED=ON - cmake --build out/debug-tls-fixed --parallel 4 --config Debug - ctest --test-dir out/debug-tls-fixed --verbose --timeout 240 -C Debug + cmake . -B out/debug-tls-fixed-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_FIXED=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-tls-fixed-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-tls-fixed-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' run: | - cmake . -B out/release-tls-fixed -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_FIXED=ON - cmake --build out/release-tls-fixed --parallel 4 --config Release - ctest --test-dir out/release-tls-fixed --verbose --timeout 240 -C Release + cmake . -B out/release-tls-fixed-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_FIXED=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-tls-fixed-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-tls-fixed-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release # Extra, Linux pthread TLS - name: Debug, Linux, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | - cmake . -B out/debug-tls-pthreads -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_PTHREADS=ON - cmake --build out/debug-tls-pthreads --parallel 4 --config Debug - ctest --test-dir out/debug-tls-pthreads --verbose --timeout 240 -C Debug + cmake . -B out/debug-tls-pthreads-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_PTHREADS=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-tls-pthreads-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-tls-pthreads-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Linux, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | - cmake . -B out/release-tls-pthreads -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON -DMI_SEE_ASM=ON - cmake --build out/release-tls-pthreads --parallel 4 --config Release - ctest --test-dir out/release-tls-pthreads --verbose --timeout 240 -C Release + cmake . -B out/release-tls-pthreads-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON -DMI_SEE_ASM=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-tls-pthreads-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-tls-pthreads-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Release, Linux, TLS pthreads, Asm if: matrix.tests == 'extra' && runner.os == 'linux' @@ -260,33 +214,26 @@ jobs: - name: Release, SIMD if: matrix.tests == 'extra' run: | - cmake . -B out/release-simd -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON - cmake --build out/release-simd --parallel 8 --config Release - ctest --test-dir out/release-simd --verbose --timeout 240 -C Release - - - name: Release, C++, SIMD - if: matrix.tests == 'basic' && runner.os != 'Windows' && runner.os != 'macOS' - run: | - cmake . -B out/release-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON -DMI_USE_CXX=ON - cmake --build out/release-cxx --parallel 8 --config Release - ctest --test-dir out/release-cxx --verbose --timeout 240 -C Release + cmake . -B out/release-simd-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-simd-${{ matrix.c_or_cxx }} --parallel 8 --config Release + ctest --test-dir out/release-simd-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, C++, clang++ - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' run: | - CC=clang CXX=clang++ cmake . -B out/debug-clang-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON - cmake --build out/debug-clang-cxx --parallel 8 --config Debug - ctest --test-dir out/debug-clang-cxx --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON + cmake --build out/debug-clang-${{ matrix.c_or_cxx }} --parallel 8 --config Debug + ctest --test-dir out/debug-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, C++, clang++ - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' run: | - CC=clang CXX=clang++ cmake . -B out/release-clang-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON -DMI_SEE_ASM=ON - cmake --build out/release-clang-cxx --parallel 8 --config Release - ctest --test-dir out/release-clang-cxx --verbose --timeout 240 -C Release + CC=clang CXX=clang++ cmake . -B out/release-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON -DMI_SEE_ASM=ON + cmake --build out/release-clang-${{ matrix.c_or_cxx }} --parallel 8 --config Release + ctest --test-dir out/release-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Release, C++, clang++, Asm - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-clang-cxx-asm @@ -295,48 +242,48 @@ jobs: - name: Debug, ASAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-asan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_TRACK=ASAN - cmake --build out/debug-asan --parallel 8 --config Debug - ctest --test-dir out/debug-asan --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-asan-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_TRACK=ASAN ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-asan-${{ matrix.c_or_cxx }} --parallel 8 --config Debug + ctest --test-dir out/debug-asan-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Debug, UBSAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-ubsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_UBSAN=ON - cmake --build out/debug-ubsan --parallel 8 --config Debug - ctest --test-dir out/debug-ubsan --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-ubsan-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_UBSAN=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-ubsan-${{ matrix.c_or_cxx }} --parallel 8 --config Debug + ctest --test-dir out/debug-ubsan-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Debug, TSAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-tsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_TSAN=ON - cmake --build out/debug-tsan --parallel 8 --config Debug - ctest --test-dir out/debug-tsan --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-tsan-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_TSAN=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-tsan-${{ matrix.c_or_cxx }} --parallel 8 --config Debug + ctest --test-dir out/debug-tsan-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Debug, Guarded if: matrix.tests == 'extra' && runner.os != 'macOS' run: | - cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=ON - cmake --build out/debug-guarded --parallel 8 --config Debug - ctest --test-dir out/debug-guarded --verbose --timeout 240 -C Debug + cmake . -B out/debug-guarded-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-guarded-${{ matrix.c_or_cxx }} --parallel 8 --config Debug + ctest --test-dir out/debug-guarded-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug env: MIMALLOC_GUARDED_SAMPLE_RATE: 10 - name: Release, Guarded if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') run: | - cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=ON - cmake --build out/release-guarded --parallel 8 --config Release - ctest --test-dir out/release-guarded --verbose --timeout 240 -C Release + cmake . -B out/release-guarded-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-guarded-${{ matrix.c_or_cxx }} --parallel 8 --config Release + ctest --test-dir out/release-guarded-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release env: MIMALLOC_GUARDED_SAMPLE_RATE: 10 - name: Debug, Secure Full if: matrix.tests == 'extra' && runner.os != 'macOS' run: | - cmake . -B out/debug-secure-full -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_SECURE=FULL - cmake --build out/debug-secure-full --parallel 8 --config Debug - ctest --test-dir out/debug-secure-full --verbose --timeout 240 -C Debug + cmake . -B out/debug-secure-full-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_SECURE=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-secure-full-${{ matrix.c_or_cxx }} --parallel 8 --config Debug + ctest --test-dir out/debug-secure-full-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug # MSYS2 mingw64 - name: mingw-ucrt64 x64, Setup @@ -355,17 +302,17 @@ jobs: shell: msys2 {0} run: | env - cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL - cmake --build out/debug-mingw-ucrt64-x64 --parallel 4 --config Debug - ctest --test-dir out/debug-mingw-ucrt64-x64 --verbose --timeout 240 -C Debug + cmake . -B out/debug-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_GCC }} + cmake --build out/debug-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: mingw-ucrt64 x64, Release if: matrix.tests == 'mingw-ucrt64' shell: msys2 {0} run: | - cmake . -B out/release-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON - cmake --build out/release-mingw-ucrt64-x64 --parallel 4 --config Release - ctest --test-dir out/release-mingw-ucrt64-x64 --verbose --timeout 240 -C Release + cmake . -B out/release-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_GCC }} + cmake --build out/release-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release # VM: FreeBSD (arm64 and x64) @@ -385,27 +332,27 @@ jobs: shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/debug-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL - cmake --build out/debug-arm64 --parallel 4 --config Debug - ctest --test-dir out/debug-arm64 --verbose --timeout 240 -C Debug + cmake . -B out/debug-arm64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-arm64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-arm64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: FreeBSD arm64, Release if: matrix.tests == 'freebsd-arm64' shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/release-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON - cmake --build out/release-arm64 --parallel 4 --config Release - ctest --test-dir out/release-arm64 --verbose --timeout 240 -C Release + cmake . -B out/release-arm64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-arm64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-arm64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: FreeBSD arm64, Secure if: matrix.tests == 'freebsd-arm64' shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/secure-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - cmake --build out/secure-arm64 --parallel 4 --config Release - ctest --test-dir out/secure-arm64 --verbose --timeout 240 -C Release + cmake . -B out/secure-arm64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/secure-arm64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/secure-arm64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: FreeBSD x64, Setup @@ -423,25 +370,25 @@ jobs: if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL - cmake --build out/debug-x64 --parallel 4 --config Debug - ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug + cmake . -B out/debug-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: FreeBSD x64, Release if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/release-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON - cmake --build out/release-x64 --parallel 4 --config Release - ctest --test-dir out/release-x64 --verbose --timeout 240 -C Release + cmake . -B out/release-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: FreeBSD x64, Secure if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/secure-x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - cmake --build out/secure-x64 --parallel 4 --config Release - ctest --test-dir out/secure-x64 --verbose --timeout 240 -C Release + cmake . -B out/secure-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/secure-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/secure-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release # VM: Alpine Linux with MUSL libc (riscv64 and x64) @@ -463,17 +410,17 @@ jobs: MI_TEST_LIGHT: 1 run: | uname -m - cmake . -B out/debug-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL - cmake --build out/debug-riscv64 --parallel 4 --config Debug - ctest --test-dir out/debug-riscv64 --verbose --timeout 240 -C Debug + cmake . -B out/debug-riscv64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-riscv64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-riscv64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Alpine riscv64, Release if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} run: | - cmake . -B out/release-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON - cmake --build out/release-riscv64 --parallel 4 --config Release - ctest --test-dir out/release-riscv64 --verbose --timeout 240 -C Release + cmake . -B out/release-riscv64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-riscv64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-riscv64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Alpine risc64, Asm if: matrix.tests == 'alpine-riscv64' @@ -486,9 +433,9 @@ jobs: if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} run: | - cmake . -B out/secure-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - cmake --build out/secure-riscv64 --parallel 4 --config Release - ctest --test-dir out/secure-riscv64 --verbose --timeout 240 -C Release + cmake . -B out/secure-riscv64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/secure-riscv64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/secure-riscv64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Alpine x64, Setup @@ -505,25 +452,25 @@ jobs: if: matrix.tests == 'alpine-x64' shell: alpine-x64.sh {0} run: | - cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL - cmake --build out/debug-x64 --parallel 4 --config Debug - ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug + cmake . -B out/debug-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Alpine x64, Release if: matrix.tests == 'alpine-x64' shell: alpine-x64.sh {0} run: | - cmake . -B out/release-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON - cmake --build out/release-x64 --parallel 4 --config Release - ctest --test-dir out/release-x64 --verbose --timeout 240 -C Release + cmake . -B out/release-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Alpine x64, Secure if: matrix.tests == 'alpine-x64' shell: alpine-x64.sh {0} run: | - cmake . -B out/secure-x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - cmake --build out/secure-x64 --parallel 4 --config Release - ctest --test-dir out/secure-x64 --verbose --timeout 240 -C Release + cmake . -B out/secure-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/secure-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/secure-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Alpine arm32, Setup if: matrix.tests == 'alpine-arm32' @@ -542,25 +489,25 @@ jobs: MI_TEST_LIGHT: 1 run: | uname -m - cmake . -B out/debug-arm32 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL - cmake --build out/debug-arm32 --parallel 4 --config Debug - ctest --test-dir out/debug-arm32 --verbose --timeout 240 -C Debug + cmake . -B out/debug-arm32-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-arm32-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-arm32-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Alpine arm32, Release if: matrix.tests == 'alpine-arm32' shell: alpine-arm32.sh {0} run: | - cmake . -B out/release-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON - cmake --build out/release-arm32 --parallel 4 --config Release - ctest --test-dir out/release-arm32 --verbose --timeout 240 -C Release + cmake . -B out/release-arm32-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/release-arm32-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/release-arm32-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Alpine arm32, Secure if: matrix.tests == 'alpine-arm32' shell: alpine-arm32.sh {0} run: | - cmake . -B out/secure-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - cmake --build out/secure-arm32 --parallel 4 --config Release - ctest --test-dir out/secure-arm32 --verbose --timeout 240 -C Release + cmake . -B out/secure-arm32-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/secure-arm32-${{ matrix.c_or_cxx }} --parallel 4 --config Release + ctest --test-dir out/secure-arm32-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release # - name: Android, Setup # if: matrix.tests == 'android-x64' From aca9e6e0d0f91619243202068ab20c0862a832dc Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 12:48:22 -0700 Subject: [PATCH 103/280] experiment codegen on x64 --- include/mimalloc/internal.h | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 754b60075..7f9c91e51 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -773,8 +773,10 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { // if the page meta data is aligned in front of pages we can find it efficiently // without needing to go through the page map (for valid pointers). static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { - mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); - const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; + // mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); + // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; + mi_page_t* const page_metas = (mi_page_t*)((uintptr_t)p & ~(MI_PAGE_META_ALIGNMENT-1)); + const uintptr_t page_idx = ((uintptr_t)p & (MI_PAGE_META_ALIGNMENT-1)) / MI_ARENA_SLICE_SIZE; mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_ALIGNED_COUNT); return &page_metas[page_idx]; } From 7cca1b5404fabed26db9faf37df474e2645b4ac2 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 13:02:14 -0700 Subject: [PATCH 104/280] Work around clang-cl missing ldar, stlr, see PR #1379 by @res2k --- include/mimalloc/atomic.h | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index 525e5e136..096285a94 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -221,9 +221,11 @@ static inline uintptr_t mi_atomic_load_explicit(_Atomic(uintptr_t) const* p, mi_ if (mo == mi_memory_order_relaxed) { return (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); } + #if !defined(__clang__) // work around __ldar missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_acquire) { return MI_MSC_XX(__ldar)((volatile const uintptr_t*)p); } + #endif else { const uintptr_t u = (uintptr_t)MI_MSC_XX(__iso_volatile_load)((volatile const intptr_t*)p); __dmb(15); // _ARM(64)_BARRIER_SY @@ -243,9 +245,11 @@ static inline void mi_atomic_store_explicit(_Atomic(uintptr_t)*p, uintptr_t x, m if (mo == mi_memory_order_relaxed) { MI_MSC_XX(__iso_volatile_store)((volatile intptr_t*)p, x); } + #if !defined(__clang__) // work around __stlr missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_release) { MI_MSC_XX(__stlr)((volatile uintptr_t*)p,x); } + #endif else { mi_atomic_exchange_explicit(p, x, mo); } @@ -263,7 +267,7 @@ static inline int64_t mi_atomic_loadi64_explicit(_Atomic(int64_t)*p, mi_memory_o if (mo == mi_memory_order_relaxed) { return __iso_volatile_load64((volatile const int64_t*)p); } - #if defined(_M_ARM64) + #if defined(_M_ARM64) && !defined(__clang__) // work around __ldar64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo <= mi_memory_order_acquire) { return __ldar64((volatile const uintptr_t*)p); } @@ -288,7 +292,7 @@ static inline void mi_atomic_storei64_explicit(_Atomic(int64_t)*p, int64_t x, mi if (mo == mi_memory_order_relaxed) { __iso_volatile_store64((volatile int64_t*)p,x); } - #if defined(_M_ARM64) + #if defined(_M_ARM64) && !defined(__clang__) // work around __stlr64 missing in clang-cl, see https://github.com/llvm/llvm-project/issues/121689 else if (mo == mi_memory_order_release) { __stlr64((volatile uint64_t*)p, (uint64_t)x); } From cfab42c30ae00dfd603632f0f9683b80a3c5e308 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 13:11:20 -0700 Subject: [PATCH 105/280] Configure monthly Dependabot updates, see PR #1376 --- .github/workflows/dependabot.yaml | 10 ++++++++++ 1 file changed, 10 insertions(+) create mode 100644 .github/workflows/dependabot.yaml diff --git a/.github/workflows/dependabot.yaml b/.github/workflows/dependabot.yaml new file mode 100644 index 000000000..97489cb60 --- /dev/null +++ b/.github/workflows/dependabot.yaml @@ -0,0 +1,10 @@ +version: 2 +updates: + - package-ecosystem: "github-actions" + directory: "/" + schedule: + interval: "monthly" + groups: + minor-and-patch: + patterns: ["*"] + update-types: ["minor", "patch"] \ No newline at end of file From 8cc8a2aa75f32e63e98c523ff1336d5837fd7807 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 13:25:03 -0700 Subject: [PATCH 106/280] Allow for C mode building w/ MSVC or clang-cl, pr #1380 by @res2k --- CMakeLists.txt | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c815b6796..319b7e99a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -186,7 +186,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel") endif() # force C++ compilation with msvc or clang-cl to use modern C++ atomics -if(CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel" OR MI_CLANG_CL) +if((CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel" OR MI_CLANG_CL) AND NOT MI_NO_USE_CXX) set(MI_USE_CXX "ON") endif() @@ -494,7 +494,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM endif() if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) - list(APPEND mi_cflags /Zc:__cplusplus) + if(MI_USE_CXX) + list(APPEND mi_cflags /Zc:__cplusplus) + endif() if(MI_OPT_ARCH AND NOT MI_CLANG_CL) if(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH_FLAGS "/arch:armv8.1") # fast atomics (LSE) From 69f9b07a4a537edbe28446c1ae71a32cc9ea5e12 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 13:44:05 -0700 Subject: [PATCH 107/280] improve free codegen --- include/mimalloc/internal.h | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 7f9c91e51..2b73fd990 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -773,10 +773,9 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { // if the page meta data is aligned in front of pages we can find it efficiently // without needing to go through the page map (for valid pointers). static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { - // mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); + mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; - mi_page_t* const page_metas = (mi_page_t*)((uintptr_t)p & ~(MI_PAGE_META_ALIGNMENT-1)); - const uintptr_t page_idx = ((uintptr_t)p & (MI_PAGE_META_ALIGNMENT-1)) / MI_ARENA_SLICE_SIZE; + const uintptr_t page_idx = ((uintptr_t)p % MI_PAGE_META_ALIGNMENT) / MI_ARENA_SLICE_SIZE; mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_ALIGNED_COUNT); return &page_metas[page_idx]; } From d3dfc3923c516a01cd0be2466f6fc6c28dc44e6c Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 13:47:50 -0700 Subject: [PATCH 108/280] fix dependabot location --- .github/{workflows => }/dependabot.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{workflows => }/dependabot.yaml (100%) diff --git a/.github/workflows/dependabot.yaml b/.github/dependabot.yaml similarity index 100% rename from .github/workflows/dependabot.yaml rename to .github/dependabot.yaml From f0194a5a34b9705364a317221cfedb8fdf875b75 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 15:01:35 -0700 Subject: [PATCH 109/280] improve free codegen --- include/mimalloc/internal.h | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 2b73fd990..fa752d3d0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -773,11 +773,17 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { // if the page meta data is aligned in front of pages we can find it efficiently // without needing to go through the page map (for valid pointers). static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { - mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; - const uintptr_t page_idx = ((uintptr_t)p % MI_PAGE_META_ALIGNMENT) / MI_ARENA_SLICE_SIZE; + const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); + mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_ALIGNED_COUNT); - return &page_metas[page_idx]; + #if MI_ARCH_ARM64 + mi_page_t* const page = &page_metas[page_idx]; + #else + mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))); + #endif + return page; + } static inline mi_page_t* _mi_aligned_ptr_page(const void* p) { From 3dfa6aacc24aca8f60c8b28e48d60a8272e6ef26 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 15:01:49 -0700 Subject: [PATCH 110/280] fix test paths --- .github/workflows/test.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ed721fc6c..2bf769333 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -91,7 +91,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-asm - path: out/release/CMakeFiles/mimalloc-static.dir/src/*.s + path: out/release-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s - name: Secure @@ -207,7 +207,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-pthreads-asm - path: out/release-tls-pthreads/CMakeFiles/mimalloc-static.dir/src/*.s + path: out/release-tls-pthreads-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s # Extra @@ -221,16 +221,16 @@ jobs: - name: Debug, C++, clang++ if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' run: | - CC=clang CXX=clang++ cmake . -B out/debug-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON - cmake --build out/debug-clang-${{ matrix.c_or_cxx }} --parallel 8 --config Debug - ctest --test-dir out/debug-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-clang-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON + cmake --build out/debug-clang-cxx --parallel 8 --config Debug + ctest --test-dir out/debug-clang-cxx --verbose --timeout 240 -C Debug - name: Release, C++, clang++ if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' run: | - CC=clang CXX=clang++ cmake . -B out/release-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON -DMI_SEE_ASM=ON - cmake --build out/release-clang-${{ matrix.c_or_cxx }} --parallel 8 --config Release - ctest --test-dir out/release-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + CC=clang CXX=clang++ cmake . -B out/release-clang-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON -DMI_SEE_ASM=ON + cmake --build out/release-clang-cxx --parallel 8 --config Release + ctest --test-dir out/release-clang-cxx --verbose --timeout 240 -C Release - name: Release, C++, clang++, Asm if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' @@ -247,11 +247,11 @@ jobs: ctest --test-dir out/debug-asan-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Debug, UBSAN - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' run: | - CC=clang CXX=clang++ cmake . -B out/debug-ubsan-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_UBSAN=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-ubsan-${{ matrix.c_or_cxx }} --parallel 8 --config Debug - ctest --test-dir out/debug-ubsan-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-ubsan-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_UBSAN=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-ubsan-cxx --parallel 8 --config Debug + ctest --test-dir out/debug-ubsan-cxx --verbose --timeout 240 -C Debug - name: Debug, TSAN if: matrix.tests == 'extra' && runner.os == 'Linux' @@ -427,7 +427,7 @@ jobs: uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: alpine-riscv64-asm - path: out/release-riscv64/CMakeFiles/mimalloc-static.dir/src/*.s + path: out/release-riscv64-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s - name: Alpine riscv64, Secure if: matrix.tests == 'alpine-riscv64' From c2259fbf779c0340e99793c38ed8af0ecb5af903 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 15:09:33 -0700 Subject: [PATCH 111/280] add fail fast false to workflow --- .github/workflows/test.yaml | 38 ++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2bf769333..118013ba9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -18,6 +18,7 @@ jobs: env: MIMALLOC_SHOW_ERRORS: 1 strategy: + fail-fast: false matrix: tests: [basic,extra] os: [windows-latest, macos-latest, ubuntu-latest, macos-14, macos-15-intel, ubuntu-22.04-arm, windows-11-arm] @@ -71,14 +72,6 @@ jobs: fi # Basic - - name: Debug - if: matrix.tests == 'basic' - run: | - env - cmake . -B out/debug-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - - name: Release if: matrix.tests == 'basic' run: | @@ -93,6 +86,13 @@ jobs: name: ${{ matrix.os }}-asm path: out/release-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Debug + if: matrix.tests == 'basic' + run: | + env + cmake . -B out/debug-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Secure if: matrix.tests == 'basic' @@ -403,17 +403,6 @@ jobs: cmake ninja - - name: Alpine riscv64, Debug - if: matrix.tests == 'alpine-riscv64' - shell: alpine-riscv64.sh {0} - env: - MI_TEST_LIGHT: 1 - run: | - uname -m - cmake . -B out/debug-riscv64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-riscv64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-riscv64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - - name: Alpine riscv64, Release if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} @@ -429,6 +418,17 @@ jobs: name: alpine-riscv64-asm path: out/release-riscv64-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Alpine riscv64, Debug + if: matrix.tests == 'alpine-riscv64' + shell: alpine-riscv64.sh {0} + env: + MI_TEST_LIGHT: 1 + run: | + uname -m + cmake . -B out/debug-riscv64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + cmake --build out/debug-riscv64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug + ctest --test-dir out/debug-riscv64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + - name: Alpine riscv64, Secure if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} From 155fef1313801772eff498f987db70ef9377a088 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 15:29:43 -0700 Subject: [PATCH 112/280] improve free codegen --- include/mimalloc/internal.h | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index fa752d3d0..297ca4cc3 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -773,14 +773,14 @@ static inline mi_page_t* _mi_checked_ptr_page(const void* p) { // if the page meta data is aligned in front of pages we can find it efficiently // without needing to go through the page map (for valid pointers). static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { - // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; - const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); + // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; + const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_ALIGNED_COUNT); - #if MI_ARCH_ARM64 - mi_page_t* const page = &page_metas[page_idx]; - #else + #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64 mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))); + #else + mi_page_t* const page = &page_metas[page_idx]; #endif return page; From e2dd60cb8faee3e89869eac5ce8bb040caee7632 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 15:51:15 -0700 Subject: [PATCH 113/280] reduce c/c++ test variants a bit --- .github/workflows/test.yaml | 194 +++++++++++++++++------------------- 1 file changed, 91 insertions(+), 103 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 118013ba9..0391ea94d 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -26,6 +26,9 @@ jobs: exclude: - os: macos-14 tests: extra + - c_or_cxx: cxx + tests: extra + os: ${{ matrix.os != 'windows-latest' && matrix.os }} # do not exclude on windows include: - os: ubuntu-latest tests: freebsd-x64 @@ -102,20 +105,6 @@ jobs: ctest --test-dir out/secure-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release # Extra Windows: x86, clang-cl, and clang - - name: Debug - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/debug-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} - cmake --build out/debug-${{ matrix.c_or_cxx }} --parallel 8 --config Debug - ctest --test-dir out/debug-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - - - name: Release - if: matrix.tests == 'extra' && runner.os == 'Windows' - run: | - cmake . -B out/release-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} - cmake --build out/release-${{ matrix.c_or_cxx }} --parallel 8 --config Release - ctest --test-dir out/release-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - - name: Debug, Win32 if: matrix.tests == 'extra' && runner.os == 'Windows' run: | @@ -176,64 +165,63 @@ jobs: - name: Debug, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' run: | - cmake . -B out/debug-tls-fixed-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_FIXED=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-tls-fixed-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-tls-fixed-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-tls-fixed -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_FIXED=ON + cmake --build out/debug-tls-fixed --parallel 4 --config Debug + ctest --test-dir out/debug-tls-fixed --verbose --timeout 240 -C Debug - name: Release, macOS, TLS fixed if: matrix.tests == 'extra' && runner.os == 'macOS' run: | - cmake . -B out/release-tls-fixed-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_FIXED=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-tls-fixed-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/release-tls-fixed-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-tls-fixed -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_FIXED=ON + cmake --build out/release-tls-fixed --parallel 4 --config Release + ctest --test-dir out/release-tls-fixed --verbose --timeout 240 -C Release # Extra, Linux pthread TLS - name: Debug, Linux, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | - cmake . -B out/debug-tls-pthreads-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_PTHREADS=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-tls-pthreads-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-tls-pthreads-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-tls-pthreads -DCMAKE_BUILD_TYPE=Debug -DMI_TLS_MODEL_PTHREADS=ON + cmake --build out/debug-tls-pthreads --parallel 4 --config Debug + ctest --test-dir out/debug-tls-pthreads --verbose --timeout 240 -C Debug - name: Release, Linux, TLS pthreads if: matrix.tests == 'extra' && runner.os == 'linux' run: | - cmake . -B out/release-tls-pthreads-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON -DMI_SEE_ASM=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-tls-pthreads-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/release-tls-pthreads-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-tls-pthreads -DCMAKE_BUILD_TYPE=Release -DMI_TLS_MODEL_PTHREADS=ON -DMI_SEE_ASM=ON + cmake --build out/release-tls-pthreads --parallel 4 --config Release + ctest --test-dir out/release-tls-pthreads --verbose --timeout 240 -C Release - name: Release, Linux, TLS pthreads, Asm if: matrix.tests == 'extra' && runner.os == 'linux' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-pthreads-asm - path: out/release-tls-pthreads-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s - + path: out/release-tls-pthreads/CMakeFiles/mimalloc-static.dir/src/*.s # Extra - name: Release, SIMD if: matrix.tests == 'extra' run: | - cmake . -B out/release-simd-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-simd-${{ matrix.c_or_cxx }} --parallel 8 --config Release - ctest --test-dir out/release-simd-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-simd -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON + cmake --build out/release-simd --parallel 8 --config Release + ctest --test-dir out/release-simd --verbose --timeout 240 -C Release - name: Debug, C++, clang++ - if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' + if: matrix.tests == 'extra' && runner.os == 'Linux' run: | CC=clang CXX=clang++ cmake . -B out/debug-clang-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_USE_CXX=ON cmake --build out/debug-clang-cxx --parallel 8 --config Debug ctest --test-dir out/debug-clang-cxx --verbose --timeout 240 -C Debug - name: Release, C++, clang++ - if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' + if: matrix.tests == 'extra' && runner.os == 'Linux' run: | CC=clang CXX=clang++ cmake . -B out/release-clang-cxx -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_USE_CXX=ON -DMI_SEE_ASM=ON cmake --build out/release-clang-cxx --parallel 8 --config Release ctest --test-dir out/release-clang-cxx --verbose --timeout 240 -C Release - name: Release, C++, clang++, Asm - if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' + if: matrix.tests == 'extra' && runner.os == 'Linux' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-clang-cxx-asm @@ -242,48 +230,48 @@ jobs: - name: Debug, ASAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-asan-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_TRACK=ASAN ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-asan-${{ matrix.c_or_cxx }} --parallel 8 --config Debug - ctest --test-dir out/debug-asan-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-asan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_TRACK=ASAN + cmake --build out/debug-asan --parallel 8 --config Debug + ctest --test-dir out/debug-asan --verbose --timeout 240 -C Debug - name: Debug, UBSAN - if: matrix.tests == 'extra' && runner.os == 'Linux' && matrix.c_or_cxx == 'cxx' + if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-ubsan-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_UBSAN=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} + CC=clang CXX=clang++ cmake . -B out/debug-ubsan-cxx -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_UBSAN=ON cmake --build out/debug-ubsan-cxx --parallel 8 --config Debug ctest --test-dir out/debug-ubsan-cxx --verbose --timeout 240 -C Debug - name: Debug, TSAN if: matrix.tests == 'extra' && runner.os == 'Linux' run: | - CC=clang CXX=clang++ cmake . -B out/debug-tsan-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_TSAN=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-tsan-${{ matrix.c_or_cxx }} --parallel 8 --config Debug - ctest --test-dir out/debug-tsan-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + CC=clang CXX=clang++ cmake . -B out/debug-tsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_TSAN=ON + cmake --build out/debug-tsan --parallel 8 --config Debug + ctest --test-dir out/debug-tsan --verbose --timeout 240 -C Debug - name: Debug, Guarded if: matrix.tests == 'extra' && runner.os != 'macOS' run: | - cmake . -B out/debug-guarded-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-guarded-${{ matrix.c_or_cxx }} --parallel 8 --config Debug - ctest --test-dir out/debug-guarded-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=ON + cmake --build out/debug-guarded --parallel 8 --config Debug + ctest --test-dir out/debug-guarded --verbose --timeout 240 -C Debug env: MIMALLOC_GUARDED_SAMPLE_RATE: 10 - name: Release, Guarded if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') run: | - cmake . -B out/release-guarded-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-guarded-${{ matrix.c_or_cxx }} --parallel 8 --config Release - ctest --test-dir out/release-guarded-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=ON + cmake --build out/release-guarded --parallel 8 --config Release + ctest --test-dir out/release-guarded --verbose --timeout 240 -C Release env: MIMALLOC_GUARDED_SAMPLE_RATE: 10 - name: Debug, Secure Full if: matrix.tests == 'extra' && runner.os != 'macOS' run: | - cmake . -B out/debug-secure-full-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_SECURE=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-secure-full-${{ matrix.c_or_cxx }} --parallel 8 --config Debug - ctest --test-dir out/debug-secure-full-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-secure-full -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_SECURE=FULL + cmake --build out/debug-secure-full --parallel 8 --config Debug + ctest --test-dir out/debug-secure-full --verbose --timeout 240 -C Debug # MSYS2 mingw64 - name: mingw-ucrt64 x64, Setup @@ -302,17 +290,17 @@ jobs: shell: msys2 {0} run: | env - cmake . -B out/debug-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_GCC }} - cmake --build out/debug-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL + cmake --build out/debug-mingw-ucrt64-x64 --parallel 4 --config Debug + ctest --test-dir out/debug-mingw-ucrt64-x64 --verbose --timeout 240 -C Debug - name: mingw-ucrt64 x64, Release if: matrix.tests == 'mingw-ucrt64' shell: msys2 {0} run: | - cmake . -B out/release-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_GCC }} - cmake --build out/release-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/release-mingw-ucrt64-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-mingw-ucrt64-x64 --parallel 4 --config Release + ctest --test-dir out/release-mingw-ucrt64-x64 --verbose --timeout 240 -C Release # VM: FreeBSD (arm64 and x64) @@ -332,27 +320,27 @@ jobs: shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/debug-arm64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-arm64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-arm64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL + cmake --build out/debug-arm64 --parallel 4 --config Debug + ctest --test-dir out/debug-arm64 --verbose --timeout 240 -C Debug - name: FreeBSD arm64, Release if: matrix.tests == 'freebsd-arm64' shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/release-arm64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-arm64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/release-arm64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-arm64 --parallel 4 --config Release + ctest --test-dir out/release-arm64 --verbose --timeout 240 -C Release - name: FreeBSD arm64, Secure if: matrix.tests == 'freebsd-arm64' shell: freebsd-arm64 {0} run: | export MI_TEST_LIGHT=1 - cmake . -B out/secure-arm64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/secure-arm64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/secure-arm64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/secure-arm64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-arm64 --parallel 4 --config Release + ctest --test-dir out/secure-arm64 --verbose --timeout 240 -C Release - name: FreeBSD x64, Setup @@ -370,25 +358,25 @@ jobs: if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/debug-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL + cmake --build out/debug-x64 --parallel 4 --config Debug + ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug - name: FreeBSD x64, Release if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/release-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/release-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-x64 --parallel 4 --config Release + ctest --test-dir out/release-x64 --verbose --timeout 240 -C Release - name: FreeBSD x64, Secure if: matrix.tests == 'freebsd-x64' shell: freebsd-x64 {0} run: | - cmake . -B out/secure-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/secure-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/secure-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/secure-x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-x64 --parallel 4 --config Release + ctest --test-dir out/secure-x64 --verbose --timeout 240 -C Release # VM: Alpine Linux with MUSL libc (riscv64 and x64) @@ -407,16 +395,16 @@ jobs: if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} run: | - cmake . -B out/release-riscv64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-riscv64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/release-riscv64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON + cmake --build out/release-riscv64 --parallel 4 --config Release + ctest --test-dir out/release-riscv64 --verbose --timeout 240 -C Release - name: Alpine risc64, Asm if: matrix.tests == 'alpine-riscv64' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: alpine-riscv64-asm - path: out/release-riscv64-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s + path: out/release-riscv64/CMakeFiles/mimalloc-static.dir/src/*.s - name: Alpine riscv64, Debug if: matrix.tests == 'alpine-riscv64' @@ -425,17 +413,17 @@ jobs: MI_TEST_LIGHT: 1 run: | uname -m - cmake . -B out/debug-riscv64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-riscv64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-riscv64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL + cmake --build out/debug-riscv64 --parallel 4 --config Debug + ctest --test-dir out/debug-riscv64 --verbose --timeout 240 -C Debug - name: Alpine riscv64, Secure if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} run: | - cmake . -B out/secure-riscv64-${{ matrix.c_or_cxx }} -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/secure-riscv64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/secure-riscv64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/secure-riscv64 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-riscv64 --parallel 4 --config Release + ctest --test-dir out/secure-riscv64 --verbose --timeout 240 -C Release - name: Alpine x64, Setup @@ -452,25 +440,25 @@ jobs: if: matrix.tests == 'alpine-x64' shell: alpine-x64.sh {0} run: | - cmake . -B out/debug-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL + cmake --build out/debug-x64 --parallel 4 --config Debug + ctest --test-dir out/debug-x64 --verbose --timeout 240 -C Debug - name: Alpine x64, Release if: matrix.tests == 'alpine-x64' shell: alpine-x64.sh {0} run: | - cmake . -B out/release-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/release-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-x64 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-x64 --parallel 4 --config Release + ctest --test-dir out/release-x64 --verbose --timeout 240 -C Release - name: Alpine x64, Secure if: matrix.tests == 'alpine-x64' shell: alpine-x64.sh {0} run: | - cmake . -B out/secure-x64-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/secure-x64-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/secure-x64-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/secure-x64 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-x64 --parallel 4 --config Release + ctest --test-dir out/secure-x64 --verbose --timeout 240 -C Release - name: Alpine arm32, Setup if: matrix.tests == 'alpine-arm32' @@ -489,25 +477,25 @@ jobs: MI_TEST_LIGHT: 1 run: | uname -m - cmake . -B out/debug-arm32-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/debug-arm32-${{ matrix.c_or_cxx }} --parallel 4 --config Debug - ctest --test-dir out/debug-arm32-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug + cmake . -B out/debug-arm32 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL + cmake --build out/debug-arm32 --parallel 4 --config Debug + ctest --test-dir out/debug-arm32 --verbose --timeout 240 -C Debug - name: Alpine arm32, Release if: matrix.tests == 'alpine-arm32' shell: alpine-arm32.sh {0} run: | - cmake . -B out/release-arm32-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/release-arm32-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/release-arm32-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/release-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake --build out/release-arm32 --parallel 4 --config Release + ctest --test-dir out/release-arm32 --verbose --timeout 240 -C Release - name: Alpine arm32, Secure if: matrix.tests == 'alpine-arm32' shell: alpine-arm32.sh {0} run: | - cmake . -B out/secure-arm32-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_DEFAULT }} - cmake --build out/secure-arm32-${{ matrix.c_or_cxx }} --parallel 4 --config Release - ctest --test-dir out/secure-arm32-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release + cmake . -B out/secure-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + cmake --build out/secure-arm32 --parallel 4 --config Release + ctest --test-dir out/secure-arm32 --verbose --timeout 240 -C Release # - name: Android, Setup # if: matrix.tests == 'android-x64' From 8006b346a10ddb2d344ab9d0524b651015a5d4dc Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 15:52:42 -0700 Subject: [PATCH 114/280] reduce c/c++ test variants a bit --- .github/workflows/test.yaml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 0391ea94d..1412627b9 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,8 +27,7 @@ jobs: - os: macos-14 tests: extra - c_or_cxx: cxx - tests: extra - os: ${{ matrix.os != 'windows-latest' && matrix.os }} # do not exclude on windows + tests: extra include: - os: ubuntu-latest tests: freebsd-x64 @@ -106,56 +105,56 @@ jobs: # Extra Windows: x86, clang-cl, and clang - name: Debug, Win32 - if: matrix.tests == 'extra' && runner.os == 'Windows' + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | cmake . -B out/debug-x86-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -A Win32 ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/debug-x86-${{ matrix.c_or_cxx }} --parallel 4 --config Debug ctest --test-dir out/debug-x86-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Win32 - if: matrix.tests == 'extra' && runner.os == 'Windows' + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | cmake . -B out/release-x86-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -A Win32 ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/release-x86-${{ matrix.c_or_cxx }} --parallel 4 --config Release ctest --test-dir out/release-x86-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Win32, clang-cl - if: matrix.tests == 'extra' && runner.os == 'Windows' + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | cmake . -B out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Debug ctest --test-dir out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Win32, clang-cl - if: matrix.tests == 'extra' && runner.os == 'Windows' + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | cmake . -B out/release-x86-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/release-x86-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Release ctest --test-dir out/release-x86-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Windows, clang-cl - if: matrix.tests == 'extra' && runner.os == 'Windows' + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | cmake . -B out/debug-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/debug-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Debug ctest --test-dir out/debug-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Windows, clang-cl - if: matrix.tests == 'extra' && runner.os == 'Windows' + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | cmake . -B out/release-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/release-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Release ctest --test-dir out/release-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Windows, clang - if: matrix.tests == 'extra' && runner.os == 'Windows' + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | cmake . -B out/debug-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/debug-clang-${{ matrix.c_or_cxx }} --parallel 4 --config Debug ctest --test-dir out/debug-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Windows, clang - if: matrix.tests == 'extra' && runner.os == 'Windows' + if: matrix.tests == 'basic' && runner.os == 'Windows' run: | cmake . -B out/release-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/release-clang-${{ matrix.c_or_cxx }} --parallel 4 --config Release From fe5a54c4c6e319dc75f9641f85ad35d6ba6da2c1 Mon Sep 17 00:00:00 2001 From: daanx Date: Fri, 28 Aug 2026 16:52:57 -0700 Subject: [PATCH 115/280] faster free_small --- CMakeLists.txt | 6 +++--- include/mimalloc/internal.h | 12 ++++++++---- include/mimalloc/types.h | 10 +++------- src/arena.c | 26 ++++++++++++++------------ src/free.c | 2 +- test/test-api.c | 16 ++++++++++++++++ 6 files changed, 45 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 98926fb18..9c6440635 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -444,11 +444,11 @@ if(MI_FREE_USE_PAGEMAP) endif() # deprecated -if(MI_OPT_FREE_SMALL) +if(MI_OPT_FREE_SMALL) if(MI_GUARDED OR MI_SECURE) message(STATUS "Cannot enable MI_OPT_FREE_SMALL in secure or guarded mode (MI_OPT_FREE_SMALL=OFF)") - elseif(MI_OPT_FREE) - message(STATUS "Cannot enable MI_OPT_FREE_SMALL with MI_OPT_FREE (MI_OPT_FREE_SMALL=OFF)") + #elseif(MI_OPT_FREE) + # message(STATUS "Cannot enable MI_OPT_FREE_SMALL with MI_OPT_FREE (MI_OPT_FREE_SMALL=OFF)") else() message(STATUS "Use aligned `mi_free_small` (MI_OPT_FREE_SMALL=ON)") list(APPEND mi_defines MI_OPT_FREE_SMALL=1) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 297ca4cc3..4d4fcb6f0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -860,10 +860,14 @@ static inline size_t mi_page_usable_block_size(const mi_page_t* page) { } static inline bool mi_page_meta_is_separated(const mi_page_t* page) { - #if MI_PAGE_META_IS_ALIGNED - MI_UNUSED_RELEASE(page); - mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); - return true; + #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED + return (page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + #else + MI_UNUSED_RELEASE(page); + mi_assert_internal(page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); + return true; + #endif #elif MI_PAGE_META_IS_SEPARATED // usually separated but can still be in front for direct OS allocations (due to size or alignment) or due to MI_PAGE_META_SMALL_IS_ALIGNED return (page->memid.memkind == MI_MEM_ARENA && page != _mi_align_down_ptr(mi_page_start(page), MI_ARENA_SLICE_ALIGN)); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 701f9a4be..d006f0543 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -148,22 +148,18 @@ terms of the MIT license. A copy of the license can be found in the file #if MI_PAGE_META_IS_SEPARATED #define MI_PAGE_META_IS_ALIGNED 1 #define MI_PAGE_META_ALIGNED_CHUNKS MI_INTPTR_SIZE -#ifdef MI_PAGE_META_SMALL_IS_ALIGNED -#undef MI_PAGE_META_SMALL_IS_ALIGNED -#endif #else #warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" #endif #endif -// Deprecated (in favor of MI_PAGE_META_IS_ALIGNED): // We can choose to only put page info of small pages at the start of the page area. // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). -#if !MI_PAGE_META_IS_ALIGNED -#if MI_OPT_FREE_SMALL && !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#if (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED #define MI_PAGE_META_SMALL_IS_ALIGNED 1 -#elif !defined(MI_PAGE_META_SMALL_IS_ALIGNED) +#else #define MI_PAGE_META_SMALL_IS_ALIGNED 0 #endif #endif diff --git a/src/arena.c b/src/arena.c index b32a52216..fec4a0410 100644 --- a/src/arena.c +++ b/src/arena.c @@ -871,7 +871,6 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ return start; } -#if !MI_PAGE_META_IS_ALIGNED // Only used for non-separate pages static size_t mi_page_block_start(size_t block_size, bool os_align) { @@ -903,7 +902,6 @@ static size_t mi_page_block_start(size_t block_size, bool os_align) } return _mi_align_up(offset,MI_MAX_ALIGN_SIZE); } -#endif // Free a page without modifying page_bin stats static void mi_arenas_page_free_prim(mi_page_t* page); @@ -980,11 +978,16 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou #if MI_PAGE_META_SMALL_IS_ALIGNED // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) - if (block_size > MI_SMALL_SIZE_MAX) + if (block_size <= MI_SMALL_SIZE_MAX) { + // put page info in front of the slice + page = (mi_page_t*)slice_start; + block_start = mi_page_block_start(block_size, os_align); + } + else #endif { - page = page_meta; page_meta_is_separate = true; + page = page_meta; block_start = 0; #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ @@ -1081,14 +1084,13 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou // mi_assert_internal(mi_page_theap(page) == _mi_heap_theap_peek(page->heap)) #if MI_PAGE_META_IS_ALIGNED - mi_atomic_store_ptr_release(mi_page_t,&page->self,page); - if (page_meta_is_separate) { - if (slice_count > 1) { - // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice - for(size_t i = 1; i < max_page_meta_count; i++) { - mi_assert_internal(page[i].block_size == 0); - mi_atomic_store_ptr_release(mi_page_t,&page[i].self,page); - } + mi_assert_internal(page_meta!=NULL); + mi_atomic_store_ptr_release(mi_page_t,&page_meta->self,page); + if (slice_count > 1) { + // at least two for large singleton blocks as guard pages can have a large offset beyond a single slice + for(size_t i = 1; i < max_page_meta_count; i++) { + mi_assert_internal(page_meta[i].block_size == 0); + mi_atomic_store_ptr_release(mi_page_t,&page_meta[i].self,page); } } #if MI_DEBUG>1 diff --git a/src/free.c b/src/free.c index cf21b6ad1..3b6388305 100644 --- a/src/free.c +++ b/src/free.c @@ -267,7 +267,7 @@ void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { void mi_free_small(void* p) mi_attr_noexcept { #if MI_PAGE_META_SMALL_IS_ALIGNED - // deprecated: We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. + // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. // If we keep page info in front of the page area for small objects, we can find the info // just by aligning down the pointer instead of looking it up in the page map. #if MI_GUARDED diff --git a/test/test-api.c b/test/test-api.c index fb5caf6ff..0378675ba 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -420,7 +420,23 @@ int main(void) { } } + CHECK_BODY("free_small3") { + for(size_t n = 1; n < MI_SMALL_SIZE_MAX/sizeof(int); n *=2) { + const size_t size = n*sizeof(int); + int* p = (int*)mi_zalloc_small(size); + p[n-1] = 42; + mi_free_small(p); + } + } + CHECK_BODY("free_small4") { + for(size_t n = 1; n < MI_SMALL_SIZE_MAX/sizeof(int); n *=2) { + const size_t size = n*sizeof(int); + int* p = (int*)mi_zalloc_small(size); + p[n-1] = 42; + mi_free(p); + } + } // --------------------------------------------------- // Returned block sizes // --------------------------------------------------- From eab1ebe6f54cfb2c011d1fad6d232cefe956c0b3 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 06:38:45 -0700 Subject: [PATCH 116/280] improve free_small --- .github/workflows/test.yaml | 27 ++++++++++++- CMakeLists.txt | 2 - include/mimalloc/types.h | 4 +- src/arena.c | 19 +++++---- src/free.c | 80 ++++++++++++++++--------------------- src/options.c | 8 +++- 6 files changed, 78 insertions(+), 62 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 1412627b9..145b9f8c0 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -27,8 +27,10 @@ jobs: - os: macos-14 tests: extra - c_or_cxx: cxx - tests: extra + tests: extra include: + - os: ubuntu-latest + tests: tsan - os: ubuntu-latest tests: freebsd-x64 - os: ubuntu-latest @@ -241,7 +243,7 @@ jobs: ctest --test-dir out/debug-ubsan-cxx --verbose --timeout 240 -C Debug - name: Debug, TSAN - if: matrix.tests == 'extra' && runner.os == 'Linux' + if: matrix.tests == 'tsan' && runner.os == 'Linux' run: | CC=clang CXX=clang++ cmake . -B out/debug-tsan -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_DEBUG_TSAN=ON cmake --build out/debug-tsan --parallel 8 --config Debug @@ -265,6 +267,20 @@ jobs: env: MIMALLOC_GUARDED_SAMPLE_RATE: 10 + - name: Debug, Use pagemap + if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') + run: | + cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_FREE_USE_PAGEMAP=ON + cmake --build out/debug-guarded --parallel 8 --config Debug + ctest --test-dir out/debug-guarded --verbose --timeout 240 -C Debug + + - name: Release, Use pagemap + if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') + run: | + cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_FREE_USE_PAGEMAP=ON + cmake --build out/release-guarded --parallel 8 --config Release + ctest --test-dir out/release-guarded --verbose --timeout 240 -C Release + - name: Debug, Secure Full if: matrix.tests == 'extra' && runner.os != 'macOS' run: | @@ -272,6 +288,13 @@ jobs: cmake --build out/debug-secure-full --parallel 8 --config Debug ctest --test-dir out/debug-secure-full --verbose --timeout 240 -C Debug + - name: Release, Secure Full + if: matrix.tests == 'extra' && runner.os != 'macOS' + run: | + cmake . -B out/release-secure-full -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SECURE=FULL + cmake --build out/release-secure-full --parallel 8 --config Release + ctest --test-dir out/release-secure-full --verbose --timeout 240 -C Release + # MSYS2 mingw64 - name: mingw-ucrt64 x64, Setup if: matrix.tests == 'mingw-ucrt64' diff --git a/CMakeLists.txt b/CMakeLists.txt index 9c6440635..cbbdef155 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -447,8 +447,6 @@ endif() if(MI_OPT_FREE_SMALL) if(MI_GUARDED OR MI_SECURE) message(STATUS "Cannot enable MI_OPT_FREE_SMALL in secure or guarded mode (MI_OPT_FREE_SMALL=OFF)") - #elseif(MI_OPT_FREE) - # message(STATUS "Cannot enable MI_OPT_FREE_SMALL with MI_OPT_FREE (MI_OPT_FREE_SMALL=OFF)") else() message(STATUS "Use aligned `mi_free_small` (MI_OPT_FREE_SMALL=ON)") list(APPEND mi_defines MI_OPT_FREE_SMALL=1) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index d006f0543..b91c89628 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -157,7 +157,9 @@ terms of the MIT license. A copy of the license can be found in the file // This can be used to have a slightly faster `mi_free_small` function for specialized // cases (like language runtime systems). #if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) -#if (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED +#if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0 +#define MI_PAGE_META_SMALL_IS_ALIGNED 0 +#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE // && !MI_GUARDED #define MI_PAGE_META_SMALL_IS_ALIGNED 1 #else #define MI_PAGE_META_SMALL_IS_ALIGNED 0 diff --git a/src/arena.c b/src/arena.c index fec4a0410..2bda823bc 100644 --- a/src/arena.c +++ b/src/arena.c @@ -872,7 +872,7 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ } // Only used for non-separate pages -static size_t mi_page_block_start(size_t block_size, bool os_align) +mi_decl_maybe_unused static size_t mi_page_block_start(size_t block_size, bool os_align) { size_t offset; #if MI_GUARDED @@ -891,6 +891,7 @@ static size_t mi_page_block_start(size_t block_size, bool os_align) else if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { // naturally align power-of-2 blocks up to MI_PAGE_MAX_START_BLOCK_ALIGN2 size (4KiB) offset = _mi_align_up(mi_page_info_size(), block_size); + if (block_size < 64) { offset += 3*block_size; } } else if (block_size != 0 && (block_size % MI_PAGE_OSPAGE_BLOCK_ALIGN2) == 0) { // also align large pages that are a multiple of MI_PAGE_OSPAGE_BLOCK_ALIGN2 (4KiB) @@ -903,6 +904,7 @@ static size_t mi_page_block_start(size_t block_size, bool os_align) return _mi_align_up(offset,MI_MAX_ALIGN_SIZE); } + // Free a page without modifying page_bin stats static void mi_arenas_page_free_prim(mi_page_t* page); @@ -995,14 +997,8 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { - if (block_size < 64) { - block_start += sizeof(mi_page_t) + 3*block_size; - } - else { - block_start += sizeof(mi_page_t); - } - block_start = _mi_align_up(block_start, MI_MAX_ALIGN_SIZE); - block_start = _mi_align_up(block_start, block_size); // to maintain natural alignment + block_start = _mi_align_up(mi_page_info_size(), block_size); // to maintain natural alignment + if (block_size < 64) { block_start += 3*block_size; } } mi_assert_internal(page->block_size == 0); _mi_memzero_aligned(page, sizeof(*page)); @@ -1019,7 +1015,10 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou block_start = mi_page_block_start(block_size, os_align); #endif } - mi_assert_internal(block_start % MI_MAX_ALIGN_SIZE == 0); + mi_assert_internal(block_size < MI_MAX_ALIGN_SIZE || block_start % MI_MAX_ALIGN_SIZE == 0); + if (_mi_is_power_of_two(block_size) && block_size <= MI_PAGE_MAX_START_BLOCK_ALIGN2) { + mi_assert_internal(block_start % block_size == 0); // natural alignment (see also alloc_aligned.c) + } // commit first block? size_t commit_size = 0; diff --git a/src/free.c b/src/free.c index 3b6388305..aaa9e2f35 100644 --- a/src/free.c +++ b/src/free.c @@ -180,35 +180,42 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, #endif mi_page_t* page; + #if MI_PAGE_META_SMALL_IS_ALIGNED + if (free_small) { page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); } + else + #endif #if MI_PAGE_META_IS_ALIGNED - page = _mi_aligned_ptr_page0(p); - if mi_unlikely(page==NULL) return false; // p==NULL => page==NULL - #if MI_DEBUG - mi_page_t* const cpage = _mi_checked_ptr_page(p); - if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + { page = _mi_aligned_ptr_page0(p); } + #else + { page = _mi_ptr_page(p); } + #endif + + if mi_unlikely(page==NULL) { + #if MI_DEBUG + if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } #endif + return false; + } + #if MI_DEBUG + mi_page_t* const cpage = _mi_checked_ptr_page(p); + if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif - #if MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE + #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED + if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + else + #elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE // for mi_free_small we can avoid a load-acquire - if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } - else { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); } - #else - MI_UNUSED(free_small); - page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); + if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } + else #endif - - mi_assert_internal(page!=NULL); - mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); - #else - MI_UNUSED(free_small); - page = _mi_ptr_page(p); - if mi_unlikely(page==NULL) { - #if MI_DEBUG - if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } - #endif - return false; - } + { page = mi_atomic_load_ptr_acquire(mi_page_t,&page->self); } #endif + + mi_assert_internal(page!=NULL); + mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); + if (free_small) { mi_assert_internal(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX) /* free small should only be called on small pages */); } *ppage = page; return true; } @@ -266,29 +273,10 @@ void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { } void mi_free_small(void* p) mi_attr_noexcept { - #if MI_PAGE_META_SMALL_IS_ALIGNED - // We can only call `mi_free_small` for pointers allocated with `mi_(heap_)malloc_small`. - // If we keep page info in front of the page area for small objects, we can find the info - // just by aligning down the pointer instead of looking it up in the page map. - #if MI_GUARDED - #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED) ignored as MI_GUARDED is defined" - mi_free(p); - #elif MI_ARENA_SLICE_ALIGN < MI_SMALL_PAGE_SIZE - #warning "MI_OPT_FREE_SMALL (MI_PAGE_META_SMALL_IS_ALIGNED) ignored as the MI_ARENA_SLICE_ALIGN is less than the small page size" - mi_free(p); - #else - mi_page_t* const page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); - mi_assert(page == mi_validate_ptr_page(p,"mi_free_small")); - mi_assert((void*)page == _mi_align_down_ptr(mi_page_start(page),MI_SMALL_PAGE_SIZE)); - mi_assert(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX)); // note: not `MI_SMALL_MAX_OBJ_SIZE` as we need to match `mi_(heap_)malloc_small` - mi_free_nonnull(p, page, NULL, true); - #endif - #else - mi_page_t* page; - if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free_small",true /* is_small? */,&page)) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); - } - #endif + mi_page_t* page; + if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free_small",true /* is_small? */,&page)) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + } } // void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { diff --git a/src/options.c b/src/options.c index a35661ece..327f36fc4 100644 --- a/src/options.c +++ b/src/options.c @@ -249,10 +249,16 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ #if MI_TSAN _mi_fprintf(out, arg, "thread santizer enabled\n"); #endif - #if MI_PAGE_META_IS_ALIGNED + #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED + _mi_fprintf(out, arg, "free: (small) aligned, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_PAGE_META_IS_ALIGNED _mi_fprintf(out, arg, "free: aligned, page size: %zu\n", sizeof(mi_page_t)); + #elif MI_PAGE_META_SMALL_IS_ALIGNED + _mi_fprintf(out, arg, "free: small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t)); #elif MI_FREE_IS_CHECKED _mi_fprintf(out, arg, "free: checked, page size: %zu\n", sizeof(mi_page_t)); + #else + _mi_fprintf(out, arg, "free: pagemap, page size: %zu\n", sizeof(mi_page_t)); #endif #if MI_ENCODE_FREELIST _mi_fprintf(out, arg, "free lists: encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); From 06fb5b0497e4327c04d8f7c3fc89e401ed916439 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 06:42:51 -0700 Subject: [PATCH 117/280] comments --- .github/workflows/test.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 145b9f8c0..ee081ad21 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -310,7 +310,7 @@ jobs: - name: mingw-ucrt64 x64, Debug if: matrix.tests == 'mingw-ucrt64' shell: msys2 {0} - run: | + run: | env cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-mingw-ucrt64-x64 --parallel 4 --config Debug @@ -401,7 +401,7 @@ jobs: ctest --test-dir out/secure-x64 --verbose --timeout 240 -C Release - # VM: Alpine Linux with MUSL libc (riscv64 and x64) + # VM: Alpine Linux with MUSL libc (riscv64, x64, and arm32) - name: Alpine riscv64, Setup if: matrix.tests == 'alpine-riscv64' uses: jirutka/setup-alpine@ae3b3ddba35054804fc4a3507b519fa7e8152050 # v1.4.1 From 2deba9e9d5884438c213e3107e61045a3440ebaf Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 06:52:00 -0700 Subject: [PATCH 118/280] add default to MI_USE_CXX option --- CMakeLists.txt | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 319b7e99a..e8b7164d8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -11,7 +11,10 @@ option(MI_OVERRIDE "Override the standard malloc interface (i.e. define option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) option(MI_GUARDED "Build with guard pages behind certain object allocations (enabled by default in a debug build)" OFF) -option(MI_USE_CXX "Use the C++ compiler to compile the library (instead of the C compiler)" OFF) + +set(MI_USE_CXX "DEFAULT" CACHE STRING "Use the C++ compiler to compile the library (instead of the C compiler)") +set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") + option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for arm64: '-march=armv8.1-a' (2016))" OFF) option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) @@ -185,9 +188,13 @@ if(CMAKE_C_COMPILER_ID MATCHES "Intel") list(APPEND mi_cflags -Wall) endif() -# force C++ compilation with msvc or clang-cl to use modern C++ atomics -if((CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel" OR MI_CLANG_CL) AND NOT MI_NO_USE_CXX) - set(MI_USE_CXX "ON") +if(MI_USE_CXX STREQUAL "DEFAULT") + if(CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel" OR MI_CLANG_CL) + # force C++ compilation with msvc or clang-cl to use modern C++ atomics + set(MI_USE_CXX "ON") + else() + set(MI_USE_CXX "OFF") + endif() endif() if(MI_OVERRIDE) @@ -391,6 +398,8 @@ if(MI_USE_CXX) endif() if(CMAKE_CXX_COMPILER_ID MATCHES "Intel" AND NOT CMAKE_CXX_COMPILER_ID MATCHES "IntelLLVM") list(APPEND mi_cflags -Kc++) + elseif(MSVC AND MSVC_VERSION GREATER_EQUAL 1914) + list(APPEND mi_cflags /Zc:__cplusplus) endif() endif() @@ -494,9 +503,6 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM endif() if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) - if(MI_USE_CXX) - list(APPEND mi_cflags /Zc:__cplusplus) - endif() if(MI_OPT_ARCH AND NOT MI_CLANG_CL) if(MI_ARCH STREQUAL "arm64") set(MI_OPT_ARCH_FLAGS "/arch:armv8.1") # fast atomics (LSE) From 9c2a0d29b53b5775aff46aeeb575a32ed43be236 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 07:15:44 -0700 Subject: [PATCH 119/280] fix guarded build assertions --- include/mimalloc/types.h | 2 +- src/free.c | 30 ++++++++++++++++++++---------- 2 files changed, 21 insertions(+), 11 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index b91c89628..308c17bd3 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -159,7 +159,7 @@ terms of the MIT license. A copy of the license can be found in the file #if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) #if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0 #define MI_PAGE_META_SMALL_IS_ALIGNED 0 -#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE // && !MI_GUARDED +#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // cannot be guarded as that may allocate large blocks for small allocations #define MI_PAGE_META_SMALL_IS_ALIGNED 1 #else #define MI_PAGE_META_SMALL_IS_ALIGNED 0 diff --git a/src/free.c b/src/free.c index aaa9e2f35..b7550e614 100644 --- a/src/free.c +++ b/src/free.c @@ -171,7 +171,7 @@ void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) // Does further checks in debug mode to see if this was a valid pointer. static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, bool free_small, mi_page_t** ppage) { - MI_UNUSED_RELEASE(msg); + MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small); #if MI_DEBUG if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); @@ -205,8 +205,8 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, #if MI_PAGE_META_SMALL_IS_ALIGNED if (free_small) { mi_assert_internal(NULL == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } else - #elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE - // for mi_free_small we can avoid a load-acquire + #elif MI_SMALL_PAGE_SIZE == MI_ARENA_SLICE_SIZE && !MI_GUARDED + // for mi_free_small we can avoid a load-acquire (but not when guarded as that may still allocate large blocks) if (free_small) { mi_assert_internal(page == mi_atomic_load_ptr_acquire(mi_page_t,&page->self)); } else #endif @@ -215,7 +215,9 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, mi_assert_internal(page!=NULL); mi_assert(cpage==page /* page_map lookup should be the same as aligned lookup */ ); + #if !MI_GUARDED if (free_small) { mi_assert_internal(page->block_size <= mi_good_size(MI_SMALL_SIZE_MAX) /* free small should only be called on small pages */); } + #endif *ppage = page; return true; } @@ -303,14 +305,22 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { mi_assert(p!=NULL); const size_t usable = _mi_page_usable_size(page,p); if mi_unlikely(size > usable) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); - mi_free(p); - return; + const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const bool is_guarded = mi_block_ptr_is_guarded(block,p); + if (!is_guarded) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the size %zu is greater than the usable size %zu\n", p, size, usable); + mi_free(p); + return; + } } - else if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); - mi_free(p); - return; + if mi_unlikely(size <= MI_SMALL_SIZE_MAX && mi_page_block_size(page) > mi_good_size(MI_SMALL_SIZE_MAX)) { + const mi_block_t* block = _mi_page_ptr_unalign(page, p); + const bool is_guarded = mi_block_ptr_is_guarded(block,p); + if (!is_guarded) { + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); + mi_free(p); + return; + } } #endif #if MI_PAGE_META_SMALL_IS_ALIGNED From 52c0fed0e2a4e3b8099681d74f2d7827d2fbfef1 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 07:17:05 -0700 Subject: [PATCH 120/280] fix test.yml --- .github/workflows/test.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index ee081ad21..14fba65df 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -310,7 +310,7 @@ jobs: - name: mingw-ucrt64 x64, Debug if: matrix.tests == 'mingw-ucrt64' shell: msys2 {0} - run: | + run: | env cmake . -B out/debug-mingw-ucrt64-x64 -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL cmake --build out/debug-mingw-ucrt64-x64 --parallel 4 --config Debug From 081728d8566a2e5ae159eebe969ea1efe8ffd6ce Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 07:30:17 -0700 Subject: [PATCH 121/280] simplify test.yml --- .github/workflows/test.yaml | 37 +++++++++++++------------------------ 1 file changed, 13 insertions(+), 24 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 14fba65df..e18252e2f 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -41,6 +41,8 @@ jobs: tests: alpine-arm32 # qemu - os: windows-latest tests: mingw-ucrt64 + - os: windows-latest + tests: clang # - os: ubuntu-latest # tests: android-x64 # - os: ubuntu-latest # slow, so disable by default @@ -51,29 +53,16 @@ jobs: - name: Checkout repository uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 - # Choose MI_USE_CXX/MI_NO_USE_CXX values - - name: Compute CMake args + # Choose MI_USE_CXX setting + - name: Set default cmake options id: cmake_args shell: bash run: | - CMAKE_CXX_OPT_C="-DMI_USE_CXX=OFF -DMI_NO_USE_CXX=ON" - CMAKE_CXX_OPT_CXX="-DMI_USE_CXX=ON -DMI_NO_USE_CXX=OFF" - # On MSVC, C++ mode is default, so there C mode explicitly disables C++. - # Everywhere else, C mode is default, so explicitly enable C if [ "${{ matrix.c_or_cxx }}" = "cxx" ]; then - CMAKE_CXX_OPT_GCC=$CMAKE_CXX_OPT_CXX - CMAKE_CXX_OPT_MSVC= + echo "CMAKE_CXX_OPT_DEFAULT=-DMI_USE_CXX=ON" >> $GITHUB_OUTPUT else # Default to C mode - CMAKE_CXX_OPT_GCC= - CMAKE_CXX_OPT_MSVC=$CMAKE_CXX_OPT_C - fi - echo "CMAKE_CXX_OPT_GCC=$CMAKE_CXX_OPT_GCC" >> $GITHUB_OUTPUT - echo "CMAKE_CXX_OPT_MSVC=$CMAKE_CXX_OPT_MSVC" >> $GITHUB_OUTPUT - if [ "${{ runner.os }}" = "Windows" ]; then - echo "CMAKE_CXX_OPT_DEFAULT=$CMAKE_CXX_OPT_MSVC" >> $GITHUB_OUTPUT - else - echo "CMAKE_CXX_OPT_DEFAULT=$CMAKE_CXX_OPT_GCC" >> $GITHUB_OUTPUT - fi + echo "CMAKE_CXX_OPT_DEFAULT=-DMI_USE_CXX=OFF" >> $GITHUB_OUTPUT + fi # Basic - name: Release @@ -121,42 +110,42 @@ jobs: ctest --test-dir out/release-x86-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Win32, clang-cl - if: matrix.tests == 'basic' && runner.os == 'Windows' + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | cmake . -B out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -A Win32 -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Debug ctest --test-dir out/debug-x86-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Win32, clang-cl - if: matrix.tests == 'basic' && runner.os == 'Windows' + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | cmake . -B out/release-x86-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -A Win32 -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/release-x86-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Release ctest --test-dir out/release-x86-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Windows, clang-cl - if: matrix.tests == 'basic' && runner.os == 'Windows' + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | cmake . -B out/debug-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/debug-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Debug ctest --test-dir out/debug-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Windows, clang-cl - if: matrix.tests == 'basic' && runner.os == 'Windows' + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | cmake . -B out/release-clang-cl-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -T ClangCl ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/release-clang-cl-${{ matrix.c_or_cxx }} --parallel 4 --config Release ctest --test-dir out/release-clang-cl-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Debug, Windows, clang - if: matrix.tests == 'basic' && runner.os == 'Windows' + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | cmake . -B out/debug-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Debug -DCMAKE_C_COMPILER=clang ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/debug-clang-${{ matrix.c_or_cxx }} --parallel 4 --config Debug ctest --test-dir out/debug-clang-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Debug - name: Release, Windows, clang - if: matrix.tests == 'basic' && runner.os == 'Windows' + if: matrix.tests == 'clang' && runner.os == 'Windows' run: | cmake . -B out/release-clang-${{ matrix.c_or_cxx }} -DCMAKE_BUILD_TYPE=Release -DCMAKE_C_COMPILER=clang ${{ steps.cmake_args.outputs.CMAKE_CXX_OPT_MSVC }} cmake --build out/release-clang-${{ matrix.c_or_cxx }} --parallel 4 --config Release From 5787e6b701135ac4b9edc6584bcf9a0e5550d595 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 07:45:08 -0700 Subject: [PATCH 122/280] allow alpine tests to fail in test.yml --- .github/workflows/test.yaml | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e18252e2f..18f4867a4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -17,6 +17,8 @@ jobs: runs-on: ${{ matrix.os }} env: MIMALLOC_SHOW_ERRORS: 1 + continue-on-error: + ${{ startsWith(matrix.tests,'alpine') || false }} strategy: fail-fast: false matrix: @@ -41,8 +43,10 @@ jobs: tests: alpine-arm32 # qemu - os: windows-latest tests: mingw-ucrt64 - - os: windows-latest + - os: windows-latest # clang-cl and clang tests: clang + - os: windows11-arm # clang-cl and clang + tests: clang # - os: ubuntu-latest # tests: android-x64 # - os: ubuntu-latest # slow, so disable by default From 6070a8d78a81c323ae50a1fc56ab01fff6fe0393 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 09:47:53 -0700 Subject: [PATCH 123/280] remove linux pthreads asm --- .github/workflows/test.yaml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 18f4867a4..662b952cb 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -185,13 +185,6 @@ jobs: cmake --build out/release-tls-pthreads --parallel 4 --config Release ctest --test-dir out/release-tls-pthreads --verbose --timeout 240 -C Release - - name: Release, Linux, TLS pthreads, Asm - if: matrix.tests == 'extra' && runner.os == 'linux' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: ${{ matrix.os }}-pthreads-asm - path: out/release-tls-pthreads/CMakeFiles/mimalloc-static.dir/src/*.s - # Extra - name: Release, SIMD if: matrix.tests == 'extra' From f0d57e89542240edd69866b9eaf0410429ea726e Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 10:14:37 -0700 Subject: [PATCH 124/280] fix test.yml --- .github/workflows/test.yaml | 2 +- include/mimalloc.h | 6 +++++- src/alloc.c | 4 ++-- src/free.c | 36 ++++++++++++++++++++++++------------ 4 files changed, 32 insertions(+), 16 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 662b952cb..7214029cf 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -45,7 +45,7 @@ jobs: tests: mingw-ucrt64 - os: windows-latest # clang-cl and clang tests: clang - - os: windows11-arm # clang-cl and clang + - os: windows-11-arm # clang-cl and clang tests: clang # - os: ubuntu-latest # tests: android-x64 diff --git a/include/mimalloc.h b/include/mimalloc.h index 229e39484..711c16902 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -133,13 +133,17 @@ mi_decl_nodiscard mi_decl_export void* mi_reallocf(void* p, size_t newsize) mi_decl_nodiscard mi_decl_export size_t mi_usable_size(const void* p) mi_attr_noexcept; mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_noexcept; -// `mi_free_size` can be more efficient (as at calls mi_free_small internally). +// `mi_free_size` can be more efficient (as it calls mi_free_small internally). mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; // `mi_free_small` is for special applications like language runtimes. // it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free` mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; +// As `mi_free_small` but `p` should be non-NULL. +void mi_free_small_nonnull(void* p) mi_attr_noexcept; + + // ------------------------------------------------------------------------------------- // Aligned allocation // Note that `alignment` always follows `size` for consistency with unaligned diff --git a/src/alloc.c b/src/alloc.c index 9e86e5eae..a12038a8a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -369,7 +369,7 @@ void* mi_expand(void* p, size_t newsize) mi_attr_noexcept { return NULL; #else if (p == NULL) return NULL; - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_expand"); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_expand"); const size_t size = _mi_page_usable_size(page,p); if (newsize > size) return NULL; return p; // it fits @@ -388,7 +388,7 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } } else { - page = mi_validate_ptr_page(p,"mi_realloc"); + page = mi_ptr_page_validate(p,"mi_realloc"); if mi_unlikely(page==NULL) { // invalid pointer if (pblock_size_pre!=NULL) { *pblock_size_pre = 0; } if (pblock_size_post!=NULL) { *pblock_size_post = 0; } diff --git a/src/free.c b/src/free.c index b7550e614..6ade886a4 100644 --- a/src/free.c +++ b/src/free.c @@ -169,7 +169,7 @@ void mi_decl_noinline _mi_free_generic(mi_page_t* page, bool is_local, void* p) // Get the page belonging to a pointer // Does further checks in debug mode to see if this was a valid pointer. -static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, bool free_small, mi_page_t** ppage) +static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const char* msg, bool free_small, bool check_p_for_null, mi_page_t** ppage) { MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small); #if MI_DEBUG @@ -190,7 +190,7 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, { page = _mi_ptr_page(p); } #endif - if mi_unlikely(page==NULL) { + if mi_unlikely(check_p_for_null && page==NULL) { #if MI_DEBUG if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } #endif @@ -222,9 +222,13 @@ static inline bool mi_validate_ptr_page_nonnull(const void* p, const char* msg, return true; } -static inline mi_page_t* mi_validate_ptr_page(const void* p, const char* msg) { +static mi_decl_forceinline bool mi_ptr_page_is_valid(const void* p, const char* msg, mi_page_t** ppage) { + return mi_ptr_page_is_valid_ex(p,msg,false /* free_small? */, true /* check_p_for_null */, ppage ); +} + +static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const char* msg) { mi_page_t* page; - return (mi_validate_ptr_page_nonnull(p,msg,false,&page) ? page : NULL); + return (mi_ptr_page_is_valid(p,msg,&page) ? page : NULL); } // Free a block @@ -259,14 +263,14 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t void mi_free(void* p) mi_attr_noexcept { mi_page_t* page; - if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free",false,&page)) { + if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) { mi_free_nonnull(p, page, NULL, true /* allow collect? */); } } void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_ufree",false,&page)) { + if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) { mi_free_nonnull(p, page, pblock_size, true /* allow collect? */); } else { @@ -276,7 +280,15 @@ void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { void mi_free_small(void* p) mi_attr_noexcept { mi_page_t* page; - if mi_likely(mi_validate_ptr_page_nonnull(p,"mi_free_small",true /* is_small? */,&page)) { + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) { + mi_free_nonnull(p, page, NULL, true /* allow collect? */); + } +} + +void mi_free_small_nonnull(void* p) mi_attr_noexcept { + mi_assert(p!=NULL); + mi_page_t* page; + if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) { mi_free_nonnull(p, page, NULL, true /* allow collect? */); } } @@ -288,7 +300,7 @@ void mi_free_small(void* p) mi_attr_noexcept { // Free a pointer that is potentially allocated in a different sub-process void _mi_free_subproc_safe(void* p) mi_attr_noexcept { mi_page_t* page; - if mi_likely(mi_validate_ptr_page_nonnull(p,"_mi_free_subproc_safe",false,&page)) { + if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { mi_free_nonnull(p, page, NULL, false /* allow collect? */); } } @@ -300,7 +312,7 @@ void _mi_free_subproc_safe(void* p) mi_attr_noexcept { void mi_free_size(void* p, size_t size) mi_attr_noexcept { MI_UNUSED_RELEASE(size); #if MI_DEBUG - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_free_size"); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_free_size"); if (page==NULL) return; mi_assert(p!=NULL); const size_t usable = _mi_page_usable_size(page,p); @@ -323,7 +335,7 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { } } #endif - #if MI_PAGE_META_SMALL_IS_ALIGNED + #if MI_PAGE_META_SMALL_IS_ALIGNED || MI_PAGE_META_IS_ALIGNED if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } @@ -531,7 +543,7 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept { if mi_unlikely(page==NULL) return 0; - mi_assert_internal(mi_validate_ptr_page(p,"_mi_page_usable_size") == page); + mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page); if mi_likely(!mi_page_has_interior_pointers(page)) { const mi_block_t* block = mi_validate_block_from_ptr(page,p); return mi_page_usable_size_of(page, block, false /* is guarded */); @@ -543,7 +555,7 @@ size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexce } mi_decl_nodiscard size_t mi_usable_size(const void* p) mi_attr_noexcept { - const mi_page_t* const page = mi_validate_ptr_page(p,"mi_usable_size"); + const mi_page_t* const page = mi_ptr_page_validate(p,"mi_usable_size"); return _mi_page_usable_size(page,p); } From 61f993bf532a0cdb54835d50f164ab8b705cc910 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 10:42:33 -0700 Subject: [PATCH 125/280] add mi_free_csize_nonnull --- include/mimalloc.h | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/include/mimalloc.h b/include/mimalloc.h index 711c16902..3c5b2f7d9 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -414,6 +414,10 @@ static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, si static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } } +static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept { + // assert(p!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } +} // ------------------------------------------------------ // Experimental From 116733a85ac6baa89188d997d0e25bfde90c77b1 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 11:02:20 -0700 Subject: [PATCH 126/280] update documentation --- doc/mimalloc-doc.h | 66 +- docs/annotated.html | 2 +- docs/bench.html | 2 +- docs/build.html | 2 +- docs/classes.html | 2 +- docs/doxygen_crawl.html | 5 +- docs/environment.html | 2 +- docs/functions.html | 2 +- docs/functions_vars.html | 2 +- docs/group__aligned.html | 2 +- docs/group__analysis.html | 2 +- docs/group__arenas.html | 2 +- docs/group__constantsize.html | 54 +- docs/group__constantsize.js | 1 + docs/group__cpp.html | 2 +- docs/group__extended.html | 50 +- docs/group__extended.js | 1 + docs/group__heap.html | 2 +- docs/group__malloc.html | 33 +- docs/group__malloc.js | 1 + docs/group__options.html | 2 +- docs/group__posix.html | 72 +- docs/group__posix.js | 2 +- docs/group__stats.html | 4 +- docs/group__subproc.html | 2 +- docs/group__theap.html | 2 +- docs/group__typed.html | 4 +- docs/group__zeroinit.html | 2 +- docs/index.html | 2 +- docs/mimalloc-doc_8h_source.html | 1116 +++++++++++++++--------------- docs/modes.html | 2 +- docs/navtreedata.js | 2 +- docs/navtreeindex0.js | 72 +- docs/navtreeindex1.js | 3 + docs/overrides.html | 2 +- docs/pages.html | 2 +- docs/search/all_9.js | 459 ++++++------ docs/search/functions_0.js | 321 ++++----- docs/tools.html | 2 +- docs/topics.html | 4 +- docs/using.html | 2 +- 41 files changed, 1256 insertions(+), 1058 deletions(-) diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index 7dc6f654c..81e68684f 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -124,9 +124,18 @@ Further information: /// Free previously allocated memory. /// The pointer `p` must have been allocated before (or be \a NULL). -/// @param p pointer to free, or \a NULL. +/// @param p pointer to the block that is freed, or \a NULL. +/// See also mi_cfree() for a checked free that validates the pointer +/// before free-ing. +/// @see mi_cfree() void mi_free(void* p); +/// Checked `free`: just as mi_free() but always checks if the pointer `p` belongs to our heap +/// @param p pointer to a block that is freed, or any other arbitrary pointer. +/// This is safe to use when a pointer might be invalid or allocated by +/// another allocator (in which case the pointer is ignored and not freed). +void mi_cfree(void* p); + /// Allocate \a size bytes. /// @param size number of bytes to allocate. /// @returns pointer to the allocated memory or \a NULL if out of memory. @@ -404,13 +413,14 @@ void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t of /// \} /// \defgroup constantsize Constant size allocation -/// Constant size allocation and freeing +/// Inlined definitions for onstant size allocation and freeing. +/// If the size is constant, it can pick statically between +/// mi_malloc() and mi_malloc_small(). /// For example: /// ``` /// long* p = (long*)mi_malloc_csize(sizeof(long)) /// ``` -/// On __v3__ these can improve performance as they will statically -/// use mi_malloc_small(), mi_free_small(), etc. when possible. +/// On __v3__ these can improve performance. /// This is especially useful for compilers and runtimes where the /// the size is often statically known. These functions can be used /// even when the size is not constant but then there is no performance benefit. @@ -431,11 +441,20 @@ void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size); /// @brief Free a pointer with a known constant size. /// @param p The pointer to the allocated block (or \a NULL ) -/// @param size The (compile time constant) size with which the pointer was allocated. +/// @param size The size with which the pointer was allocated. /// @see mi_malloc_csize() /// @see mi_free_size() +/// @see mi_free_size_nonnull() void mi_free_csize(void* p, size_t size); +/// @brief Free a non-null pointer with a known constant size. +/// @param p The pointer to the allocated block (cannot be \a NULL !) +/// @param size The size with which the pointer was allocated. +/// @see mi_malloc_csize() +/// @see mi_free_size() +/// @see mi_free_size_nonnull() +void mi_free_csize_nonnull(void* p, size_t size); + /// \} /// \defgroup zeroinit Zero initialized re-allocation @@ -1056,12 +1075,13 @@ void* mi_zalloc_small(size_t size); /// __v3__: Can be used to free an object that was allocated with mi_malloc_small() or /// any allocation with a size < MI_SMALL_SIZE_MAX() -/// @param p Pointer that was returned from #mi_malloc_small et al. +/// @param p Pointer that was returned from #mi_malloc_small et al. (or \a NULL ) /// @details /// This function is meant for use in run-time systems for best /// performance and does not check if the pointer was _indeed_ allocated /// as a small object -- use with care! /// +/// @see mi_free_small_nonnull() /// @see mi_malloc_small() /// @see mi_zalloc_small() /// @see mi_heap_malloc_small() @@ -1070,6 +1090,23 @@ void* mi_zalloc_small(size_t size); /// @see mi_theap_zalloc_small() void mi_free_small(void* p); +/// __v3__: Can be used to free an object that was allocated with mi_malloc_small() or +/// any allocation with a size < MI_SMALL_SIZE_MAX() +/// @param p Pointer that was returned from #mi_malloc_small et al. Cannot be \a NULL ! +/// @details +/// This function is meant for use in run-time systems for best +/// performance and does not check if the pointer was _indeed_ allocated +/// as a small object or not NULL -- use with care! +/// +/// @see mi_free_small() +/// @see mi_malloc_small() +/// @see mi_zalloc_small() +/// @see mi_heap_malloc_small() +/// @see mi_heap_zalloc_small() +/// @see mi_theap_malloc_small() +/// @see mi_theap_zalloc_small() +void mi_free_small_nonnull(void* p); + /// \} @@ -1406,9 +1443,6 @@ void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize); /// /// \{ -/// Checked `free`: just as `mi_free` but always checks if the pointer `p` belongs to our heap -/// and is safe to use when a pointer might be invalid or allocated by another allocator. -void mi_cfree(void* p); void* mi__expand(void* p, size_t newsize); size_t mi_malloc_size(const void* p); @@ -1437,15 +1471,25 @@ int mi_reallocarr(void* p, size_t count, size_t size); void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment); void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset); -/// @brief Free an object that was allocated with a known size. +/// @brief Free a block that was allocated with a known size. /// @param p The pointer that was allocated with \a size bytes (or \a NULL ) /// @param size The size in bytes. -/// __v3__: this can improve performance for objects with a \size < MI_SMALL_SIZE_MAX() +/// __v3__: this can improve performance for objects with a \a size < MI_SMALL_SIZE_MAX() /// as it will use mi_free_small() internally. Always use this if possible. /// @see mi_free_tp() /// @see mi_free_csize() +/// @see mi_free_size_nonnull() void mi_free_size(void* p, size_t size); +/// @brief Free a non-NULL pointer that was allocated with a known size. +/// @param p The pointer that was allocated with \a size bytes (cannot be \a NULL !) +/// @param size The size in bytes. +/// __v3__: this can improve performance for objects with a \a size < MI_SMALL_SIZE_MAX() +/// as it will use mi_free_small_nonnull() internally. Always use this if possible. +/// @see mi_free_tp() +/// @see mi_free_csize() +void mi_free_size_nonnull(void* p, size_t size); + /// @brief Free an object that was allocated aligned. /// @param p The pointer to the object (or \a NULL ) /// @param size The size of the object as allocated. diff --git a/docs/annotated.html b/docs/annotated.html index 87d0fb2f6..cd3d28680 100644 --- a/docs/annotated.html +++ b/docs/annotated.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/bench.html b/docs/bench.html index 33217d900..e365f4a82 100644 --- a/docs/bench.html +++ b/docs/bench.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/build.html b/docs/build.html index 78e1cc335..2de2ebca0 100644 --- a/docs/build.html +++ b/docs/build.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/classes.html b/docs/classes.html index acb1e25d5..6ea4e4725 100644 --- a/docs/classes.html +++ b/docs/classes.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/doxygen_crawl.html b/docs/doxygen_crawl.html index aeea689b4..3c2434f1e 100644 --- a/docs/doxygen_crawl.html +++ b/docs/doxygen_crawl.html @@ -98,6 +98,7 @@ + @@ -113,6 +114,7 @@ + @@ -174,6 +176,7 @@ + @@ -250,9 +253,9 @@ - + diff --git a/docs/environment.html b/docs/environment.html index 4aafebc69..be9356a2f 100644 --- a/docs/environment.html +++ b/docs/environment.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/functions.html b/docs/functions.html index ff9b06858..fcb66777d 100644 --- a/docs/functions.html +++ b/docs/functions.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/functions_vars.html b/docs/functions_vars.html index caa47689a..72a3e8398 100644 --- a/docs/functions_vars.html +++ b/docs/functions_vars.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__aligned.html b/docs/group__aligned.html index 3de1f8731..43ab98c71 100644 --- a/docs/group__aligned.html +++ b/docs/group__aligned.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__analysis.html b/docs/group__analysis.html index 62d1187be..5cb834f2f 100644 --- a/docs/group__analysis.html +++ b/docs/group__analysis.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__arenas.html b/docs/group__arenas.html index b8fef1965..e9cf0a6c5 100644 --- a/docs/group__arenas.html +++ b/docs/group__arenas.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__constantsize.html b/docs/group__constantsize.html index dc7e32ef6..069784fd2 100644 --- a/docs/group__constantsize.html +++ b/docs/group__constantsize.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -110,7 +110,7 @@
-

Constant size allocation and freeing For example: +

Inlined definitions for onstant size allocation and freeing. More...

+ + +

@@ -127,12 +127,15 @@

void mi_free_csize (void *p, size_t size)
 Free a pointer with a known constant size.
 
void mi_free_csize_nonnull (void *p, size_t size)
 Free a non-null pointer with a known constant size.
 

Detailed Description

-

Constant size allocation and freeing For example:

-
long* p = (long*)mi_malloc_csize(sizeof(long))
+

Inlined definitions for onstant size allocation and freeing.

+

If the size is constant, it can pick statically between mi_malloc() and mi_malloc_small(). For example:

long* p = (long*)mi_malloc_csize(sizeof(long))
mi_malloc_csize
void * mi_malloc_csize(size_t size)
Allocate a constant size object.
-

On v3 these can improve performance as they will statically use mi_malloc_small(), mi_free_small(), etc. when possible. This is especially useful for compilers and runtimes where the the size is often statically known. These functions can be used even when the size is not constant but then there is no performance benefit.

+

On v3 these can improve performance. This is especially useful for compilers and runtimes where the the size is often statically known. These functions can be used even when the size is not constant but then there is no performance benefit.

Function Documentation

◆ mi_free_csize()

@@ -157,13 +160,50 @@

Parameters
- +
pThe pointer to the allocated block (or NULL )
sizeThe (compile time constant) size with which the pointer was allocated.
sizeThe size with which the pointer was allocated.
See also
mi_malloc_csize()
-mi_free_size()
+mi_free_size() +
+mi_free_size_nonnull()
+ +

+
+ +

◆ mi_free_csize_nonnull()

+ +
+
+ + + + + + + + + + + +
void mi_free_csize_nonnull (void * p,
size_t size )
+
+ +

Free a non-null pointer with a known constant size.

+
Parameters
+ + + +
pThe pointer to the allocated block (cannot be NULL !)
sizeThe size with which the pointer was allocated.
+
+
+
See also
mi_malloc_csize()
+
+mi_free_size()
+
+mi_free_size_nonnull()
diff --git a/docs/group__constantsize.js b/docs/group__constantsize.js index bfaeb70ff..a3b6430e1 100644 --- a/docs/group__constantsize.js +++ b/docs/group__constantsize.js @@ -1,6 +1,7 @@ var group__constantsize = [ [ "mi_free_csize", "group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5", null ], + [ "mi_free_csize_nonnull", "group__constantsize.html#ga38a93d373a9dc8edc8bca83edb6af5c5", null ], [ "mi_malloc_csize", "group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571", null ], [ "mi_theap_malloc_csize", "group__constantsize.html#ga105bd667da91a15d113fe79be04d625c", null ], [ "mi_theap_zalloc_csize", "group__constantsize.html#ga1addf755829380581dd9c3c9175e376f", null ], diff --git a/docs/group__cpp.html b/docs/group__cpp.html index 42c217152..ddc0ac046 100644 --- a/docs/group__cpp.html +++ b/docs/group__cpp.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__extended.html b/docs/group__extended.html index e4eb5ef4b..7d3c8c67c 100644 --- a/docs/group__extended.html +++ b/docs/group__extended.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -171,6 +171,9 @@ void mi_free_small (void *p)  v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()
  +void mi_free_small_nonnull (void *p) + v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()
+ 

Detailed Description

Internal functionality.

@@ -310,12 +313,53 @@

mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()

Parameters
- +
pPointer that was returned from mi_malloc_small et al.
pPointer that was returned from mi_malloc_small et al. (or NULL )

This function is meant for use in run-time systems for best performance and does not check if the pointer was indeed allocated as a small object – use with care!

-
See also
mi_malloc_small()
+
See also
mi_free_small_nonnull()
+
+mi_malloc_small()
+
+mi_zalloc_small()
+
+mi_heap_malloc_small()
+
+mi_heap_zalloc_small()
+
+mi_theap_malloc_small()
+
+mi_theap_zalloc_small()
+ +

+
+ +

◆ mi_free_small_nonnull()

+ +
+
+ + + + + + + +
void mi_free_small_nonnull (void * p)
+
+ +

v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a size < MI_SMALL_SIZE_MAX()

+
Parameters
+ + +
pPointer that was returned from mi_malloc_small et al. Cannot be NULL !
+
+
+

This function is meant for use in run-time systems for best performance and does not check if the pointer was indeed allocated as a small object or not NULL – use with care!

+
See also
mi_free_small()
+
+mi_malloc_small()
mi_zalloc_small()
diff --git a/docs/group__extended.js b/docs/group__extended.js index dd2242214..2934ab5be 100644 --- a/docs/group__extended.js +++ b/docs/group__extended.js @@ -6,6 +6,7 @@ var group__extended = [ "mi_output_fun", "group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d", null ], [ "mi_collect", "group__extended.html#ga421430e2226d7d468529cec457396756", null ], [ "mi_free_small", "group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610", null ], + [ "mi_free_small_nonnull", "group__extended.html#ga22890f54e5be433837069c8e97fdf379", null ], [ "mi_is_redirected", "group__extended.html#gaad25050b19f30cd79397b227e0157a3f", null ], [ "mi_malloc_small", "group__extended.html#ga7f050bc6b897da82692174f5fce59cde", null ], [ "mi_register_deferred_free", "group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece", null ], diff --git a/docs/group__heap.html b/docs/group__heap.html index 50a85dc5c..2d06f93fd 100644 --- a/docs/group__heap.html +++ b/docs/group__heap.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__malloc.html b/docs/group__malloc.html index 6b7796979..50f3615c3 100644 --- a/docs/group__malloc.html +++ b/docs/group__malloc.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -118,6 +118,9 @@ void mi_free (void *p)  Free previously allocated memory.
  +void mi_cfree (void *p) + Checked free: just as mi_free() but always checks if the pointer p belongs to our heap.
+  void * mi_malloc (size_t size)  Allocate size bytes.
  @@ -191,6 +194,31 @@

Returns
pointer to the allocated memory of size*count bytes, or NULL if either out of memory or when count*size overflows.

Returns a unique pointer if called with either size or count of 0.

See also
mi_zalloc()
+
+
+ +

◆ mi_cfree()

+ +
+
+ + + + + + + +
void mi_cfree (void * p)
+
+ +

Checked free: just as mi_free() but always checks if the pointer p belongs to our heap.

+
Parameters
+ + +
ppointer to a block that is freed, or any other arbitrary pointer. This is safe to use when a pointer might be invalid or allocated by another allocator (in which case the pointer is ignored and not freed).
+
+
+
@@ -242,10 +270,11 @@

Parameters
- +
ppointer to free, or NULL.
ppointer to the block that is freed, or NULL. See also mi_cfree() for a checked free that validates the pointer before free-ing.
+
See also
mi_cfree()

diff --git a/docs/group__malloc.js b/docs/group__malloc.js index 9fd99dfe4..28d281767 100644 --- a/docs/group__malloc.js +++ b/docs/group__malloc.js @@ -1,6 +1,7 @@ var group__malloc = [ [ "mi_calloc", "group__malloc.html#ga6686568014b54d1e6c7ac64a076e4f56", null ], + [ "mi_cfree", "group__malloc.html#ga705dc7a64bffacfeeb0141501a5c35d7", null ], [ "mi_expand", "group__malloc.html#ga19299856216cfbb08e2628593654dfb0", null ], [ "mi_free", "group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95", null ], [ "mi_good_size", "group__malloc.html#gac057927cd06c854b45fe7847e921bd47", null ], diff --git a/docs/group__options.html b/docs/group__options.html index 2824760be..5ad478ad4 100644 --- a/docs/group__options.html +++ b/docs/group__options.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__posix.html b/docs/group__posix.html index 96a40e751..5193821c8 100644 --- a/docs/group__posix.html +++ b/docs/group__posix.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -115,9 +115,6 @@ - - - @@ -157,8 +154,11 @@ - + + + + @@ -301,25 +301,6 @@

- - - -

◆ mi_cfree()

- -
-
-

Functions

void mi_cfree (void *p)
 Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to use when a pointer might be invalid or allocated by another allocator.
 
void * mi__expand (void *p, size_t newsize)
 
size_t mi_malloc_size (const void *p)
void * mi_aligned_offset_recalloc (void *p, size_t newcount, size_t size, size_t alignment, size_t offset)
 
void mi_free_size (void *p, size_t size)
 Free an object that was allocated with a known size.
 Free a block that was allocated with a known size.
 
void mi_free_size_nonnull (void *p, size_t size)
 Free a non-NULL pointer that was allocated with a known size.
 
void mi_free_size_aligned (void *p, size_t size, size_t alignment)
 Free an object that was allocated aligned.
 
- - - - - - -
void mi_cfree (void * p)
-
- -

Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to use when a pointer might be invalid or allocated by another allocator.

-
@@ -388,17 +369,19 @@

-

Free an object that was allocated with a known size.

+

Free a block that was allocated with a known size.

Parameters
- +
pThe pointer that was allocated with size bytes (or NULL )
sizeThe size in bytes. v3: this can improve performance for objects with a \size < MI_SMALL_SIZE_MAX() as it will use mi_free_small() internally. Always use this if possible.
sizeThe size in bytes. v3: this can improve performance for objects with a size < MI_SMALL_SIZE_MAX() as it will use mi_free_small() internally. Always use this if possible.
See also
mi_free_tp()
-mi_free_csize()
+mi_free_csize() +
+mi_free_size_nonnull()

@@ -431,11 +414,44 @@

pThe pointer to the object (or NULL ) sizeThe size of the object as allocated. - alignmentThe requested alignment at the allocation. Currently just defers to mi_free_size() + alignmentThe requested alignment at the allocation. Currently just defers to mi_free_size() +

+
+ +

◆ mi_free_size_nonnull()

+ +
+
+ + + + + + + + + + + +
void mi_free_size_nonnull (void * p,
size_t size )
+
+ +

Free a non-NULL pointer that was allocated with a known size.

+
Parameters
+ + + +
pThe pointer that was allocated with size bytes (cannot be NULL !)
sizeThe size in bytes. v3: this can improve performance for objects with a size < MI_SMALL_SIZE_MAX() as it will use mi_free_small_nonnull() internally. Always use this if possible.
+
+
+
See also
mi_free_tp()
+
+mi_free_csize()
+
diff --git a/docs/group__posix.js b/docs/group__posix.js index ea4ed269e..fa782adcb 100644 --- a/docs/group__posix.js +++ b/docs/group__posix.js @@ -5,11 +5,11 @@ var group__posix = [ "mi_aligned_alloc", "group__posix.html#ga430ed1513f0571ff83be00ec58a98ee0", null ], [ "mi_aligned_offset_recalloc", "group__posix.html#ga16570deddd559001b44953eedbad0084", null ], [ "mi_aligned_recalloc", "group__posix.html#gaf82cbb4b4f24acf723348628451798d3", null ], - [ "mi_cfree", "group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7", null ], [ "mi_dupenv_s", "group__posix.html#gab41369c1a1da7504013a7a0b1d4dd958", null ], [ "mi_free_aligned", "group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9", null ], [ "mi_free_size", "group__posix.html#gae01389eedab8d67341ff52e2aad80ebb", null ], [ "mi_free_size_aligned", "group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc", null ], + [ "mi_free_size_nonnull", "group__posix.html#ga796602dae2a89797026f72e0c0240f32", null ], [ "mi_malloc_good_size", "group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071", null ], [ "mi_malloc_size", "group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de", null ], [ "mi_malloc_usable_size", "group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17", null ], diff --git a/docs/group__stats.html b/docs/group__stats.html index 529d18d89..2555bcac6 100644 --- a/docs/group__stats.html +++ b/docs/group__stats.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -260,7 +260,7 @@

if (mi_stats_get(&stats)) {

...
}
-
mi_stats_t_decl
#define mi_stats_t_decl(name)
Helper to declare a properly initialized mi_stats_t() local variable as name where the size and versi...
Definition mimalloc-doc.h:1099
+
mi_stats_t_decl
#define mi_stats_t_decl(name)
Helper to declare a properly initialized mi_stats_t() local variable as name where the size and versi...
Definition mimalloc-doc.h:1136
mi_stats_get
bool mi_stats_get(mi_stats_t *stats)
Get the statistics for the current subprocess aggregated over all its heaps.
diff --git a/docs/group__subproc.html b/docs/group__subproc.html index f3830c87f..88e1e8976 100644 --- a/docs/group__subproc.html +++ b/docs/group__subproc.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__theap.html b/docs/group__theap.html index e8abeb2c7..f84216e6a 100644 --- a/docs/group__theap.html +++ b/docs/group__theap.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/group__typed.html b/docs/group__typed.html index bbd77cafb..8042c1872 100644 --- a/docs/group__typed.html +++ b/docs/group__typed.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -155,7 +155,7 @@

Detailed Description

Typed allocation macros.

For example:

int* p = mi_malloc_tp(int)
-
mi_malloc_tp
#define mi_malloc_tp(tp)
Allocate a block of type tp.
Definition mimalloc-doc.h:364
+
mi_malloc_tp
#define mi_malloc_tp(tp)
Allocate a block of type tp.
Definition mimalloc-doc.h:373

On v3 these can improve performance as they use the mi_<malloc>_csize api's that in turn can use mi_malloc_small(), mi_free_small(), etc.

Macro Definition Documentation

diff --git a/docs/group__zeroinit.html b/docs/group__zeroinit.html index ecfa05d49..fde3d387d 100644 --- a/docs/group__zeroinit.html +++ b/docs/group__zeroinit.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/index.html b/docs/index.html index 25354e879..ba454f3e5 100644 --- a/docs/index.html +++ b/docs/index.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/mimalloc-doc_8h_source.html b/docs/mimalloc-doc_8h_source.html index 37d74e056..8552c6ea8 100644 --- a/docs/mimalloc-doc_8h_source.html +++ b/docs/mimalloc-doc_8h_source.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -119,531 +119,538 @@
10
123
124
-
128void mi_free(void* p);
-
129
-
134void* mi_malloc(size_t size);
-
135
-
140void* mi_zalloc(size_t size);
-
141
-
151void* mi_calloc(size_t count, size_t size);
-
152
-
166void* mi_realloc(void* p, size_t newsize);
-
167
-
181void* mi_expand(void* p, size_t newsize);
-
182
-
192void* mi_mallocn(size_t count, size_t size);
-
193
-
203void* mi_reallocn(void* p, size_t count, size_t size);
-
204
-
221void* mi_reallocf(void* p, size_t newsize);
-
222
-
223
-
232char* mi_strdup(const char* s);
-
233
-
243char* mi_strndup(const char* s, size_t n);
-
244
-
257char* mi_realpath(const char* fname, char* resolved_name);
-
258
-
272size_t mi_usable_size(void* p);
-
273
-
283size_t mi_good_size(size_t size);
-
284
-
286
-
287
-
288// ------------------------------------------------------
-
289// Aligned allocation
-
290// ------------------------------------------------------
-
291
-
299
-
317void* mi_malloc_aligned(size_t size, size_t alignment);
-
318void* mi_zalloc_aligned(size_t size, size_t alignment);
-
319void* mi_calloc_aligned(size_t count, size_t size, size_t alignment);
-
320void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment);
-
321
-
333void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset);
-
334void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset);
-
335void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset);
-
336void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
-
337
-
339
-
340
-
350
-
364#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp)))
-
365
-
372#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp)))
-
373
-
375#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp)))
-
376
-
378#define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp)))
-
379
-
381#define mi_mallocn_tp(tp,count) ((tp*)mi_mallocn(count,sizeof(tp)))
+
131void mi_free(void* p);
+
132
+
137void mi_cfree(void* p);
+
138
+
143void* mi_malloc(size_t size);
+
144
+
149void* mi_zalloc(size_t size);
+
150
+
160void* mi_calloc(size_t count, size_t size);
+
161
+
175void* mi_realloc(void* p, size_t newsize);
+
176
+
190void* mi_expand(void* p, size_t newsize);
+
191
+
201void* mi_mallocn(size_t count, size_t size);
+
202
+
212void* mi_reallocn(void* p, size_t count, size_t size);
+
213
+
230void* mi_reallocf(void* p, size_t newsize);
+
231
+
232
+
241char* mi_strdup(const char* s);
+
242
+
252char* mi_strndup(const char* s, size_t n);
+
253
+
266char* mi_realpath(const char* fname, char* resolved_name);
+
267
+
281size_t mi_usable_size(void* p);
+
282
+
292size_t mi_good_size(size_t size);
+
293
+
295
+
296
+
297// ------------------------------------------------------
+
298// Aligned allocation
+
299// ------------------------------------------------------
+
300
+
308
+
326void* mi_malloc_aligned(size_t size, size_t alignment);
+
327void* mi_zalloc_aligned(size_t size, size_t alignment);
+
328void* mi_calloc_aligned(size_t count, size_t size, size_t alignment);
+
329void* mi_realloc_aligned(void* p, size_t newsize, size_t alignment);
+
330
+
342void* mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset);
+
343void* mi_zalloc_aligned_at(size_t size, size_t alignment, size_t offset);
+
344void* mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset);
+
345void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
+
346
+
348
+
349
+
359
+
373#define mi_malloc_tp(tp) ((tp*)mi_malloc_csize(sizeof(tp)))
+
374
+
381#define mi_free_tp(tp,p) (mi_free_csize(p,sizeof(tp)))
382
-
384#define mi_reallocn_tp(p,tp,count) ((tp*)mi_reallocn(p,count,sizeof(tp)))
+
384#define mi_zalloc_tp(tp) ((tp*)mi_zalloc_csize(sizeof(tp)))
385
-
387#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
+
387#define mi_calloc_tp(tp,count) ((tp*)mi_calloc(count,sizeof(tp)))
388
-
390#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
+
390#define mi_mallocn_tp(tp,count) ((tp*)mi_mallocn(count,sizeof(tp)))
391
-
393#define mi_heap_calloc_tp(tp,hp,count) ((tp*)mi_heap_calloc(hp,count,sizeof(tp)))
+
393#define mi_reallocn_tp(p,tp,count) ((tp*)mi_reallocn(p,count,sizeof(tp)))
394
-
396#define mi_heap_mallocn_tp(tp,hp,count) ((tp*)mi_heap_mallocn(hp,count,sizeof(tp)))
+
396#define mi_heap_malloc_tp(tp,hp) ((tp*)mi_heap_malloc(hp,sizeof(tp)))
397
-
399#define mi_heap_reallocn_tp(tp,hp,p,count) ((tp*)mi_heap_reallocn(p,count,sizeof(tp)))
+
399#define mi_heap_zalloc_tp(tp,hp) ((tp*)mi_heap_zalloc(hp,sizeof(tp)))
400
-
402#define mi_heap_recalloc_tp(tp,hp,p,count) ((tp*)mi_heap_recalloc(p,count,sizeof(tp)))
+
402#define mi_heap_calloc_tp(tp,hp,count) ((tp*)mi_heap_calloc(hp,count,sizeof(tp)))
403
-
405
-
418
-
424void* mi_malloc_csize(size_t size);
-
425
-
426void* mi_zalloc_csize(size_t size);
-
427
-
428void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size);
-
429
-
430void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size);
-
431
-
437void mi_free_csize(void* p, size_t size);
-
438
-
440
-
450
-
461void* mi_recalloc(void* p, size_t count, size_t size);
-
462
-
463void* mi_rezalloc(void* p, size_t newsize);
-
464void* mi_recalloc(void* p, size_t newcount, size_t size) ;
-
465
-
466void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment);
-
467void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
-
468void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment);
-
469void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
-
470
-
471void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize);
-
472void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size);
-
473
-
474void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
-
475void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
-
476void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment);
-
477void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
-
478
-
480
+
405#define mi_heap_mallocn_tp(tp,hp,count) ((tp*)mi_heap_mallocn(hp,count,sizeof(tp)))
+
406
+
408#define mi_heap_reallocn_tp(tp,hp,p,count) ((tp*)mi_heap_reallocn(p,count,sizeof(tp)))
+
409
+
411#define mi_heap_recalloc_tp(tp,hp,p,count) ((tp*)mi_heap_recalloc(p,count,sizeof(tp)))
+
412
+
414
+
428
+
434void* mi_malloc_csize(size_t size);
+
435
+
436void* mi_zalloc_csize(size_t size);
+
437
+
438void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size);
+
439
+
440void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size);
+
441
+
448void mi_free_csize(void* p, size_t size);
+
449
+
456void mi_free_csize_nonnull(void* p, size_t size);
+
457
+
459
+
469
+
480void* mi_recalloc(void* p, size_t count, size_t size);
481
-
482
+
482void* mi_rezalloc(void* p, size_t newsize);
+
483void* mi_recalloc(void* p, size_t newcount, size_t size) ;
+
484
+
485void* mi_rezalloc_aligned(void* p, size_t newsize, size_t alignment);
+
486void* mi_rezalloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t offset);
+
487void* mi_recalloc_aligned(void* p, size_t newcount, size_t size, size_t alignment);
+
488void* mi_recalloc_aligned_at(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
+
489
+
490void* mi_heap_rezalloc(mi_heap_t* heap, void* p, size_t newsize);
+
491void* mi_heap_recalloc(mi_heap_t* heap, void* p, size_t newcount, size_t size);
+
492
+
493void* mi_heap_rezalloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
+
494void* mi_heap_rezalloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
+
495void* mi_heap_recalloc_aligned(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment);
+
496void* mi_heap_recalloc_aligned_at(mi_heap_t* heap, void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
+
497
+
499
500
-
506struct mi_heap_s;
-
507
-
513typedef struct mi_heap_s mi_heap_t;
-
514
-
516mi_heap_t* mi_heap_new();
-
517
-
524void mi_heap_delete(mi_heap_t* heap);
-
525
-
532void mi_heap_destroy(mi_heap_t* heap);
+
501
+
519
+
525struct mi_heap_s;
+
526
+
532typedef struct mi_heap_s mi_heap_t;
533
-
538mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
-
539
-
543mi_heap_t* mi_heap_get_default();
+
535mi_heap_t* mi_heap_new();
+
536
+
543void mi_heap_delete(mi_heap_t* heap);
544
-
551mi_heap_t* mi_heap_get_backing();
+
551void mi_heap_destroy(mi_heap_t* heap);
552
-
556void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
-
557
-
563mi_heap_t* mi_heap_main(void);
-
564
-
570mi_heap_t* mi_heap_of(const void* p);
+
557mi_heap_t* mi_heap_set_default(mi_heap_t* heap);
+
558
+
562mi_heap_t* mi_heap_get_default();
+
563
+
570mi_heap_t* mi_heap_get_backing();
571
-
576bool mi_heap_contains(const mi_heap_t* heap, const void* p);
-
577
-
581bool mi_any_heap_contains(const void* p);
-
582
+
575void mi_heap_set_numa_affinity(mi_heap_t* heap, int numa_node);
+
576
+
582mi_heap_t* mi_heap_main(void);
583
-
589bool mi_is_in_heap_region(const void* p);
+
589mi_heap_t* mi_heap_of(const void* p);
590
-
596bool mi_check_owned(const void* p);
-
597
-
604bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
-
605
-
612bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
-
613
-
615void mi_heap_collect(mi_heap_t* heap, bool force);
+
595bool mi_heap_contains(const mi_heap_t* heap, const void* p);
+
596
+
600bool mi_any_heap_contains(const void* p);
+
601
+
602
+
608bool mi_is_in_heap_region(const void* p);
+
609
+
615bool mi_check_owned(const void* p);
616
-
619void* mi_heap_malloc(mi_heap_t* heap, size_t size);
-
620
-
624void* mi_heap_malloc_small(mi_heap_t* heap, size_t size);
-
625
-
628void* mi_heap_zalloc(mi_heap_t* heap, size_t size);
-
629
-
633void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size);
-
634
-
637void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size);
-
638
-
641void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size);
-
642
-
645char* mi_heap_strdup(mi_heap_t* heap, const char* s);
-
646
-
649char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n);
-
650
-
653char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name);
-
654
-
655void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize);
-
656void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size);
-
657void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize);
-
658
-
659void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
-
660void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
-
661void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
-
662void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
-
663void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment);
-
664void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset);
-
665void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
-
666void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
-
667
+
623bool mi_heap_contains_block(mi_heap_t* heap, const void* p);
+
624
+
631bool mi_heap_check_owned(mi_heap_t* heap, const void* p);
+
632
+
634void mi_heap_collect(mi_heap_t* heap, bool force);
+
635
+
638void* mi_heap_malloc(mi_heap_t* heap, size_t size);
+
639
+
643void* mi_heap_malloc_small(mi_heap_t* heap, size_t size);
+
644
+
647void* mi_heap_zalloc(mi_heap_t* heap, size_t size);
+
648
+
652void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size);
+
653
+
656void* mi_heap_calloc(mi_heap_t* heap, size_t count, size_t size);
+
657
+
660void* mi_heap_mallocn(mi_heap_t* heap, size_t count, size_t size);
+
661
+
664char* mi_heap_strdup(mi_heap_t* heap, const char* s);
+
665
+
668char* mi_heap_strndup(mi_heap_t* heap, const char* s, size_t n);
669
-
670
-
671
-
676
+
672char* mi_heap_realpath(mi_heap_t* heap, const char* fname, char* resolved_name);
+
673
+
674void* mi_heap_realloc(mi_heap_t* heap, void* p, size_t newsize);
+
675void* mi_heap_reallocn(mi_heap_t* heap, void* p, size_t count, size_t size);
+
676void* mi_heap_reallocf(mi_heap_t* heap, void* p, size_t newsize);
677
-
-
680typedef struct mi_heap_area_s {
-
681 void* blocks;
-
682 size_t reserved;
-
683 size_t committed;
-
684 size_t used;
-
685 size_t block_size;
-
686 size_t full_block_size;
-
687 int heap_tag;
-
688} mi_heap_area_t;
-
+
678void* mi_heap_malloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
+
679void* mi_heap_malloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
+
680void* mi_heap_zalloc_aligned(mi_heap_t* heap, size_t size, size_t alignment);
+
681void* mi_heap_zalloc_aligned_at(mi_heap_t* heap, size_t size, size_t alignment, size_t offset);
+
682void* mi_heap_calloc_aligned(mi_heap_t* heap, size_t count, size_t size, size_t alignment);
+
683void* mi_heap_calloc_aligned_at(mi_heap_t* heap, size_t count, size_t size, size_t alignment, size_t offset);
+
684void* mi_heap_realloc_aligned(mi_heap_t* heap, void* p, size_t newsize, size_t alignment);
+
685void* mi_heap_realloc_aligned_at(mi_heap_t* heap, void* p, size_t newsize, size_t alignment, size_t offset);
+
686
+
688
689
-
697typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
-
698
-
713bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
-
714
-
730bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
-
731
+
690
+
695
+
696
+
+
699typedef struct mi_heap_area_s {
+
700 void* blocks;
+
701 size_t reserved;
+
702 size_t committed;
+
703 size_t used;
+
704 size_t block_size;
+
705 size_t full_block_size;
+
706 int heap_tag;
+
707} mi_heap_area_t;
+
+
708
+
716typedef bool (mi_block_visit_fun)(const mi_heap_t* heap, const mi_heap_area_t* area, void* block, size_t block_size, void* arg);
+
717
+
732bool mi_heap_visit_blocks(const mi_heap_t* heap, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
733
-
734// ------------------------------------------------------
-
735// Arenas
-
736// ------------------------------------------------------
-
737
-
745
-
747typedef int mi_arena_id_t;
-
748
-
757int mi_reserve_os_memory(size_t size, bool commit, bool allow_large);
-
758
-
766int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
+
749bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun* visitor, void* arg);
+
750
+
752
+
753// ------------------------------------------------------
+
754// Arenas
+
755// ------------------------------------------------------
+
756
+
764
+
766typedef int mi_arena_id_t;
767
-
780int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs);
-
781
-
794int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs);
-
795
-
803int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id);
-
804
-
807size_t mi_arena_min_alignment(void);
-
808
-
820bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node);
-
821
-
832bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id);
-
833
-
834
-
836void mi_debug_show_arenas(void);
-
837
-
842void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
-
843
-
844
-
845
-
849mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
-
850
-
862mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id);
+
776int mi_reserve_os_memory(size_t size, bool commit, bool allow_large);
+
777
+
785int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t* arena_id);
+
786
+
799int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs);
+
800
+
813int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs);
+
814
+
822int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t* arena_id);
+
823
+
826size_t mi_arena_min_alignment(void);
+
827
+
839bool mi_manage_os_memory(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node);
+
840
+
851bool mi_manage_os_memory_ex(void* start, size_t size, bool is_committed, bool is_large, bool is_zero, int numa_node, bool exclusive, mi_arena_id_t* arena_id);
+
852
+
853
+
855void mi_debug_show_arenas(void);
+
856
+
861void* mi_arena_area(mi_arena_id_t arena_id, size_t* size);
+
862
863
-
865
-
866
-
875
-
880typedef void* mi_subproc_id_t;
-
881
-
883mi_subproc_id_t mi_subproc_main(void);
+
864
+
868mi_heap_t* mi_heap_new_in_arena(mi_arena_id_t arena_id);
+
869
+
881mi_heap_t* mi_heap_new_ex(int heap_tag, bool allow_destroy, mi_arena_id_t arena_id);
+
882
884
-
886mi_subproc_id_t mi_subproc_current(void);
-
887
-
890mi_subproc_id_t mi_subproc_new(void);
-
891
-
895void mi_subproc_destroy(mi_subproc_id_t subproc);
-
896
-
901void mi_subproc_delete(mi_subproc_id_t subproc);
-
902
-
906void mi_subproc_add_current_thread(mi_subproc_id_t subproc);
-
907
-
911typedef bool (mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
-
912
-
918bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
-
919
+
885
+
894
+
899typedef void* mi_subproc_id_t;
+
900
+
902mi_subproc_id_t mi_subproc_main(void);
+
903
+
905mi_subproc_id_t mi_subproc_current(void);
+
906
+
909mi_subproc_id_t mi_subproc_new(void);
+
910
+
914void mi_subproc_destroy(mi_subproc_id_t subproc);
+
915
+
920void mi_subproc_delete(mi_subproc_id_t subproc);
921
-
922
-
923// ------------------------------------------------------
-
924// Extended functionality
-
925// ------------------------------------------------------
+
925void mi_subproc_add_current_thread(mi_subproc_id_t subproc);
926
+
930typedef bool (mi_heap_visit_fun)(mi_heap_t* heap, void* arg);
931
-
935#define MI_SMALL_SIZE_MAX (128*sizeof(void*))
-
936
-
940int mi_version(void);
+
937bool mi_subproc_visit_heaps(mi_subproc_id_t subproc, mi_heap_visit_fun* visitor, void* arg);
+
938
+
940
941
-
949void mi_collect(bool force);
+
942// ------------------------------------------------------
+
943// Extended functionality
+
944// ------------------------------------------------------
+
945
950
-
956void mi_thread_set_in_threadpool(void);
-
957
-
962bool mi_is_redirected();
-
963
-
967void mi_thread_init(void);
-
968
-
973void mi_thread_done(void);
-
974
-
981typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
+
954#define MI_SMALL_SIZE_MAX (128*sizeof(void*))
+
955
+
959int mi_version(void);
+
960
+
968void mi_collect(bool force);
+
969
+
975void mi_thread_set_in_threadpool(void);
+
976
+
981bool mi_is_redirected();
982
-
998void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg);
-
999
-
1005typedef void (mi_output_fun)(const char* msg, void* arg);
-
1006
-
1013void mi_register_output(mi_output_fun* out, void* arg);
-
1014
-
1020typedef void (mi_error_fun)(int err, void* arg);
-
1021
-
1037void mi_register_error(mi_error_fun* errfun, void* arg);
-
1038
-
1046void* mi_malloc_small(size_t size);
-
1047
-
1055void* mi_zalloc_small(size_t size);
-
1056
-
1071void mi_free_small(void* p);
-
1072
-
1074
+
986void mi_thread_init(void);
+
987
+
992void mi_thread_done(void);
+
993
+
1000typedef void (mi_deferred_free_fun)(bool force, unsigned long long heartbeat, void* arg);
+
1001
+
1017void mi_register_deferred_free(mi_deferred_free_fun* deferred_free, void* arg);
+
1018
+
1024typedef void (mi_output_fun)(const char* msg, void* arg);
+
1025
+
1032void mi_register_output(mi_output_fun* out, void* arg);
+
1033
+
1039typedef void (mi_error_fun)(int err, void* arg);
+
1040
+
1056void mi_register_error(mi_error_fun* errfun, void* arg);
+
1057
+
1065void* mi_malloc_small(size_t size);
+
1066
+
1074void* mi_zalloc_small(size_t size);
1075
-
1076
-
1077
-
1078
-
1079// ------------------------------------------------------
-
1080// Statistics
-
1081// ------------------------------------------------------
-
1082
-
1086
-
1088#define MI_STAT_VERSION 4
-
1089
-
1099#define mi_stats_t_decl(name)
-
1100
-
-
1102struct mi_stats_s {
-
1104 size_t size;
-
1106 size_t version;
-
1107};
-
-
1108
-
1110typedef struct mi_stats_s mi_stats_t;
+
1091void mi_free_small(void* p);
+
1092
+
1108void mi_free_small_nonnull(void* p);
+
1109
1111
1112
-
1115void mi_process_info_print(void);
-
1116
-
1121void mi_process_info_print_out(mi_output_fun* out, void* arg);
-
1122
-
1136void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults);
+
1113
+
1114
+
1115
+
1116// ------------------------------------------------------
+
1117// Statistics
+
1118// ------------------------------------------------------
+
1119
+
1123
+
1125#define MI_STAT_VERSION 4
+
1126
+
1136#define mi_stats_t_decl(name)
1137
-
1138
-
1143void mi_stats_print(void* out);
-
1144
-
1150void mi_stats_print_out(mi_output_fun* out, void* arg);
-
1151
-
1157bool mi_stats_get(mi_stats_t* stats);
-
1158
-
1164char* mi_stats_get_json(size_t buf_size, char* buf);
-
1165
-
1167size_t mi_stats_get_bin_size(size_t bin);
-
1168
-
1175bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats);
-
1176
-
1183char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
-
1184
-
1189void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg);
-
1190
-
1194void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
+
+
1139struct mi_stats_s {
+
1141 size_t size;
+
1143 size_t version;
+
1144};
+
+
1145
+
1147typedef struct mi_stats_s mi_stats_t;
+
1148
+
1149
+
1152void mi_process_info_print(void);
+
1153
+
1158void mi_process_info_print_out(mi_output_fun* out, void* arg);
+
1159
+
1173void mi_process_info(size_t* elapsed_msecs, size_t* user_msecs, size_t* system_msecs, size_t* current_rss, size_t* peak_rss, size_t* current_commit, size_t* peak_commit, size_t* page_faults);
+
1174
+
1175
+
1180void mi_stats_print(void* out);
+
1181
+
1187void mi_stats_print_out(mi_output_fun* out, void* arg);
+
1188
+
1194bool mi_stats_get(mi_stats_t* stats);
1195
-
1202bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats);
-
1203
-
1210char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
-
1211
-
1216void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
-
1217
-
1218
-
1223void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
-
1224
-
1231char* mi_stats_as_json( mi_stats_t* stats, size_t buf_size, char* buf);
+
1201char* mi_stats_get_json(size_t buf_size, char* buf);
+
1202
+
1204size_t mi_stats_get_bin_size(size_t bin);
+
1205
+
1212bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats);
+
1213
+
1220char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
+
1221
+
1226void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg);
+
1227
+
1231void mi_heap_stats_merge_to_subproc(mi_heap_t* heap);
1232
-
1234void mi_stats_reset(void);
-
1235
-
1237void mi_stats_merge(void);
-
1238
+
1239bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats);
1240
-
1241// ------------------------------------------------------
-
1242// Runtime Options
-
1243// ------------------------------------------------------
-
1244
-
1249
-
1252void mi_options_print(void);
-
1253
-
1259void mi_options_print_out(mi_output_fun* out, void* arg);
-
1260
+
1247char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf); // use mi_free to free the result if the input buf == NULL
+
1248
+
1253void mi_subproc_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
+
1254
+
1255
+
1260void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun* out, void* arg);
1261
-
-
1263typedef enum mi_option_e {
-
1264 // stable options
-
1265 mi_option_show_errors,
-
1266 mi_option_show_stats,
-
1267 mi_option_verbose,
-
1268 mi_option_max_errors,
-
1269 mi_option_max_warnings,
-
1270
-
1271 // advanced options
-
1272 mi_option_reserve_huge_os_pages,
-
1273 mi_option_reserve_huge_os_pages_at,
-
1274 mi_option_reserve_os_memory,
-
1275 mi_option_allow_large_os_pages,
-
1276 mi_option_purge_decommits,
-
1277 mi_option_arena_reserve,
-
1278 mi_option_os_tag,
-
1279 mi_option_retry_on_oom,
-
1280 mi_option_generic_collect,
-
1281 mi_option_allow_thp,
-
1282
-
1283 // guard pages
-
1284 mi_option_guarded_min,
-
1285 mi_option_guarded_max,
-
1286 mi_option_guarded_precise,
-
1287 mi_option_guarded_sample_rate,
-
1288 mi_option_guarded_sample_seed,
-
1289
-
1290 // experimental options
-
1291 mi_option_eager_commit,
-
1292 mi_option_eager_commit_delay,
-
1293 mi_option_arena_eager_commit,
-
1294 mi_option_abandoned_page_purge,
-
1295 mi_option_purge_delay,
-
1296 mi_option_use_numa_nodes,
-
1297 mi_option_disallow_os_alloc,
-
1298 mi_option_max_segment_reclaim,
-
1299 mi_option_destroy_on_exit,
-
1300 mi_option_arena_purge_mult,
-
1301 mi_option_abandoned_reclaim_on_free,
-
1302 mi_option_purge_extend_delay,
-
1303 mi_option_disallow_arena_alloc,
-
1304 mi_option_visit_abandoned,
-
1305 mi_option_target_segments_per_thread,
-
1306
-
1307 // v3 options
-
1308 mi_option_page_reclaim_on_free,
-
1309 mi_option_page_full_retain,
-
1310 mi_option_page_max_candidates,
-
1311 mi_option_max_vabits,
-
1312 mi_option_pagemap_commit,
-
1313 mi_option_page_commit_on_demand,
-
1314 mi_option_page_max_reclaim,
-
1315 mi_option_page_cross_thread_max_reclaim,
-
1316 mi_option_minimal_purge_size,
-
1317 mi_option_arena_max_object_size,
-
1318
-
1319 _mi_option_last
-
1320} mi_option_t;
+
1268char* mi_stats_as_json( mi_stats_t* stats, size_t buf_size, char* buf);
+
1269
+
1271void mi_stats_reset(void);
+
1272
+
1274void mi_stats_merge(void);
+
1275
+
1277
+
1278// ------------------------------------------------------
+
1279// Runtime Options
+
1280// ------------------------------------------------------
+
1281
+
1286
+
1289void mi_options_print(void);
+
1290
+
1296void mi_options_print_out(mi_output_fun* out, void* arg);
+
1297
+
1298
+
+
1300typedef enum mi_option_e {
+
1301 // stable options
+
1302 mi_option_show_errors,
+
1303 mi_option_show_stats,
+
1304 mi_option_verbose,
+
1305 mi_option_max_errors,
+
1306 mi_option_max_warnings,
+
1307
+
1308 // advanced options
+
1309 mi_option_reserve_huge_os_pages,
+
1310 mi_option_reserve_huge_os_pages_at,
+
1311 mi_option_reserve_os_memory,
+
1312 mi_option_allow_large_os_pages,
+
1313 mi_option_purge_decommits,
+
1314 mi_option_arena_reserve,
+
1315 mi_option_os_tag,
+
1316 mi_option_retry_on_oom,
+
1317 mi_option_generic_collect,
+
1318 mi_option_allow_thp,
+
1319
+
1320 // guard pages
+
1321 mi_option_guarded_min,
+
1322 mi_option_guarded_max,
+
1323 mi_option_guarded_precise,
+
1324 mi_option_guarded_sample_rate,
+
1325 mi_option_guarded_sample_seed,
+
1326
+
1327 // experimental options
+
1328 mi_option_eager_commit,
+
1329 mi_option_eager_commit_delay,
+
1330 mi_option_arena_eager_commit,
+
1331 mi_option_abandoned_page_purge,
+
1332 mi_option_purge_delay,
+
1333 mi_option_use_numa_nodes,
+
1334 mi_option_disallow_os_alloc,
+
1335 mi_option_max_segment_reclaim,
+
1336 mi_option_destroy_on_exit,
+
1337 mi_option_arena_purge_mult,
+
1338 mi_option_abandoned_reclaim_on_free,
+
1339 mi_option_purge_extend_delay,
+
1340 mi_option_disallow_arena_alloc,
+
1341 mi_option_visit_abandoned,
+
1342 mi_option_target_segments_per_thread,
+
1343
+
1344 // v3 options
+
1345 mi_option_page_reclaim_on_free,
+
1346 mi_option_page_full_retain,
+
1347 mi_option_page_max_candidates,
+
1348 mi_option_max_vabits,
+
1349 mi_option_pagemap_commit,
+
1350 mi_option_page_commit_on_demand,
+
1351 mi_option_page_max_reclaim,
+
1352 mi_option_page_cross_thread_max_reclaim,
+
1353 mi_option_minimal_purge_size,
+
1354 mi_option_arena_max_object_size,
+
1355
+
1356 _mi_option_last
+
1357} mi_option_t;
-
1321
-
1322
-
1323bool mi_option_is_enabled(mi_option_t option);
-
1324void mi_option_enable(mi_option_t option);
-
1325void mi_option_disable(mi_option_t option);
-
1326void mi_option_set_enabled(mi_option_t option, bool enable);
-
1327void mi_option_set_enabled_default(mi_option_t option, bool enable);
-
1328
-
1329long mi_option_get(mi_option_t option);
-
1330long mi_option_get_clamp(mi_option_t option, long min, long max);
-
1331size_t mi_option_get_size(mi_option_t option);
-
1332
-
1333void mi_option_set(mi_option_t option, long value);
-
1334void mi_option_set_default(mi_option_t option, long value);
-
1335
-
1337
-
1350
-
1352struct mi_theap_s;
-
1353
-
1355typedef struct mi_theap_s mi_theap_t;
-
1356
-
1363mi_theap_t* mi_heap_theap(mi_heap_t* heap);
-
1364
-
1366void mi_theap_collect(mi_theap_t* theap, bool force);
-
1367
-
1372mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
-
1373
-
1376mi_theap_t* mi_theap_get_default();
-
1377
-
1380void* mi_theap_malloc(mi_theap_t* theap, size_t size);
-
1381
-
1385void* mi_theap_malloc_small(mi_theap_t* theap, size_t size);
-
1386
-
1389void* mi_theap_zalloc(mi_theap_t* theap, size_t size);
+
1358
+
1359
+
1360bool mi_option_is_enabled(mi_option_t option);
+
1361void mi_option_enable(mi_option_t option);
+
1362void mi_option_disable(mi_option_t option);
+
1363void mi_option_set_enabled(mi_option_t option, bool enable);
+
1364void mi_option_set_enabled_default(mi_option_t option, bool enable);
+
1365
+
1366long mi_option_get(mi_option_t option);
+
1367long mi_option_get_clamp(mi_option_t option, long min, long max);
+
1368size_t mi_option_get_size(mi_option_t option);
+
1369
+
1370void mi_option_set(mi_option_t option, long value);
+
1371void mi_option_set_default(mi_option_t option, long value);
+
1372
+
1374
+
1387
+
1389struct mi_theap_s;
1390
-
1394void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size);
-
1395
-
1396void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size);
-
1397void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment);
-
1398void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize);
-
1399
+
1392typedef struct mi_theap_s mi_theap_t;
+
1393
+
1400mi_theap_t* mi_heap_theap(mi_heap_t* heap);
1401
-
1402
-
1408
-
1411void mi_cfree(void* p);
-
1412void* mi__expand(void* p, size_t newsize);
-
1413
-
1414size_t mi_malloc_size(const void* p);
-
1415size_t mi_malloc_good_size(size_t size);
-
1416size_t mi_malloc_usable_size(const void *p);
-
1417
-
1418int mi_posix_memalign(void** p, size_t alignment, size_t size);
-
1419int mi__posix_memalign(void** p, size_t alignment, size_t size);
-
1420void* mi_memalign(size_t alignment, size_t size);
-
1421void* mi_valloc(size_t size);
-
1422void* mi_pvalloc(size_t size);
-
1423void* mi_aligned_alloc(size_t alignment, size_t size);
-
1424
-
1425unsigned short* mi_wcsdup(const unsigned short* s);
-
1426unsigned char* mi_mbsdup(const unsigned char* s);
-
1427int mi_dupenv_s(char** buf, size_t* size, const char* name);
-
1428int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name);
-
1429
-
1432void* mi_reallocarray(void* p, size_t count, size_t size);
-
1433
-
1435int mi_reallocarr(void* p, size_t count, size_t size);
+
1403void mi_theap_collect(mi_theap_t* theap, bool force);
+
1404
+
1409mi_theap_t* mi_theap_set_default(mi_theap_t* theap);
+
1410
+
1413mi_theap_t* mi_theap_get_default();
+
1414
+
1417void* mi_theap_malloc(mi_theap_t* theap, size_t size);
+
1418
+
1422void* mi_theap_malloc_small(mi_theap_t* theap, size_t size);
+
1423
+
1426void* mi_theap_zalloc(mi_theap_t* theap, size_t size);
+
1427
+
1431void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size);
+
1432
+
1433void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size);
+
1434void* mi_theap_malloc_aligned(mi_theap_t* theap, size_t size, size_t alignment);
+
1435void* mi_theap_realloc(mi_theap_t* theap, void* p, size_t newsize);
1436
-
1437void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment);
-
1438void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
+
1438
1439
-
1447void mi_free_size(void* p, size_t size);
-
1448
-
1454void mi_free_size_aligned(void* p, size_t size, size_t alignment);
-
1455
-
1456void mi_free_aligned(void* p, size_t alignment);
-
1457
-
1459
-
1472
-
1474void* mi_new(std::size_t n) noexcept(false);
-
1475
-
1477void* mi_new_n(size_t count, size_t size) noexcept(false);
-
1478
-
1480void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false);
-
1481
-
1483void* mi_new_nothrow(size_t n);
-
1484
-
1486void* mi_new_aligned_nothrow(size_t n, size_t alignment);
-
1487
-
1489void* mi_new_realloc(void* p, size_t newsize);
-
1490
-
1492void* mi_new_reallocn(void* p, size_t newcount, size_t size);
-
1493
-
1501template<class T> struct mi_stl_allocator { }
-
1502
-
1504
+
1445
+
1446void* mi__expand(void* p, size_t newsize);
+
1447
+
1448size_t mi_malloc_size(const void* p);
+
1449size_t mi_malloc_good_size(size_t size);
+
1450size_t mi_malloc_usable_size(const void *p);
+
1451
+
1452int mi_posix_memalign(void** p, size_t alignment, size_t size);
+
1453int mi__posix_memalign(void** p, size_t alignment, size_t size);
+
1454void* mi_memalign(size_t alignment, size_t size);
+
1455void* mi_valloc(size_t size);
+
1456void* mi_pvalloc(size_t size);
+
1457void* mi_aligned_alloc(size_t alignment, size_t size);
+
1458
+
1459unsigned short* mi_wcsdup(const unsigned short* s);
+
1460unsigned char* mi_mbsdup(const unsigned char* s);
+
1461int mi_dupenv_s(char** buf, size_t* size, const char* name);
+
1462int mi_wdupenv_s(unsigned short** buf, size_t* size, const unsigned short* name);
+
1463
+
1466void* mi_reallocarray(void* p, size_t count, size_t size);
+
1467
+
1469int mi_reallocarr(void* p, size_t count, size_t size);
+
1470
+
1471void* mi_aligned_recalloc(void* p, size_t newcount, size_t size, size_t alignment);
+
1472void* mi_aligned_offset_recalloc(void* p, size_t newcount, size_t size, size_t alignment, size_t offset);
+
1473
+
1482void mi_free_size(void* p, size_t size);
+
1483
+
1491void mi_free_size_nonnull(void* p, size_t size);
+
1492
+
1498void mi_free_size_aligned(void* p, size_t size, size_t alignment);
+
1499
+
1500void mi_free_aligned(void* p, size_t alignment);
+
1501
+
1503
+
1516
+
1518void* mi_new(std::size_t n) noexcept(false);
+
1519
+
1521void* mi_new_n(size_t count, size_t size) noexcept(false);
+
1522
+
1524void* mi_new_aligned(std::size_t n, std::align_val_t alignment) noexcept(false);
+
1525
+
1527void* mi_new_nothrow(size_t n);
+
1528
+
1530void* mi_new_aligned_nothrow(size_t n, size_t alignment);
+
1531
+
1533void* mi_new_realloc(void* p, size_t newsize);
+
1534
+
1536void* mi_new_reallocn(void* p, size_t newcount, size_t size);
+
1537
+
1545template<class T> struct mi_stl_allocator { }
+
1546
+
1548
mi_malloc_aligned_at
void * mi_malloc_aligned_at(size_t size, size_t alignment, size_t offset)
Allocate size bytes aligned by alignment at a specified offset.
mi_calloc_aligned
void * mi_calloc_aligned(size_t count, size_t size, size_t alignment)
mi_realloc_aligned
void * mi_realloc_aligned(void *p, size_t newsize, size_t alignment)
@@ -652,17 +659,17 @@
mi_calloc_aligned_at
void * mi_calloc_aligned_at(size_t count, size_t size, size_t alignment, size_t offset)
mi_zalloc_aligned
void * mi_zalloc_aligned(size_t size, size_t alignment)
mi_realloc_aligned_at
void * mi_realloc_aligned_at(void *p, size_t newsize, size_t alignment, size_t offset)
-
mi_heap_area_t::heap_tag
int heap_tag
heap tag associated with this area (see mi_heap_new_ex)
Definition mimalloc-doc.h:687
-
mi_heap_area_t::block_size
size_t block_size
size in bytes of one block
Definition mimalloc-doc.h:685
-
mi_heap_area_t::committed
size_t committed
current committed bytes of this area
Definition mimalloc-doc.h:683
-
mi_heap_area_t::full_block_size
size_t full_block_size
size in bytes of a full block including padding and metadata.
Definition mimalloc-doc.h:686
-
mi_heap_area_t::used
size_t used
bytes in use by allocated blocks
Definition mimalloc-doc.h:684
-
mi_heap_area_t::blocks
void * blocks
start of the area containing heap blocks
Definition mimalloc-doc.h:681
-
mi_heap_area_t::reserved
size_t reserved
bytes reserved for this area
Definition mimalloc-doc.h:682
+
mi_heap_area_t::heap_tag
int heap_tag
heap tag associated with this area (see mi_heap_new_ex)
Definition mimalloc-doc.h:706
+
mi_heap_area_t::block_size
size_t block_size
size in bytes of one block
Definition mimalloc-doc.h:704
+
mi_heap_area_t::committed
size_t committed
current committed bytes of this area
Definition mimalloc-doc.h:702
+
mi_heap_area_t::full_block_size
size_t full_block_size
size in bytes of a full block including padding and metadata.
Definition mimalloc-doc.h:705
+
mi_heap_area_t::used
size_t used
bytes in use by allocated blocks
Definition mimalloc-doc.h:703
+
mi_heap_area_t::blocks
void * blocks
start of the area containing heap blocks
Definition mimalloc-doc.h:700
+
mi_heap_area_t::reserved
size_t reserved
bytes reserved for this area
Definition mimalloc-doc.h:701
mi_abandoned_visit_blocks
bool mi_abandoned_visit_blocks(mi_subproc_id_t subproc_id, int heap_tag, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
v1,v2: Visit all areas and blocks in abandoned heaps.
-
mi_block_visit_fun
bool mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg)
Visitor function passed to mi_heap_visit_blocks()
Definition mimalloc-doc.h:697
+
mi_block_visit_fun
bool mi_block_visit_fun(const mi_heap_t *heap, const mi_heap_area_t *area, void *block, size_t block_size, void *arg)
Visitor function passed to mi_heap_visit_blocks()
Definition mimalloc-doc.h:716
mi_heap_visit_blocks
bool mi_heap_visit_blocks(const mi_heap_t *heap, bool visit_blocks, mi_block_visit_fun *visitor, void *arg)
Visit all areas and blocks in a heap.
-
mi_heap_area_t
An area of heap space contains blocks of a single size.
Definition mimalloc-doc.h:680
+
mi_heap_area_t
An area of heap space contains blocks of a single size.
Definition mimalloc-doc.h:699
mi_reserve_os_memory
int mi_reserve_os_memory(size_t size, bool commit, bool allow_large)
Reserve OS memory for use by mimalloc.
mi_reserve_huge_os_pages_interleave
int mi_reserve_huge_os_pages_interleave(size_t pages, size_t numa_nodes, size_t timeout_msecs)
Reserve pages of huge OS pages (1GiB) evenly divided over numa_nodes nodes, but stops after at most t...
mi_reserve_os_memory_ex
int mi_reserve_os_memory_ex(size_t size, bool commit, bool allow_large, bool exclusive, mi_arena_id_t *arena_id)
Reserve OS memory to be managed in an arena.
@@ -672,12 +679,13 @@
mi_reserve_huge_os_pages_at_ex
int mi_reserve_huge_os_pages_at_ex(size_t pages, int numa_node, size_t timeout_msecs, bool exclusive, mi_arena_id_t *arena_id)
Reserve huge OS pages (1GiB) into a single arena.
mi_reserve_huge_os_pages_at
int mi_reserve_huge_os_pages_at(size_t pages, int numa_node, size_t timeout_msecs)
Reserve pages of huge OS pages (1GiB) at a specific numa_node, but stops after at most timeout_msecs ...
mi_debug_show_arenas
void mi_debug_show_arenas(void)
Show all current arena's.
-
mi_arena_id_t
int mi_arena_id_t
Each arena has an associated identifier.
Definition mimalloc-doc.h:747
+
mi_arena_id_t
int mi_arena_id_t
Each arena has an associated identifier.
Definition mimalloc-doc.h:766
mi_arena_area
void * mi_arena_area(mi_arena_id_t arena_id, size_t *size)
Return the start and size of an arena.
mi_arena_min_alignment
size_t mi_arena_min_alignment(void)
Return the minimal alignment required for managed OS memory.
mi_heap_new_in_arena
mi_heap_t * mi_heap_new_in_arena(mi_arena_id_t arena_id)
Create a new heap that only allocates in the specified arena.
mi_theap_malloc_csize
void * mi_theap_malloc_csize(mi_theap_t *theap, size_t size)
mi_theap_zalloc_csize
void * mi_theap_zalloc_csize(mi_theap_t *theap, size_t size)
+
mi_free_csize_nonnull
void mi_free_csize_nonnull(void *p, size_t size)
Free a non-null pointer with a known constant size.
mi_free_csize
void mi_free_csize(void *p, size_t size)
Free a pointer with a known constant size.
mi_malloc_csize
void * mi_malloc_csize(size_t size)
Allocate a constant size object.
mi_zalloc_csize
void * mi_zalloc_csize(size_t size)
@@ -688,19 +696,20 @@
mi_new_aligned_nothrow
void * mi_new_aligned_nothrow(size_t n, size_t alignment)
like mi_malloc_aligned, but when out of memory, use std::get_new_handler but return NULL on failure.
mi_new_reallocn
void * mi_new_reallocn(void *p, size_t newcount, size_t size)
like mi_reallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc excepti...
mi_new_n
void * mi_new_n(size_t count, size_t size) noexcept(false)
like mi_mallocn(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exceptio...
-
mi_stl_allocator
std::allocator implementation for mimalloc for use in STL containers. For example:
Definition mimalloc-doc.h:1501
+
mi_stl_allocator
std::allocator implementation for mimalloc for use in STL containers. For example:
Definition mimalloc-doc.h:1545
mi_thread_done
void mi_thread_done(void)
Uninitialize mimalloc on a thread.
-
mi_deferred_free_fun
void mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
Type of deferred free functions.
Definition mimalloc-doc.h:981
+
mi_free_small_nonnull
void mi_free_small_nonnull(void *p)
v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a ...
+
mi_deferred_free_fun
void mi_deferred_free_fun(bool force, unsigned long long heartbeat, void *arg)
Type of deferred free functions.
Definition mimalloc-doc.h:1000
mi_free_small
void mi_free_small(void *p)
v3: Can be used to free an object that was allocated with mi_malloc_small() or any allocation with a ...
mi_register_deferred_free
void mi_register_deferred_free(mi_deferred_free_fun *deferred_free, void *arg)
Register a deferred free function.
mi_collect
void mi_collect(bool force)
Eagerly free memory.
mi_zalloc_small
void * mi_zalloc_small(size_t size)
Allocate a zero initialized small object.
mi_malloc_small
void * mi_malloc_small(size_t size)
Allocate a small object.
-
mi_error_fun
void mi_error_fun(int err, void *arg)
Type of error callback functions.
Definition mimalloc-doc.h:1020
+
mi_error_fun
void mi_error_fun(int err, void *arg)
Type of error callback functions.
Definition mimalloc-doc.h:1039
mi_register_error
void mi_register_error(mi_error_fun *errfun, void *arg)
Register an error callback function.
mi_is_redirected
bool mi_is_redirected()
Is the C runtime malloc API redirected?
mi_thread_set_in_threadpool
void mi_thread_set_in_threadpool(void)
v3: Communicate that a thread is in a threadpool.
-
mi_output_fun
void mi_output_fun(const char *msg, void *arg)
Type of output functions.
Definition mimalloc-doc.h:1005
+
mi_output_fun
void mi_output_fun(const char *msg, void *arg)
Type of output functions.
Definition mimalloc-doc.h:1024
mi_register_output
void mi_register_output(mi_output_fun *out, void *arg)
Register an output function.
mi_version
int mi_version(void)
Return the mimalloc version.
mi_thread_init
void mi_thread_init(void)
Initialize mimalloc on a thread.
@@ -712,7 +721,7 @@
mi_heap_delete
void mi_heap_delete(mi_heap_t *heap)
Delete a previously allocated heap.
mi_heap_malloc_aligned
void * mi_heap_malloc_aligned(mi_heap_t *heap, size_t size, size_t alignment)
mi_heap_set_default
mi_heap_t * mi_heap_set_default(mi_heap_t *heap)
v1,v2: Set the default heap to use in the current thread for mi_malloc() et al.
-
mi_heap_t
struct mi_heap_s mi_heap_t
Type of first-class heaps.
Definition mimalloc-doc.h:513
+
mi_heap_t
struct mi_heap_s mi_heap_t
Type of first-class heaps.
Definition mimalloc-doc.h:532
mi_heap_of
mi_heap_t * mi_heap_of(const void *p)
v3: return the heap that contains the given pointer.
mi_heap_zalloc_aligned_at
void * mi_heap_zalloc_aligned_at(mi_heap_t *heap, size_t size, size_t alignment, size_t offset)
mi_heap_realpath
char * mi_heap_realpath(mi_heap_t *heap, const char *fname, char *resolved_name)
Resolve a file path name using a specific heap to allocate the result.
@@ -749,6 +758,7 @@
mi_reallocf
void * mi_reallocf(void *p, size_t newsize)
Re-allocate memory to newsize bytes,.
mi_mallocn
void * mi_mallocn(size_t count, size_t size)
Allocate count elements of size bytes.
mi_calloc
void * mi_calloc(size_t count, size_t size)
Allocate zero-initialized count elements of size bytes.
+
mi_cfree
void mi_cfree(void *p)
Checked free: just as mi_free() but always checks if the pointer p belongs to our heap.
mi_reallocn
void * mi_reallocn(void *p, size_t count, size_t size)
Re-allocate memory to count elements of size bytes.
mi_realpath
char * mi_realpath(const char *fname, char *resolved_name)
Resolve a file path name.
mi_good_size
size_t mi_good_size(size_t size)
Return the probable allocation block size for a given required size.
@@ -767,53 +777,53 @@
mi_option_disable
void mi_option_disable(mi_option_t option)
mi_options_print
void mi_options_print(void)
Print out all runtime parameters for mimalloc. Also printed with MIMALLOC_VERBOSE=1 at startup.
mi_option_set
void mi_option_set(mi_option_t option, long value)
-
mi_option_t
mi_option_t
Runtime options.
Definition mimalloc-doc.h:1263
-
mi_option_guarded_min
@ mi_option_guarded_min
only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
Definition mimalloc-doc.h:1284
-
mi_option_abandoned_reclaim_on_free
@ mi_option_abandoned_reclaim_on_free
v1,v2: allow to reclaim an abandoned segment on a free (=1)
Definition mimalloc-doc.h:1301
-
mi_option_purge_extend_delay
@ mi_option_purge_extend_delay
v1,v2: extend purge delay on each subsequent delay (=1)
Definition mimalloc-doc.h:1302
-
mi_option_allow_thp
@ mi_option_allow_thp
allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the proces...
Definition mimalloc-doc.h:1281
-
mi_option_show_stats
@ mi_option_show_stats
Print statistics on termination.
Definition mimalloc-doc.h:1266
-
mi_option_use_numa_nodes
@ mi_option_use_numa_nodes
0 = use all available numa nodes, otherwise use at most N nodes.
Definition mimalloc-doc.h:1296
-
mi_option_page_max_reclaim
@ mi_option_page_max_reclaim
v3: don't reclaim pages of the same originating theap if we already own N pages (in that size class) ...
Definition mimalloc-doc.h:1314
-
mi_option_abandoned_page_purge
@ mi_option_abandoned_page_purge
v1,v2: immediately purge delayed purges on thread termination
Definition mimalloc-doc.h:1294
-
mi_option_eager_commit_delay
@ mi_option_eager_commit_delay
v2: the first N segments per thread are not eagerly committed (but per page in the segment on demand)
Definition mimalloc-doc.h:1292
-
mi_option_eager_commit
@ mi_option_eager_commit
v1,v2: eager commit segments? (after eager_commit_delay segments) (enabled by default).
Definition mimalloc-doc.h:1291
-
mi_option_guarded_precise
@ mi_option_guarded_precise
disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard pa...
Definition mimalloc-doc.h:1286
-
mi_option_visit_abandoned
@ mi_option_visit_abandoned
allow visiting heap blocks from abandoned threads (=0)
Definition mimalloc-doc.h:1304
-
mi_option_os_tag
@ mi_option_os_tag
tag used for OS logging (macOS only for now) (=100)
Definition mimalloc-doc.h:1278
-
mi_option_arena_max_object_size
@ mi_option_arena_max_object_size
v3: set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
Definition mimalloc-doc.h:1317
-
mi_option_minimal_purge_size
@ mi_option_minimal_purge_size
v3: set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled....
Definition mimalloc-doc.h:1316
-
_mi_option_last
@ _mi_option_last
Definition mimalloc-doc.h:1319
-
mi_option_destroy_on_exit
@ mi_option_destroy_on_exit
if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
Definition mimalloc-doc.h:1299
-
mi_option_page_cross_thread_max_reclaim
@ mi_option_page_cross_thread_max_reclaim
v3: don't reclaim pages across threads if we already own N pages (in that size class) (=16)
Definition mimalloc-doc.h:1315
-
mi_option_page_commit_on_demand
@ mi_option_page_commit_on_demand
v3: commit page memory on-demand (=0)
Definition mimalloc-doc.h:1313
-
mi_option_guarded_sample_seed
@ mi_option_guarded_sample_seed
can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
Definition mimalloc-doc.h:1288
-
mi_option_verbose
@ mi_option_verbose
Print verbose messages.
Definition mimalloc-doc.h:1267
-
mi_option_allow_large_os_pages
@ mi_option_allow_large_os_pages
allow large (2 or 4 MiB) OS pages, implies eager commit.
Definition mimalloc-doc.h:1275
-
mi_option_arena_purge_mult
@ mi_option_arena_purge_mult
multiplier for purge_delay for the purging delay for arenas (=10)
Definition mimalloc-doc.h:1300
-
mi_option_target_segments_per_thread
@ mi_option_target_segments_per_thread
v1,v2: experimental (=0)
Definition mimalloc-doc.h:1305
-
mi_option_generic_collect
@ mi_option_generic_collect
collect heaps every N (=10000) generic allocation calls
Definition mimalloc-doc.h:1280
-
mi_option_retry_on_oom
@ mi_option_retry_on_oom
retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
Definition mimalloc-doc.h:1279
-
mi_option_guarded_max
@ mi_option_guarded_max
only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
Definition mimalloc-doc.h:1285
-
mi_option_purge_decommits
@ mi_option_purge_decommits
should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
Definition mimalloc-doc.h:1276
-
mi_option_reserve_huge_os_pages_at
@ mi_option_reserve_huge_os_pages_at
Reserve N huge OS pages at a specific NUMA node N.
Definition mimalloc-doc.h:1273
-
mi_option_pagemap_commit
@ mi_option_pagemap_commit
v3: commit the full pagemap (to always catch invalid pointer uses) (=0)
Definition mimalloc-doc.h:1312
-
mi_option_max_segment_reclaim
@ mi_option_max_segment_reclaim
v2: max. percentage of the abandoned segments can be reclaimed per try (=10%)
Definition mimalloc-doc.h:1298
-
mi_option_max_vabits
@ mi_option_max_vabits
v3: max user space virtual address bits to consider (=48)
Definition mimalloc-doc.h:1311
-
mi_option_arena_reserve
@ mi_option_arena_reserve
initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use ...
Definition mimalloc-doc.h:1277
-
mi_option_page_reclaim_on_free
@ mi_option_page_reclaim_on_free
v3: reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from...
Definition mimalloc-doc.h:1308
-
mi_option_reserve_huge_os_pages
@ mi_option_reserve_huge_os_pages
reserve N huge OS pages (1GiB pages) at startup
Definition mimalloc-doc.h:1272
-
mi_option_page_max_candidates
@ mi_option_page_max_candidates
v3: max candidate pages to consider for allocation (=4)
Definition mimalloc-doc.h:1310
-
mi_option_disallow_os_alloc
@ mi_option_disallow_os_alloc
1 = do not use OS memory for allocation (but only programmatically reserved arenas)
Definition mimalloc-doc.h:1297
-
mi_option_purge_delay
@ mi_option_purge_delay
memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all...
Definition mimalloc-doc.h:1295
-
mi_option_disallow_arena_alloc
@ mi_option_disallow_arena_alloc
1 = do not use arena's for allocation (except if using specific arena id's)
Definition mimalloc-doc.h:1303
-
mi_option_guarded_sample_rate
@ mi_option_guarded_sample_rate
1 out of N allocations in the min/max range will be guarded (=1000)
Definition mimalloc-doc.h:1287
-
mi_option_max_errors
@ mi_option_max_errors
issue at most N error messages
Definition mimalloc-doc.h:1268
-
mi_option_page_full_retain
@ mi_option_page_full_retain
v3: retain N full (small) pages per size class (=2). Use -1 for infinite (as in v1,...
Definition mimalloc-doc.h:1309
-
mi_option_max_warnings
@ mi_option_max_warnings
issue at most N warning messages
Definition mimalloc-doc.h:1269
-
mi_option_show_errors
@ mi_option_show_errors
Print error messages.
Definition mimalloc-doc.h:1265
-
mi_option_reserve_os_memory
@ mi_option_reserve_os_memory
reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use m...
Definition mimalloc-doc.h:1274
-
mi_option_arena_eager_commit
@ mi_option_arena_eager_commit
eager commit arenas? Use 2 to enable just on overcommit systems (=2)
Definition mimalloc-doc.h:1293
+
mi_option_t
mi_option_t
Runtime options.
Definition mimalloc-doc.h:1300
+
mi_option_guarded_min
@ mi_option_guarded_min
only used when building with MI_GUARDED: minimal rounded object size for guarded objects (=0)
Definition mimalloc-doc.h:1321
+
mi_option_abandoned_reclaim_on_free
@ mi_option_abandoned_reclaim_on_free
v1,v2: allow to reclaim an abandoned segment on a free (=1)
Definition mimalloc-doc.h:1338
+
mi_option_purge_extend_delay
@ mi_option_purge_extend_delay
v1,v2: extend purge delay on each subsequent delay (=1)
Definition mimalloc-doc.h:1339
+
mi_option_allow_thp
@ mi_option_allow_thp
allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the proces...
Definition mimalloc-doc.h:1318
+
mi_option_show_stats
@ mi_option_show_stats
Print statistics on termination.
Definition mimalloc-doc.h:1303
+
mi_option_use_numa_nodes
@ mi_option_use_numa_nodes
0 = use all available numa nodes, otherwise use at most N nodes.
Definition mimalloc-doc.h:1333
+
mi_option_page_max_reclaim
@ mi_option_page_max_reclaim
v3: don't reclaim pages of the same originating theap if we already own N pages (in that size class) ...
Definition mimalloc-doc.h:1351
+
mi_option_abandoned_page_purge
@ mi_option_abandoned_page_purge
v1,v2: immediately purge delayed purges on thread termination
Definition mimalloc-doc.h:1331
+
mi_option_eager_commit_delay
@ mi_option_eager_commit_delay
v2: the first N segments per thread are not eagerly committed (but per page in the segment on demand)
Definition mimalloc-doc.h:1329
+
mi_option_eager_commit
@ mi_option_eager_commit
v1,v2: eager commit segments? (after eager_commit_delay segments) (enabled by default).
Definition mimalloc-doc.h:1328
+
mi_option_guarded_precise
@ mi_option_guarded_precise
disregard minimal alignment requirement to always place guarded blocks exactly in front of a guard pa...
Definition mimalloc-doc.h:1323
+
mi_option_visit_abandoned
@ mi_option_visit_abandoned
allow visiting heap blocks from abandoned threads (=0)
Definition mimalloc-doc.h:1341
+
mi_option_os_tag
@ mi_option_os_tag
tag used for OS logging (macOS only for now) (=100)
Definition mimalloc-doc.h:1315
+
mi_option_arena_max_object_size
@ mi_option_arena_max_object_size
v3: set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit).
Definition mimalloc-doc.h:1354
+
mi_option_minimal_purge_size
@ mi_option_minimal_purge_size
v3: set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled....
Definition mimalloc-doc.h:1353
+
_mi_option_last
@ _mi_option_last
Definition mimalloc-doc.h:1356
+
mi_option_destroy_on_exit
@ mi_option_destroy_on_exit
if set, release all memory on exit; sometimes used for dynamic unloading but can be unsafe
Definition mimalloc-doc.h:1336
+
mi_option_page_cross_thread_max_reclaim
@ mi_option_page_cross_thread_max_reclaim
v3: don't reclaim pages across threads if we already own N pages (in that size class) (=16)
Definition mimalloc-doc.h:1352
+
mi_option_page_commit_on_demand
@ mi_option_page_commit_on_demand
v3: commit page memory on-demand (=0)
Definition mimalloc-doc.h:1350
+
mi_option_guarded_sample_seed
@ mi_option_guarded_sample_seed
can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0)
Definition mimalloc-doc.h:1325
+
mi_option_verbose
@ mi_option_verbose
Print verbose messages.
Definition mimalloc-doc.h:1304
+
mi_option_allow_large_os_pages
@ mi_option_allow_large_os_pages
allow large (2 or 4 MiB) OS pages, implies eager commit.
Definition mimalloc-doc.h:1312
+
mi_option_arena_purge_mult
@ mi_option_arena_purge_mult
multiplier for purge_delay for the purging delay for arenas (=10)
Definition mimalloc-doc.h:1337
+
mi_option_target_segments_per_thread
@ mi_option_target_segments_per_thread
v1,v2: experimental (=0)
Definition mimalloc-doc.h:1342
+
mi_option_generic_collect
@ mi_option_generic_collect
collect heaps every N (=10000) generic allocation calls
Definition mimalloc-doc.h:1317
+
mi_option_retry_on_oom
@ mi_option_retry_on_oom
retry on out-of-memory for N milli seconds (=400), set to 0 to disable retries. (only on windows)
Definition mimalloc-doc.h:1316
+
mi_option_guarded_max
@ mi_option_guarded_max
only used when building with MI_GUARDED: maximal rounded object size for guarded objects (=0)
Definition mimalloc-doc.h:1322
+
mi_option_purge_decommits
@ mi_option_purge_decommits
should a memory purge decommit? (=1). Set to 0 to use memory reset on a purge (instead of decommit)
Definition mimalloc-doc.h:1313
+
mi_option_reserve_huge_os_pages_at
@ mi_option_reserve_huge_os_pages_at
Reserve N huge OS pages at a specific NUMA node N.
Definition mimalloc-doc.h:1310
+
mi_option_pagemap_commit
@ mi_option_pagemap_commit
v3: commit the full pagemap (to always catch invalid pointer uses) (=0)
Definition mimalloc-doc.h:1349
+
mi_option_max_segment_reclaim
@ mi_option_max_segment_reclaim
v2: max. percentage of the abandoned segments can be reclaimed per try (=10%)
Definition mimalloc-doc.h:1335
+
mi_option_max_vabits
@ mi_option_max_vabits
v3: max user space virtual address bits to consider (=48)
Definition mimalloc-doc.h:1348
+
mi_option_arena_reserve
@ mi_option_arena_reserve
initial memory size for arena reservation (= 1 GiB on 64-bit) (internally, this value is in KiB; use ...
Definition mimalloc-doc.h:1314
+
mi_option_page_reclaim_on_free
@ mi_option_page_reclaim_on_free
v3: reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from...
Definition mimalloc-doc.h:1345
+
mi_option_reserve_huge_os_pages
@ mi_option_reserve_huge_os_pages
reserve N huge OS pages (1GiB pages) at startup
Definition mimalloc-doc.h:1309
+
mi_option_page_max_candidates
@ mi_option_page_max_candidates
v3: max candidate pages to consider for allocation (=4)
Definition mimalloc-doc.h:1347
+
mi_option_disallow_os_alloc
@ mi_option_disallow_os_alloc
1 = do not use OS memory for allocation (but only programmatically reserved arenas)
Definition mimalloc-doc.h:1334
+
mi_option_purge_delay
@ mi_option_purge_delay
memory purging is delayed by N milli seconds; use 0 for immediate purging or -1 for no purging at all...
Definition mimalloc-doc.h:1332
+
mi_option_disallow_arena_alloc
@ mi_option_disallow_arena_alloc
1 = do not use arena's for allocation (except if using specific arena id's)
Definition mimalloc-doc.h:1340
+
mi_option_guarded_sample_rate
@ mi_option_guarded_sample_rate
1 out of N allocations in the min/max range will be guarded (=1000)
Definition mimalloc-doc.h:1324
+
mi_option_max_errors
@ mi_option_max_errors
issue at most N error messages
Definition mimalloc-doc.h:1305
+
mi_option_page_full_retain
@ mi_option_page_full_retain
v3: retain N full (small) pages per size class (=2). Use -1 for infinite (as in v1,...
Definition mimalloc-doc.h:1346
+
mi_option_max_warnings
@ mi_option_max_warnings
issue at most N warning messages
Definition mimalloc-doc.h:1306
+
mi_option_show_errors
@ mi_option_show_errors
Print error messages.
Definition mimalloc-doc.h:1302
+
mi_option_reserve_os_memory
@ mi_option_reserve_os_memory
reserve specified amount of OS memory in an arena at startup (internally, this value is in KiB; use m...
Definition mimalloc-doc.h:1311
+
mi_option_arena_eager_commit
@ mi_option_arena_eager_commit
eager commit arenas? Use 2 to enable just on overcommit systems (=2)
Definition mimalloc-doc.h:1330
mi_malloc_usable_size
size_t mi_malloc_usable_size(const void *p)
mi_free_aligned
void mi_free_aligned(void *p, size_t alignment)
mi_aligned_offset_recalloc
void * mi_aligned_offset_recalloc(void *p, size_t newcount, size_t size, size_t alignment, size_t offset)
@@ -823,9 +833,9 @@
mi_pvalloc
void * mi_pvalloc(size_t size)
mi__expand
void * mi__expand(void *p, size_t newsize)
mi_wdupenv_s
int mi_wdupenv_s(unsigned short **buf, size_t *size, const unsigned short *name)
-
mi_cfree
void mi_cfree(void *p)
Checked free: just as mi_free but always checks if the pointer p belongs to our heap and is safe to u...
mi_memalign
void * mi_memalign(size_t alignment, size_t size)
mi_free_size_aligned
void mi_free_size_aligned(void *p, size_t size, size_t alignment)
Free an object that was allocated aligned.
+
mi_free_size_nonnull
void mi_free_size_nonnull(void *p, size_t size)
Free a non-NULL pointer that was allocated with a known size.
mi_mbsdup
unsigned char * mi_mbsdup(const unsigned char *s)
mi_reallocarr
int mi_reallocarr(void *p, size_t count, size_t size)
Corresponds to reallocarr in NetBSD.
mi_malloc_good_size
size_t mi_malloc_good_size(size_t size)
@@ -834,10 +844,10 @@
mi_posix_memalign
int mi_posix_memalign(void **p, size_t alignment, size_t size)
mi__posix_memalign
int mi__posix_memalign(void **p, size_t alignment, size_t size)
mi_reallocarray
void * mi_reallocarray(void *p, size_t count, size_t size)
Correspond s to reallocarray in FreeBSD.
-
mi_free_size
void mi_free_size(void *p, size_t size)
Free an object that was allocated with a known size.
+
mi_free_size
void mi_free_size(void *p, size_t size)
Free a block that was allocated with a known size.
mi_aligned_recalloc
void * mi_aligned_recalloc(void *p, size_t newcount, size_t size, size_t alignment)
-
mi_stats_s::version
size_t version
initialize with `MI_STAT_VERSION (so linking dynamically with a separately compiled mimalloc is safe)
Definition mimalloc-doc.h:1106
-
mi_stats_s::size
size_t size
initialize with sizeof(mi_stats_t) (so linking dynamically with a separately compiled mimalloc is saf...
Definition mimalloc-doc.h:1104
+
mi_stats_s::version
size_t version
initialize with `MI_STAT_VERSION (so linking dynamically with a separately compiled mimalloc is safe)
Definition mimalloc-doc.h:1143
+
mi_stats_s::size
size_t size
initialize with sizeof(mi_stats_t) (so linking dynamically with a separately compiled mimalloc is saf...
Definition mimalloc-doc.h:1141
mi_process_info_print_out
void mi_process_info_print_out(mi_output_fun *out, void *arg)
Print out all process info.
mi_subproc_heap_stats_print_out
void mi_subproc_heap_stats_print_out(mi_subproc_id_t subproc_id, mi_output_fun *out, void *arg)
v3: Print statistics for a given subprocess with each heap separately printed.
mi_heap_stats_get
bool mi_heap_stats_get(mi_heap_t *heap, mi_stats_t *stats)
v3: Return statistics for a given heap.
@@ -857,11 +867,11 @@
mi_stats_get_bin_size
size_t mi_stats_get_bin_size(size_t bin)
v3: Return the block size for the given bin.
mi_process_info_print
void mi_process_info_print(void)
Print out all process info.
mi_heap_stats_print_out
void mi_heap_stats_print_out(mi_heap_t *heap, mi_output_fun *out, void *arg)
v3: Show the heap statistics as JSON.
-
mi_stats_s
Statistics. See include/mimalloc-stats.h for the full definition.
Definition mimalloc-doc.h:1102
+
mi_stats_s
Statistics. See include/mimalloc-stats.h for the full definition.
Definition mimalloc-doc.h:1139
mi_subproc_main
mi_subproc_id_t mi_subproc_main(void)
Get the main sub-process identifier.
-
mi_heap_visit_fun
bool mi_heap_visit_fun(mi_heap_t *heap, void *arg)
The type of a heap visitor function.
Definition mimalloc-doc.h:911
+
mi_heap_visit_fun
bool mi_heap_visit_fun(mi_heap_t *heap, void *arg)
The type of a heap visitor function.
Definition mimalloc-doc.h:930
mi_subproc_new
mi_subproc_id_t mi_subproc_new(void)
Create a fresh sub-process (with no associated threads yet).
-
mi_subproc_id_t
void * mi_subproc_id_t
A process can associate threads with sub-processes.
Definition mimalloc-doc.h:880
+
mi_subproc_id_t
void * mi_subproc_id_t
A process can associate threads with sub-processes.
Definition mimalloc-doc.h:899
mi_subproc_delete
void mi_subproc_delete(mi_subproc_id_t subproc)
v1,v2: Delete a previously created sub-process.
mi_subproc_current
mi_subproc_id_t mi_subproc_current(void)
Get the current sub-process identifier of this thread.
mi_subproc_destroy
void mi_subproc_destroy(mi_subproc_id_t subproc)
v3: Destroy a previously created sub-process.
@@ -873,7 +883,7 @@
mi_theap_get_default
mi_theap_t * mi_theap_get_default()
Get the default theap that is used for mi_malloc() et al. for the current thread.
mi_heap_theap
mi_theap_t * mi_heap_theap(mi_heap_t *heap)
Return the thread-local theap for the given heap.
mi_theap_malloc_aligned
void * mi_theap_malloc_aligned(mi_theap_t *theap, size_t size, size_t alignment)
-
mi_theap_t
struct mi_theap_s mi_theap_t
Type of thread-local heaps.
Definition mimalloc-doc.h:1355
+
mi_theap_t
struct mi_theap_s mi_theap_t
Type of thread-local heaps.
Definition mimalloc-doc.h:1392
mi_theap_collect
void mi_theap_collect(mi_theap_t *theap, bool force)
Release outstanding resources in a specific theap.
mi_theap_zalloc
void * mi_theap_zalloc(mi_theap_t *theap, size_t size)
Allocate zero-initialized in a specific heap.
mi_theap_zalloc_small
void * mi_theap_zalloc_small(mi_theap_t *theap, size_t size)
Allocate a small object in a specific theap. size must be smaller or equal to MI_SMALL_SIZE_MAX().
diff --git a/docs/modes.html b/docs/modes.html index a2c488c29..7e29f1c65 100644 --- a/docs/modes.html +++ b/docs/modes.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/navtreedata.js b/docs/navtreedata.js index 91658f29a..dc276b57b 100644 --- a/docs/navtreedata.js +++ b/docs/navtreedata.js @@ -47,7 +47,7 @@ var NAVTREE = var NAVTREEINDEX = [ "annotated.html", -"group__theap.html#ga7905a73865733da8a43339a500533180" +"group__theap.html#ga11943128dc354313c4189e72f4299dbb" ]; var SYNCONMSG = 'click to disable panel synchronisation'; diff --git a/docs/navtreeindex0.js b/docs/navtreeindex0.js index 24e0f8478..fd485b71e 100644 --- a/docs/navtreeindex0.js +++ b/docs/navtreeindex0.js @@ -43,11 +43,12 @@ var NAVTREEINDEX0 = "group__arenas.html#gaa1a59464dbf3bc972d34fca0f86668fc":[7,7,2], "group__arenas.html#gaaf2d9976576d5efd5544be12848af949":[7,7,5], "group__constantsize.html":[7,3], -"group__constantsize.html#ga105bd667da91a15d113fe79be04d625c":[7,3,2], -"group__constantsize.html#ga1addf755829380581dd9c3c9175e376f":[7,3,3], +"group__constantsize.html#ga105bd667da91a15d113fe79be04d625c":[7,3,3], +"group__constantsize.html#ga1addf755829380581dd9c3c9175e376f":[7,3,4], +"group__constantsize.html#ga38a93d373a9dc8edc8bca83edb6af5c5":[7,3,1], "group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5":[7,3,0], -"group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571":[7,3,1], -"group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec":[7,3,4], +"group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571":[7,3,2], +"group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec":[7,3,5], "group__cpp.html":[7,14], "group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54":[7,14,5], "group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a":[7,14,1], @@ -58,22 +59,23 @@ var NAVTREEINDEX0 = "group__cpp.html#gadd11b85c15d21d308386844b5233856c":[7,14,4], "group__cpp.html#structmi__stl__allocator":[7,14,0], "group__extended.html":[7,9], -"group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf":[7,9,11], +"group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf":[7,9,12], "group__extended.html#ga1ea64283508718d9d645c38efc2f4305":[7,9,0], +"group__extended.html#ga22890f54e5be433837069c8e97fdf379":[7,9,6], "group__extended.html#ga292a45f7dbc7cd23c5352ce1f0002816":[7,9,1], "group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610":[7,9,5], -"group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece":[7,9,8], +"group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece":[7,9,9], "group__extended.html#ga421430e2226d7d468529cec457396756":[7,9,4], -"group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e":[7,9,15], -"group__extended.html#ga7f050bc6b897da82692174f5fce59cde":[7,9,7], +"group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e":[7,9,16], +"group__extended.html#ga7f050bc6b897da82692174f5fce59cde":[7,9,8], "group__extended.html#ga83fc6a688b322261e1c2deab000b0591":[7,9,2], -"group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45":[7,9,9], -"group__extended.html#gaad25050b19f30cd79397b227e0157a3f":[7,9,6], -"group__extended.html#gada899104276aac52539eb4de7c020f1f":[7,9,13], +"group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45":[7,9,10], +"group__extended.html#gaad25050b19f30cd79397b227e0157a3f":[7,9,7], +"group__extended.html#gada899104276aac52539eb4de7c020f1f":[7,9,14], "group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d":[7,9,3], -"group__extended.html#gae5b17ff027cd2150b43a33040250cf3f":[7,9,10], -"group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057":[7,9,14], -"group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17":[7,9,12], +"group__extended.html#gae5b17ff027cd2150b43a33040250cf3f":[7,9,11], +"group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057":[7,9,15], +"group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17":[7,9,13], "group__heap.html":[7,5], "group__heap.html#ga012c5c8abe22b10043de39ff95909541":[7,5,18], "group__heap.html#ga0994f44cc595f9c1840a9af162d98597":[7,5,35], @@ -113,20 +115,21 @@ var NAVTREEINDEX0 = "group__heap.html#gae7cd171425bee04c683c65a3701f0b4a":[7,5,25], "group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa":[7,5,17], "group__malloc.html":[7,0], -"group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583":[7,0,6], -"group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee":[7,0,12], -"group__malloc.html#ga19299856216cfbb08e2628593654dfb0":[7,0,1], -"group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889":[7,0,10], -"group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d":[7,0,11], -"group__malloc.html#ga4dc3a4067037b151a64629fe8a332641":[7,0,7], -"group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6":[7,0,5], +"group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583":[7,0,7], +"group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee":[7,0,13], +"group__malloc.html#ga19299856216cfbb08e2628593654dfb0":[7,0,2], +"group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889":[7,0,11], +"group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d":[7,0,12], +"group__malloc.html#ga4dc3a4067037b151a64629fe8a332641":[7,0,8], +"group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6":[7,0,6], "group__malloc.html#ga6686568014b54d1e6c7ac64a076e4f56":[7,0,0], -"group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467":[7,0,8], -"group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd":[7,0,9], -"group__malloc.html#gac057927cd06c854b45fe7847e921bd47":[7,0,3], -"group__malloc.html#gae1dd97b542420c87ae085e822b1229e8":[7,0,4], -"group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8":[7,0,13], -"group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95":[7,0,2], +"group__malloc.html#ga705dc7a64bffacfeeb0141501a5c35d7":[7,0,1], +"group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467":[7,0,9], +"group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd":[7,0,10], +"group__malloc.html#gac057927cd06c854b45fe7847e921bd47":[7,0,4], +"group__malloc.html#gae1dd97b542420c87ae085e822b1229e8":[7,0,5], +"group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8":[7,0,14], +"group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95":[7,0,3], "group__options.html":[7,11], "group__options.html#ga04180ae41b0d601421dd62ced40ca050":[7,11,2], "group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2":[7,11,12], @@ -189,7 +192,7 @@ var NAVTREEINDEX0 = "group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5":[7,11,0,22], "group__posix.html":[7,13], "group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17":[7,13,12], -"group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9":[7,13,7], +"group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9":[7,13,6], "group__posix.html#ga16570deddd559001b44953eedbad0084":[7,13,3], "group__posix.html#ga430ed1513f0571ff83be00ec58a98ee0":[7,13,2], "group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de":[7,13,11], @@ -197,18 +200,18 @@ var NAVTREEINDEX0 = "group__posix.html#ga644bebccdbb2821542dd8c7e7641f476":[7,13,16], "group__posix.html#ga66bcfeb4faedbb42b796bc680821ef84":[7,13,0], "group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1":[7,13,21], -"group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7":[7,13,5], "group__posix.html#ga726867f13fd29ca36064954c0285b1d8":[7,13,14], -"group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc":[7,13,9], +"group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc":[7,13,8], +"group__posix.html#ga796602dae2a89797026f72e0c0240f32":[7,13,9], "group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0":[7,13,13], "group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5":[7,13,17], "group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071":[7,13,10], "group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf":[7,13,20], -"group__posix.html#gab41369c1a1da7504013a7a0b1d4dd958":[7,13,6], +"group__posix.html#gab41369c1a1da7504013a7a0b1d4dd958":[7,13,5], "group__posix.html#gacff84f226ba9feb2031b8992e5579447":[7,13,15], "group__posix.html#gad5a69c8fea96aa2b7a7c818c2130090a":[7,13,1], "group__posix.html#gadfeccb72748a2f6305474a37d9d57bce":[7,13,18], -"group__posix.html#gae01389eedab8d67341ff52e2aad80ebb":[7,13,8], +"group__posix.html#gae01389eedab8d67341ff52e2aad80ebb":[7,13,7], "group__posix.html#gaf82cbb4b4f24acf723348628451798d3":[7,13,4], "group__stats.html":[7,10], "group__stats.html#aa74fc9c40dfaef5e06dec4019c19de68":[7,10,0,1], @@ -246,8 +249,5 @@ var NAVTREEINDEX0 = "group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c":[7,8,2], "group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93":[7,8,8], "group__theap.html":[7,12], -"group__theap.html#ga04420ce52785019412397e4de24a984a":[7,12,9], -"group__theap.html#ga11943128dc354313c4189e72f4299dbb":[7,12,8], -"group__theap.html#ga1fc65302a212daaecb21abda96ff247c":[7,12,7], -"group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5":[7,12,4] +"group__theap.html#ga04420ce52785019412397e4de24a984a":[7,12,9] }; diff --git a/docs/navtreeindex1.js b/docs/navtreeindex1.js index 79286c5a6..3b01ba053 100644 --- a/docs/navtreeindex1.js +++ b/docs/navtreeindex1.js @@ -1,5 +1,8 @@ var NAVTREEINDEX1 = { +"group__theap.html#ga11943128dc354313c4189e72f4299dbb":[7,12,8], +"group__theap.html#ga1fc65302a212daaecb21abda96ff247c":[7,12,7], +"group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5":[7,12,4], "group__theap.html#ga7905a73865733da8a43339a500533180":[7,12,1], "group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36":[7,12,6], "group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8":[7,12,0], diff --git a/docs/overrides.html b/docs/overrides.html index d5736734f..d7f7e0909 100644 --- a/docs/overrides.html +++ b/docs/overrides.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/pages.html b/docs/pages.html index c1d698676..5b13b6dfd 100644 --- a/docs/pages.html +++ b/docs/pages.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/search/all_9.js b/docs/search/all_9.js index 54d47ba9a..bf716a23b 100644 --- a/docs/search/all_9.js +++ b/docs/search/all_9.js @@ -19,7 +19,7 @@ var searchData= ['mi_5fcalloc_5faligned_16',['mi_calloc_aligned',['../group__aligned.html#ga424ef386fb1f9f8e0a86ab53f16eaaf1',1,'mimalloc-doc.h']]], ['mi_5fcalloc_5faligned_5fat_17',['mi_calloc_aligned_at',['../group__aligned.html#ga977f96bd2c5c141bcd70e6685c90d6c3',1,'mimalloc-doc.h']]], ['mi_5fcalloc_5ftp_18',['mi_calloc_tp',['../group__typed.html#gae80c47c9d4cab10961fff1a8ac98fc07',1,'mimalloc-doc.h']]], - ['mi_5fcfree_19',['mi_cfree',['../group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7',1,'mimalloc-doc.h']]], + ['mi_5fcfree_19',['mi_cfree',['../group__malloc.html#ga705dc7a64bffacfeeb0141501a5c35d7',1,'mimalloc-doc.h']]], ['mi_5fcheck_5fowned_20',['mi_check_owned',['../group__heap.html#ga628c237489c2679af84a4d0d143b3dd5',1,'mimalloc-doc.h']]], ['mi_5fcollect_21',['mi_collect',['../group__extended.html#ga421430e2226d7d468529cec457396756',1,'mimalloc-doc.h']]], ['mi_5fdebug_5fshow_5farenas_22',['mi_debug_show_arenas',['../group__arenas.html#ga98cda17aa7adeda8b213382af8e7810b',1,'mimalloc-doc.h']]], @@ -30,231 +30,234 @@ var searchData= ['mi_5ffree_27',['mi_free',['../group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95',1,'mimalloc-doc.h']]], ['mi_5ffree_5faligned_28',['mi_free_aligned',['../group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9',1,'mimalloc-doc.h']]], ['mi_5ffree_5fcsize_29',['mi_free_csize',['../group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_30',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_5faligned_31',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsmall_32',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], - ['mi_5ffree_5ftp_33',['mi_free_tp',['../group__typed.html#gafc80000c300f02bdd4fe695eb53ca70a',1,'mimalloc-doc.h']]], - ['mi_5fgood_5fsize_34',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], - ['mi_5fheap_5farea_5ft_35',['mi_heap_area_t',['../group__analysis.html#structmi__heap__area__t',1,'']]], - ['mi_5fheap_5fcalloc_36',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_37',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_5fat_38',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5ftp_39',['mi_heap_calloc_tp',['../group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcheck_5fowned_40',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcollect_41',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_42',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_5fblock_43',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdelete_44',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdestroy_45',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fbacking_46',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fdefault_47',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmain_48',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_49',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_50',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_5fat_51',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5fsmall_52',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5ftp_53',['mi_heap_malloc_tp',['../group__typed.html#ga02d3e85bf26aa3120132f2552c408a88',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_54',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_5ftp_55',['mi_heap_mallocn_tp',['../group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_56',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fex_57',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fin_5farena_58',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fof_59',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_60',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_61',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_5fat_62',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocf_63',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_64',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_5ftp_65',['mi_heap_reallocn_tp',['../group__typed.html#ga817e315c24133294caad4d50470ff313',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealpath_66',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_67',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_68',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_5fat_69',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5ftp_70',['mi_heap_recalloc_tp',['../group__typed.html#ga7bf06ad296e20d1d74c90d64294eecec',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_71',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_72',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_5fat_73',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fdefault_74',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fnuma_5faffinity_75',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_76',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_5fjson_77',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_78',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fprint_5fout_79',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrdup_80',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrndup_81',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ft_82',['mi_heap_t',['../group__heap.html#ga34a47cde5a5b38c29f1aa3c5e76943c2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ftheap_83',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5fblocks_84',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5ffun_85',['mi_heap_visit_fun',['../group__subproc.html#ga7dcbc5863df9989b45e7051999448a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_86',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_87',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_5fat_88',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5fsmall_89',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5ftp_90',['mi_heap_zalloc_tp',['../group__typed.html#ga0aaa1aa747d8863e844c37e424ed096c',1,'mimalloc-doc.h']]], - ['mi_5fis_5fin_5fheap_5fregion_91',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], - ['mi_5fis_5fredirected_92',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_93',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_94',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_5fat_95',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fcsize_96',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fgood_5fsize_97',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsize_98',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsmall_99',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5ftp_100',['mi_malloc_tp',['../group__typed.html#ga0619a62c5fd886f1016030abe91f0557',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fusable_5fsize_101',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_102',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_5ftp_103',['mi_mallocn_tp',['../group__typed.html#gae5cb6e0fafc9f23169c5622e077afe8b',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_104',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_5fex_105',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], - ['mi_5fmbsdup_106',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], - ['mi_5fmemalign_107',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], - ['mi_5fnew_108',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_109',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_5fnothrow_110',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fn_111',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fnothrow_112',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], - ['mi_5fnew_5frealloc_113',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], - ['mi_5fnew_5freallocn_114',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], - ['mi_5foption_5fabandoned_5fpage_5fpurge_115',['mi_option_abandoned_page_purge',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11e62ed69200a489a5be955582078c0c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fabandoned_5freclaim_5fon_5ffree_116',['mi_option_abandoned_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca009e4b5684922ce664d73d2a8e1698d9',1,'mimalloc-doc.h']]], - ['mi_5foption_5fallow_5flarge_5fos_5fpages_117',['mi_option_allow_large_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7cc4804ced69004fa42a9a136a9ba556',1,'mimalloc-doc.h']]], - ['mi_5foption_5fallow_5fthp_118',['mi_option_allow_thp',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca04c45931ce2c82a20d08ca1471e74087',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5feager_5fcommit_119',['mi_option_arena_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5fmax_5fobject_5fsize_120',['mi_option_arena_max_object_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4dee58d60bc95ce0eeef759c61fd849b',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5fpurge_5fmult_121',['mi_option_arena_purge_mult',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8236501f1ab45d26e6fd885d191a2b5e',1,'mimalloc-doc.h']]], - ['mi_5foption_5farena_5freserve_122',['mi_option_arena_reserve',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cab1c88e23ae290bbeec824038a97959de',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdestroy_5fon_5fexit_123',['mi_option_destroy_on_exit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca6364331e305e7d3c0218b058ff3afc88',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisable_124',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisallow_5farena_5falloc_125',['mi_option_disallow_arena_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeae1696100e4057ffc4182730cc04e40',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisallow_5fos_5falloc_126',['mi_option_disallow_os_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadcfb5a09580361b1be65901d2d812de6',1,'mimalloc-doc.h']]], - ['mi_5foption_5feager_5fcommit_127',['mi_option_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1e8de72c93da7ff22d91e1e27b52ac2b',1,'mimalloc-doc.h']]], - ['mi_5foption_5feager_5fcommit_5fdelay_128',['mi_option_eager_commit_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca17a190c25be381142d87e0468c4c068c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fenable_129',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], - ['mi_5foption_5fgeneric_5fcollect_130',['mi_option_generic_collect',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8b5815a577955fe8125f2b964f41ee7f',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_131',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fclamp_132',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fsize_133',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fmax_134',['mi_option_guarded_max',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca96077da4e5c841edeec0c4d6e45a5ffa',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fmin_135',['mi_option_guarded_min',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca00734c528aae52784654a8db996ae62b',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fprecise_136',['mi_option_guarded_precise',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1eaf89bb1698f988cec27b66e546760c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fsample_5frate_137',['mi_option_guarded_sample_rate',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeb3c75b1a838eb42106750979d0888ec',1,'mimalloc-doc.h']]], - ['mi_5foption_5fguarded_5fsample_5fseed_138',['mi_option_guarded_sample_seed',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7b1e41ece69f9abbe5ba81d9ea7a3dc0',1,'mimalloc-doc.h']]], - ['mi_5foption_5fis_5fenabled_139',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5ferrors_140',['mi_option_max_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caec6ecbe29d46a48205ed8823a8a52a6a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fsegment_5freclaim_141',['mi_option_max_segment_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa9ad9005d7017c8c30ad2d6ba31db909',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fvabits_142',['mi_option_max_vabits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caabf8c9793d421ce28361b6d93c432de9',1,'mimalloc-doc.h']]], - ['mi_5foption_5fmax_5fwarnings_143',['mi_option_max_warnings',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caf9595921087e942602ee079158762665',1,'mimalloc-doc.h']]], - ['mi_5foption_5fminimal_5fpurge_5fsize_144',['mi_option_minimal_purge_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5635112095936356425d9a01910afff7',1,'mimalloc-doc.h']]], - ['mi_5foption_5fos_5ftag_145',['mi_option_os_tag',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4b74ae2a69e445de6c2361b73c1d14bf',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fcommit_5fon_5fdemand_146',['mi_option_page_commit_on_demand',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca679d9ee91221bffba4d5aafe488a8d7f',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fcross_5fthread_5fmax_5freclaim_147',['mi_option_page_cross_thread_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca671e806447c15a53c7d593afdd8d45c5',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5ffull_5fretain_148',['mi_option_page_full_retain',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeede7d0652e391940357e1c054d29829',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fmax_5fcandidates_149',['mi_option_page_max_candidates',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cad13382757101aa4fbc4984fa0f917133',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5fmax_5freclaim_150',['mi_option_page_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11a13739488294318ffc03eea378f5a4',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpage_5freclaim_5fon_5ffree_151',['mi_option_page_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cac16e3f838e861338fb2357b430f6974c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpagemap_5fcommit_152',['mi_option_pagemap_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa88f56e3bccce85542dc2b3297024236',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fdecommits_153',['mi_option_purge_decommits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca9d15c5e3d2115eef681c17e4dd5ab9a4',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fdelay_154',['mi_option_purge_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadd351e615acd8563529c20a347be7290',1,'mimalloc-doc.h']]], - ['mi_5foption_5fpurge_5fextend_5fdelay_155',['mi_option_purge_extend_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca02005f164bdf03f5f00c5be726adf487',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fhuge_5fos_5fpages_156',['mi_option_reserve_huge_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fhuge_5fos_5fpages_5fat_157',['mi_option_reserve_huge_os_pages_at',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c',1,'mimalloc-doc.h']]], - ['mi_5foption_5freserve_5fos_5fmemory_158',['mi_option_reserve_os_memory',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4999c828cf79a0fb2de65d23f7333',1,'mimalloc-doc.h']]], - ['mi_5foption_5fretry_5fon_5foom_159',['mi_option_retry_on_oom',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8f51df355bf6651db899e6085b54865e',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_160',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fdefault_161',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_162',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_5fdefault_163',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], - ['mi_5foption_5fshow_5ferrors_164',['mi_option_show_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4822e5c00732c5984b32a032837f0',1,'mimalloc-doc.h']]], - ['mi_5foption_5fshow_5fstats_165',['mi_option_show_stats',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0957ef73b2550764b4840edf48422fda',1,'mimalloc-doc.h']]], - ['mi_5foption_5ft_166',['mi_option_t',['../group__options.html#gafebf7ed116adb38ae5218bc3ce06884c',1,'mimalloc-doc.h']]], - ['mi_5foption_5ftarget_5fsegments_5fper_5fthread_167',['mi_option_target_segments_per_thread',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca874f36a7a1e51f7baee4a78b2830b643',1,'mimalloc-doc.h']]], - ['mi_5foption_5fuse_5fnuma_5fnodes_168',['mi_option_use_numa_nodes',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0ac33a18f6b659fcfaf44efb0bab1b74',1,'mimalloc-doc.h']]], - ['mi_5foption_5fverbose_169',['mi_option_verbose',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7c8b7bf5281c581bad64f5daa6442777',1,'mimalloc-doc.h']]], - ['mi_5foption_5fvisit_5fabandoned_170',['mi_option_visit_abandoned',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca38c67733a3956a1f4eeaca89fab9e78e',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_171',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_5fout_172',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], - ['mi_5foutput_5ffun_173',['mi_output_fun',['../group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d',1,'mimalloc-doc.h']]], - ['mi_5fposix_5fmemalign_174',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_175',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_176',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_5fout_177',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], - ['mi_5fpvalloc_178',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], - ['mi_5frealloc_179',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_180',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_5fat_181',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], - ['mi_5freallocarr_182',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], - ['mi_5freallocarray_183',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], - ['mi_5freallocf_184',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], - ['mi_5freallocn_185',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], - ['mi_5freallocn_5ftp_186',['mi_reallocn_tp',['../group__typed.html#ga1158b49a55dfa81f58a4426a7578f523',1,'mimalloc-doc.h']]], - ['mi_5frealpath_187',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_188',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_189',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_5fat_190',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], - ['mi_5fregister_5fdeferred_5ffree_191',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], - ['mi_5fregister_5ferror_192',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], - ['mi_5fregister_5foutput_193',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_194',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_195',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_196',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_197',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_5fex_198',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_199',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_200',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_5fat_201',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], - ['mi_5fsmall_5fsize_5fmax_202',['MI_SMALL_SIZE_MAX',['../group__extended.html#ga1ea64283508718d9d645c38efc2f4305',1,'mimalloc-doc.h']]], - ['mi_5fstat_5fversion_203',['MI_STAT_VERSION',['../group__stats.html#gac4411cf3eed23719e511b9e1128888e0',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fas_5fjson_204',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_205',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fbin_5fsize_206',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fjson_207',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fmerge_208',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_209',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_5fout_210',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], - ['mi_5fstats_5freset_211',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fs_212',['mi_stats_s',['../group__stats.html#structmi__stats__s',1,'']]], - ['mi_5fstats_5ft_5fdecl_213',['mi_stats_t_decl',['../group__stats.html#ga961d0f9f0327ebc9bd8a0b84f6dcb410',1,'mimalloc-doc.h']]], - ['mi_5fstl_5fallocator_214',['mi_stl_allocator',['../group__cpp.html#structmi__stl__allocator',1,'']]], - ['mi_5fstrdup_215',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], - ['mi_5fstrndup_216',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fadd_5fcurrent_5fthread_217',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fcurrent_218',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdelete_219',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdestroy_220',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_221',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fid_5ft_222',['mi_subproc_id_t',['../group__subproc.html#ga8c0bcd1fee27c7641e9c3c0d991b3b7d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fmain_223',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fnew_224',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_225',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_5fjson_226',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fprint_5fout_227',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fvisit_5fheaps_228',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcalloc_229',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcollect_230',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fget_5fdefault_231',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_232',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5faligned_233',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fcsize_234',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fsmall_235',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5frealloc_236',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fset_5fdefault_237',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5ft_238',['mi_theap_t',['../group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_239',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fcsize_240',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fsmall_241',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fdone_242',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], - ['mi_5fthread_5finit_243',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fset_5fin_5fthreadpool_244',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], - ['mi_5fusable_5fsize_245',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], - ['mi_5fvalloc_246',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], - ['mi_5fversion_247',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], - ['mi_5fwcsdup_248',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], - ['mi_5fwdupenv_5fs_249',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_250',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_251',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_5fat_252',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fcsize_253',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fsmall_254',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5ftp_255',['mi_zalloc_tp',['../group__typed.html#gac77a61bdaf680a803785fe307820b48c',1,'mimalloc-doc.h']]], - ['modes_256',['Build Modes',['../modes.html',1,'']]] + ['mi_5ffree_5fcsize_5fnonnull_30',['mi_free_csize_nonnull',['../group__constantsize.html#ga38a93d373a9dc8edc8bca83edb6af5c5',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_31',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5faligned_32',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5fnonnull_33',['mi_free_size_nonnull',['../group__posix.html#ga796602dae2a89797026f72e0c0240f32',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_34',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_5fnonnull_35',['mi_free_small_nonnull',['../group__extended.html#ga22890f54e5be433837069c8e97fdf379',1,'mimalloc-doc.h']]], + ['mi_5ffree_5ftp_36',['mi_free_tp',['../group__typed.html#gafc80000c300f02bdd4fe695eb53ca70a',1,'mimalloc-doc.h']]], + ['mi_5fgood_5fsize_37',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], + ['mi_5fheap_5farea_5ft_38',['mi_heap_area_t',['../group__analysis.html#structmi__heap__area__t',1,'']]], + ['mi_5fheap_5fcalloc_39',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_40',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_5fat_41',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5ftp_42',['mi_heap_calloc_tp',['../group__typed.html#ga953f22101bb354a8b6de2a6d6fda9b98',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcheck_5fowned_43',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcollect_44',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_45',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_5fblock_46',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdelete_47',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdestroy_48',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fbacking_49',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fdefault_50',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmain_51',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_52',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_53',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_5fat_54',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5fsmall_55',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5ftp_56',['mi_heap_malloc_tp',['../group__typed.html#ga02d3e85bf26aa3120132f2552c408a88',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_57',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_5ftp_58',['mi_heap_mallocn_tp',['../group__typed.html#gaa58690658cc6f5dd54c99a9971d4aa7d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_59',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fex_60',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fin_5farena_61',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fof_62',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_63',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_64',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_5fat_65',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocf_66',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_67',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_5ftp_68',['mi_heap_reallocn_tp',['../group__typed.html#ga817e315c24133294caad4d50470ff313',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealpath_69',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_70',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_71',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_5fat_72',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5ftp_73',['mi_heap_recalloc_tp',['../group__typed.html#ga7bf06ad296e20d1d74c90d64294eecec',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_74',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_75',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_5fat_76',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fdefault_77',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fnuma_5faffinity_78',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_79',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_5fjson_80',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_81',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fprint_5fout_82',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrdup_83',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrndup_84',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ft_85',['mi_heap_t',['../group__heap.html#ga34a47cde5a5b38c29f1aa3c5e76943c2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ftheap_86',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5fblocks_87',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5ffun_88',['mi_heap_visit_fun',['../group__subproc.html#ga7dcbc5863df9989b45e7051999448a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_89',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_90',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_5fat_91',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5fsmall_92',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5ftp_93',['mi_heap_zalloc_tp',['../group__typed.html#ga0aaa1aa747d8863e844c37e424ed096c',1,'mimalloc-doc.h']]], + ['mi_5fis_5fin_5fheap_5fregion_94',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], + ['mi_5fis_5fredirected_95',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_96',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_97',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_5fat_98',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fcsize_99',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fgood_5fsize_100',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsize_101',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsmall_102',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5ftp_103',['mi_malloc_tp',['../group__typed.html#ga0619a62c5fd886f1016030abe91f0557',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fusable_5fsize_104',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_105',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_5ftp_106',['mi_mallocn_tp',['../group__typed.html#gae5cb6e0fafc9f23169c5622e077afe8b',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_107',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_5fex_108',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], + ['mi_5fmbsdup_109',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], + ['mi_5fmemalign_110',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], + ['mi_5fnew_111',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_112',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_5fnothrow_113',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fn_114',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fnothrow_115',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], + ['mi_5fnew_5frealloc_116',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], + ['mi_5fnew_5freallocn_117',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], + ['mi_5foption_5fabandoned_5fpage_5fpurge_118',['mi_option_abandoned_page_purge',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11e62ed69200a489a5be955582078c0c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fabandoned_5freclaim_5fon_5ffree_119',['mi_option_abandoned_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca009e4b5684922ce664d73d2a8e1698d9',1,'mimalloc-doc.h']]], + ['mi_5foption_5fallow_5flarge_5fos_5fpages_120',['mi_option_allow_large_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7cc4804ced69004fa42a9a136a9ba556',1,'mimalloc-doc.h']]], + ['mi_5foption_5fallow_5fthp_121',['mi_option_allow_thp',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca04c45931ce2c82a20d08ca1471e74087',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5feager_5fcommit_122',['mi_option_arena_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafd0c5ddbc4b59fd8b5216871728167a5',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5fmax_5fobject_5fsize_123',['mi_option_arena_max_object_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4dee58d60bc95ce0eeef759c61fd849b',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5fpurge_5fmult_124',['mi_option_arena_purge_mult',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8236501f1ab45d26e6fd885d191a2b5e',1,'mimalloc-doc.h']]], + ['mi_5foption_5farena_5freserve_125',['mi_option_arena_reserve',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cab1c88e23ae290bbeec824038a97959de',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdestroy_5fon_5fexit_126',['mi_option_destroy_on_exit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca6364331e305e7d3c0218b058ff3afc88',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisable_127',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisallow_5farena_5falloc_128',['mi_option_disallow_arena_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeae1696100e4057ffc4182730cc04e40',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisallow_5fos_5falloc_129',['mi_option_disallow_os_alloc',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadcfb5a09580361b1be65901d2d812de6',1,'mimalloc-doc.h']]], + ['mi_5foption_5feager_5fcommit_130',['mi_option_eager_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1e8de72c93da7ff22d91e1e27b52ac2b',1,'mimalloc-doc.h']]], + ['mi_5foption_5feager_5fcommit_5fdelay_131',['mi_option_eager_commit_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca17a190c25be381142d87e0468c4c068c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fenable_132',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], + ['mi_5foption_5fgeneric_5fcollect_133',['mi_option_generic_collect',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8b5815a577955fe8125f2b964f41ee7f',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_134',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fclamp_135',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fsize_136',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fmax_137',['mi_option_guarded_max',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca96077da4e5c841edeec0c4d6e45a5ffa',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fmin_138',['mi_option_guarded_min',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca00734c528aae52784654a8db996ae62b',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fprecise_139',['mi_option_guarded_precise',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca1eaf89bb1698f988cec27b66e546760c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fsample_5frate_140',['mi_option_guarded_sample_rate',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeb3c75b1a838eb42106750979d0888ec',1,'mimalloc-doc.h']]], + ['mi_5foption_5fguarded_5fsample_5fseed_141',['mi_option_guarded_sample_seed',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7b1e41ece69f9abbe5ba81d9ea7a3dc0',1,'mimalloc-doc.h']]], + ['mi_5foption_5fis_5fenabled_142',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5ferrors_143',['mi_option_max_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caec6ecbe29d46a48205ed8823a8a52a6a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fsegment_5freclaim_144',['mi_option_max_segment_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa9ad9005d7017c8c30ad2d6ba31db909',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fvabits_145',['mi_option_max_vabits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caabf8c9793d421ce28361b6d93c432de9',1,'mimalloc-doc.h']]], + ['mi_5foption_5fmax_5fwarnings_146',['mi_option_max_warnings',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caf9595921087e942602ee079158762665',1,'mimalloc-doc.h']]], + ['mi_5foption_5fminimal_5fpurge_5fsize_147',['mi_option_minimal_purge_size',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca5635112095936356425d9a01910afff7',1,'mimalloc-doc.h']]], + ['mi_5foption_5fos_5ftag_148',['mi_option_os_tag',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca4b74ae2a69e445de6c2361b73c1d14bf',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fcommit_5fon_5fdemand_149',['mi_option_page_commit_on_demand',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca679d9ee91221bffba4d5aafe488a8d7f',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fcross_5fthread_5fmax_5freclaim_150',['mi_option_page_cross_thread_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca671e806447c15a53c7d593afdd8d45c5',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5ffull_5fretain_151',['mi_option_page_full_retain',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caeede7d0652e391940357e1c054d29829',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fmax_5fcandidates_152',['mi_option_page_max_candidates',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cad13382757101aa4fbc4984fa0f917133',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5fmax_5freclaim_153',['mi_option_page_max_reclaim',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca11a13739488294318ffc03eea378f5a4',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpage_5freclaim_5fon_5ffree_154',['mi_option_page_reclaim_on_free',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cac16e3f838e861338fb2357b430f6974c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpagemap_5fcommit_155',['mi_option_pagemap_commit',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa88f56e3bccce85542dc2b3297024236',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fdecommits_156',['mi_option_purge_decommits',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca9d15c5e3d2115eef681c17e4dd5ab9a4',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fdelay_157',['mi_option_purge_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cadd351e615acd8563529c20a347be7290',1,'mimalloc-doc.h']]], + ['mi_5foption_5fpurge_5fextend_5fdelay_158',['mi_option_purge_extend_delay',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca02005f164bdf03f5f00c5be726adf487',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fhuge_5fos_5fpages_159',['mi_option_reserve_huge_os_pages',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caca7ed041be3b0b9d0b82432c7bf41af2',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fhuge_5fos_5fpages_5fat_160',['mi_option_reserve_huge_os_pages_at',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884caa13e7926d4339d2aa6fbf61d4473fd5c',1,'mimalloc-doc.h']]], + ['mi_5foption_5freserve_5fos_5fmemory_161',['mi_option_reserve_os_memory',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4999c828cf79a0fb2de65d23f7333',1,'mimalloc-doc.h']]], + ['mi_5foption_5fretry_5fon_5foom_162',['mi_option_retry_on_oom',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca8f51df355bf6651db899e6085b54865e',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_163',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fdefault_164',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_165',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_5fdefault_166',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], + ['mi_5foption_5fshow_5ferrors_167',['mi_option_show_errors',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884cafbf4822e5c00732c5984b32a032837f0',1,'mimalloc-doc.h']]], + ['mi_5foption_5fshow_5fstats_168',['mi_option_show_stats',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0957ef73b2550764b4840edf48422fda',1,'mimalloc-doc.h']]], + ['mi_5foption_5ft_169',['mi_option_t',['../group__options.html#gafebf7ed116adb38ae5218bc3ce06884c',1,'mimalloc-doc.h']]], + ['mi_5foption_5ftarget_5fsegments_5fper_5fthread_170',['mi_option_target_segments_per_thread',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca874f36a7a1e51f7baee4a78b2830b643',1,'mimalloc-doc.h']]], + ['mi_5foption_5fuse_5fnuma_5fnodes_171',['mi_option_use_numa_nodes',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca0ac33a18f6b659fcfaf44efb0bab1b74',1,'mimalloc-doc.h']]], + ['mi_5foption_5fverbose_172',['mi_option_verbose',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca7c8b7bf5281c581bad64f5daa6442777',1,'mimalloc-doc.h']]], + ['mi_5foption_5fvisit_5fabandoned_173',['mi_option_visit_abandoned',['../group__options.html#ggafebf7ed116adb38ae5218bc3ce06884ca38c67733a3956a1f4eeaca89fab9e78e',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_174',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_5fout_175',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], + ['mi_5foutput_5ffun_176',['mi_output_fun',['../group__extended.html#gadf31cea7d0332a81c8b882cbbdbadb8d',1,'mimalloc-doc.h']]], + ['mi_5fposix_5fmemalign_177',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_178',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_179',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_5fout_180',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], + ['mi_5fpvalloc_181',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], + ['mi_5frealloc_182',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_183',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_5fat_184',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], + ['mi_5freallocarr_185',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], + ['mi_5freallocarray_186',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], + ['mi_5freallocf_187',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], + ['mi_5freallocn_188',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], + ['mi_5freallocn_5ftp_189',['mi_reallocn_tp',['../group__typed.html#ga1158b49a55dfa81f58a4426a7578f523',1,'mimalloc-doc.h']]], + ['mi_5frealpath_190',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_191',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_192',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_5fat_193',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], + ['mi_5fregister_5fdeferred_5ffree_194',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], + ['mi_5fregister_5ferror_195',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], + ['mi_5fregister_5foutput_196',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_197',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_198',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_199',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_200',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_5fex_201',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_202',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_203',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_5fat_204',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], + ['mi_5fsmall_5fsize_5fmax_205',['MI_SMALL_SIZE_MAX',['../group__extended.html#ga1ea64283508718d9d645c38efc2f4305',1,'mimalloc-doc.h']]], + ['mi_5fstat_5fversion_206',['MI_STAT_VERSION',['../group__stats.html#gac4411cf3eed23719e511b9e1128888e0',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fas_5fjson_207',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_208',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fbin_5fsize_209',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fjson_210',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fmerge_211',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_212',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_5fout_213',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], + ['mi_5fstats_5freset_214',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fs_215',['mi_stats_s',['../group__stats.html#structmi__stats__s',1,'']]], + ['mi_5fstats_5ft_5fdecl_216',['mi_stats_t_decl',['../group__stats.html#ga961d0f9f0327ebc9bd8a0b84f6dcb410',1,'mimalloc-doc.h']]], + ['mi_5fstl_5fallocator_217',['mi_stl_allocator',['../group__cpp.html#structmi__stl__allocator',1,'']]], + ['mi_5fstrdup_218',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], + ['mi_5fstrndup_219',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fadd_5fcurrent_5fthread_220',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fcurrent_221',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdelete_222',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdestroy_223',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_224',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fid_5ft_225',['mi_subproc_id_t',['../group__subproc.html#ga8c0bcd1fee27c7641e9c3c0d991b3b7d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fmain_226',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fnew_227',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_228',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_5fjson_229',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fprint_5fout_230',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fvisit_5fheaps_231',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcalloc_232',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcollect_233',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fget_5fdefault_234',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_235',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5faligned_236',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fcsize_237',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fsmall_238',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5frealloc_239',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fset_5fdefault_240',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5ft_241',['mi_theap_t',['../group__theap.html#ga8722f6ac4f638b4692e06ddfddc844f8',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_242',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fcsize_243',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fsmall_244',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fdone_245',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], + ['mi_5fthread_5finit_246',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fset_5fin_5fthreadpool_247',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], + ['mi_5fusable_5fsize_248',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], + ['mi_5fvalloc_249',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], + ['mi_5fversion_250',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], + ['mi_5fwcsdup_251',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], + ['mi_5fwdupenv_5fs_252',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_253',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_254',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_5fat_255',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fcsize_256',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fsmall_257',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5ftp_258',['mi_zalloc_tp',['../group__typed.html#gac77a61bdaf680a803785fe307820b48c',1,'mimalloc-doc.h']]], + ['modes_259',['Build Modes',['../modes.html',1,'']]] ]; diff --git a/docs/search/functions_0.js b/docs/search/functions_0.js index d16b716ee..6d7a877dc 100644 --- a/docs/search/functions_0.js +++ b/docs/search/functions_0.js @@ -12,7 +12,7 @@ var searchData= ['mi_5fcalloc_9',['mi_calloc',['../group__malloc.html#ga6686568014b54d1e6c7ac64a076e4f56',1,'mimalloc-doc.h']]], ['mi_5fcalloc_5faligned_10',['mi_calloc_aligned',['../group__aligned.html#ga424ef386fb1f9f8e0a86ab53f16eaaf1',1,'mimalloc-doc.h']]], ['mi_5fcalloc_5faligned_5fat_11',['mi_calloc_aligned_at',['../group__aligned.html#ga977f96bd2c5c141bcd70e6685c90d6c3',1,'mimalloc-doc.h']]], - ['mi_5fcfree_12',['mi_cfree',['../group__posix.html#ga705dc7a64bffacfeeb0141501a5c35d7',1,'mimalloc-doc.h']]], + ['mi_5fcfree_12',['mi_cfree',['../group__malloc.html#ga705dc7a64bffacfeeb0141501a5c35d7',1,'mimalloc-doc.h']]], ['mi_5fcheck_5fowned_13',['mi_check_owned',['../group__heap.html#ga628c237489c2679af84a4d0d143b3dd5',1,'mimalloc-doc.h']]], ['mi_5fcollect_14',['mi_collect',['../group__extended.html#ga421430e2226d7d468529cec457396756',1,'mimalloc-doc.h']]], ['mi_5fdebug_5fshow_5farenas_15',['mi_debug_show_arenas',['../group__arenas.html#ga98cda17aa7adeda8b213382af8e7810b',1,'mimalloc-doc.h']]], @@ -21,162 +21,165 @@ var searchData= ['mi_5ffree_18',['mi_free',['../group__malloc.html#gaf2c7b89c327d1f60f59e68b9ea644d95',1,'mimalloc-doc.h']]], ['mi_5ffree_5faligned_19',['mi_free_aligned',['../group__posix.html#ga0d28d5cf61e6bfbb18c63092939fe5c9',1,'mimalloc-doc.h']]], ['mi_5ffree_5fcsize_20',['mi_free_csize',['../group__constantsize.html#ga7a421e302074e598527c3eb052a6a3c5',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_21',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsize_5faligned_22',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], - ['mi_5ffree_5fsmall_23',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], - ['mi_5fgood_5fsize_24',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_25',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_26',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcalloc_5faligned_5fat_27',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcheck_5fowned_28',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcollect_29',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_30',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fcontains_5fblock_31',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdelete_32',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fdestroy_33',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fbacking_34',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fget_5fdefault_35',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmain_36',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_37',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_38',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5faligned_5fat_39',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmalloc_5fsmall_40',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fmallocn_41',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_42',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fex_43',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fnew_5fin_5farena_44',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fof_45',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_46',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_47',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealloc_5faligned_5fat_48',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocf_49',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5freallocn_50',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frealpath_51',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_52',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_53',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frecalloc_5faligned_5fat_54',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_55',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_56',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], - ['mi_5fheap_5frezalloc_5faligned_5fat_57',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fdefault_58',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fset_5fnuma_5faffinity_59',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_60',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fget_5fjson_61',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_62',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstats_5fprint_5fout_63',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrdup_64',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fstrndup_65',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], - ['mi_5fheap_5ftheap_66',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fvisit_5fblocks_67',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_68',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_69',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5faligned_5fat_70',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], - ['mi_5fheap_5fzalloc_5fsmall_71',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], - ['mi_5fis_5fin_5fheap_5fregion_72',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], - ['mi_5fis_5fredirected_73',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_74',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_75',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5faligned_5fat_76',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fcsize_77',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fgood_5fsize_78',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsize_79',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fsmall_80',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], - ['mi_5fmalloc_5fusable_5fsize_81',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], - ['mi_5fmallocn_82',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_83',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], - ['mi_5fmanage_5fos_5fmemory_5fex_84',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], - ['mi_5fmbsdup_85',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], - ['mi_5fmemalign_86',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], - ['mi_5fnew_87',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_88',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], - ['mi_5fnew_5faligned_5fnothrow_89',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fn_90',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], - ['mi_5fnew_5fnothrow_91',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], - ['mi_5fnew_5frealloc_92',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], - ['mi_5fnew_5freallocn_93',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], - ['mi_5foption_5fdisable_94',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], - ['mi_5foption_5fenable_95',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_96',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fclamp_97',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], - ['mi_5foption_5fget_5fsize_98',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], - ['mi_5foption_5fis_5fenabled_99',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_100',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fdefault_101',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_102',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], - ['mi_5foption_5fset_5fenabled_5fdefault_103',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_104',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], - ['mi_5foptions_5fprint_5fout_105',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], - ['mi_5fposix_5fmemalign_106',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_107',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_108',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], - ['mi_5fprocess_5finfo_5fprint_5fout_109',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], - ['mi_5fpvalloc_110',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], - ['mi_5frealloc_111',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_112',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], - ['mi_5frealloc_5faligned_5fat_113',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], - ['mi_5freallocarr_114',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], - ['mi_5freallocarray_115',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], - ['mi_5freallocf_116',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], - ['mi_5freallocn_117',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], - ['mi_5frealpath_118',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_119',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_120',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], - ['mi_5frecalloc_5faligned_5fat_121',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], - ['mi_5fregister_5fdeferred_5ffree_122',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], - ['mi_5fregister_5ferror_123',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], - ['mi_5fregister_5foutput_124',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_125',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_126',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_127',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_128',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], - ['mi_5freserve_5fos_5fmemory_5fex_129',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_130',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_131',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], - ['mi_5frezalloc_5faligned_5fat_132',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fas_5fjson_133',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_134',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fbin_5fsize_135',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fget_5fjson_136',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fmerge_137',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_138',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], - ['mi_5fstats_5fprint_5fout_139',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], - ['mi_5fstats_5freset_140',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], - ['mi_5fstrdup_141',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], - ['mi_5fstrndup_142',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fadd_5fcurrent_5fthread_143',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fcurrent_144',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdelete_145',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fdestroy_146',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_147',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fmain_148',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fnew_149',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_150',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fget_5fjson_151',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fstats_5fprint_5fout_152',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], - ['mi_5fsubproc_5fvisit_5fheaps_153',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcalloc_154',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fcollect_155',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fget_5fdefault_156',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_157',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5faligned_158',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fcsize_159',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fmalloc_5fsmall_160',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5frealloc_161',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fset_5fdefault_162',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_163',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fcsize_164',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], - ['mi_5ftheap_5fzalloc_5fsmall_165',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fdone_166',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], - ['mi_5fthread_5finit_167',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], - ['mi_5fthread_5fset_5fin_5fthreadpool_168',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], - ['mi_5fusable_5fsize_169',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], - ['mi_5fvalloc_170',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], - ['mi_5fversion_171',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], - ['mi_5fwcsdup_172',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], - ['mi_5fwdupenv_5fs_173',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_174',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_175',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5faligned_5fat_176',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fcsize_177',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], - ['mi_5fzalloc_5fsmall_178',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]] + ['mi_5ffree_5fcsize_5fnonnull_21',['mi_free_csize_nonnull',['../group__constantsize.html#ga38a93d373a9dc8edc8bca83edb6af5c5',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_22',['mi_free_size',['../group__posix.html#gae01389eedab8d67341ff52e2aad80ebb',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5faligned_23',['mi_free_size_aligned',['../group__posix.html#ga72e9d7ffb5fe94d69bc722c8506e27bc',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsize_5fnonnull_24',['mi_free_size_nonnull',['../group__posix.html#ga796602dae2a89797026f72e0c0240f32',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_25',['mi_free_small',['../group__extended.html#ga2a8469764c38dcaf878f36e5f50e3610',1,'mimalloc-doc.h']]], + ['mi_5ffree_5fsmall_5fnonnull_26',['mi_free_small_nonnull',['../group__extended.html#ga22890f54e5be433837069c8e97fdf379',1,'mimalloc-doc.h']]], + ['mi_5fgood_5fsize_27',['mi_good_size',['../group__malloc.html#gac057927cd06c854b45fe7847e921bd47',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_28',['mi_heap_calloc',['../group__heap.html#gac0098aaf231d3e9586c73136d5df95da',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_29',['mi_heap_calloc_aligned',['../group__heap.html#gacafcc26df827c7a7de5e850217566108',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcalloc_5faligned_5fat_30',['mi_heap_calloc_aligned_at',['../group__heap.html#gaa42ec2079989c4374f2c331d9b35f4e4',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcheck_5fowned_31',['mi_heap_check_owned',['../group__heap.html#ga0d67c1789faaa15ff366c024fcaf6377',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcollect_32',['mi_heap_collect',['../group__heap.html#ga7922f7495cde30b1984d0e6072419298',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_33',['mi_heap_contains',['../group__heap.html#gac988496aea8ffc5c8bf92a8b10165fa0',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fcontains_5fblock_34',['mi_heap_contains_block',['../group__heap.html#gaa862aa8ed8d57d84cae41fc1022d71af',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdelete_35',['mi_heap_delete',['../group__heap.html#ga2ab1af8d438819b55319c7ef51d1e409',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fdestroy_36',['mi_heap_destroy',['../group__heap.html#ga9f9c0844edb9717f4feacd79116b8e0d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fbacking_37',['mi_heap_get_backing',['../group__heap.html#gac6ac9f0e7be9ab4ff70acfc8dad1235a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fget_5fdefault_38',['mi_heap_get_default',['../group__heap.html#ga14c667a6e2c5d28762d8cb7d4e057909',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmain_39',['mi_heap_main',['../group__heap.html#ga8fc318a65dc79ee8778c6a0e52d12477',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_40',['mi_heap_malloc',['../group__heap.html#gab374e206c7034e0d899fb934e4f4a863',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_41',['mi_heap_malloc_aligned',['../group__heap.html#ga33f4f05b7fea7af2113c62a4bf882cc5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5faligned_5fat_42',['mi_heap_malloc_aligned_at',['../group__heap.html#gae7ffc045c3996497a7f3a5f6fe7b8aaa',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmalloc_5fsmall_43',['mi_heap_malloc_small',['../group__heap.html#ga012c5c8abe22b10043de39ff95909541',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fmallocn_44',['mi_heap_mallocn',['../group__heap.html#gab0f755c0b21c387fe8e9024200faa372',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_45',['mi_heap_new',['../group__heap.html#gaa718bb226ec0546ba6d1b6cb32179f3a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fex_46',['mi_heap_new_ex',['../group__arenas.html#ga3ae360583f4351aa5267ee7e43008faf',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fnew_5fin_5farena_47',['mi_heap_new_in_arena',['../group__arenas.html#gaaf2d9976576d5efd5544be12848af949',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fof_48',['mi_heap_of',['../group__heap.html#ga424b69f285109b2515631d59f5eecb1b',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_49',['mi_heap_realloc',['../group__heap.html#gac5252d6a2e510bd349e4fcb452e6a93a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_50',['mi_heap_realloc_aligned',['../group__heap.html#gaccf8c249872f30bf1c2493a09197d734',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealloc_5faligned_5fat_51',['mi_heap_realloc_aligned_at',['../group__heap.html#ga6df988a7219d5707f010d5f3eb0dc3f5',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocf_52',['mi_heap_reallocf',['../group__heap.html#gae7cd171425bee04c683c65a3701f0b4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5freallocn_53',['mi_heap_reallocn',['../group__heap.html#gaccf7bfe10ce510a000d3547d9cf7fa29',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frealpath_54',['mi_heap_realpath',['../group__heap.html#ga55545a3ec6da29c5b4f62e540ecac1e2',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_55',['mi_heap_recalloc',['../group__zeroinit.html#gad1a0d325d930eeb80f25e3fea37aacde',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_56',['mi_heap_recalloc_aligned',['../group__zeroinit.html#ga87ddd674bf1c67237d780d0b9e0f0f32',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frecalloc_5faligned_5fat_57',['mi_heap_recalloc_aligned_at',['../group__zeroinit.html#ga07b5bcbaf00d0d2e598c232982588496',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_58',['mi_heap_rezalloc',['../group__zeroinit.html#ga8d8b7ebb24b513cd84d1a696048da60d',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_59',['mi_heap_rezalloc_aligned',['../group__zeroinit.html#ga5129f6dc46ee1613d918820a8a0533a7',1,'mimalloc-doc.h']]], + ['mi_5fheap_5frezalloc_5faligned_5fat_60',['mi_heap_rezalloc_aligned_at',['../group__zeroinit.html#ga2bafa79c3f98ea74882349d44cffa5d9',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fdefault_61',['mi_heap_set_default',['../group__heap.html#ga349b677dec7da5eacdbc7a385bd62a4a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fset_5fnuma_5faffinity_62',['mi_heap_set_numa_affinity',['../group__heap.html#gaab93abbc0624f570e2900f95cac18591',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_63',['mi_heap_stats_get',['../group__stats.html#ga2654195cef93ed3613e98b59a19a296f',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fget_5fjson_64',['mi_heap_stats_get_json',['../group__stats.html#gacb42dede4c38c38f2014b5450c5fcba8',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fmerge_5fto_5fsubproc_65',['mi_heap_stats_merge_to_subproc',['../group__stats.html#ga5a34328f846c661aa560f4769cfc15f1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstats_5fprint_5fout_66',['mi_heap_stats_print_out',['../group__stats.html#gafea365d7842f5f5bc1f235bbc303d585',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrdup_67',['mi_heap_strdup',['../group__heap.html#ga5754e09ccc51dd6bc73885bb6ea21b7a',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fstrndup_68',['mi_heap_strndup',['../group__heap.html#gad224df78f1fbee942df8adf023e12cf3',1,'mimalloc-doc.h']]], + ['mi_5fheap_5ftheap_69',['mi_heap_theap',['../group__theap.html#ga7905a73865733da8a43339a500533180',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fvisit_5fblocks_70',['mi_heap_visit_blocks',['../group__analysis.html#gae69fcff3324f8e419af850e34cf54c46',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_71',['mi_heap_zalloc',['../group__heap.html#gabebc796399619d964d8db77aa835e8c1',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_72',['mi_heap_zalloc_aligned',['../group__heap.html#ga6466bde8b5712aa34e081a8317f9f471',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5faligned_5fat_73',['mi_heap_zalloc_aligned_at',['../group__heap.html#ga484e3d01cd174f78c7e53370e5a7c819',1,'mimalloc-doc.h']]], + ['mi_5fheap_5fzalloc_5fsmall_74',['mi_heap_zalloc_small',['../group__heap.html#ga0994f44cc595f9c1840a9af162d98597',1,'mimalloc-doc.h']]], + ['mi_5fis_5fin_5fheap_5fregion_75',['mi_is_in_heap_region',['../group__heap.html#ga5f071b10d4df1c3658e04e7fd67a94e6',1,'mimalloc-doc.h']]], + ['mi_5fis_5fredirected_76',['mi_is_redirected',['../group__extended.html#gaad25050b19f30cd79397b227e0157a3f',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_77',['mi_malloc',['../group__malloc.html#gae1dd97b542420c87ae085e822b1229e8',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_78',['mi_malloc_aligned',['../group__aligned.html#ga69578ff1a98ca16e1dcd02c0995cd65c',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5faligned_5fat_79',['mi_malloc_aligned_at',['../group__aligned.html#ga2022f71b95a7cd6cce1b6e07752ae8ca',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fcsize_80',['mi_malloc_csize',['../group__constantsize.html#ga9cd44d925d34f7180922e12bbf594571',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fgood_5fsize_81',['mi_malloc_good_size',['../group__posix.html#ga9d23ac7885fed7413c11d8e0ffa31071',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsize_82',['mi_malloc_size',['../group__posix.html#ga4531c9e775bb3ae12db57c1ba8a5d7de',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fsmall_83',['mi_malloc_small',['../group__extended.html#ga7f050bc6b897da82692174f5fce59cde',1,'mimalloc-doc.h']]], + ['mi_5fmalloc_5fusable_5fsize_84',['mi_malloc_usable_size',['../group__posix.html#ga06d07cf357bbac5c73ba5d0c0c421e17',1,'mimalloc-doc.h']]], + ['mi_5fmallocn_85',['mi_mallocn',['../group__malloc.html#ga61f46bade3db76ca24aaafedc40de7b6',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_86',['mi_manage_os_memory',['../group__arenas.html#ga4c6486a1fdcd7a423b5f25fe4be8e0cf',1,'mimalloc-doc.h']]], + ['mi_5fmanage_5fos_5fmemory_5fex_87',['mi_manage_os_memory_ex',['../group__arenas.html#ga41ce8525d77bbb60f618fa1029994f6e',1,'mimalloc-doc.h']]], + ['mi_5fmbsdup_88',['mi_mbsdup',['../group__posix.html#ga7b82a44094fdec4d2084eb4288a979b0',1,'mimalloc-doc.h']]], + ['mi_5fmemalign_89',['mi_memalign',['../group__posix.html#ga726867f13fd29ca36064954c0285b1d8',1,'mimalloc-doc.h']]], + ['mi_5fnew_90',['mi_new',['../group__cpp.html#ga633d96e3bc7011f960df9f3b2731fc6a',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_91',['mi_new_aligned',['../group__cpp.html#ga79c54da0b4b4ce9fcc11d2f6ef6675f8',1,'mimalloc-doc.h']]], + ['mi_5fnew_5faligned_5fnothrow_92',['mi_new_aligned_nothrow',['../group__cpp.html#ga92ae00b6dd64406c7e64557711ec04b7',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fn_93',['mi_new_n',['../group__cpp.html#gadd11b85c15d21d308386844b5233856c',1,'mimalloc-doc.h']]], + ['mi_5fnew_5fnothrow_94',['mi_new_nothrow',['../group__cpp.html#ga5cb4f120d1f7296074256215aa9a9e54',1,'mimalloc-doc.h']]], + ['mi_5fnew_5frealloc_95',['mi_new_realloc',['../group__cpp.html#ga6867d89baf992728e0cc20a1f47db4d0',1,'mimalloc-doc.h']]], + ['mi_5fnew_5freallocn_96',['mi_new_reallocn',['../group__cpp.html#gaace912ce086682d56f3ce9f7638d9d67',1,'mimalloc-doc.h']]], + ['mi_5foption_5fdisable_97',['mi_option_disable',['../group__options.html#gaebf6ff707a2e688ebb1a2296ca564054',1,'mimalloc-doc.h']]], + ['mi_5foption_5fenable_98',['mi_option_enable',['../group__options.html#ga04180ae41b0d601421dd62ced40ca050',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_99',['mi_option_get',['../group__options.html#ga7e8af195cc81d3fa64ccf2662caa565a',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fclamp_100',['mi_option_get_clamp',['../group__options.html#ga96ad9c406338bd314cfe878cfc9bf723',1,'mimalloc-doc.h']]], + ['mi_5foption_5fget_5fsize_101',['mi_option_get_size',['../group__options.html#ga274db5a6ac87cc24ef0b23e7006ed02c',1,'mimalloc-doc.h']]], + ['mi_5foption_5fis_5fenabled_102',['mi_option_is_enabled',['../group__options.html#ga459ad98f18b3fc9275474807fe0ca188',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_103',['mi_option_set',['../group__options.html#gaf84921c32375e25754dc2ee6a911fa60',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fdefault_104',['mi_option_set_default',['../group__options.html#ga7ef623e440e6e5545cb08c94e71e4b90',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_105',['mi_option_set_enabled',['../group__options.html#ga9a13d05fcb77489cb06d4d017ebd8bed',1,'mimalloc-doc.h']]], + ['mi_5foption_5fset_5fenabled_5fdefault_106',['mi_option_set_enabled_default',['../group__options.html#ga65518b69ec5d32336b50e07f74b3f629',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_107',['mi_options_print',['../group__options.html#gaecfcd3ed90af99932670d881de1fa8f1',1,'mimalloc-doc.h']]], + ['mi_5foptions_5fprint_5fout_108',['mi_options_print_out',['../group__options.html#ga1dd50c7533baec4cc6c1a2e9adf05dd2',1,'mimalloc-doc.h']]], + ['mi_5fposix_5fmemalign_109',['mi_posix_memalign',['../group__posix.html#gacff84f226ba9feb2031b8992e5579447',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_110',['mi_process_info',['../group__stats.html#ga7d862c2affd5790381da14eb102a364d',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_111',['mi_process_info_print',['../group__stats.html#gaf887e7aa75f13242c4e7ab95d8390536',1,'mimalloc-doc.h']]], + ['mi_5fprocess_5finfo_5fprint_5fout_112',['mi_process_info_print_out',['../group__stats.html#ga03ac88d3efee2ae7c34f5340502f4732',1,'mimalloc-doc.h']]], + ['mi_5fpvalloc_113',['mi_pvalloc',['../group__posix.html#ga644bebccdbb2821542dd8c7e7641f476',1,'mimalloc-doc.h']]], + ['mi_5frealloc_114',['mi_realloc',['../group__malloc.html#ga0621af6a5e3aa384e6a1b548958bf583',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_115',['mi_realloc_aligned',['../group__aligned.html#ga5d7a46d054b4d7abe9d8d2474add2edf',1,'mimalloc-doc.h']]], + ['mi_5frealloc_5faligned_5fat_116',['mi_realloc_aligned_at',['../group__aligned.html#gad06dcf2bb8faadb2c8ea61ee5d24bbf6',1,'mimalloc-doc.h']]], + ['mi_5freallocarr_117',['mi_reallocarr',['../group__posix.html#ga7e1934d60a3e697950eeb48e042bfad5',1,'mimalloc-doc.h']]], + ['mi_5freallocarray_118',['mi_reallocarray',['../group__posix.html#gadfeccb72748a2f6305474a37d9d57bce',1,'mimalloc-doc.h']]], + ['mi_5freallocf_119',['mi_reallocf',['../group__malloc.html#ga4dc3a4067037b151a64629fe8a332641',1,'mimalloc-doc.h']]], + ['mi_5freallocn_120',['mi_reallocn',['../group__malloc.html#ga8bddfb4a1270a0854bbcf44cb3980467',1,'mimalloc-doc.h']]], + ['mi_5frealpath_121',['mi_realpath',['../group__malloc.html#ga94c3afcc086e85d75a57e9f76b9b71dd',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_122',['mi_recalloc',['../group__zeroinit.html#ga23a0fbb452b5dce8e31fab1a1958cacc',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_123',['mi_recalloc_aligned',['../group__zeroinit.html#ga3e2169b48683aa0ab64f813fd68d839e',1,'mimalloc-doc.h']]], + ['mi_5frecalloc_5faligned_5fat_124',['mi_recalloc_aligned_at',['../group__zeroinit.html#gaae25e4ddedd4e0fb61b1a8bd5d452750',1,'mimalloc-doc.h']]], + ['mi_5fregister_5fdeferred_5ffree_125',['mi_register_deferred_free',['../group__extended.html#ga3460a6ca91af97be4058f523d3cb8ece',1,'mimalloc-doc.h']]], + ['mi_5fregister_5ferror_126',['mi_register_error',['../group__extended.html#gaa1d55e0e894be240827e5d87ec3a1f45',1,'mimalloc-doc.h']]], + ['mi_5fregister_5foutput_127',['mi_register_output',['../group__extended.html#gae5b17ff027cd2150b43a33040250cf3f',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_128',['mi_reserve_huge_os_pages_at',['../group__arenas.html#ga7795a13d20087447281858d2c771cca1',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5fat_5fex_129',['mi_reserve_huge_os_pages_at_ex',['../group__arenas.html#ga591aab1c2bc2ca920e33f0f9f9cb5c52',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fhuge_5fos_5fpages_5finterleave_130',['mi_reserve_huge_os_pages_interleave',['../group__arenas.html#ga3132f521fb756fc0e8ec0b74fb58df50',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_131',['mi_reserve_os_memory',['../group__arenas.html#ga00ec3324b6b2591c7fe3677baa30a767',1,'mimalloc-doc.h']]], + ['mi_5freserve_5fos_5fmemory_5fex_132',['mi_reserve_os_memory_ex',['../group__arenas.html#ga32f519797fd9a81acb4f52d36e6d751b',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_133',['mi_rezalloc',['../group__zeroinit.html#gadfd34cd7b4f2bbda7ae06367a6360756',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_134',['mi_rezalloc_aligned',['../group__zeroinit.html#ga4d02404fe1e7db00beb65f185e012caa',1,'mimalloc-doc.h']]], + ['mi_5frezalloc_5faligned_5fat_135',['mi_rezalloc_aligned_at',['../group__zeroinit.html#ga6843a88285bbfcc3bdfccc60aafd1270',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fas_5fjson_136',['mi_stats_as_json',['../group__stats.html#ga6ad266c70a0eabf6c26248aed1af6fa1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_137',['mi_stats_get',['../group__stats.html#gaa26d5060ad21b1331a150bb92f6f7086',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fbin_5fsize_138',['mi_stats_get_bin_size',['../group__stats.html#gadecd549d1a7216af596322ac27e78b0d',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fget_5fjson_139',['mi_stats_get_json',['../group__stats.html#gadaa3be15899cc82e314fa7d3f8e2afbd',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fmerge_140',['mi_stats_merge',['../group__stats.html#ga854b1de8cb067c7316286c28b2fcd3d1',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_141',['mi_stats_print',['../group__stats.html#ga2d126e5c62d3badc35445e5d84166df2',1,'mimalloc-doc.h']]], + ['mi_5fstats_5fprint_5fout_142',['mi_stats_print_out',['../group__stats.html#ga537f13b299ddf801e49a5a94fde02c79',1,'mimalloc-doc.h']]], + ['mi_5fstats_5freset_143',['mi_stats_reset',['../group__stats.html#ga3bb8468b8cfcc6e2a61d98aee85c5f99',1,'mimalloc-doc.h']]], + ['mi_5fstrdup_144',['mi_strdup',['../group__malloc.html#ga245ac90ebc2cfdd17de599e5fea59889',1,'mimalloc-doc.h']]], + ['mi_5fstrndup_145',['mi_strndup',['../group__malloc.html#ga486d0d26b3b3794f6d1cdb41a9aed92d',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fadd_5fcurrent_5fthread_146',['mi_subproc_add_current_thread',['../group__subproc.html#gadbc53414eb68b275588ec001ce1ddc7c',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fcurrent_147',['mi_subproc_current',['../group__subproc.html#gaabc4db6585bf16099e43e60d64364950',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdelete_148',['mi_subproc_delete',['../group__subproc.html#gaa7d263e9429bac9ac8345c9d25de610e',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fdestroy_149',['mi_subproc_destroy',['../group__subproc.html#gada12e4d9bd382935c398ec324d0391f9',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fheap_5fstats_5fprint_5fout_150',['mi_subproc_heap_stats_print_out',['../group__stats.html#ga19f1a04bd972e11e4f3b586f25229d5b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fmain_151',['mi_subproc_main',['../group__subproc.html#ga2ecba0d7ebdc99e71bb985c4a1609806',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fnew_152',['mi_subproc_new',['../group__subproc.html#ga8068cac328e41fa2170faef707315243',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_153',['mi_subproc_stats_get',['../group__stats.html#ga5ec3f9058eec804924711ab38ba64b51',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fget_5fjson_154',['mi_subproc_stats_get_json',['../group__stats.html#gad1cf918f08b6506e65ae134423477a0b',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fstats_5fprint_5fout_155',['mi_subproc_stats_print_out',['../group__stats.html#ga59ec89969837db414a6bce8b58915624',1,'mimalloc-doc.h']]], + ['mi_5fsubproc_5fvisit_5fheaps_156',['mi_subproc_visit_heaps',['../group__subproc.html#gafd2136e7f167b5cd02b6ab00bed0ac93',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcalloc_157',['mi_theap_calloc',['../group__theap.html#gadbf8f27bfe9edf7834e19d32cab63edc',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fcollect_158',['mi_theap_collect',['../group__theap.html#ga9377a43095b8dd14996613ceb73d9a32',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fget_5fdefault_159',['mi_theap_get_default',['../group__theap.html#ga78afbb07c1e0cd333c7539cf7403dfd5',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_160',['mi_theap_malloc',['../group__theap.html#gac7308e029c59e6675dd5ab5ec54f304a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5faligned_161',['mi_theap_malloc_aligned',['../group__theap.html#ga84503fc2bd83fbfb7f789b7d80d76f36',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fcsize_162',['mi_theap_malloc_csize',['../group__constantsize.html#ga105bd667da91a15d113fe79be04d625c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fmalloc_5fsmall_163',['mi_theap_malloc_small',['../group__theap.html#ga1fc65302a212daaecb21abda96ff247c',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5frealloc_164',['mi_theap_realloc',['../group__theap.html#ga11943128dc354313c4189e72f4299dbb',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fset_5fdefault_165',['mi_theap_set_default',['../group__theap.html#ga04420ce52785019412397e4de24a984a',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_166',['mi_theap_zalloc',['../group__theap.html#gaa7fd973dcd8c75eb05707bdeffbc06f7',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fcsize_167',['mi_theap_zalloc_csize',['../group__constantsize.html#ga1addf755829380581dd9c3c9175e376f',1,'mimalloc-doc.h']]], + ['mi_5ftheap_5fzalloc_5fsmall_168',['mi_theap_zalloc_small',['../group__theap.html#gaa85fccf9d40af3e826c5932aded1d534',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fdone_169',['mi_thread_done',['../group__extended.html#ga0ae4581e85453456a0d658b2b98bf7bf',1,'mimalloc-doc.h']]], + ['mi_5fthread_5finit_170',['mi_thread_init',['../group__extended.html#gaf8e73efc2cbca9ebfdfb166983a04c17',1,'mimalloc-doc.h']]], + ['mi_5fthread_5fset_5fin_5fthreadpool_171',['mi_thread_set_in_threadpool',['../group__extended.html#gada899104276aac52539eb4de7c020f1f',1,'mimalloc-doc.h']]], + ['mi_5fusable_5fsize_172',['mi_usable_size',['../group__malloc.html#ga089c859d9eddc5f9b4bd946cd53cebee',1,'mimalloc-doc.h']]], + ['mi_5fvalloc_173',['mi_valloc',['../group__posix.html#ga50cafb9722020402f065de93799f64ca',1,'mimalloc-doc.h']]], + ['mi_5fversion_174',['mi_version',['../group__extended.html#gae9ddddcf14a58d0224ef3c8778e5b057',1,'mimalloc-doc.h']]], + ['mi_5fwcsdup_175',['mi_wcsdup',['../group__posix.html#gaa9fd7f25c9ac3a20e89b33bd6e383fcf',1,'mimalloc-doc.h']]], + ['mi_5fwdupenv_5fs_176',['mi_wdupenv_s',['../group__posix.html#ga6ac6a6a8f3c96f1af24bb8d0439cbbd1',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_177',['mi_zalloc',['../group__malloc.html#gae6e38c4403247a7b40d80419e093bfb8',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_178',['mi_zalloc_aligned',['../group__aligned.html#gaac7d0beb782f9b9ac31f47492b130f82',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5faligned_5fat_179',['mi_zalloc_aligned_at',['../group__aligned.html#ga7c1778805ce50ebbf02ccbd5e39d5dba',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fcsize_180',['mi_zalloc_csize',['../group__constantsize.html#gadca6d340ee15b5c70695ed82d95c6cec',1,'mimalloc-doc.h']]], + ['mi_5fzalloc_5fsmall_181',['mi_zalloc_small',['../group__extended.html#ga51c47637e81df0e2f13a2d7a2dec123e',1,'mimalloc-doc.h']]] ]; diff --git a/docs/tools.html b/docs/tools.html index fa662b9fa..d969be621 100644 --- a/docs/tools.html +++ b/docs/tools.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
diff --git a/docs/topics.html b/docs/topics.html index bbbae22f2..efbd5b033 100644 --- a/docs/topics.html +++ b/docs/topics.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
@@ -112,7 +112,7 @@  Basic AllocationThe basic allocation interface  Aligned AllocationAllocating aligned memory blocks  Typed MacrosTyped allocation macros - Constant size allocationConstant size allocation and freeing For example: + Constant size allocationInlined definitions for onstant size allocation and freeing  Zero initialized re-allocationRe-allocation of zero initialized blocks  HeapsFirst-class heaps  Heap IntrospectionWalk areas and blocks of the heap at runtime diff --git a/docs/using.html b/docs/using.html index 87f479dce..ee8b7f7e6 100644 --- a/docs/using.html +++ b/docs/using.html @@ -34,7 +34,7 @@ Logo -
mi-malloc 3.4/2.4/1.9 +
mi-malloc 3.5/2.5/1.15
From 430db36fe56128ce05300a4b805b6264f070c8c1 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 11:11:38 -0700 Subject: [PATCH 127/280] improve mi_free_size error messages --- src/free.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/free.c b/src/free.c index 6ade886a4..91670c709 100644 --- a/src/free.c +++ b/src/free.c @@ -329,7 +329,7 @@ void mi_free_size(void* p, size_t size) mi_attr_noexcept { const mi_block_t* block = _mi_page_ptr_unalign(page, p); const bool is_guarded = mi_block_ptr_is_guarded(block,p); if (!is_guarded) { - _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n", p, size, mi_page_block_size(page)); + _mi_error_message(EINVAL, "pointer %p is freed with mi_free_size but the given size %zu is less than the allocated block size %zu\n (maybe a `new[]` was matched with `delete` instead of `delete[]`?)\n", p, size, mi_page_block_size(page)); mi_free(p); return; } From 8c1d13f1210c9ff07cee6a991507deb9be69f830 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 11:19:48 -0700 Subject: [PATCH 128/280] improve codegen on aarch64 --- src/free.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/free.c b/src/free.c index 91670c709..68c233eef 100644 --- a/src/free.c +++ b/src/free.c @@ -240,7 +240,7 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t const mi_threadid_t ptid = mi_page_xthread_id(page); const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid); - if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` + if (xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page mi_block_t* const block = mi_validate_block_from_ptr(page,p); mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); From 8b5ddc49b526e64022458b2a221f58b43cb97558 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 11:33:03 -0700 Subject: [PATCH 129/280] test mi_free_small_nonnull codegen --- src/free.c | 26 ++++++++++++++------------ 1 file changed, 14 insertions(+), 12 deletions(-) diff --git a/src/free.c b/src/free.c index 68c233eef..80c85b898 100644 --- a/src/free.c +++ b/src/free.c @@ -245,19 +245,21 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t mi_block_t* const block = mi_validate_block_from_ptr(page,p); mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); } - else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` - // page is local, but is full or contains (inner) aligned blocks; use generic path - mi_free_generic_local(page, p); - } - // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap) - else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` - // blocks are aligned (and not a full page); push on the thread_free list - mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); - } else { - // page is full or contains (inner) aligned blocks; use generic multi-thread path - mi_free_generic_mt(page, p, allow_collect); + if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` + // page is local, but is full or contains (inner) aligned blocks; use generic path + mi_free_generic_local(page, p); + } + // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap) + else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` + // blocks are aligned (and not a full page); push on the thread_free list + mi_block_t* const block = mi_validate_block_from_ptr(page,p); + mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); + } + else { + // page is full or contains (inner) aligned blocks; use generic multi-thread path + mi_free_generic_mt(page, p, allow_collect); + } } } From 066d25a3b78c2278768df938106a99b3ed97bf67 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 12:19:26 -0700 Subject: [PATCH 130/280] fix mi_free_small_nonnull codegen on aarch64 --- CMakeLists.txt | 3 +++ src/free.c | 28 +++++++++++++--------------- 2 files changed, 16 insertions(+), 15 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3accd81e4..14c0d3292 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -624,6 +624,9 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU" AND NOT MI_CLANG_CL) if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wno-static-in-inline -mno-outline) endif() + if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND MI_ARCH STREQUAL "arm64") + list(APPEND mi_cflags -Xclang -fno-split-cold-code) + endif() endif() if(CMAKE_C_COMPILER_ID MATCHES "Intel") diff --git a/src/free.c b/src/free.c index 80c85b898..91670c709 100644 --- a/src/free.c +++ b/src/free.c @@ -240,26 +240,24 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t const mi_threadid_t ptid = mi_page_xthread_id(page); const mi_threadid_t xtid = (_mi_prim_thread_id() ^ ptid); - if (xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` + if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page mi_block_t* const block = mi_validate_block_from_ptr(page,p); mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); } + else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` + // page is local, but is full or contains (inner) aligned blocks; use generic path + mi_free_generic_local(page, p); + } + // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap) + else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` + // blocks are aligned (and not a full page); push on the thread_free list + mi_block_t* const block = mi_validate_block_from_ptr(page,p); + mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); + } else { - if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` - // page is local, but is full or contains (inner) aligned blocks; use generic path - mi_free_generic_local(page, p); - } - // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap) - else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` - // blocks are aligned (and not a full page); push on the thread_free list - mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); - } - else { - // page is full or contains (inner) aligned blocks; use generic multi-thread path - mi_free_generic_mt(page, p, allow_collect); - } + // page is full or contains (inner) aligned blocks; use generic multi-thread path + mi_free_generic_mt(page, p, allow_collect); } } From dc48dd6d00f1a425c817f0300f6748b31cd55c74 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 12:24:47 -0700 Subject: [PATCH 131/280] fix arm64 compile --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 14c0d3292..89c40aa0f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -624,7 +624,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU" AND NOT MI_CLANG_CL) if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wno-static-in-inline -mno-outline) endif() - if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND MI_ARCH STREQUAL "arm64") + if(CMAKE_C_COMPILER_ID MATCHES "AppleClang" AND MI_ARCH STREQUAL "arm64") list(APPEND mi_cflags -Xclang -fno-split-cold-code) endif() endif() From 6e337d247cc76238f29c90fbdaa919532c434263 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 12:31:06 -0700 Subject: [PATCH 132/280] improve mi_free codegen --- .github/workflows/test.yaml | 2 +- src/free.c | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 7214029cf..75015f6db 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -80,7 +80,7 @@ jobs: if: matrix.tests == 'basic' && runner.os == 'Linux' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: - name: ${{ matrix.os }}-asm + name: ${{ matrix.os }}-${{ matrix.c_or_cxx }}-asm path: out/release-${{ matrix.c_or_cxx }}/CMakeFiles/mimalloc-static.dir/src/*.s - name: Debug diff --git a/src/free.c b/src/free.c index 91670c709..c3fa86896 100644 --- a/src/free.c +++ b/src/free.c @@ -245,12 +245,12 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t mi_block_t* const block = mi_validate_block_from_ptr(page,p); mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); } - else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` + else if mi_likely(xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` // page is local, but is full or contains (inner) aligned blocks; use generic path mi_free_generic_local(page, p); } // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap) - else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` + else if mi_likely((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` // blocks are aligned (and not a full page); push on the thread_free list mi_block_t* const block = mi_validate_block_from_ptr(page,p); mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); From 7e0b273fb6065eebeeb445ca69cf76737cb2bb71 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 12:33:23 -0700 Subject: [PATCH 133/280] fix arm64 compile --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 89c40aa0f..b779df968 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -624,7 +624,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU" AND NOT MI_CLANG_CL) if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wno-static-in-inline -mno-outline) endif() - if(CMAKE_C_COMPILER_ID MATCHES "AppleClang" AND MI_ARCH STREQUAL "arm64") + if(CMAKE_C_COMPILER_ID STREQUAL "AppleClang" AND MI_ARCH STREQUAL "arm64") list(APPEND mi_cflags -Xclang -fno-split-cold-code) endif() endif() From 4df1f0dd03254ed81441b835ddc001e714572b7f Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 12:34:53 -0700 Subject: [PATCH 134/280] fix arm64 compile --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b779df968..14c0d3292 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -624,7 +624,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU" AND NOT MI_CLANG_CL) if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wno-static-in-inline -mno-outline) endif() - if(CMAKE_C_COMPILER_ID STREQUAL "AppleClang" AND MI_ARCH STREQUAL "arm64") + if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND MI_ARCH STREQUAL "arm64") list(APPEND mi_cflags -Xclang -fno-split-cold-code) endif() endif() From 75a968ab46959907aeb1eb575b2d0ef46f6972f2 Mon Sep 17 00:00:00 2001 From: daanx Date: Sat, 29 Aug 2026 12:38:51 -0700 Subject: [PATCH 135/280] fix arm64 compile --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 14c0d3292..b779df968 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -624,7 +624,7 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU" AND NOT MI_CLANG_CL) if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") list(APPEND mi_cflags -Wno-static-in-inline -mno-outline) endif() - if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND MI_ARCH STREQUAL "arm64") + if(CMAKE_C_COMPILER_ID STREQUAL "AppleClang" AND MI_ARCH STREQUAL "arm64") list(APPEND mi_cflags -Xclang -fno-split-cold-code) endif() endif() From 93fb234604648453f832f382c90550e4827e852c Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 08:44:13 -0700 Subject: [PATCH 136/280] remove superfluous mi_likely --- src/free.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/free.c b/src/free.c index c3fa86896..91670c709 100644 --- a/src/free.c +++ b/src/free.c @@ -245,12 +245,12 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t mi_block_t* const block = mi_validate_block_from_ptr(page,p); mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); } - else if mi_likely(xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` + else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` // page is local, but is full or contains (inner) aligned blocks; use generic path mi_free_generic_local(page, p); } // free-ing in a page owned by a theap in another thread, or an abandoned page (not belonging to a theap) - else if mi_likely((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` + else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` // blocks are aligned (and not a full page); push on the thread_free list mi_block_t* const block = mi_validate_block_from_ptr(page,p); mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); From a1036ba3eb553beca71dfe9674682735a3b6c1b9 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 08:50:50 -0700 Subject: [PATCH 137/280] ensure MI_OPT_ARCH=OFF is respected on arm64 (issue #1382) --- CMakeLists.txt | 15 ++++++++++++--- 1 file changed, 12 insertions(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e8b7164d8..a3eb421d2 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,7 +15,9 @@ option(MI_GUARDED "Build with guard pages behind certain object alloca set(MI_USE_CXX "DEFAULT" CACHE STRING "Use the C++ compiler to compile the library (instead of the C compiler)") set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") -option(MI_OPT_ARCH "Only for optimized builds: turn on architecture specific optimizations (for arm64: '-march=armv8.1-a' (2016))" OFF) +set(MI_OPT_ARCH "DEFAULT" CACHE STRING "Only for optimized builds: turn on architecture specific optimizations (for arm64: '-march=armv8.1-a' (2016))") +set_property(CACHE MI_OPT_ARCH PROPERTY STRINGS "OFF" "ON" "DEFAULT") + option(MI_SEE_ASM "Generate assembly files" OFF) option(MI_OSX_INTERPOSE "Use interpose to override standard malloc on macOS" ON) option(MI_OSX_ZONE "Use malloc zone to override standard malloc on macOS" ON) @@ -158,10 +160,17 @@ message(STATUS "Architecture: ${MI_ARCH}") # (${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE if(MI_NO_USE_CXX) set(MI_USE_CXX "OFF") endif() + if(MI_NO_OPT_ARCH) set(MI_OPT_ARCH "OFF") -elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH "ON") # enable armv8.1-a by default on arm64 unless MI_NO_OPT_ARCH is set +endif() + +if(MI_OPT_ARCH STREQUAL "DEFAULT") + if(MI_ARCH STREQUAL "arm64") + set(MI_OPT_ARCH "ON") # enable armv8.1-a by default on arm64 unless MI_NO_OPT_ARCH is set + else() + set(MI_OPT_ARCH "OFF") + endif() endif() From 0f80c396e68a418857775874c18520442fbac41a Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 10:06:14 -0700 Subject: [PATCH 138/280] Allow specicic architecture to be passed to MI_OPT_ARCH; default to armv8.1 by default again, except on Apple which defaults to armv8.3 (M1+), (issue #1382) --- CMakeLists.txt | 96 ++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 69 insertions(+), 27 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d091927b2..5bdc5b85d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -15,8 +15,8 @@ option(MI_GUARDED "Build with guard pages behind certain object alloca set(MI_USE_CXX "DEFAULT" CACHE STRING "Use the C++ compiler to compile the library (instead of the C compiler)") set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") -set(MI_OPT_ARCH "DEFAULT" CACHE STRING "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.3-a' (2016), for riscV: '-march=rv64gcb_zacas' (2023))") -set_property(CACHE MI_OPT_ARCH PROPERTY STRINGS "OFF" "ON" "DEFAULT") +set(MI_OPT_ARCH "DEFAULT" CACHE STRING "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016), for riscV: '-march=rv64gcb_zacas' (2023))") +set_property(CACHE MI_OPT_ARCH PROPERTY STRINGS "OFF" "ON" "avx2" "armv8.1" "armv8.3" "rva22" "rva23" "rv64gcb_zacas" "DEFAULT") option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) @@ -137,6 +137,17 @@ if(EXISTS "${CMAKE_SOURCE_DIR}/.git/index") endif() endif() +# case insensitive comparison +function(STRIEQUAL str1 str2 is_equal) + string(TOUPPER "${str1}" str1_upper) + string(TOUPPER "${str2}" str2_upper) + if(str1_upper STREQUAL str2_upper) + set(${is_equal} TRUE PARENT_SCOPE) + else() + set(${is_equal} FALSE PARENT_SCOPE) + endif() +endfunction() + # ----------------------------------------------------------------------------- # Convenience: set default build type and compiler depending on the build directory # ----------------------------------------------------------------------------- @@ -164,7 +175,6 @@ endif() # Determine architecture -set(MI_OPT_ARCH_FLAGS "") set(MI_ARCH "unknown") if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86|i[3456]86)$" OR CMAKE_GENERATOR_PLATFORM MATCHES "^(x86|Win32)$") set(MI_ARCH "x86") @@ -196,7 +206,7 @@ endif() if(MI_OPT_ARCH STREQUAL "DEFAULT") if(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH "ON") # enable armv8.3-a by default (has LDAPR) on arm6 + set(MI_OPT_ARCH "ON") # enable by default for arm64 (defaults to armv8.1-a which has fast atomics) else() set(MI_OPT_ARCH "OFF") endif() @@ -663,33 +673,64 @@ if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel") endif() # Compiler and architecture specific flags -if(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") - if(MI_OPT_ARCH) - if(APPLE AND CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" AND CMAKE_OSX_ARCHITECTURES) # to support multi-arch binaries (#999) - if("arm64" IN_LIST CMAKE_OSX_ARCHITECTURES) - list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_arm64;-march=armv8.3-a") - endif() - if("x86_64" IN_LIST CMAKE_OSX_ARCHITECTURES) - list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_x86_64;-march=haswell;-Xarch_x86_64;-mavx2") - endif() - elseif(MI_ARCH STREQUAL "x64") - set(MI_OPT_ARCH_FLAGS "-march=haswell;-mavx2") # fast bit scan (since 2013) - elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH_FLAGS "-march=armv8.3-a") # fast atomics (since 2016) - elseif(MI_ARCH STREQUAL "riscv64") - set(MI_OPT_ARCH_FLAGS "-march=rv64gcb_zacas") # fast bit scan and atomics (since 2023) TODO: add zalasr (2025) when possible. - elseif(MI_ARCH STREQUAL "riscv32") - set(MI_OPT_ARCH_FLAGS "-march=rv32gcb_zacas") # fast bit scan and atomics (since 2023) +if(MI_OPT_ARCH MATCHES "^(1|ON|YES|TRUE|Y)$") + if(APPLE AND CMAKE_OSX_ARCHITECTURES AND CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang" ) + set(MI_OPT_ARCH "ON") + elseif(MI_ARCH STREQUAL "arm64") + if(APPLE) + set(MI_OPT_ARCH "armv8.3") # M1+ supports arm8.3-a (which includes LDAPR (load-aquire)) + else() + set(MI_OPT_ARCH "armv8.1") # enable armv8.1-a by default (which has fast atomics) endif() + elseif(MI_ARCH STREQUAL "x64") + set(MI_OPT_ARCH "avx2") + elseif(MI_ARCH STREQUAL "riscv64") + # set(MI_OPT_ARCH "rva23") # best as it has zalasr (2025) + set(MI_OPT_ARCH "rv64gcb_zacas") # fast bit scan and atomics (since 2023) + elseif(MI_ARCH STREQUAL "riscv32") + set(MI_OPT_ARCH "rv32gcb_zacas") endif() endif() -if (MSVC AND MSVC_VERSION GREATER_EQUAL 1914) # vs2017+ - if(MI_OPT_ARCH AND NOT MI_CLANG_CL) - if(MI_ARCH STREQUAL "x64") +# Compiler and architecture specific flags +set(MI_OPT_ARCH_FLAGS "") +if(NOT MI_OPT_ARCH MATCHES "^(0|OFF|NO|FALSE|N)$") + # apple support for multi-arch binaries (#999) + if(APPLE AND CMAKE_OSX_ARCHITECTURES AND CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang") + if("arm64" IN_LIST CMAKE_OSX_ARCHITECTURES) + list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_arm64;-march=armv8.3-a") + endif() + if("x86_64" IN_LIST CMAKE_OSX_ARCHITECTURES) + list(APPEND MI_OPT_ARCH_FLAGS "-Xarch_x86_64;-march=haswell;-Xarch_x86_64;-mavx2") + endif() + + # clang/gcc + elseif(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") + if(MI_OPT_ARCH STREQUAL "avx2") + set(MI_OPT_ARCH_FLAGS "-march=haswell;-mavx2") # fast bit scan (since 2013) + elseif(MI_OPT_ARCH MATCHES "^armv[89](\.[0-9])?$") + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}-a") # fast atomics + elseif(MI_OPT_ARCH MATCHES "^rva[0-9][0-9]$") + if(MI_ARCH STREQUAL "riscv64") + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}u64") + else() + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}u32") + endif() + elseif(MI_OPT_ARCH MATCHES "^rv*$") + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}") + else() + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}") + message(STATUS "warning: unknown MI_OPT_ARCH=${MI_OPT_ARCH} passed as '-march=${MI_OPT_ARCH}'") + endif() + + # msvc + elseif (MSVC AND MSVC_VERSION GREATER_EQUAL 1914 AND NOT MI_CLANG_CL) # vs2017+ + if(MI_OPT_ARCH STREQUAL "avx2") set(MI_OPT_ARCH_FLAGS "/arch:AVX2") - elseif(MI_ARCH STREQUAL "arm64") - set(MI_OPT_ARCH_FLAGS "/arch:armv8.3") # fast atomics (LDAPR) + elseif(MI_OPT_ARCH MATCHES "^armv[89](\.[0-9])?$") + set(MI_OPT_ARCH_FLAGS "/arch:${MI_OPT_ARCH}") # fast atomics + else() + message(STATUS "warning: unknown MI_OPT_ARCH=${MI_OPT_ARCH} passed as '/arch:${MI_OPT_ARCH}'") endif() endif() endif() @@ -700,7 +741,7 @@ endif() if(MI_OPT_ARCH_FLAGS) list(APPEND mi_cflags ${MI_OPT_ARCH_FLAGS}) - message(STATUS "Architecture specific optimization is enabled (with ${MI_OPT_ARCH_FLAGS}) (MI_OPT_ARCH=ON)") + message(STATUS "Architecture specific optimization is enabled (with ${MI_OPT_ARCH_FLAGS}) (MI_OPT_ARCH=${MI_OPT_ARCH})") if (MI_OPT_SIMD) list(APPEND mi_defines "MI_OPT_SIMD=1") message(STATUS "SIMD instructions are enabled (MI_OPT_SIMD=ON)") @@ -709,6 +750,7 @@ elseif(MI_OPT_SIMD) message(STATUS "SIMD instructions are not enabled (either MI_OPT_ARCH=OFF or this architecture has no SIMD support)") endif() + # extra needed libraries # we prefer -l test over `find_library` as sometimes core libraries From c42fb1af4fd3dc3bb4380c7919c9d569c0225839 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 10:13:43 -0700 Subject: [PATCH 139/280] fix MI_OPT_ARCH on arm32 --- CMakeLists.txt | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 5bdc5b85d..4e4949dd1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -682,13 +682,17 @@ if(MI_OPT_ARCH MATCHES "^(1|ON|YES|TRUE|Y)$") else() set(MI_OPT_ARCH "armv8.1") # enable armv8.1-a by default (which has fast atomics) endif() + elseif(MI_ARCH_STREQUAL "arm32") + set(MI_OPT_ARCH "armv8.1") # enable armv8.1-a by default (which has fast atomics) elseif(MI_ARCH STREQUAL "x64") set(MI_OPT_ARCH "avx2") elseif(MI_ARCH STREQUAL "riscv64") # set(MI_OPT_ARCH "rva23") # best as it has zalasr (2025) set(MI_OPT_ARCH "rv64gcb_zacas") # fast bit scan and atomics (since 2023) elseif(MI_ARCH STREQUAL "riscv32") - set(MI_OPT_ARCH "rv32gcb_zacas") + set(MI_OPT_ARCH "rv32gcb_zacas") + else() + set(MI_OPT_ARCH OFF) endif() endif() From 27816291daf04f44eaa25af441a8fbf99df15e74 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 10:14:55 -0700 Subject: [PATCH 140/280] fix MI_OPT_ARCH on arm32 --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 4e4949dd1..d8c0f3b60 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -682,7 +682,7 @@ if(MI_OPT_ARCH MATCHES "^(1|ON|YES|TRUE|Y)$") else() set(MI_OPT_ARCH "armv8.1") # enable armv8.1-a by default (which has fast atomics) endif() - elseif(MI_ARCH_STREQUAL "arm32") + elseif(MI_ARCH STREQUAL "arm32") set(MI_OPT_ARCH "armv8.1") # enable armv8.1-a by default (which has fast atomics) elseif(MI_ARCH STREQUAL "x64") set(MI_OPT_ARCH "avx2") From a51fe32bc9dd892f012894ab779f92b05dc9490c Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 10:40:13 -0700 Subject: [PATCH 141/280] add rva23 to ci tests --- .github/workflows/test.yaml | 20 +++++++++++++++++++- 1 file changed, 19 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 75015f6db..bf04f34a4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -394,11 +394,12 @@ jobs: with: arch: riscv64 shell-name: alpine-riscv64.sh + apk-tools-url: https://gitlab.alpinelinux.org/api/v4/projects/5/packages/generic/v3.0.7/x86_64/apk.static#!sha256!c07bf5356eacc9dd7a8c56bc537f46702f007170287403299d52a04264e74b3c packages: | build-base cmake ninja - + - name: Alpine riscv64, Release if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} @@ -433,6 +434,23 @@ jobs: cmake --build out/secure-riscv64 --parallel 4 --config Release ctest --test-dir out/secure-riscv64 --verbose --timeout 240 -C Release + - name: Alpine riscv64, Release rva23 + if: matrix.tests == 'alpine-riscv64' + shell: alpine-riscv64.sh {0} + run: | + cmake . -B out/release-riscv64-rva23 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rva23 -DMI_SEE_ASM=ON + cmake --build out/release-riscv64-rva23 --parallel 4 --config Release + ctest --test-dir out/release-riscv64-rva23 --verbose --timeout 240 -C Release + + - name: Alpine risc64, Asm rva23 + if: matrix.tests == 'alpine-riscv64' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: alpine-riscv64-rva23-asm + path: out/release-riscv64-rva23/CMakeFiles/mimalloc-static.dir/src/*.s + + + - name: Alpine x64, Setup if: matrix.tests == 'alpine-x64' From 784f0579765e416796c174158463f35e7b93e1ff Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Sun, 30 Aug 2026 19:31:21 +0200 Subject: [PATCH 142/280] Use generator expression to get path for generated object file --- CMakeLists.txt | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d8c0f3b60..bbd6a7d92 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -964,11 +964,7 @@ if (MI_BUILD_OBJECT) ) # Copy the generated object file (`static.o`) to the output directory (as `mimalloc.o`) - if(CMAKE_GENERATOR MATCHES "^Visual Studio.*$") - set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/mimalloc-obj.dir/$/static${CMAKE_C_OUTPUT_EXTENSION}") - else() - set(mimalloc-obj-static "${CMAKE_CURRENT_BINARY_DIR}/CMakeFiles/mimalloc-obj.dir/src/static.c${CMAKE_C_OUTPUT_EXTENSION}") - endif() + set(mimalloc-obj-static "$") set(mimalloc-obj-out "${CMAKE_CURRENT_BINARY_DIR}/${mi_libname}${CMAKE_C_OUTPUT_EXTENSION}") add_custom_command(OUTPUT ${mimalloc-obj-out} DEPENDS mimalloc-obj mimalloc-static COMMAND "${CMAKE_COMMAND}" -E copy "${mimalloc-obj-static}" "${mimalloc-obj-out}") add_custom_target(mimalloc-obj-target ALL DEPENDS ${mimalloc-obj-out}) From 9b6a914e44e7522a9cd36644504b2844235b4c91 Mon Sep 17 00:00:00 2001 From: Frank Richter Date: Sun, 30 Aug 2026 19:33:35 +0200 Subject: [PATCH 143/280] Improve behavior on multi-config generators --- CMakeLists.txt | 45 ++++++++++++++++++++++++++++++++++----------- 1 file changed, 34 insertions(+), 11 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index bbd6a7d92..14253fde0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -153,7 +153,7 @@ endfunction() # ----------------------------------------------------------------------------- message(STATUS "") -if(NOT CMAKE_BUILD_TYPE) +if(NOT CMAKE_BUILD_TYPE AND NOT CMAKE_CONFIGURATION_TYPES) if ("${CMAKE_BINARY_DIR}" MATCHES ".*((D|d)ebug|asan|tsan|ubsan|valgrind)$") message(STATUS "No build type selected, default to 'Debug'") set(CMAKE_BUILD_TYPE "Debug") @@ -229,7 +229,7 @@ if(MI_USE_CXX STREQUAL "DEFAULT") if(CMAKE_C_COMPILER_ID MATCHES "MSVC|Intel" OR MI_CLANG_CL) # force C++ compilation with msvc or clang-cl to use modern C++ atomics set(MI_USE_CXX "ON") - elseif(MI_DEBUG_UBSAN AND CMAKE_BUILD_TYPE MATCHES "Debug") # ubsan needs C++ + elseif(MI_DEBUG_UBSAN AND (CMAKE_BUILD_TYPE MATCHES "Debug" OR NOT CMAKE_CONFIGURATION_TYPES)) # ubsan needs C++ set(MI_USE_CXX "ON") else() set(MI_USE_CXX "OFF") @@ -398,10 +398,16 @@ elseif(MI_DEBUG_INTERNAL) endif() if(MI_DEBUG STREQUAL "DEFAULT") - if(CMAKE_BUILD_TYPE MATCHES "Debug") - set(MI_DEBUG "INTERNAL") + if(CMAKE_CONFIGURATION_TYPES) + # Multi-config generators: Choose debug flags using generator expression + list(APPEND mi_defines $,MI_DEBUG=2 MI_GUARDED=1,MI_DEBUG=0>) + set(MI_DEBUG OFF) # don't go into further MI_DEBUG checks below... else() - set(MI_DEBUG "OFF") + if(CMAKE_BUILD_TYPE MATCHES "Debug") + set(MI_DEBUG "INTERNAL") + else() + set(MI_DEBUG "OFF") + endif() endif() endif() @@ -829,12 +835,27 @@ endif() if(MI_TRACK STREQUAL "ASAN") set(mi_libname "${mi_libname}-asan") endif() -string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) -list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE_LC}") #todo: multi-config project needs $ ? -if(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$") - list(APPEND mi_defines MI_BUILD_RELEASE) +if (CMAKE_CONFIGURATION_TYPES) + # multi-config builds + + # generator expression lower-case build type + set(_mi_gx_build_type_lc $>) + # generator expression for "is this a release build?" + set(_mi_gx_release $,$,$,$>) + + list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${_mi_gx_build_type_lc}") + list(APPEND mi_defines $<${_mi_gx_release}:MI_BUILD_RELEASE>) + # append build type (e.g. -debug) if not a release version + set(mi_libname "${mi_libname}$<$:-${_mi_gx_build_type_lc}>") else() - set(mi_libname "${mi_libname}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version + # single-config builds + string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) + list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${CMAKE_BUILD_TYPE_LC}") #todo: multi-config project needs $ ? + if(CMAKE_BUILD_TYPE_LC MATCHES "^(release|relwithdebinfo|minsizerel|none)$") + list(APPEND mi_defines MI_BUILD_RELEASE) + else() + set(mi_libname "${mi_libname}-${CMAKE_BUILD_TYPE_LC}") #append build type (e.g. -debug) if not a release version + endif() endif() string(REGEX REPLACE "^mimalloc[-]?" "" mi_libsuffix "${mi_libname}") @@ -855,7 +876,9 @@ endif() message(STATUS "") message(STATUS "Library name : ${mi_libname}") message(STATUS "Version : ${mi_version}.${mi_version_patch}") -message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") +if(CMAKE_BUILD_TYPE_LC) + message(STATUS "Build type : ${CMAKE_BUILD_TYPE_LC}") +endif() if(MI_USE_CXX) message(STATUS "C++ Compiler : ${CMAKE_CXX_COMPILER}") else() From fcb80d1263316aae1dbe90360f344158aa42c080 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 10:46:19 -0700 Subject: [PATCH 144/280] fix rv* architecture match --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index d8c0f3b60..e948b8b85 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -720,7 +720,7 @@ if(NOT MI_OPT_ARCH MATCHES "^(0|OFF|NO|FALSE|N)$") else() set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}u32") endif() - elseif(MI_OPT_ARCH MATCHES "^rv*$") + elseif(MI_OPT_ARCH MATCHES "^rv.*$") set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}") else() set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}") From c0dfd631f0fa512eab2b2142f6be839fe016cc6a Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 10:47:30 -0700 Subject: [PATCH 145/280] fix rv* architecture match --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e948b8b85..e79000694 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -712,7 +712,7 @@ if(NOT MI_OPT_ARCH MATCHES "^(0|OFF|NO|FALSE|N)$") elseif(CMAKE_C_COMPILER_ID MATCHES "AppleClang|Clang|GNU|Intel" AND NOT CMAKE_SYSTEM_NAME MATCHES "Haiku") if(MI_OPT_ARCH STREQUAL "avx2") set(MI_OPT_ARCH_FLAGS "-march=haswell;-mavx2") # fast bit scan (since 2013) - elseif(MI_OPT_ARCH MATCHES "^armv[89](\.[0-9])?$") + elseif(MI_OPT_ARCH MATCHES "^armv[89].*$") set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}-a") # fast atomics elseif(MI_OPT_ARCH MATCHES "^rva[0-9][0-9]$") if(MI_ARCH STREQUAL "riscv64") @@ -731,7 +731,7 @@ if(NOT MI_OPT_ARCH MATCHES "^(0|OFF|NO|FALSE|N)$") elseif (MSVC AND MSVC_VERSION GREATER_EQUAL 1914 AND NOT MI_CLANG_CL) # vs2017+ if(MI_OPT_ARCH STREQUAL "avx2") set(MI_OPT_ARCH_FLAGS "/arch:AVX2") - elseif(MI_OPT_ARCH MATCHES "^armv[89](\.[0-9])?$") + elseif(MI_OPT_ARCH MATCHES "^armv[89].*$") set(MI_OPT_ARCH_FLAGS "/arch:${MI_OPT_ARCH}") # fast atomics else() message(STATUS "warning: unknown MI_OPT_ARCH=${MI_OPT_ARCH} passed as '/arch:${MI_OPT_ARCH}'") From 3fd9b1c590be612368d8da98abea0b1f88098357 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 10:48:05 -0700 Subject: [PATCH 146/280] fix assertion warning --- include/mimalloc/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 4d4fcb6f0..910665929 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -776,7 +776,7 @@ static inline mi_page_t* _mi_aligned_ptr_page0(const void* p) { mi_page_t* const page_metas = (mi_page_t*)_mi_align_down_ptr(p,MI_PAGE_META_ALIGNMENT); // const ptrdiff_t page_idx = ((uint8_t*)p - (uint8_t*)page_metas)/MI_ARENA_SLICE_SIZE; const uintptr_t page_idx = ((uintptr_t)p / MI_ARENA_SLICE_SIZE) % (MI_PAGE_META_ALIGNMENT / MI_ARENA_SLICE_SIZE); - mi_assert_internal(page_idx >= 0 && page_idx <= MI_PAGE_META_ALIGNED_COUNT); + mi_assert_internal(page_idx <= MI_PAGE_META_ALIGNED_COUNT); #if MI_ARCH_X64 || MI_ARCH_X86 || MI_ARCH_RISCV // better code on x64/x86/riscv64 mi_page_t* const page = (mi_page_t*)((uintptr_t)page_metas | (page_idx * sizeof(mi_page_t))); #else From 4df4e21ca9921b8cf8a1d3a720fc79f6e9768478 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 11:13:09 -0700 Subject: [PATCH 147/280] add zalasr test to risv --- .github/workflows/test.yaml | 34 ++++++++++++++++------------------ CMakeLists.txt | 35 +++++++++++++++++++---------------- 2 files changed, 35 insertions(+), 34 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index bf04f34a4..5551570bd 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -399,6 +399,22 @@ jobs: build-base cmake ninja + + - name: Alpine riscv64, Release zalasr + if: matrix.tests == 'alpine-riscv64' + shell: alpine-riscv64.sh {0} + run: | + cmake . -B out/release-riscv64-zalasr -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rv64gcb-zacas-zalasr -DMI_SEE_ASM=ON + cmake --build out/release-riscv64-zalasr --parallel 4 --config Release + ctest --test-dir out/release-riscv64-zalasr --verbose --timeout 240 -C Release + + - name: Alpine risc64, Asm zalasr + if: matrix.tests == 'alpine-riscv64' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: alpine-riscv64-zalasr-asm + path: out/release-riscv64-zalasr/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Alpine riscv64, Release if: matrix.tests == 'alpine-riscv64' @@ -434,24 +450,6 @@ jobs: cmake --build out/secure-riscv64 --parallel 4 --config Release ctest --test-dir out/secure-riscv64 --verbose --timeout 240 -C Release - - name: Alpine riscv64, Release rva23 - if: matrix.tests == 'alpine-riscv64' - shell: alpine-riscv64.sh {0} - run: | - cmake . -B out/release-riscv64-rva23 -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rva23 -DMI_SEE_ASM=ON - cmake --build out/release-riscv64-rva23 --parallel 4 --config Release - ctest --test-dir out/release-riscv64-rva23 --verbose --timeout 240 -C Release - - - name: Alpine risc64, Asm rva23 - if: matrix.tests == 'alpine-riscv64' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: alpine-riscv64-rva23-asm - path: out/release-riscv64-rva23/CMakeFiles/mimalloc-static.dir/src/*.s - - - - - name: Alpine x64, Setup if: matrix.tests == 'alpine-x64' uses: jirutka/setup-alpine@ae3b3ddba35054804fc4a3507b519fa7e8152050 # v1.4.1 diff --git a/CMakeLists.txt b/CMakeLists.txt index e79000694..beea3a57c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -16,7 +16,7 @@ set(MI_USE_CXX "DEFAULT" CACHE STRING "Use the C++ compiler to compile the libra set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") set(MI_OPT_ARCH "DEFAULT" CACHE STRING "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016), for riscV: '-march=rv64gcb_zacas' (2023))") -set_property(CACHE MI_OPT_ARCH PROPERTY STRINGS "OFF" "ON" "avx2" "armv8.1" "armv8.3" "rva22" "rva23" "rv64gcb_zacas" "DEFAULT") +set_property(CACHE MI_OPT_ARCH PROPERTY STRINGS "OFF" "ON" "avx2" "armv8.1" "armv8.3" "rva22" "rva23" "rv64gcb_zacas" "rv64gcb_zacas_zalasr" "DEFAULT") option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) @@ -175,6 +175,18 @@ endif() # Determine architecture +if(CMAKE_SIZEOF_VOID_P==16) + set(MI_ARCH_BITS "128") +elseif(CMAKE_SIZEOF_VOID_P==8) + set(MI_ARCH_BITS "64") +elseif(CMAKE_SIZEOF_VOID_P==4) + set(MI_ARCH_BITS "32") +elseif(CMAKE_SIZEOF_VOID_P==2) + set(MI_ARCH_BITS "16") +else() + set(MI_ARCH_BITS "") +endif() + set(MI_ARCH "unknown") if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(x86|i[3456]86)$" OR CMAKE_GENERATOR_PLATFORM MATCHES "^(x86|Win32)$") set(MI_ARCH "x86") @@ -185,14 +197,11 @@ elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|armv[89].?|ARM64)$" OR CM elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(arm|armv[34567].?|ARM)$") set(MI_ARCH "arm32") elseif(CMAKE_SYSTEM_PROCESSOR MATCHES "^(riscv|riscv32|riscv64)$") - if(CMAKE_SIZEOF_VOID_P==4) - set(MI_ARCH "riscv32") - else() - set(MI_ARCH "riscv64") - endif() + set(MI_ARCH "riscv${MI_ARCH_BITS}") else() set(MI_ARCH ${CMAKE_SYSTEM_PROCESSOR}) endif() + message(STATUS "Architecture: ${MI_ARCH}") # (${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_GENERATOR_PLATFORM}, ${CMAKE_GENERATOR})") # negative overrides (mainly to support vcpkg features) @@ -686,11 +695,9 @@ if(MI_OPT_ARCH MATCHES "^(1|ON|YES|TRUE|Y)$") set(MI_OPT_ARCH "armv8.1") # enable armv8.1-a by default (which has fast atomics) elseif(MI_ARCH STREQUAL "x64") set(MI_OPT_ARCH "avx2") - elseif(MI_ARCH STREQUAL "riscv64") - # set(MI_OPT_ARCH "rva23") # best as it has zalasr (2025) - set(MI_OPT_ARCH "rv64gcb_zacas") # fast bit scan and atomics (since 2023) - elseif(MI_ARCH STREQUAL "riscv32") - set(MI_OPT_ARCH "rv32gcb_zacas") + elseif(MI_ARCH MATCHES "riscv") + set(MI_OPT_ARCH "rv${MI_ARCH_BITS}gcb_zacas") # fast bit scan and atomics (since 2023) + # set(MI_OPT_ARCH "rv${MI_ARCH_BITS}gcb_zacas_zalasr") # fast bit scan and atomics (since 2025) else() set(MI_OPT_ARCH OFF) endif() @@ -715,11 +722,7 @@ if(NOT MI_OPT_ARCH MATCHES "^(0|OFF|NO|FALSE|N)$") elseif(MI_OPT_ARCH MATCHES "^armv[89].*$") set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}-a") # fast atomics elseif(MI_OPT_ARCH MATCHES "^rva[0-9][0-9]$") - if(MI_ARCH STREQUAL "riscv64") - set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}u64") - else() - set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}u32") - endif() + set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}u${MI_ARCH_BITS}") elseif(MI_OPT_ARCH MATCHES "^rv.*$") set(MI_OPT_ARCH_FLAGS "-march=${MI_OPT_ARCH}") else() From 9f3fc2be3c04c7fb260dabf85bf27bb5d7da39bc Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 11:16:52 -0700 Subject: [PATCH 148/280] add clang to riscv CI --- .github/workflows/test.yaml | 19 ++++++++++++++++++- 1 file changed, 18 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 5551570bd..78c0980b1 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -399,12 +399,13 @@ jobs: build-base cmake ninja + clang - name: Alpine riscv64, Release zalasr if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} run: | - cmake . -B out/release-riscv64-zalasr -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rv64gcb-zacas-zalasr -DMI_SEE_ASM=ON + cmake . -B out/release-riscv64-zalasr -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rv64gcb_zacas_zalasr -DMI_SEE_ASM=ON cmake --build out/release-riscv64-zalasr --parallel 4 --config Release ctest --test-dir out/release-riscv64-zalasr --verbose --timeout 240 -C Release @@ -414,7 +415,23 @@ jobs: with: name: alpine-riscv64-zalasr-asm path: out/release-riscv64-zalasr/CMakeFiles/mimalloc-static.dir/src/*.s + + - name: Alpine riscv64, Release clang + if: matrix.tests == 'alpine-riscv64' + shell: alpine-riscv64.sh {0} + run: | + CC=clang CXX=clang++ cmake . -B out/release-riscv64-clang -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON + cmake --build out/release-riscv64-clang --parallel 4 --config Release + ctest --test-dir out/release-riscv64-clang --verbose --timeout 240 -C Release + + - name: Alpine risc64, Asm clang + if: matrix.tests == 'alpine-riscv64' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: alpine-riscv64-clang-asm + path: out/release-riscv64-clang/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Alpine riscv64, Release if: matrix.tests == 'alpine-riscv64' From 17aa0af8e25c88632abca51f7e53c5e5096198bd Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 11:18:57 -0700 Subject: [PATCH 149/280] add clang++ to riscv CI --- .github/workflows/test.yaml | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 78c0980b1..dd2b1922b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -400,21 +400,7 @@ jobs: cmake ninja clang - - - name: Alpine riscv64, Release zalasr - if: matrix.tests == 'alpine-riscv64' - shell: alpine-riscv64.sh {0} - run: | - cmake . -B out/release-riscv64-zalasr -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rv64gcb_zacas_zalasr -DMI_SEE_ASM=ON - cmake --build out/release-riscv64-zalasr --parallel 4 --config Release - ctest --test-dir out/release-riscv64-zalasr --verbose --timeout 240 -C Release - - - name: Alpine risc64, Asm zalasr - if: matrix.tests == 'alpine-riscv64' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: alpine-riscv64-zalasr-asm - path: out/release-riscv64-zalasr/CMakeFiles/mimalloc-static.dir/src/*.s + clang++ - name: Alpine riscv64, Release clang if: matrix.tests == 'alpine-riscv64' @@ -431,6 +417,21 @@ jobs: name: alpine-riscv64-clang-asm path: out/release-riscv64-clang/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Alpine riscv64, Release zalasr + if: matrix.tests == 'alpine-riscv64' + shell: alpine-riscv64.sh {0} + run: | + CC=clang CXX=clang++ cmake . -B out/release-riscv64-zalasr -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rv64gcb_zacas_zalasr -DMI_SEE_ASM=ON + cmake --build out/release-riscv64-zalasr --parallel 4 --config Release + ctest --test-dir out/release-riscv64-zalasr --verbose --timeout 240 -C Release + + - name: Alpine risc64, Asm zalasr + if: matrix.tests == 'alpine-riscv64' + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: alpine-riscv64-zalasr-asm + path: out/release-riscv64-zalasr/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Alpine riscv64, Release From 5a4f73da5495c891bccc9b71773be06e9f4eef45 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 11:25:32 -0700 Subject: [PATCH 150/280] remove clang++ from riscv CI --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index dd2b1922b..c1f8ee662 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -400,7 +400,6 @@ jobs: cmake ninja clang - clang++ - name: Alpine riscv64, Release clang if: matrix.tests == 'alpine-riscv64' From 30de2b461bb375eb9e63d2c6bf142310a0b52ca9 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 11:30:51 -0700 Subject: [PATCH 151/280] fix sizeof(void*) test in cmake --- CMakeLists.txt | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index beea3a57c..e9ec76282 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -175,13 +175,13 @@ endif() # Determine architecture -if(CMAKE_SIZEOF_VOID_P==16) +if(CMAKE_SIZEOF_VOID_P EQUAL 16) set(MI_ARCH_BITS "128") -elseif(CMAKE_SIZEOF_VOID_P==8) +elseif(CMAKE_SIZEOF_VOID_P EQUAL 8) set(MI_ARCH_BITS "64") -elseif(CMAKE_SIZEOF_VOID_P==4) +elseif(CMAKE_SIZEOF_VOID_P EQUAL 4) set(MI_ARCH_BITS "32") -elseif(CMAKE_SIZEOF_VOID_P==2) +elseif(CMAKE_SIZEOF_VOID_P EQUAL 2) set(MI_ARCH_BITS "16") else() set(MI_ARCH_BITS "") @@ -202,7 +202,7 @@ else() set(MI_ARCH ${CMAKE_SYSTEM_PROCESSOR}) endif() -message(STATUS "Architecture: ${MI_ARCH}") # (${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_GENERATOR_PLATFORM}, ${CMAKE_GENERATOR})") +message(STATUS "Architecture: ${MI_ARCH} (${MI_ARCH_BITS} bit)") # (${CMAKE_SYSTEM_PROCESSOR}, ${CMAKE_GENERATOR_PLATFORM}, ${CMAKE_GENERATOR})") # negative overrides (mainly to support vcpkg features) if(MI_NO_USE_CXX) From 9cf99c79ae1a73bfbdd155080640c1f16c036cd1 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 30 Aug 2026 11:50:27 -0700 Subject: [PATCH 152/280] fix __riscv macro misspelling --- .github/workflows/test.yaml | 21 ++------------------- include/mimalloc/prim-tls.h | 2 +- 2 files changed, 3 insertions(+), 20 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index c1f8ee662..d95821599 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -405,7 +405,7 @@ jobs: if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} run: | - CC=clang CXX=clang++ cmake . -B out/release-riscv64-clang -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON + CC=clang CXX=clang++ cmake . -B out/release-riscv64-clang -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rv64gcb_zacas_zalasr -DMI_SEE_ASM=ON cmake --build out/release-riscv64-clang --parallel 4 --config Release ctest --test-dir out/release-riscv64-clang --verbose --timeout 240 -C Release @@ -415,24 +415,7 @@ jobs: with: name: alpine-riscv64-clang-asm path: out/release-riscv64-clang/CMakeFiles/mimalloc-static.dir/src/*.s - - - name: Alpine riscv64, Release zalasr - if: matrix.tests == 'alpine-riscv64' - shell: alpine-riscv64.sh {0} - run: | - CC=clang CXX=clang++ cmake . -B out/release-riscv64-zalasr -G Ninja -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=rv64gcb_zacas_zalasr -DMI_SEE_ASM=ON - cmake --build out/release-riscv64-zalasr --parallel 4 --config Release - ctest --test-dir out/release-riscv64-zalasr --verbose --timeout 240 -C Release - - - name: Alpine risc64, Asm zalasr - if: matrix.tests == 'alpine-riscv64' - uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 - with: - name: alpine-riscv64-zalasr-asm - path: out/release-riscv64-zalasr/CMakeFiles/mimalloc-static.dir/src/*.s - - - + - name: Alpine riscv64, Release if: matrix.tests == 'alpine-riscv64' shell: alpine-riscv64.sh {0} diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index b0b5c137a..13f5860f9 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -66,7 +66,7 @@ static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); // #if (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__aarch64__)) /* aarch64 for older gcc versions (issue #851) */ \ || (defined(__GNUC__) && (__GNUC__ >= 7) && defined(__riscv)) \ || (defined(__GNUC__) && (__GNUC__ >= 11) && defined(__x86_64__)) \ - || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__) || defined(__riscv__))) + || (defined(__clang_major__) && (__clang_major__ >= 14) && (defined(__aarch64__) || defined(__x86_64__) || defined(__riscv))) #define MI_USE_BUILTIN_THREAD_POINTER 1 #endif #endif From f60b4cfe335e56b47a6d1a60fa04d6fb39f1dc66 Mon Sep 17 00:00:00 2001 From: lenamonj Date: Mon, 31 Aug 2026 02:28:29 -0400 Subject: [PATCH 153/280] fix: mi_theap_zalloc_csize zeroes its large branch --- include/mimalloc.h | 2 +- test/test-api.c | 18 ++++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 3c5b2f7d9..358120fff 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -409,7 +409,7 @@ static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, si if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_malloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } } static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_zalloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_zalloc_small(theap,size); } else { return mi_theap_zalloc(theap,size); } } static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } diff --git a/test/test-api.c b/test/test-api.c index 0378675ba..09a663c33 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -330,6 +330,24 @@ int main(void) { } result = ok; }; + CHECK_BODY("zalloc-csize-large") { + // mi_theap_zalloc_csize must zero its large branch too: it used to + // delegate sizes above MI_SMALL_SIZE_MAX to the plain malloc path. + mi_theap_t* theap = mi_theap_get_default(); + size_t size = MI_SMALL_SIZE_MAX + 64; + bool ok = true; + for (int round = 0; round < 16 && ok; round++) { + uint8_t* junk = (uint8_t*)mi_theap_malloc(theap, size); + memset(junk, 0xAB, size); + mi_free(junk); + uint8_t* z = (uint8_t*)mi_theap_zalloc_csize(theap, size); + for (size_t i = 0; i < size; i++) { + if (z[i] != 0) { ok = false; break; } + } + mi_free(z); + } + result = ok; + }; CHECK_BODY("zalloc-aligned-small1") { size_t zalloc_size = MI_SMALL_SIZE_MAX / 2; uint8_t* p = (uint8_t*)mi_zalloc_aligned(zalloc_size, MI_MAX_ALIGN_SIZE * 2); From e73740d2eb3809ca779a9d3ace55906482456d65 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 31 Aug 2026 07:48:25 -0700 Subject: [PATCH 154/280] fix signature of mi_free_small_nonnull --- include/mimalloc.h | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index 358120fff..922e71afd 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -137,11 +137,11 @@ mi_decl_nodiscard mi_decl_export size_t mi_good_size(size_t size) mi_attr_no mi_decl_export void mi_free_size(void* p, size_t size) mi_attr_noexcept; // `mi_free_small` is for special applications like language runtimes. -// it should only be used to free objects from `mi_(heap_)(m|z)alloc_small` and is potentially a tiny bit faster than `mi_free` +// it should only be used to free objects from `mi_*alloc_small` and is potentially a tiny bit faster than `mi_free` mi_decl_export void mi_free_small(void* p) mi_attr_noexcept; -// As `mi_free_small` but `p` should be non-NULL. -void mi_free_small_nonnull(void* p) mi_attr_noexcept; +// As `mi_free_small` but `p` should not be a NULL pointer. +mi_decl_export void mi_free_small_nonnull(void* p) mi_attr_noexcept; // ------------------------------------------------------------------------------------- From 451aff1be6270d36f2a901486eb9f113d63ffd65 Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 31 Aug 2026 07:49:23 -0700 Subject: [PATCH 155/280] add v3 to docs --- doc/mimalloc-doc.h | 6 +++--- docs/group__constantsize.html | 20 ++++++++++---------- docs/group__posix.html | 4 ++-- docs/group__typed.html | 6 +++--- docs/mimalloc-doc_8h_source.html | 6 +++--- 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/doc/mimalloc-doc.h b/doc/mimalloc-doc.h index 81e68684f..bc34e6ce7 100644 --- a/doc/mimalloc-doc.h +++ b/doc/mimalloc-doc.h @@ -426,7 +426,7 @@ void* mi_realloc_aligned_at(void* p, size_t newsize, size_t alignment, size_t of /// even when the size is not constant but then there is no performance benefit. /// \{ -/// @brief Allocate a constant size object. +/// @brief __v3__: Allocate a constant size object. /// @param size A (compile-time) constant size. /// @return A pointer to an allocated block of size \a size bytes. /// @see mi_free_csize() @@ -439,7 +439,7 @@ void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size); void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size); -/// @brief Free a pointer with a known constant size. +/// @brief __v3__: Free a pointer with a known constant size. /// @param p The pointer to the allocated block (or \a NULL ) /// @param size The size with which the pointer was allocated. /// @see mi_malloc_csize() @@ -447,7 +447,7 @@ void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size); /// @see mi_free_size_nonnull() void mi_free_csize(void* p, size_t size); -/// @brief Free a non-null pointer with a known constant size. +/// @brief __v3__: Free a non-null pointer with a known constant size. /// @param p The pointer to the allocated block (cannot be \a NULL !) /// @param size The size with which the pointer was allocated. /// @see mi_malloc_csize() diff --git a/docs/group__constantsize.html b/docs/group__constantsize.html index 069784fd2..5be895f99 100644 --- a/docs/group__constantsize.html +++ b/docs/group__constantsize.html @@ -116,7 +116,7 @@

Functions

void * mi_malloc_csize (size_t size) - Allocate a constant size object.
+ v3: Allocate a constant size object.
  void * mi_zalloc_csize (size_t size)   @@ -125,16 +125,16 @@ void * mi_theap_zalloc_csize (mi_theap_t *theap, size_t size)   void mi_free_csize (void *p, size_t size) - Free a pointer with a known constant size.
+ v3: Free a pointer with a known constant size.
  void mi_free_csize_nonnull (void *p, size_t size) - Free a non-null pointer with a known constant size.
+ v3: Free a non-null pointer with a known constant size.
 

Detailed Description

Inlined definitions for onstant size allocation and freeing.

If the size is constant, it can pick statically between mi_malloc() and mi_malloc_small(). For example:

long* p = (long*)mi_malloc_csize(sizeof(long))
-
mi_malloc_csize
void * mi_malloc_csize(size_t size)
Allocate a constant size object.
+
mi_malloc_csize
void * mi_malloc_csize(size_t size)
v3: Allocate a constant size object.

On v3 these can improve performance. This is especially useful for compilers and runtimes where the the size is often statically known. These functions can be used even when the size is not constant but then there is no performance benefit.

Function Documentation

@@ -156,7 +156,7 @@

-

Free a pointer with a known constant size.

+

v3: Free a pointer with a known constant size.

Parameters
@@ -164,7 +164,7 @@

See also
mi_malloc_csize()
+
See also
mi_malloc_csize()
mi_free_size()
@@ -191,7 +191,7 @@

-

Free a non-null pointer with a known constant size.

+

v3: Free a non-null pointer with a known constant size.

Parameters

pThe pointer to the allocated block (or NULL )
@@ -199,7 +199,7 @@

See also
mi_malloc_csize()
+
See also
mi_malloc_csize()
mi_free_size()
@@ -222,7 +222,7 @@

-

Allocate a constant size object.

+

v3: Allocate a constant size object.

Parameters

pThe pointer to the allocated block (cannot be NULL !)
@@ -230,7 +230,7 @@

Returns
A pointer to an allocated block of size size bytes.
-
See also
mi_free_csize()
+
See also
mi_free_csize()
mi_malloc()
diff --git a/docs/group__posix.html b/docs/group__posix.html index 5193821c8..e97ba3f22 100644 --- a/docs/group__posix.html +++ b/docs/group__posix.html @@ -379,7 +379,7 @@

See also
mi_free_tp()
-mi_free_csize()
+mi_free_csize()
mi_free_size_nonnull()

@@ -450,7 +450,7 @@

See also
mi_free_tp()
-mi_free_csize()
+mi_free_csize() diff --git a/docs/group__typed.html b/docs/group__typed.html index 8042c1872..4b16d9c36 100644 --- a/docs/group__typed.html +++ b/docs/group__typed.html @@ -204,11 +204,11 @@

Parameters

sizeA (compile-time) constant size.
- +
tpThe type of the object that is freed.
pA pointer to an object of type tp (or NULL ) Can be more efficient as it internally uses mi_free_csize()
pA pointer to an object of type tp (or NULL ) Can be more efficient as it internally uses mi_free_csize()
-
See also
mi_free_csize()
+
See also
mi_free_csize()
mi_free_small()
@@ -406,7 +406,7 @@

Returns
A pointer to an object of type tp, or NULL if out of memory.

Example:

int* p = mi_malloc_tp(int)
-

Using this is more safe, but can also be more efficient since the allocation size is a compile-time constant.

See also
mi_malloc_csize()
+

Using this is more safe, but can also be more efficient since the allocation size is a compile-time constant.

See also
mi_malloc_csize()
mi_malloc_small()
diff --git a/docs/mimalloc-doc_8h_source.html b/docs/mimalloc-doc_8h_source.html index 8552c6ea8..b02dec412 100644 --- a/docs/mimalloc-doc_8h_source.html +++ b/docs/mimalloc-doc_8h_source.html @@ -685,9 +685,9 @@
mi_heap_new_in_arena
mi_heap_t * mi_heap_new_in_arena(mi_arena_id_t arena_id)
Create a new heap that only allocates in the specified arena.
mi_theap_malloc_csize
void * mi_theap_malloc_csize(mi_theap_t *theap, size_t size)
mi_theap_zalloc_csize
void * mi_theap_zalloc_csize(mi_theap_t *theap, size_t size)
-
mi_free_csize_nonnull
void mi_free_csize_nonnull(void *p, size_t size)
Free a non-null pointer with a known constant size.
-
mi_free_csize
void mi_free_csize(void *p, size_t size)
Free a pointer with a known constant size.
-
mi_malloc_csize
void * mi_malloc_csize(size_t size)
Allocate a constant size object.
+
mi_free_csize_nonnull
void mi_free_csize_nonnull(void *p, size_t size)
v3: Free a non-null pointer with a known constant size.
+
mi_free_csize
void mi_free_csize(void *p, size_t size)
v3: Free a pointer with a known constant size.
+
mi_malloc_csize
void * mi_malloc_csize(size_t size)
v3: Allocate a constant size object.
mi_zalloc_csize
void * mi_zalloc_csize(size_t size)
mi_new_nothrow
void * mi_new_nothrow(size_t n)
like mi_malloc, but when out of memory, use std::get_new_handler but return NULL on failure.
mi_new
void * mi_new(std::size_t n) noexcept(false)
like mi_malloc(), but when out of memory, use std::get_new_handler and raise std::bad_alloc exception...
From 43b9da620311641850670281293af5fbb33c7e2f Mon Sep 17 00:00:00 2001 From: daanx Date: Mon, 31 Aug 2026 10:47:35 -0700 Subject: [PATCH 156/280] refactor mi_page_malloc_zero --- include/mimalloc/internal.h | 44 ++++++++++----------- src/alloc.c | 78 ++++++++++++++++++++++--------------- 2 files changed, 68 insertions(+), 54 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 910665929..ac5dec471 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1435,32 +1435,34 @@ static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) { #endif } -static inline void _mi_memcpy(void* dst, const void* src, size_t n) { +static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?) mi_rep_movsb(dst, src, n); + return dst; } else { - memcpy(dst, src, n); + return memcpy(dst, src, n); } } -static inline void _mi_memset(void* dst, int val, size_t n) { +static inline void* _mi_memset(void* dst, int val, size_t n) { if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127 mi_rep_stosb(dst, (uint8_t)val, n); + return dst; } else { - memset(dst, val, n); + return memset(dst, val, n); } } #else -static inline void _mi_memcpy(void* dst, const void* src, size_t n) { - memcpy(dst, src, n); +static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { + return memcpy(dst, src, n); } -static inline void _mi_memset(void* dst, int val, size_t n) { - memset(dst, val, n); +static inline void* _mi_memset(void* dst, int val, size_t n) { + return memset(dst, val, n); } #endif @@ -1473,42 +1475,40 @@ static inline void _mi_memset(void* dst, int val, size_t n) { #if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) // On GCC/CLang we provide a hint that the pointers are word aligned. -static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { +static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); - _mi_memcpy(adst, asrc, n); + return _mi_memcpy(adst, asrc, n); } -static inline void _mi_memset_aligned(void* dst, int val, size_t n) { +static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); - _mi_memset(adst, val, n); + return _mi_memset(adst, val, n); } #else // Default fallback on `_mi_memcpy` -static inline void _mi_memcpy_aligned(void* dst, const void* src, size_t n) { +static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - _mi_memcpy(dst, src, n); + return _mi_memcpy(dst, src, n); } -static inline void _mi_memset_aligned(void* dst, int val, size_t n) { +static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - _mi_memset(dst, val, n); + return _mi_memset(dst, val, n); } #endif -static inline void _mi_memzero(void* dst, size_t n) { - _mi_memset(dst, 0, n); +static inline void* _mi_memzero(void* dst, size_t n) { + return _mi_memset(dst, 0, n); } -static inline void _mi_memzero_aligned(void* dst, size_t n) { - _mi_memset_aligned(dst, 0, n); +static inline void* _mi_memzero_aligned(void* dst, size_t n) { + return _mi_memset_aligned(dst, 0, n); } - - #endif // MI_INTERNAL_H diff --git a/src/alloc.c b/src/alloc.c index a12038a8a..945ea0ca3 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -26,6 +26,33 @@ terms of the MIT license. A copy of the license can be found in the file // Allocation // ------------------------------------------------------ +#if MI_PADDING +static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_block_t* block, size_t size) mi_attr_noexcept { + const size_t bsize = mi_page_usable_block_size(page); + mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); + ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); + #if (MI_DEBUG>=2) + mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); + #endif + mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess + padding->canary = mi_ptr_encode_canary(page,block,page->keys); + padding->delta = (uint32_t)(delta); + #if MI_PADDING_CHECK_BYTES + if (!mi_page_is_huge(page)) { + uint8_t* fill = (uint8_t*)padding - delta; + const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes + for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } + } + #endif +} +#else +static mi_decl_maybe_unused mi_decl_noinline void* mi_block_zero(mi_block_t* block, size_t bsize) mi_attr_noexcept { + // const size_t bsize = mi_page_usable_block_size(page); + _mi_memzero_aligned(block, bsize); + return block; +} +#endif + // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: in release mode the (inlined) routine is about 7 instructions with a single test. @@ -41,7 +68,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ mi_block_t* const block = page->free; const mi_used_t used = page->used; #if defined(__GNUC__) - __asm("" : : : "memory"); // alway load used before the test + __asm("" : : : "memory"); // always load the `used` field before the test #endif if (block == NULL) { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); @@ -52,7 +79,9 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // pop from the free list mi_block_t* next = mi_block_next(page,block); mi_track_mem_undefined(block,sizeof(*block)); - block->next = 0; // don't leak internal data + #if MI_SECURE + if (!zero) block->next = 0; // don't leak internal data + #endif page->free = next; page->used = used+1; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); @@ -70,6 +99,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ const size_t bsize = mi_page_usable_block_size(page); mi_track_mem_undefined(block, bsize); + // track per-block statistics #if (MI_STAT>0) if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { mi_theap_stat_increase(theap, malloc_normal, bsize); @@ -82,44 +112,28 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ } #endif + // in debug mode initialize with 0xD0 + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN + if mi_likely(!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } + #endif + // zero the block? note: we need to zero the full block size (issue #63) - if mi_likely(!zero) { - // #if MI_SECURE - // block->next = 0; // don't leak internal data - // #endif - #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN - if (!mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } - #endif - } - else { + if mi_unlikely(zero) { if (!page->free_is_zero) { - _mi_memzero_aligned(block, bsize); + _mi_memzero_aligned(block,bsize); } else { - block->next = 0; - mi_track_mem_defined(block, bsize); - } + block->next = 0; + mi_track_mem_defined(block, bsize); + } } - + + // setup padding (must come after zero'ing) #if MI_PADDING // && !MI_TRACK_ENABLED - mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); - ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); - #if (MI_DEBUG>=2) - mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); - #endif - mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess - padding->canary = mi_ptr_encode_canary(page,block,page->keys); - padding->delta = (uint32_t)(delta); - #if MI_PADDING_CHECK_BYTES - if (!mi_page_is_huge(page)) { - uint8_t* fill = (uint8_t*)padding - delta; - const size_t maxpad = (delta > MI_MAX_ALIGN_SIZE ? MI_MAX_ALIGN_SIZE : delta); // set at most N initial padding bytes - for (size_t i = 0; i < maxpad; i++) { fill[i] = MI_DEBUG_PADDING; } - } - #endif + mi_page_block_setup_padding(page,block,size); #endif - return block; + return block; } // extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`. From 1bc0403a3ee9736a454911cfc3c06cc8060b083e Mon Sep 17 00:00:00 2001 From: Daan Date: Mon, 31 Aug 2026 14:39:11 -0700 Subject: [PATCH 157/280] remove dead code --- src/free.c | 4 ---- 1 file changed, 4 deletions(-) diff --git a/src/free.c b/src/free.c index 91670c709..5d033aab4 100644 --- a/src/free.c +++ b/src/free.c @@ -293,10 +293,6 @@ void mi_free_small_nonnull(void* p) mi_attr_noexcept { } } -// void _mi_free_subproc_safe_in_page_nonnull(void* p, mi_page_t* page) mi_attr_noexcept { -// mi_free_nonnull(p, page, NULL, false /* allow collect? */); -// } - // Free a pointer that is potentially allocated in a different sub-process void _mi_free_subproc_safe(void* p) mi_attr_noexcept { mi_page_t* page; From 92cf63ee9907574728cf64758b35361281f501d9 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 1 Sep 2026 05:48:22 -0700 Subject: [PATCH 158/280] bump to v1.15.1 --- .github/workflows/release.yaml | 4 ++-- cmake/mimalloc-config-version.cmake | 2 +- include/mimalloc.h | 2 +- readme.md | 11 +++++++---- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index e495a23dc..a65308a6c 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ permissions: contents: write env: - RELEASE: Release v3.5.0 + RELEASE: Release v3.5.1 FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true name: Release @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - branch: [v3.5.0, v2.5.0, v1.15.0] # [dev,dev2,dev3] + branch: [v3.5.1, v2.5.1, v1.15.1] # [dev,dev2,dev3] # we build on the oldest ubuntu version for better binary compatibility. os: [windows-latest, macOS-latest, macos-15-intel, ubuntu-22.04, ubuntu-22.04-arm] diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index aebbc1d6d..040238b9d 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 1) set(mi_version_minor 15) -set(mi_version_patch 0) +set(mi_version_patch 1) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/include/mimalloc.h b/include/mimalloc.h index 39d8e8a05..19facfd05 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 11500 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 11501 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes diff --git a/readme.md b/readme.md index a92903b27..deede14d2 100644 --- a/readme.md +++ b/readme.md @@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.5.0` (2026-08-18) recommended. -Latest v2 release: `v2.5.0` (2026-08-18) stable. -Latest v1 release: `v1.15.0` (2026-08-18) legacy. +Latest release : `v3.5.1` (2026-09-01) recommended. +Latest v2 release: `v2.5.1` (2026-09-01) stable, legacy. +Latest v1 release: `v1.15.1` (2026-09-01) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -89,7 +89,10 @@ New development is mostly on v3, while v1 and v2 are maintained with security an __Send PR's against this version if possible.__ ### Releases - +* 2026-09-01, `v1.15.1`, `v2.5.1`, `v3.5.1`: (v3) Yet better performance for `free` calls. + (v3) Added `mi_free_small(_nonnull)` for runtimes, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. + (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. Improved cache + behavior for small allocations. Improved riscV suppor, add riscV to CI. Various small build fixes. * 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) slightly better performance for `free` using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for faster load-acquire, increase retired page count from 1 to 3, faster double free From 3bfdc2f4b9bea052e943a038d064b84d659d53d4 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 1 Sep 2026 07:39:19 -0700 Subject: [PATCH 159/280] add MI_ALLOW_THP=[OFF,ON,FULL] and use the default of FULL for Linux (never splitting THP pages). Add getting the best performance section to the readme --- CMakeLists.txt | 21 ++++++- include/mimalloc/internal.h | 1 + readme.md | 112 +++++++++++++++++++++++++++--------- src/options.c | 5 +- src/os.c | 11 +++- src/prim/unix/prim.c | 20 ++++--- 6 files changed, 129 insertions(+), 41 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e9ec76282..c47a3d05a 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,9 @@ set_property(CACHE MI_DEBUG PROPERTY STRINGS "OFF" "ON" "INTERNAL" "FULL" "DEFAU set(MI_TRACK "OFF" CACHE STRING "Enable tracking: OFF, ASAN (address sanitizer), VALGRIND, ETW (Windows event tracing). Adds a small overhead.") set_property(CACHE MI_TRACK PROPERTY STRINGS "OFF" "ASAN" "VALGRIND" "ETW") +set(MI_ALLOW_THP "DEFAULT" CACHE STRING "Enable transparent huge pages on Linux: OFF, ON, FULL, DEFAULT. 'FULL' enables THP and also avoids splitting up huge pages during purging.") +set_property(CACHE MI_ALLOW_THP PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) @@ -531,7 +534,23 @@ endif() if(CMAKE_SYSTEM_NAME MATCHES "Linux|Android") if(MI_NO_THP) - message(STATUS "Disable transparent huge pages support (MI_NO_THP=ON)") + set(MI_ALLOW_THP "OFF") + endif() + if(MI_ALLOW_THP STREQUAL "DEFAULT") + if(CMAKE_SYSTEM_NAME MATCHES "Android") + set(MI_ALLOW_THP "OFF") + else() + set(MI_ALLOW_THP "FULL") + endif() + endif() + if(MI_ALLOW_THP STREQUAL "FULL") + message(STATUS "Enable full transparent huge pages support, where purging avoids splitting huge pages (MI_ALLOW_THP=FULL)") + list(APPEND mi_defines MI_DEFAULT_ALLOW_THP=2) + elseif (MI_ALLOW_THP) + message(STATUS "Enable transparent huge pages support (MI_ALLOW_THP=${MI_ALLOW_THP}) (also consider using FULL for better performance)") + list(APPEND mi_defines MI_DEFAULT_ALLOW_THP=1) + else() + message(STATUS "Disable transparent huge pages support (MI_ALLOW_THP=${MI_ALLOW_THP})") list(APPEND mi_defines MI_DEFAULT_ALLOW_THP=0) endif() endif() diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index ac5dec471..fdd5c78be 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -206,6 +206,7 @@ size_t _mi_os_guard_page_size(void); size_t _mi_os_good_alloc_size(size_t size); bool _mi_os_has_overcommit(void); bool _mi_os_has_virtual_reserve(void); +bool _mi_os_canuse_thp(void); size_t _mi_os_virtual_address_bits(void); size_t _mi_os_minimal_purge_size(void); diff --git a/readme.md b/readme.md index a92903b27..fac213453 100644 --- a/readme.md +++ b/readme.md @@ -369,14 +369,15 @@ Advanced options: - `MIMALLOC_PURGE_DECOMMITS=1`: By default "purging" memory means unused memory is decommitted (`MEM_DECOMMIT` on Windows, `MADV_DONTNEED` (which decresease rss immediately) on `mmap` systems). Set this to 0 to instead "reset" unused memory on a purge (`MEM_RESET` on Windows, generally `MADV_FREE` (which does not decrease rss immediately) on `mmap` systems). - Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it tries to keep virtual - address ranges and decommits within those ranges (to make the underlying physical memory available to other processes). + Mimalloc generally does not "free" OS memory but only "purges" OS memory, in other words, it will keep virtual + address ranges, but decommits within those ranges (to make the underlying physical memory available to other processes). Further options for large workloads and services: -- `MIMALLOC_ALLOW_THP=1`: By default always allow transparent huge pages (THP) on Linux systems. On Android only this is - by default off. When set to `0`, THP is disabled for the process that mimalloc runs in. If enabled, mimalloc also sets - the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when purging memory. +- `MIMALLOC_ALLOW_THP=[0,1,2]`: When set to `0`, the use of transparent huge pages (THP) is disabled for the process that mimalloc + runs in, and otherwise it is enabled whenever THP is [enabled in the OS][linux-thp] (`[always]` or `[madvise]`). If set to 2 (default on Linux), + mimalloc also sets the `MIMALLOC_MINIMAL_PURGE_SIZE` in v3 to 2MiB to avoid potentially breaking up transparent huge pages when + purging memory which is better for performance (but may increase the rss!). On Android the default setting is 0 to reduce rss. - `MIMALLOC_USE_NUMA_NODES=N`: pretend there are at most `N` NUMA nodes. If not set, the actual NUMA nodes are detected at runtime. Setting `N` to 1 may avoid problems in some virtual environments. Also, setting it to a lower number than the actual NUMA nodes is fine and will only cause threads to potentially allocate more memory across actual NUMA @@ -403,6 +404,7 @@ Use caution when using `fork` in combination with either large or huge OS pages: for all pages in the original process including the huge OS pages. When any memory is now written in that area, the OS will copy the entire 1GiB huge page (or 2MiB large page) which can cause the memory usage to grow in large increments. +[linux-thp]: https://docs.redhat.com/en/documentation/red_hat_enterprise_linux/7/html/performance_tuning_guide/sect-red_hat_enterprise_linux-performance_tuning_guide-configuring_transparent_huge_pages [linux-huge]: https://access.redhat.com/documentation/en-us/red_hat_enterprise_linux/5/html/tuning_and_optimizing_red_hat_enterprise_linux_for_oracle_9i_and_10g_databases/sect-oracle_9i_and_10g_tuning_guide-large_memory_optimization_big_pages_and_huge_pages-configuring_huge_pages_in_red_hat_enterprise_linux_4_or_5 [windows-huge]: https://docs.microsoft.com/en-us/sql/database-engine/configure-windows/enable-the-lock-pages-in-memory-option-windows?view=sql-server-2017 @@ -419,7 +421,7 @@ to make mimalloc more robust against exploits. In particular: - The free lists are initialized in a random order and allocation randomly chooses between extension and reuse within a page to mitigate against attacks that rely on a predicable allocation order. Similarly, the larger heap blocks allocated by mimalloc from the OS are also address randomized. -- If enabling `-DMI_SECURE_FULL=ON` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving +- If using `-DMI_SECURE=FULL` there will also be guard pages at the end of each (64KiB) mimalloc page (thus interleaving valid block data with inaccessible gaps). This setting is not recommended in general as it is more expensive and can lead to reaching the maximum VMA limit on Linux systems if the heap gets too large. @@ -427,7 +429,7 @@ As always, evaluate with care as part of an overall security strategy as all of ## Debug Mode -When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`), +When _mimalloc_ is built using debug mode, (`-DCMAKE_BUILD_TYPE=Debug`) (or when using `-DMI_DEBUG=ON` explicitly), various checks are done at runtime to catch development errors. - Statistics are maintained in detail for each object size. They can be shown using `MIMALLOC_SHOW_STATS=1` at runtime. @@ -466,6 +468,33 @@ many system calls. Therefore, there are various environment variables (and optio Overriding the standard `malloc` (and `new`) can be done either _dynamically_ or _statically_. + +## Static override + +On Unix like systems, you can statically link with _mimalloc_ to override the standard +malloc interface. The recommended way is to link the final program with the +_mimalloc_ single object file (`mimalloc.o`). We use +an object file instead of a library file as linkers give preference to +that over archives to resolve symbols. To ensure that the standard +malloc interface resolves to the _mimalloc_ library, link it as the first +object file. For example: + +``` +> gcc -o myprogram mimalloc.o myfile1.c ... +``` + +Another way to override statically that works on all platforms, is to +link statically to mimalloc (as shown in the introduction) and include a +header file in each source file that re-defines `malloc` etc. to `mi_malloc`. +This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works +reliably though if all sources are +under your control or otherwise mixing of pointers from different heaps may occur! + +Note: recently we also enabled static overloading on Windows. In that case you need +to link with the static CRT _release_ runtime (`/MT`) and link with the static +`mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). + + ## Dynamic override This is the recommended way to override the standard malloc interface. @@ -543,30 +572,55 @@ first in the import table. In such cases the [`minject`](bin) tool can be used to patch the executable's import tables. -## Static override +# Getting the best performance -On Unix-like systems, you can also statically link with _mimalloc_ to override the standard -malloc interface. The recommended way is to link the final program with the -_mimalloc_ single object file (`mimalloc.o`). We use -an object file instead of a library file as linkers give preference to -that over archives to resolve symbols. To ensure that the standard -malloc interface resolves to the _mimalloc_ library, link it as the first -object file. For example: +Consider the following build options (v3): -``` -> gcc -o myprogram mimalloc.o myfile1.c ... -``` +- `-DMI_OPT_ARCH=ON` (by default ON on arm64): enables architecture specific optimizations. + This uses `-march=haswell;-march=avx2` on x64 (available since 2013), `-march=armv8.1` on arm64 (2016), + and `-march=rv64gcb_zacas` on riscV (2023). + Each benefits mimalloc with better bit operations and/or native atomic operations (but also, if your + emulator or hardware does not support these extensions the program will fail with an illegal instruction). + On Apple arm64 the default is `-march=armv8.3` (which is supported on the M1 and better) which includes + efficient load-acquire instructions which mimalloc uses. For riscV the corresponding `zalasr` extension + can be enabled as `-DMI_OPT_ARCH=rv64gcb_zacas_zalasr` when your hardware supports it. -Another way to override statically that works on all platforms, is to -link statically to mimalloc (as shown in the introduction) and include a -header file in each source file that re-defines `malloc` etc. to `mi_malloc`. -This is provided by [`mimalloc-override.h`](include/mimalloc-override.h). This only works -reliably though if all sources are -under your control or otherwise mixing of pointers from different heaps may occur! +- `-DMI_ALLOW_THP=OFF`: disabling THP can reduce the rss of certain programs significantly. + +- `-DMI_ALLOW_THP=FULL`: enabling THP can increase performance of certain programs significantly. This setting + still disables THP if the OS THP setting is configured as `[never]`. + +- `-DMI_FREE_IS_CHECKED=ON` (by default ON in secure mode): slightly slower performance but checks validity + of every pointer passed to `mi_free`. Sometimes needed when overriding malloc/free process wide. + Note also that `mi_cfree` (checked free) always checks the validity of pointers passed to it which can + also be used if appropiate without needing to check all pointers process wide. + +In the source code: + +- Use `mi_zalloc` for zero initialized allocations. + +- Use the typed macros `mi_malloc_tp` etc. as these are both more safe and also use `mi_malloc_csize` internally. + +- Use `mi_malloc_csize`/`mi_free_csize` etc. when the allocated or free'd size is a constant as these are inlined + and statically redirect to `mi_malloc_small`/`mi_free_small` when possible. + These are also used by the C++ `mimalloc-new-delete.h` sized overrides. + You can use `mi_free_csize_nonnull` when the pointer is guaranteed to be not `NULL` as well. + +- Use `mi_malloc_small` when the allocation size is guaranteed to be less than `MI_SMALL_SIZE_MAX` (1KiB) for faster allocations. + +- Use `mi_free_small` when the pointer was returned from `mi_malloc_small` (and guaranteed to have a + size of less than `MI_SMALL_SIZE_MAX`). Use `mi_free_small_nonnull` when the pointer is guaranteed to be not `NULL` as well. + +For run-time systems and compilers (like Koka, Lean, etc.), we can do slightly better still. + +- If the allocation size is statically known, use `mi_malloc_csize` and `mi_free_csize` etc. when possible. + (or directly `mi_malloc_small`/`mi_free_small`). + +- If the runtime already carries thread local state, it may be faster to get the default `theap` for each thread + up-front (`mi_theap_get_default()`) and use the very fast `mi_theap_malloc(_small)` etc. passing the theap + pointer directly. This avoids having mimalloc look up the thread local pointer all the time. Whether this is + faster depends a bit on the OS implementation of thread local variables. -Note: recently we also enabled static overloading on Windows. In that case you need -to link with the static CRT _release_ runtime (`/MT`) and link with the static -`mimalloc(-debug).obj` (to take precendence over the definitions in the CRT library). # Tools @@ -662,9 +716,11 @@ use a tool like [TraceControl] that is specialized for analyzing mimalloc traces [TraceControl]: https://github.com/xinglonghe/TraceControl + # Performance -Last update: 2021-01-30 +Last update: 2021-01-30. +(Note 2026-09-01: this is a while ago now, many of these allocators have improved over time as well. YMMV.) We tested _mimalloc_ against many other top allocators over a wide range of benchmarks, ranging from various real world programs to diff --git a/src/options.c b/src/options.c index 327f36fc4..da0d6669c 100644 --- a/src/options.c +++ b/src/options.c @@ -104,7 +104,8 @@ int mi_version(void) { #if defined(__ANDROID__) #define MI_DEFAULT_ALLOW_THP 0 #else -#define MI_DEFAULT_ALLOW_THP 1 +// #define MI_DEFAULT_ALLOW_THP 1 // allow THP but purging may split up THP pages +#define MI_DEFAULT_ALLOW_THP 2 // allow THP and set the minimal purge size to 2MiB to avoid breaking them up #endif #endif @@ -171,7 +172,7 @@ static mi_option_desc_t mi_options[_mi_option_last] = MI_OPTION_UNINIT, MI_OPTION(page_cross_thread_max_reclaim) }, // don't reclaim (small) pages across threads if we already own N pages in that size class { MI_DEFAULT_ALLOW_THP, MI_OPTION_UNINIT, MI_OPTION(allow_thp) }, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. - { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if `mi_option_allow_thp==2`). + { 0, MI_OPTION_UNINIT, MI_OPTION(minimal_purge_size) }, // set minimal purge size (in KiB) (=0). Using 0 resolves to either 64 (or 2048 if THP is enabled). { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation diff --git a/src/os.c b/src/os.c index c74bc736a..a727cc844 100644 --- a/src/os.c +++ b/src/os.c @@ -41,6 +41,9 @@ bool _mi_os_has_virtual_reserve(void) { return mi_os_mem_config.has_virtual_reserve; } +bool _mi_os_canuse_thp(void) { + return mi_os_mem_config.has_transparent_huge_pages; +} // OS (small) page size size_t _mi_os_page_size(void) { @@ -55,13 +58,19 @@ size_t _mi_os_large_page_size(void) { // minimal purge size. Can be larger than the page size if transparent huge pages are enabled. size_t _mi_os_minimal_purge_size(void) { size_t minsize = mi_option_get_size(mi_option_minimal_purge_size); - if (minsize != 0) { + if (minsize != 0) { + // set by user return _mi_align_up(minsize, _mi_os_page_size()); } else if (mi_os_mem_config.has_transparent_huge_pages && mi_option_get(mi_option_allow_thp) == 2) { + // don't break up THP pages; + // we don't do this by default as that can lead to increased memory usage (see issue #1282). + // on the other hand, breaking up THP pages can lead to performance degradation and perhaps + // allow_thp should be 2 by default (and the user can set the OS THP setting to [never]). return _mi_os_large_page_size(); } else { + // OS page size by default return _mi_os_page_size(); } } diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index bcb065b2d..1838901d5 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -463,15 +463,17 @@ static void* unix_mmap(void* addr, size_t size, size_t try_alignment, int protec #if !defined(MI_NO_THP) if (p != NULL && allow_large && mi_option_is_enabled(mi_option_allow_thp) && _mi_os_canuse_large_page(size, try_alignment)) { #if defined(MADV_HUGEPAGE) - // Many Linux systems don't allow MAP_HUGETLB but they support instead - // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE - // though since properly aligned allocations will already use large pages if available - // in that case -- in particular for our large regions (in `memory.c`). - // However, some systems only allow THP if called with explicit `madvise`, so - // when large OS pages are enabled for mimalloc, we call `madvise` anyways. - if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { - // *is_large = true; // possibly - }; + if (_mi_os_canuse_thp()) { + // Many Linux systems don't allow MAP_HUGETLB but they support instead + // transparent huge pages (THP). Generally, it is not required to call `madvise` with MADV_HUGE + // though since properly aligned allocations will already use large pages if available + // in that case -- in particular for our large regions (in `memory.c`). + // However, some systems only allow THP if called with explicit `madvise`, so + // when large OS pages are enabled for mimalloc, we call `madvise` anyways. + if (unix_madvise(p, size, MADV_HUGEPAGE) == 0) { + // *is_large = true; // possibly + }; + } #elif defined(__sun) struct memcntl_mha cmd = {0}; cmd.mha_pagesize = _mi_os_large_page_size(); From dfe481f28e5c32511e77ed1aa7dba13ee1e582b2 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 1 Sep 2026 07:47:00 -0700 Subject: [PATCH 160/280] bump vcpkg version to v3.5.1 --- contrib/vcpkg/portfile.cmake | 2 +- contrib/vcpkg/vcpkg.json | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/contrib/vcpkg/portfile.cmake b/contrib/vcpkg/portfile.cmake index b5d219595..46dc48554 100644 --- a/contrib/vcpkg/portfile.cmake +++ b/contrib/vcpkg/portfile.cmake @@ -3,7 +3,7 @@ vcpkg_from_github( REPO microsoft/mimalloc HEAD_REF master - # The "REF" can be a commit hash, branch name (dev3), or a version (v3.5.0). + # The "REF" can be a commit hash, branch name (dev3), or a version (v3.5.1). REF "v${VERSION}" # The sha512 is the hash of the tar.gz bundle. diff --git a/contrib/vcpkg/vcpkg.json b/contrib/vcpkg/vcpkg.json index 39b0fadaf..f66b0ef8e 100644 --- a/contrib/vcpkg/vcpkg.json +++ b/contrib/vcpkg/vcpkg.json @@ -1,6 +1,6 @@ { "name": "mimalloc", - "version": "3.5.0", + "version": "3.5.1", "port-version": 0, "description": "Compact general purpose allocator with excellent performance", "homepage": "https://github.com/microsoft/mimalloc", From f16b45cdad7b26d2ce56cc3e4f418b081f6df237 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 1 Sep 2026 07:59:58 -0700 Subject: [PATCH 161/280] update readme --- readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/readme.md b/readme.md index deede14d2..e77633269 100644 --- a/readme.md +++ b/readme.md @@ -89,7 +89,7 @@ New development is mostly on v3, while v1 and v2 are maintained with security an __Send PR's against this version if possible.__ ### Releases -* 2026-09-01, `v1.15.1`, `v2.5.1`, `v3.5.1`: (v3) Yet better performance for `free` calls. +* 2026-09-01, `v3.5.1`, `v2.5.1`, `v1.15.1`: (v3) Yet better performance for `free` calls. (v3) Added `mi_free_small(_nonnull)` for runtimes, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. Improved cache behavior for small allocations. Improved riscV suppor, add riscV to CI. Various small build fixes. From cb5aedc947b1959fbe7c3de220d34134261309c8 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 1 Sep 2026 08:05:48 -0700 Subject: [PATCH 162/280] update release notes --- doc/release-notes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 1e59f4a53..867484268 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,7 +1,7 @@ Binary and source releases for mimalloc. -- __v3__: recommended: latest mimalloc design that tends to use less memory then v2 but should perform similarly. -- __v2__: stable: most widely used. +- __v3__: recommended: latest mimalloc design that tends to use less memory then v2. +- __v2__: stable legacy. - __v1__: legacy. Notes: @@ -10,6 +10,6 @@ Notes: - Generally it is recommended to download sources (or use `vcpkg` etc.) and build mimalloc as part of your project. - Source releases can also be downloaded directly from github by the tag. - For example . + For example . - Binary releases include a release-, debug-, and secure build. - Linux binaries are built on Ubuntu 22. From 366340080f985b87b4e30b0eb7f366b76d4da385 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:27:21 +0000 Subject: [PATCH 163/280] Bump the minor-and-patch group with 2 updates Bumps the minor-and-patch group with 2 updates: [softprops/action-gh-release](https://github.com/softprops/action-gh-release) and [vmactions/freebsd-vm](https://github.com/vmactions/freebsd-vm). Updates `softprops/action-gh-release` from 3.0.2 to 3.0.3 - [Release notes](https://github.com/softprops/action-gh-release/releases) - [Changelog](https://github.com/softprops/action-gh-release/blob/master/CHANGELOG.md) - [Commits](https://github.com/softprops/action-gh-release/compare/3d0d9888cb7fd7b750713d6e236d1fcb99157228...efb35369e0ad2afab669f228072c1b0d510eae64) Updates `vmactions/freebsd-vm` from 1.5.3 to 1.5.5 - [Release notes](https://github.com/vmactions/freebsd-vm/releases) - [Commits](https://github.com/vmactions/freebsd-vm/compare/83b151f58c6047089f4c80eb5ba2039d158ce093...f0552d3b69211736abd97f02ff3d4674c56b73b1) --- updated-dependencies: - dependency-name: softprops/action-gh-release dependency-version: 3.0.3 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch - dependency-name: vmactions/freebsd-vm dependency-version: 1.5.5 dependency-type: direct:production update-type: version-update:semver-patch dependency-group: minor-and-patch ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yaml | 2 +- .github/workflows/test.yaml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a65308a6c..96769471d 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -45,7 +45,7 @@ jobs: # Create a release: https://github.com/softprops/action-gh-release (MIT license) - name: Release - uses: softprops/action-gh-release@3d0d9888cb7fd7b750713d6e236d1fcb99157228 # v3.0.2 + uses: softprops/action-gh-release@efb35369e0ad2afab669f228072c1b0d510eae64 # v3.0.3 if: ${{ env.RELEASE != '' && env.RELEASE != 'no' }} with: name: ${{ env.RELEASE }} diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d95821599..e3df3293b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -314,7 +314,7 @@ jobs: # VM: FreeBSD (arm64 and x64) - name: FreeBSD arm64, Setup if: matrix.tests == 'freebsd-arm64' - uses: vmactions/freebsd-vm@83b151f58c6047089f4c80eb5ba2039d158ce093 # v1.5.3 + uses: vmactions/freebsd-vm@f0552d3b69211736abd97f02ff3d4674c56b73b1 # v1.5.5 with: arch: arm64 custom-shell-name: freebsd-arm64 @@ -353,7 +353,7 @@ jobs: - name: FreeBSD x64, Setup if: matrix.tests == 'freebsd-x64' - uses: vmactions/freebsd-vm@83b151f58c6047089f4c80eb5ba2039d158ce093 # v1.5.3 + uses: vmactions/freebsd-vm@f0552d3b69211736abd97f02ff3d4674c56b73b1 # v1.5.5 with: arch: x86_64 usesh: true From aac758791fa43709f7bbdbf25389b99258416d01 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:27:26 +0000 Subject: [PATCH 164/280] Bump actions/stale from 10.4.0 to 11.0.0 Bumps [actions/stale](https://github.com/actions/stale) from 10.4.0 to 11.0.0. - [Release notes](https://github.com/actions/stale/releases) - [Changelog](https://github.com/actions/stale/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/stale/compare/1e223db275d687790206a7acac4d1a11bd6fe629...4391f3da665fdf50b6810c1a66712fb9ba21aa93) --- updated-dependencies: - dependency-name: actions/stale dependency-version: 11.0.0 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/stale.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yaml b/.github/workflows/stale.yaml index 71a91a887..7079b90d0 100644 --- a/.github/workflows/stale.yaml +++ b/.github/workflows/stale.yaml @@ -11,7 +11,7 @@ jobs: issues: write pull-requests: write steps: - - uses: actions/stale@1e223db275d687790206a7acac4d1a11bd6fe629 # v10.4.0 + - uses: actions/stale@4391f3da665fdf50b6810c1a66712fb9ba21aa93 # v11.0.0 with: days-before-issue-stale: 360 days-before-issue-close: 14 From c1915b35fe23b5768a89d826267f7a6d5c06fe1a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 1 Sep 2026 15:27:32 +0000 Subject: [PATCH 165/280] Bump actions/checkout from 6.1.0 to 7.0.1 Bumps [actions/checkout](https://github.com/actions/checkout) from 6.1.0 to 7.0.1. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/d23441a48e516b6c34aea4fa41551a30e30af803...3d3c42e5aac5ba805825da76410c181273ba90b1) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: 7.0.1 dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/release.yaml | 2 +- .github/workflows/test.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a65308a6c..0cb885bbb 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -26,7 +26,7 @@ jobs: steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 with: ref: ${{ matrix.branch }} fetch-tags: true diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d95821599..e376dfbaa 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -55,7 +55,7 @@ jobs: steps: # Checkout: https://github.com/actions/checkout - name: Checkout repository - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0 + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 # Choose MI_USE_CXX setting - name: Set default cmake options From 6c45ba091cdda559b844651c93980df239ce9f72 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 1 Sep 2026 22:09:23 -0700 Subject: [PATCH 166/280] wip: initial fast detailed stats --- include/mimalloc/bits.h | 8 +++ include/mimalloc/internal.h | 44 ++++++++++----- include/mimalloc/prim-tls.h | 2 +- include/mimalloc/types.h | 25 ++++++-- readme.md | 4 +- src/alloc.c | 12 ++-- src/arena.c | 8 ++- src/free.c | 79 +++++++++++++------------- src/heap.c | 4 +- src/init.c | 6 +- src/page.c | 110 +++++++++++++++++++++++++++++++----- src/theap.c | 14 ++--- test/test-api.c | 2 + 13 files changed, 225 insertions(+), 93 deletions(-) diff --git a/include/mimalloc/bits.h b/include/mimalloc/bits.h index c7ce294f9..5aa0d8375 100644 --- a/include/mimalloc/bits.h +++ b/include/mimalloc/bits.h @@ -18,6 +18,14 @@ terms of the MIT license. A copy of the license can be found in the file #include // bool #include // LONG_MAX +#if defined(__cplusplus) +#define mi_decl_externc extern "C" +#define mi_init_struct_zero { } +#else +#define mi_decl_externc +#define mi_init_struct_zero { 0 } +#endif + // ------------------------------------------------------ // Size of a pointer. // We assume that `sizeof(void*)==sizeof(intptr_t)` diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index fdd5c78be..a0168a8f5 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -101,15 +101,6 @@ terms of the MIT license. A copy of the license can be found in the file #define __has_builtin(x) 0 #endif -#if defined(__cplusplus) -#define mi_decl_externc extern "C" -#define mi_init_struct_zero { } -#else -#define mi_decl_externc -#define mi_init_struct_zero { 0 } -#endif - - #if defined(__EMSCRIPTEN__) && !defined(__wasi__) #define __wasi__ #endif @@ -285,6 +276,7 @@ void _mi_page_free_collect(mi_page_t* page, bool force); void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq); +void _mi_page_update_stats(mi_page_t* page); size_t _mi_page_stats_bin(const mi_page_t* page); // for stats size_t _mi_bin_size(size_t bin); // for stats @@ -661,7 +653,7 @@ static inline bool mi_theap_is_detached(mi_theap_t* theap) { static inline bool mi_theap_matches_thread(mi_theap_t* theap) { const mi_threadid_t tid = _mi_thread_id(); - return (theap==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); + return (theap==NULL || theap->tld==NULL || theap->tld->thread_id == tid || mi_theap_is_detached(theap)); } /* ----------------------------------------------------------- @@ -905,11 +897,33 @@ static inline size_t mi_page_committed(const mi_page_t* page) { return (slice_committed == 0 ? mi_page_size(page) : slice_committed - mi_page_slice_offset_of(page,0)); } +static inline size_t mi_page_used(const mi_page_t* page) { + mi_assert_internal(page != NULL); + return mi_xused_used_count(page->xused); +} + +static inline void mi_page_used_reset(mi_page_t* page) { + page->xused.used_alloc = page->xused.used_alloc & ~0xFFFF; +} + +static inline size_t mi_page_alloc_count(const mi_page_t* page) { + return mi_xused_alloc_count(page->xused); +} + +static inline size_t mi_page_last_used(const mi_page_t* page) { + #if MI_INTPTR_SIZE >= 8 + return (page->xused.used_alloc >> 32) & 0xFFFF; + #else + return page->xlast_used; + #endif +} + + // are all blocks in a page freed? // note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. static inline bool mi_page_all_free(const mi_page_t* page) { mi_assert_internal(page != NULL); - return (page->used == 0); + return (mi_page_used(page)==0); } // are there immediately available blocks, i.e. blocks available on the free list. @@ -928,7 +942,7 @@ static inline bool mi_page_is_expandable(const mi_page_t* page) { static inline bool mi_page_is_full(const mi_page_t* page) { - const bool full = (page->reserved == page->used); + const bool full = (page->reserved == mi_page_used(page)); mi_assert_internal(!full || page->free == NULL); return full; } @@ -937,14 +951,14 @@ static inline bool mi_page_is_full(const mi_page_t* page) { static inline bool mi_page_is_mostly_used(const mi_page_t* page) { if (page==NULL) return true; uint16_t frac = page->reserved / 8U; - return (page->reserved - page->used <= frac); + return (page->reserved - mi_page_used(page) <= frac); } // is more than (n-1)/n'th of a page in use? static inline bool mi_page_is_used_at_frac(const mi_page_t* page, uint16_t n) { if (page==NULL) return true; uint16_t frac = page->reserved / n; - return (page->reserved - page->used <= frac); + return (page->reserved - mi_page_used(page) <= frac); } @@ -1116,7 +1130,7 @@ static inline mi_block_t* mi_page_thread_free(const mi_page_t* page) { // are there any available blocks? static inline bool mi_page_has_any_available(const mi_page_t* page) { mi_assert_internal(page != NULL && page->reserved > 0); - return (page->used < page->reserved || (mi_page_thread_free(page) != NULL)); + return (mi_page_used(page) < page->reserved || (mi_page_thread_free(page) != NULL)); } // Owned? diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 13f5860f9..29d710968 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -414,7 +414,7 @@ static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) { mi_heap_t* const heap = mi_page_heap(page); mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); - if (theap==NULL) return NULL; + if (theap==NULL || theap->tld==NULL /* heap destroy */) return NULL; if (theap->heap != heap) return NULL; // should never happen, but can happen for a free across subprocesses, which can happen during pthread tls storage deallocation mi_assert_internal(!_mi_is_empty_theap(theap) && mi_theap_matches_thread(theap)); // note: for pages allocated by a detached theap, the returned theap may not be detached diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 308c17bd3..54a52dde8 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -84,7 +84,7 @@ terms of the MIT license. A copy of the license can be found in the file #if (MI_DEBUG>0) #define MI_STAT 2 #else -#define MI_STAT 0 +#define MI_STAT 2 #endif #endif @@ -392,8 +392,21 @@ typedef size_t mi_page_flags_t; // This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`. typedef uintptr_t mi_thread_free_t; -// Convenience -typedef size_t mi_used_t; +typedef union mi_used_s { + uintptr_t used_alloc; + struct { + uint16_t used_count; + uint16_t alloc_count; + #if MI_INTPTR_SIZE >= 8 + uint16_t last_used; + uint16_t padding; + #endif + } debug; +} mi_used_t; + +static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); } +static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); } +static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; } // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: @@ -428,7 +441,10 @@ typedef struct mi_page_s { #endif _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) - mi_used_t used; // number of blocks in use (including blocks in `thread_free`) + mi_used_t xused; // number of blocks in use (including blocks in `thread_free`) (and the allocated count for statistics) + #if MI_INTPTR_SIZE < 8 + int32_t xlast_used; // for statistics; on 64-bit platforms it is the upper 32-bits of xused. + #endif mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) size_t block_size; // const: size available in each block (always `>0`) @@ -455,7 +471,6 @@ typedef struct mi_page_s { #endif } mi_page_t; - // ------------------------------------------------------ // Object sizes // ------------------------------------------------------ diff --git a/readme.md b/readme.md index dcfc524b1..f356d7954 100644 --- a/readme.md +++ b/readme.md @@ -92,11 +92,11 @@ New development is mostly on v3, while v1 and v2 are maintained with security an * 2026-09-01: Added a readme section on [getting the best performance](#getting-the-best-performance). * 2026-09-01, `v3.5.1`, `v2.5.1`, `v1.15.1`: (v3) Yet better performance for `free` calls. - (v3) Added `mi_free_small(_nonnull)` for runtimes, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. + (v3) Added `mi_free_small(_nonnull)` for fast small allocations, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. (v3) Default to `-DMI_ALLOW_THP=FULL` on Linux, where we never split transparent huge pages (THP) during purging (#1282). This may increase rss but can also improve performance. Use `-DMI_ALLOW_THP=OFF` to disable the use of THP. - (v3) Improved cache behavior for small allocations. Improved riscV suppor, add riscV to CI. Various small build fixes. + (v3) Improved cache behavior for small allocations. Improved riscV support. Various small build fixes. * 2026-08-18, `v1.15.0`, `v2.5.0`, `v3.5.0`: (v3) slightly better performance for `free` using aligned chunks, cleanup cmake options, require armv8.3 (with MI_OPT_ARCH) for faster load-acquire, increase retired page count from 1 to 3, faster double free diff --git a/src/alloc.c b/src/alloc.c index 945ea0ca3..a4f710a43 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -62,13 +62,15 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ mi_assert_internal(mi_page_block_size(page) >= size); mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); } // check the free list mi_block_t* const block = page->free; - const mi_used_t used = page->used; + mi_used_t xused = page->xused; + xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count #if defined(__GNUC__) - __asm("" : : : "memory"); // always load the `used` field before the test + __asm("" : : : "memory"); // always load the `used` field before the test #endif if (block == NULL) { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); @@ -83,7 +85,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ if (!zero) block->next = 0; // don't leak internal data #endif page->free = next; - page->used = used+1; + page->xused = xused; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); @@ -100,7 +102,9 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ mi_track_mem_undefined(block, bsize); // track per-block statistics - #if (MI_STAT>0) + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + + #if 0 && (MI_STAT>0) if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { mi_theap_stat_increase(theap, malloc_normal, bsize); #if (MI_STAT>1) diff --git a/src/arena.c b/src/arena.c index 2bda823bc..5a18c94cc 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1304,6 +1304,8 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { mi_heap_t* const heap = mi_page_heap(page); mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); mi_theapx_stat_decrease(heap, current_theapx, pages, 1); + _mi_page_free_collect(page,false); + _mi_page_update_stats(page); mi_arenas_page_free_prim(page); } @@ -1983,7 +1985,7 @@ static void mi_debug_color(char* buf, size_t* k, mi_ansi_color_t color) { static int mi_page_commit_usage(mi_page_t* page) { const size_t committed_size = mi_page_committed(page); - const size_t used_size = page->used * mi_page_block_size(page); + const size_t used_size = mi_page_used(page) * mi_page_block_size(page); return (int)(used_size * 100 / committed_size); } @@ -2554,7 +2556,7 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mi_page_is_owned(page)); - if (page->used==0) { + if (mi_page_used(page)==0) { // free the page _mi_arenas_page_free(page, theap); } @@ -2563,7 +2565,7 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are _mi_page_unguard_all(page); // remove potential interior guard pages #endif // destroy the page - page->used=0; // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case + page->xused = mi_xused_used_reset(page->xused); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case _mi_arenas_page_free(page, theap); } else { diff --git a/src/free.c b/src/free.c index 5d033aab4..e9f153808 100644 --- a/src/free.c +++ b/src/free.c @@ -16,7 +16,7 @@ terms of the MIT license. A copy of the license can be found in the file mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block); static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); -static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); +// static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); // ------------------------------------------------------ @@ -25,28 +25,29 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); // regular free of a (thread local) block pointer // fast path written carefully to prevent spilling on the stack -static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool track_stats, bool check_full) +static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool was_guarded, bool check_full) { // checks size_t usable_size; if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; if mi_unlikely(!mi_check_double_free(page,block)) return; // usually checked with padding - if (track_stats) { - mi_stat_free(page, block); - mi_track_free_size(block, usable_size); - } + // mi_stat_free(page, block); + mi_track_free_size(block, usable_size); + #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN const size_t dbgsize = (usable_size > MI_MiB ? MI_MiB : usable_size); _mi_memset_aligned(block, MI_DEBUG_FREED, dbgsize); #endif // actual free: push on the local free list - const mi_used_t used = page->used - 1; + mi_used_t xused = page->xused; + xused.used_alloc--; // decrement used count mi_block_set_next(page, block, page->local_free); - page->used = used; + page->xused = xused; page->local_free = block; - if mi_unlikely(used==0) { + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + if mi_unlikely(mi_xused_used_count(xused) == 0) { // is used count zero ? if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page) _mi_page_retire(page); } @@ -57,16 +58,16 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool } // Forward declaration for multi-threaded collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept; +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept; // Free a block multi-threaded -static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_collect) mi_attr_noexcept +static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was_guarded, bool allow_reclaim) mi_attr_noexcept { size_t usable_size; if mi_unlikely(!mi_check_padding_on_free(page, block, was_guarded, &usable_size)) return; // checking padding is safe for mt // adjust stats (after padding check ) - mi_stat_free(page, block); // stat_free may access the padding + // mi_stat_free(page, block); // stat_free may access the padding mi_track_free_size(block, usable_size); // _mi_padding_shrink(page, block, sizeof(mi_block_t)); @@ -81,19 +82,16 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was mi_thread_free_t tf_new; mi_thread_free_t tf_old = mi_atomic_load_relaxed(&page->xthread_free); do { - mi_block_set_next(page, block, mi_tf_block(tf_old)); - const bool new_owned = (allow_collect ? true : mi_tf_is_owned(tf_old)); // if allow collection then always try to claim it if the page is abandoned - tf_new = mi_tf_create(block, new_owned); + mi_block_set_next(page, block, mi_tf_block(tf_old)); + tf_new = mi_tf_create(block, true /* try to own it */ ); } while (!mi_atomic_cas_weak_acq_rel(&page->xthread_free, &tf_old, tf_new)); // todo: release is enough? // and atomically try to collect the page if it was abandoned - if (allow_collect) { - const bool is_owned_now = !mi_tf_is_owned(tf_old); - if (is_owned_now) { - mi_assert_internal(mi_page_is_abandoned(page)); - mi_free_try_collect_mt(page,block); - } - } + const bool is_owned_now = !mi_tf_is_owned(tf_old); + if (is_owned_now) { + mi_assert_internal(mi_page_is_abandoned(page)); + mi_free_try_collect_mt(page,block,allow_reclaim); + } } @@ -149,15 +147,15 @@ static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_ mi_assert_internal(p!=NULL && page != NULL); mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_local(page, block, was_guarded, true /* track stats */, true /* check for a full page */); + mi_free_block_local(page, block, was_guarded, true /* check for a full page */); } // free a pointer owned by another thread (page parameter comes first for better codegen) -static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_collect) mi_attr_noexcept { +static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); const bool was_guarded = mi_block_check_unguard(page, block, p); - mi_free_block_mt(page, block, was_guarded, allow_collect); + mi_free_block_mt(page, block, was_guarded, allow_reclaim); } // generic free (for runtime integration) @@ -233,7 +231,7 @@ static mi_decl_forceinline mi_page_t* mi_ptr_page_validate(const void* p, const // Free a block // Fast path written carefully to prevent register spilling on the stack -static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_collect) +static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t* pblock_size, bool allow_reclaim) { mi_assert_internal(p!=NULL && page!=NULL); if (pblock_size!=NULL) { *pblock_size = mi_page_block_size(page); } @@ -243,7 +241,7 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t if mi_likely(xtid == 0) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) == 0` // thread-local, aligned, and not a full page mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_local(page, block, false /* was guarded */, true /* track stats */, false /* no need to check if the page is full */); + mi_free_block_local(page, block, false /* was guarded */, false /* no need to check if the page is full */); } else if (xtid <= MI_PAGE_FLAG_MASK) { // `tid == mi_page_thread_id(page) && mi_page_flags(page) != 0` // page is local, but is full or contains (inner) aligned blocks; use generic path @@ -253,25 +251,25 @@ static mi_decl_forceinline void mi_free_nonnull(void* p, mi_page_t* page, size_t else if ((xtid & MI_PAGE_FLAG_MASK) == 0) { // `tid != mi_page_thread_id(page) && mi_page_flags(page) == 0` // blocks are aligned (and not a full page); push on the thread_free list mi_block_t* const block = mi_validate_block_from_ptr(page,p); - mi_free_block_mt(page,block,false /* was_guarded */, allow_collect); + mi_free_block_mt(page,block,false /* was_guarded */, allow_reclaim); } else { // page is full or contains (inner) aligned blocks; use generic multi-thread path - mi_free_generic_mt(page, p, allow_collect); + mi_free_generic_mt(page, p, allow_reclaim); } } void mi_free(void* p) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_ptr_page_is_valid(p,"mi_free",&page)) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_ptr_page_is_valid(p,"mi_ufree",&page)) { - mi_free_nonnull(p, page, pblock_size, true /* allow collect? */); + mi_free_nonnull(p, page, pblock_size, true /* allow reclaim? */); } else { if (pblock_size!=NULL) { *pblock_size = 0; } @@ -281,7 +279,7 @@ void mi_ufree(void* p, size_t* pblock_size) mi_attr_noexcept { void mi_free_small(void* p) mi_attr_noexcept { mi_page_t* page; if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small",true /* is_small? */,true /*check p for null*/, &page)) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } @@ -289,15 +287,16 @@ void mi_free_small_nonnull(void* p) mi_attr_noexcept { mi_assert(p!=NULL); mi_page_t* page; if mi_likely(mi_ptr_page_is_valid_ex(p,"mi_free_small_nonnull",true /* is_small? */,false /*check p for null*/, &page)) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); } } // Free a pointer that is potentially allocated in a different sub-process void _mi_free_subproc_safe(void* p) mi_attr_noexcept { mi_page_t* page; - if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { - mi_free_nonnull(p, page, NULL, false /* allow collect? */); + if mi_likely(mi_ptr_page_is_valid(p,"_mi_free_subproc_safe",&page)) { + // const bool allow_reclaim = (_mi_subproc() == mi_page_subproc(page)); + mi_free_nonnull(p, page, NULL, false /* allow reclaim */); } } @@ -358,7 +357,7 @@ void mi_free_aligned(void* p, size_t alignment) mi_attr_noexcept { bool mi_cfree(void* p) mi_attr_noexcept { mi_page_t* const page = _mi_checked_ptr_page(p); if mi_likely(page!=NULL) { - mi_free_nonnull(p, page, NULL, true /* allow collect? */); + mi_free_nonnull(p, page, NULL, true /* allow reclaim? */); return true; } else { @@ -483,12 +482,12 @@ static mi_decl_noinline bool mi_abandoned_page_try_reclaim(mi_page_t* page, long // We freed a block in an abandoned page (that was not owned). Try to collect -static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free) mi_attr_noexcept +static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mt_free, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mt_free != NULL); - // mi_assert_internal(_mi_subproc() == mi_page_subproc(page)); // never collect across subprocesses + mi_assert_internal(!allow_reclaim || _mi_subproc() == mi_page_subproc(page)); // never reclaim across subprocesses // we own the page now, and it is safe to collect the thread atomic free list if (page->block_size <= MI_SMALL_SIZE_MAX) { @@ -503,7 +502,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* _mi_page_free_collect(page,false /* no force */); mt_free = NULL; // expected page->xthread_free value after collection } - const long reclaim_on_free = _mi_option_get_fast(mi_option_page_reclaim_on_free); + const long reclaim_on_free = allow_reclaim && _mi_option_get_fast(mi_option_page_reclaim_on_free); #if MI_DEBUG > 1 if (mi_page_is_singleton(page)) { mi_assert_internal(mi_page_all_free(page)); } if (mi_page_is_full(page)) { mi_assert(mi_page_is_mostly_used(page)); } @@ -749,7 +748,7 @@ mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, co #endif // only maintain stats for smaller objects if requested -#if (MI_STAT>0) +#if 0 && (MI_STAT>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(block); mi_theap_t* theap = _mi_theap_default(); diff --git a/src/heap.c b/src/heap.c index 3e7611b5e..28e75ba61 100644 --- a/src/heap.c +++ b/src/heap.c @@ -287,7 +287,7 @@ bool mi_check_owned(const void* p) { bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc_threshold) mi_attr_noexcept { if (p==NULL) return false; const mi_page_t* const page = _mi_safe_ptr_page(p); // Get the page containing this pointer - if (page==NULL || page->used==page->capacity || page->capacity < page->reserved) return false; + if (page==NULL || mi_page_used(page)==page->capacity || page->capacity < page->reserved) return false; // If the page is the head of the queue, it is currently being used for // allocations; we skip it to avoid immediate thrashing. if (page->prev == NULL) return false; @@ -300,5 +300,5 @@ bool mi_unsafe_heap_page_is_under_utilized(mi_heap_t* heap, void* p, size_t perc // check utilization if (page->capacity==0) return false; if (perc_threshold>=100) return true; - return (perc_threshold >= ((100UL*page->used) / page->capacity)); + return (perc_threshold >= ((100UL*mi_page_used(page)) / page->capacity)); } diff --git a/src/init.c b/src/init.c index 68e94aaa2..bbc2b7356 100644 --- a/src/init.c +++ b/src/init.c @@ -19,7 +19,10 @@ static const mi_page_t mi_page_empty = { #endif MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free - 0, // used + {0}, // xused + #if MI_INTPTR_SIZE < 8 + 0 // xlast_used + #endif NULL, // local_free 0, // block_size 0, // page_offset @@ -627,6 +630,7 @@ static void mi_process_done_once(void) { _mi_thread_locals_done(); if (subproc_main->heap_main != NULL) { if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { + _mi_theap_collect_abandon(subproc_main->theap_meta); // update stats of all pages in theap_meta _mi_theap_merge_stats(subproc_main->theap_meta); _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free mi_heap_stats_merge_to_subproc(subproc_main->heap_main); diff --git a/src/page.c b/src/page.c index 02ca72044..4ea3fd0ae 100644 --- a/src/page.c +++ b/src/page.c @@ -83,7 +83,7 @@ static bool mi_page_list_is_valid(mi_page_t* page, mi_block_t* p) { static bool mi_page_is_valid_init(mi_page_t* page) { mi_assert_internal(mi_page_block_size(page) > 0); - mi_assert_internal(page->used <= page->capacity); + mi_assert_internal(mi_page_used(page) <= page->capacity); mi_assert_internal(page->capacity <= page->reserved); mi_assert_internal(page->heap!=NULL); @@ -114,8 +114,9 @@ static bool mi_page_is_valid_init(mi_page_t* page) { #endif size_t free_count = mi_page_list_count(page, page->free) + mi_page_list_count(page, page->local_free); - mi_assert_internal(page->used + free_count == page->capacity); + mi_assert_internal(mi_page_used(page) + free_count == page->capacity); + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); return true; } @@ -142,6 +143,80 @@ bool _mi_page_is_valid(mi_page_t* page) { } #endif +#if MI_STAT +// Update stats for a page; +// `theap` can be NULL in which case the page's heap stats are updated. +void _mi_page_update_stats(mi_page_t* page) { + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + + // get stat counts + const mi_used_t xused = page->xused; + const size_t used = mi_xused_used_count(xused); + const size_t alloc_count = mi_xused_alloc_count(xused); + const size_t last_used = mi_page_last_used(page); + mi_assert_internal(last_used + alloc_count >= used); + const size_t free_count = last_used + alloc_count - used; + if (alloc_count + free_count == 0) { + mi_assert_internal(last_used == used); + return; + } + + // reset the alloc_count and set last_used to used + #if MI_INTPTR_SIZE >= 8 + page->xused.used_alloc = (used << 32) | used; + #else + page->xused.used_alloc = used; + page->xlast_used = used; + #endif + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + + // update stats + mi_heap_t* const heap = mi_page_heap(page); + mi_theap_t* const theap = _mi_page_associated_theap_peek(page); + mi_assert_internal(theap == NULL || (theap->tld != NULL && _mi_thread_id() == theap->tld->thread_id)); + + const size_t bsize = mi_page_usable_block_size(page); + if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { + const size_t bin = _mi_bin(bsize); + // allocations + if (alloc_count > 0) { + mi_theapx_stat_increase(heap, theap, malloc_normal, alloc_count*bsize); + #if MI_STAT>1 + mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, 1); + mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); + mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); + #endif + } + // frees + if (free_count > 0) { + mi_theapx_stat_decrease(heap, theap, malloc_normal, free_count*bsize); + #if (MI_STAT > 1) + mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count); + #endif + } + } + else { + // allocations + mi_assert_internal(alloc_count<=1); + mi_assert_internal(free_count<=1); + if (alloc_count > 0) { + mi_theapx_stat_increase(heap, theap, malloc_huge, alloc_count*bsize); + mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); + #if MI_STAT>1 + mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); + #endif + } + // frees + if (free_count > 0) { + mi_theapx_stat_decrease(heap, theap, malloc_huge, free_count*bsize); + } + } +} +#else +void _mi_page_update_stats(mi_page_t* page) { + MI_UNUSED(page); +} +#endif /* ----------------------------------------------------------- Page collect the `local_free` and `thread_free` lists @@ -167,7 +242,7 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) return; // the thread-free items cannot be freed } // if `count > page->used` there was another kind memory corruption (either in the page meta-data or in the linked list) - else if mi_unlikely(count > page->used) { + else if mi_unlikely(count > mi_page_used(page)) { _mi_error_message(EFAULT, "corrupted meta-data in thread-free list\n"); return; // the thread-free items cannot be freed } @@ -178,8 +253,8 @@ static void mi_page_thread_collect_to_local(mi_page_t* page, mi_block_t* head) // update counts now mi_assert_internal(count <= UINT16_MAX); - mi_assert_internal(page->used >= (uint16_t)count); - page->used = page->used - (uint16_t)count; + mi_assert_internal(mi_page_used(page) >= count); + page->xused.used_alloc -= count; // page->used = page->used - (uint16_t)count; } // Collect the local `thread_free` list using an atomic exchange. @@ -200,6 +275,10 @@ static void mi_page_thread_free_collect(mi_page_t* page) mi_page_thread_collect_to_local(page, head); } +static inline bool mi_page_should_update_stats(const mi_page_t* page) { + return (mi_xused_alloc_count(page->xused) > 0x7FFF); +} + // returns `true` if after collection `mi_page_immediate_available` is true. static inline bool mi_page_free_quick_collect(mi_page_t* page) { if mi_likely(page->free != NULL) return true; @@ -207,7 +286,8 @@ static inline bool mi_page_free_quick_collect(mi_page_t* page) { // move local_free to free page->free = page->local_free; page->local_free = NULL; - page->free_is_zero = false; + page->free_is_zero = false; + if (mi_page_should_update_stats(page)) { _mi_page_update_stats(page); } return true; } @@ -236,9 +316,9 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - } + } } - + _mi_page_update_stats(page); mi_assert_internal(!force || page->local_free == NULL); } @@ -258,9 +338,10 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; + if (mi_page_should_update_stats(page)) { _mi_page_update_stats(page); } } } - if (page->used == 1) { + if (mi_page_used(page) == 1) { // all elements are free'd since we skipped the `head` element itself mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head); mi_assert_internal(mi_block_next(page,head) == NULL); @@ -449,6 +530,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index < theap->page_retired_min) theap->page_retired_min = index; if (index > theap->page_retired_max) theap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); + _mi_page_update_stats(page); return; // don't free after all } } @@ -736,7 +818,8 @@ mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(page->theap == mi_page_theap(page)); mi_assert_internal(page->capacity == 0); mi_assert_internal(page->free == NULL); - mi_assert_internal(page->used == 0); + mi_assert_internal(mi_page_used(page) == 0); + mi_assert_internal(page->xused.used_alloc == 0); mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(page->xthread_free == 1); mi_assert_internal(page->next == NULL); @@ -809,7 +892,7 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, page_candidate = page; } // prefer to reuse fuller pages (in the hope the less used page gets freed) - else if (page->used >= page_candidate->used && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { + else if (mi_page_used(page) >= mi_page_used(page_candidate) && !mi_page_is_mostly_used(page)) { // && !mi_page_is_expandable(page)) { page_candidate = page; } // if we find a non-expandable candidate, or searched for N pages, return with the best candidate @@ -939,8 +1022,8 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page mi_assert_internal(mi_page_is_abandoned(page)); mi_page_set_theap(page, NULL); #endif - mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); - mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); + // mi_theap_stat_increase(theap, malloc_huge, mi_page_block_size(page)); + // mi_theap_stat_counter_increase(theap, malloc_huge_count, 1); } return page; } @@ -1078,6 +1161,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) { mi_page_to_full(page, mi_page_queue_of(page)); } + _mi_page_update_stats(page); return p; } diff --git a/src/theap.c b/src/theap.c index d9fb4309d..d74ec4beb 100644 --- a/src/theap.c +++ b/src/theap.c @@ -115,7 +115,7 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag } void _mi_theap_merge_stats(mi_theap_t* theap) { - mi_assert_internal(mi_theap_is_initialized(theap)); + mi_assert_internal(mi_theap_is_initialized(theap)); mi_heap_t* const heap = _mi_theap_heap(theap); _mi_stats_merge_into(&heap->stats, &theap->stats); } @@ -545,7 +545,7 @@ void _mi_heap_area_init(mi_heap_area_t* area, mi_page_t* page) { area->reserved = page->reserved * bsize; area->committed = page->capacity * bsize; area->blocks = mi_page_start(page); - area->used = page->used; // number of blocks in use (#553) + area->used = mi_page_used(page); // number of blocks in use (#553) area->block_size = ubsize; area->full_block_size = bsize; area->reserved1 = page; @@ -571,7 +571,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi _mi_page_free_collect(page,true); // collect both thread_delayed and local_free mi_assert_internal(page->local_free == NULL); - if (page->used == 0) return true; + if (mi_page_used(page) == 0) return true; size_t psize; uint8_t* const pstart = mi_page_area(page, &psize); @@ -581,13 +581,13 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi // optimize page with one block if (page->capacity == 1) { - mi_assert_internal(page->used == 1 && page->free == NULL); + mi_assert_internal(mi_page_used(page) == 1 && page->free == NULL); return visitor(heap, area, pstart, ubsize, arg); } mi_assert(bsize <= UINT32_MAX); // optimize full pages - if (page->used == page->capacity) { + if (mi_page_used(page) == page->capacity) { uint8_t* block = pstart; for (size_t i = 0; i < page->capacity; i++) { if (!visitor(heap, area, block, ubsize, arg)) return false; @@ -631,7 +631,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi size_t bit = blockidx - (bitidx * MI_INTPTR_BITS); free_map[bitidx] |= ((uintptr_t)1 << bit); } - mi_assert_internal(page->capacity == (free_count + page->used)); + mi_assert_internal(page->capacity == (free_count + mi_page_used(page))); // walk through all blocks skipping the free ones #if MI_DEBUG>1 @@ -663,7 +663,7 @@ bool _mi_theap_area_visit_blocks(const mi_heap_area_t* area, mi_page_t* page, mi block += bsize * MI_INTPTR_BITS; } } - mi_assert_internal(page->used == used_count); + mi_assert_internal(mi_page_used(page) == used_count); return true; } diff --git a/test/test-api.c b/test/test-api.c index 09a663c33..d9b677e09 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -377,6 +377,7 @@ int main(void) { ptr = mi_rezalloc_aligned(ptr, n, alignment); assert(mem_has_vals((uint8_t*)ptr,n/2,123)); result = mem_is_zero((uint8_t*)ptr + n/2, n/2); + mi_free(ptr); } // --------------------------------------------------- @@ -414,6 +415,7 @@ int main(void) { shared_ptr = mi_realloc(shared_ptr, i * 64); } } + mi_free(shared_ptr); } // --------------------------------------------------- From 2dcd92a57d2ed344c4c7eea7653880e3cbc526b7 Mon Sep 17 00:00:00 2001 From: Antony Hosking Date: Wed, 2 Sep 2026 20:28:07 +1000 Subject: [PATCH 167/280] _mi_subproc: fall back to the main subproc when tld->subproc is not yet set _mi_subproc() defends against partially-initialized thread state (issue #1289) by checking theap and theap->tld, but not the third member of the same initialization progression: a window where the theap and its tld exist while tld->subproc has not been set yet. In that window it returns NULL instead of falling back to _mi_subproc_main(), and the first caller that dereferences the result for stat accounting crashes (observed: a malloc-override shared library whose constructor allocates before process init completes faults at offsetof(mi_subproc_t, stats.commit_calls) in mi_subproc_stat_counter_increase(NULL, commit_calls, 1)). Extend the guard with the missing disjunct so the existing fallback covers the whole window it already defends; behavior for initialized threads is unchanged. Co-Authored-By: Claude Fable 5 --- src/subproc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/subproc.c b/src/subproc.c index 7cce9edd7..c8c3202fe 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -102,7 +102,7 @@ bool _mi_subproc_is_main(mi_subproc_t* subproc) { mi_subproc_t* _mi_subproc(void) { mi_theap_t* theap = _mi_theap_default(); - if (theap == NULL || theap->tld == NULL) { // see issue #1289 + if (theap == NULL || theap->tld == NULL || theap->tld->subproc == NULL) { // see issue #1289 return _mi_subproc_main(); } else { From 7159900e27b953a2df531c6a8967e047f377c572 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 07:29:06 -0700 Subject: [PATCH 168/280] improve efficient stats --- include/mimalloc-stats.h | 4 +- include/mimalloc/prim-tls.h | 4 +- src/arena.c | 9 +++- src/init.c | 2 +- src/page.c | 95 ++++++++++++++++++++++--------------- src/stats.c | 14 ++++-- src/theap.c | 3 +- 7 files changed, 81 insertions(+), 50 deletions(-) diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index f8be41aa6..9b297f6b7 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -79,7 +79,9 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNTER(pages_reclaim_on_free) \ MI_STAT_COUNTER(pages_reabandon_full) \ MI_STAT_COUNTER(pages_unabandon_busy_wait) \ - MI_STAT_COUNTER(heaps_delete_wait) + MI_STAT_COUNTER(heaps_delete_wait) \ + MI_STAT_COUNTER(pages_stat_updates) /* calls to successful page_stat_update */ \ + MI_STAT_COUNTER(pages_stat_update_count) /* total free/allocs */ \ // Size bins for chunks typedef enum mi_chunkbin_e { diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 29d710968..9d7eda399 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -35,7 +35,7 @@ static inline mi_theap_t* _mi_theap_cached(void); // static inline bool _mi_thread_is_initialized(void); // a thread is initialized if it has a default theap static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap); // get the thread local theap belonging to a heap static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap); // get the theap but don't update _mi_theap_cached -static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) +static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page); // get the theap associated with a page (used in `mi_free_collect_mt`) // Default TLS model @@ -411,7 +411,7 @@ static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { // Find the associated theap or NULL if it does not exist (during shutdown) // Should be fast as it is called in `free.c:mi_free_try_collect`. -static inline mi_theap_t* _mi_page_associated_theap_peek(mi_page_t* page) { +static inline mi_theap_t* _mi_page_associated_theap_peek(const mi_page_t* page) { mi_heap_t* const heap = mi_page_heap(page); mi_theap_t* const theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); if (theap==NULL || theap->tld==NULL /* heap destroy */) return NULL; diff --git a/src/arena.c b/src/arena.c index 5a18c94cc..fba97dd73 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1304,8 +1304,8 @@ void _mi_arenas_page_free(mi_page_t* page, mi_theap_t* current_theapx) { mi_heap_t* const heap = mi_page_heap(page); mi_theapx_stat_decrease(heap, current_theapx, page_bins[_mi_page_stats_bin(page)], 1); mi_theapx_stat_decrease(heap, current_theapx, pages, 1); - _mi_page_free_collect(page,false); - _mi_page_update_stats(page); + _mi_page_free_collect(page,false); // update used count for cross-thread free's + _mi_page_update_stats(page); // and update the stats mi_arenas_page_free_prim(page); } @@ -1323,6 +1323,11 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { mi_assert_internal(mi_theap_matches_thread(current_theapx)); // mi_assert_internal(current_theap == _mi_page_associated_theap(page)); + // note: somewhat expensive to update here, but might be good as then we attribute + // the current allocations/frees to the current thread/theap. Otherwise it might be + // reclaimed later in another thread/theap and those allocations/frees get attributed there... + _mi_page_update_stats(page); + // add to abandoned? mi_heap_t* heap = mi_page_heap(page); if (page->memid.memkind==MI_MEM_ARENA && !mi_page_is_full(page)) { diff --git a/src/init.c b/src/init.c index bbc2b7356..f4d5b9f88 100644 --- a/src/init.c +++ b/src/init.c @@ -630,7 +630,7 @@ static void mi_process_done_once(void) { _mi_thread_locals_done(); if (subproc_main->heap_main != NULL) { if (mi_option_is_enabled(mi_option_show_stats) || mi_option_is_enabled(mi_option_verbose)) { - _mi_theap_collect_abandon(subproc_main->theap_meta); // update stats of all pages in theap_meta + mi_theap_collect(subproc_main->theap_meta, false /* force */); // update stats of all pages in theap_meta _mi_theap_merge_stats(subproc_main->theap_meta); _mi_theap_merge_stats(_mi_theap_default()); // _mi_thread_locals_done can free mi_heap_stats_merge_to_subproc(subproc_main->heap_main); diff --git a/src/page.c b/src/page.c index 4ea3fd0ae..6899908ea 100644 --- a/src/page.c +++ b/src/page.c @@ -144,36 +144,14 @@ bool _mi_page_is_valid(mi_page_t* page) { #endif #if MI_STAT -// Update stats for a page; -// `theap` can be NULL in which case the page's heap stats are updated. -void _mi_page_update_stats(mi_page_t* page) { - mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); - - // get stat counts - const mi_used_t xused = page->xused; - const size_t used = mi_xused_used_count(xused); - const size_t alloc_count = mi_xused_alloc_count(xused); - const size_t last_used = mi_page_last_used(page); - mi_assert_internal(last_used + alloc_count >= used); - const size_t free_count = last_used + alloc_count - used; - if (alloc_count + free_count == 0) { - mi_assert_internal(last_used == used); - return; - } - - // reset the alloc_count and set last_used to used - #if MI_INTPTR_SIZE >= 8 - page->xused.used_alloc = (used << 32) | used; - #else - page->xused.used_alloc = used; - page->xlast_used = used; - #endif - mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); - +// Merge stats from the page into the corresponding theap or heap. +static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_t free_count ) { // update stats mi_heap_t* const heap = mi_page_heap(page); mi_theap_t* const theap = _mi_page_associated_theap_peek(page); mi_assert_internal(theap == NULL || (theap->tld != NULL && _mi_thread_id() == theap->tld->thread_id)); + mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1); + mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count); const size_t bsize = mi_page_usable_block_size(page); if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { @@ -182,7 +160,7 @@ void _mi_page_update_stats(mi_page_t* page) { if (alloc_count > 0) { mi_theapx_stat_increase(heap, theap, malloc_normal, alloc_count*bsize); #if MI_STAT>1 - mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, 1); + mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count); mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); #endif @@ -212,12 +190,43 @@ void _mi_page_update_stats(mi_page_t* page) { } } } + +// Update stats for a page +void _mi_page_update_stats(mi_page_t* page) { + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + + // get stat counts + const mi_used_t xused = page->xused; + const size_t used = mi_xused_used_count(xused); + const size_t alloc_count = mi_xused_alloc_count(xused); + const size_t last_used = mi_page_last_used(page); + mi_assert_internal(last_used + alloc_count >= used); + const size_t free_count = last_used + alloc_count - used; + if (alloc_count + free_count == 0) { + mi_assert_internal(last_used == used); + return; + } + + // reset the alloc_count and set last_used to used + #if MI_INTPTR_SIZE >= 8 + page->xused.used_alloc = (used << 32) | used; + #else + page->xused.used_alloc = used; + page->xlast_used = used; + #endif + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + + mi_page_merge_stats(page, alloc_count, free_count); +} + + #else void _mi_page_update_stats(mi_page_t* page) { MI_UNUSED(page); } #endif + /* ----------------------------------------------------------- Page collect the `local_free` and `thread_free` lists ----------------------------------------------------------- */ @@ -275,8 +284,15 @@ static void mi_page_thread_free_collect(mi_page_t* page) mi_page_thread_collect_to_local(page, head); } -static inline bool mi_page_should_update_stats(const mi_page_t* page) { - return (mi_xused_alloc_count(page->xused) > 0x7FFF); + +// When we collect the local free list, the `mi_page_alloc_count` can +// potentially keep growing by repeated allocation/free in the same page. +// When it gets too large where it could overflow the 16 bits, we update +// the stats to reset the allocated counter. +static inline void mi_page_maybe_update_stats(mi_page_t* page) { + if mi_unlikely(mi_xused_alloc_count(page->xused) > 0x7FFF) { + _mi_page_update_stats(page); + } } // returns `true` if after collection `mi_page_immediate_available` is true. @@ -287,7 +303,7 @@ static inline bool mi_page_free_quick_collect(mi_page_t* page) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - if (mi_page_should_update_stats(page)) { _mi_page_update_stats(page); } + mi_page_maybe_update_stats(page); return true; } @@ -317,8 +333,8 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { page->local_free = NULL; page->free_is_zero = false; } - } - _mi_page_update_stats(page); + mi_page_maybe_update_stats(page); + } mi_assert_internal(!force || page->local_free == NULL); } @@ -338,7 +354,7 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - if (mi_page_should_update_stats(page)) { _mi_page_update_stats(page); } + mi_page_maybe_update_stats(page); } } if (mi_page_used(page) == 1) { @@ -364,6 +380,7 @@ void _mi_theap_page_reclaim(mi_theap_t* theap, mi_page_t* page) mi_page_set_theap(page,theap); _mi_page_free_collect(page, false); // ensure used count is up to date + mi_page_queue_t* pq = mi_theap_page_queue_of(theap, page); mi_page_queue_push_at_end(theap, pq, page); mi_assert_expensive(_mi_page_is_valid(page)); @@ -466,6 +483,7 @@ static void mi_page_to_full(mi_page_t* page, mi_page_queue_t* pq) { // put full pages in a theap local queue (this is for theaps that cannot abandon, for example, if the theap can be destroyed) mi_page_queue_enqueue_from(&mi_page_theap(page)->pages[MI_BIN_FULL], pq, page); _mi_page_free_collect(page, false); // try to collect right away in case another thread freed just before MI_USE_DELAYED_FREE was set + _mi_page_update_stats(page); // we must update stats here as mi_theap_collect does not normally visit full pages } } @@ -530,7 +548,7 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index < theap->page_retired_min) theap->page_retired_min = index; if (index > theap->page_retired_max) theap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); - _mi_page_update_stats(page); + // _mi_page_update_stats(page); return; // don't free after all } } @@ -1158,10 +1176,11 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size mi_assert_internal(p != NULL); // move full pages to the full queue - if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE && mi_page_is_full(page)) { - mi_page_to_full(page, mi_page_queue_of(page)); - } - _mi_page_update_stats(page); + if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) { + if (mi_page_is_full(page)) { + mi_page_to_full(page, mi_page_queue_of(page)); + } + } return p; } diff --git a/src/stats.c b/src/stats.c index 9220bfec2..5b1080158 100644 --- a/src/stats.c +++ b/src/stats.c @@ -214,8 +214,10 @@ static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64 _mi_fprintf(out, arg, "%24s", ""); } else { - mi_print_amount(-unit, 1, out, arg); - mi_print_count((stat->total / -unit), 0, out, arg); + // mi_print_amount(-unit, 1, out, arg); + //mi_print_count((stat->total / -unit), 0, out, arg); + _mi_fprintf(out, arg, "%12s", ""); + mi_print_count(-unit, 0, out, arg); } } if (stat->current != 0) { @@ -370,12 +372,12 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg); #endif #if MI_STAT - mi_stat_print(&stats->malloc_normal, "binned", (stats->malloc_normal_count.total == 0 ? -1 : 1), out, arg); - mi_stat_print(&stats->malloc_huge, "huge", (stats->malloc_huge_count.total == 0 ? -1 : 1), out, arg); + mi_stat_print(&stats->malloc_normal, "binned", -stats->malloc_normal_count.total, out, arg); + mi_stat_print(&stats->malloc_huge, "huge", -stats->malloc_huge_count.total, out, arg); mi_stat_count_t total = { 0,0,0 }; mi_stat_count_add_mt(&total, &stats->malloc_normal); mi_stat_count_add_mt(&total, &stats->malloc_huge); - mi_stat_print_ex(&total, "total", 1, out, arg, ""); + mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, ""); #if MI_STAT>1 mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg); #endif @@ -397,6 +399,8 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg); mi_stat_counter_print(&stats->pages_extended, "extended", out, arg); mi_stat_counter_print(&stats->pages_retire, "retire", out, arg); + mi_stat_counter_print(&stats->pages_stat_updates, "stat upds", out, arg); + mi_stat_average_print(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg); mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); _mi_fprintf(out, arg, "\n"); } diff --git a/src/theap.c b/src/theap.c index d74ec4beb..62091083b 100644 --- a/src/theap.c +++ b/src/theap.c @@ -99,7 +99,8 @@ static bool mi_theap_page_collect(mi_theap_t* theap, mi_page_queue_t* pq, mi_pag MI_UNUSED(theap); mi_assert_expensive(mi_theap_page_is_valid(theap, pq, page, NULL, NULL)); mi_collect_t collect = *((mi_collect_t*)arg_collect); - _mi_page_free_collect(page, collect >= MI_FORCE); + _mi_page_free_collect(page, collect >= MI_FORCE); // update used count + _mi_page_update_stats(page); if (mi_page_all_free(page)) { // no more used blocks, possibly free the page. if (collect >= MI_FORCE || page->retire_expire == 0) { // either forced/abandon, or not already retired From 2987eba6df8a80e908636757b87838003abbc262 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 07:41:37 -0700 Subject: [PATCH 169/280] add MI_STAT option to cmake --- CMakeLists.txt | 22 ++++++++++++++++++++++ src/alloc.c | 18 +++++------------- src/page.c | 20 ++++++++------------ src/stats.c | 10 ++++++---- 4 files changed, 41 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c47a3d05a..788040394 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -29,6 +29,9 @@ set_property(CACHE MI_DEBUG PROPERTY STRINGS "OFF" "ON" "INTERNAL" "FULL" "DEFAU set(MI_TRACK "OFF" CACHE STRING "Enable tracking: OFF, ASAN (address sanitizer), VALGRIND, ETW (Windows event tracing). Adds a small overhead.") set_property(CACHE MI_TRACK PROPERTY STRINGS "OFF" "ASAN" "VALGRIND" "ETW") +set(MI_STAT "DEFAULT" CACHE STRING "Enable statistics.") +set_property(CACHE MI_STAT PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + set(MI_ALLOW_THP "DEFAULT" CACHE STRING "Enable transparent huge pages on Linux: OFF, ON, FULL, DEFAULT. 'FULL' enables THP and also avoids splitting up huge pages during purging.") set_property(CACHE MI_ALLOW_THP PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") @@ -473,6 +476,25 @@ else() message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, OFF, SHARED, or ON)") endif() +if(MI_STAT STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_STAT "FULL") + else() + set(MI_STAT "ON") + endif() +endif() + +if(MI_STAT STREQUAL "FULL") + message(STATUS "Enable detailed statistics including precise requested bytes (expensive) (MI_STAT=FULL)") + list(APPEND mi_defines MI_STAT=2) +elseif(MI_STAT) + message(STATUS "Enable detailed statistics (MI_STAT=${MI_STAT})") + list(APPEND mi_defines MI_STAT=1) +else() + message(STATUS "Disable detailed statistics (MI_STAT=${MI_STAT})") + list(APPEND mi_defines MI_STAT=0) +endif() + if(CMAKE_SYSTEM_NAME STREQUAL "WASI") message(STATUS "Enabling MI_FREE_USE_PAGEMAP since the target (WASI) does not support large OS aligned allocations well") set(MI_FREE_USE_PAGEMAP ON) diff --git a/src/alloc.c b/src/alloc.c index a4f710a43..9f8ed8971 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -89,6 +89,10 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); + #if MI_STAT>1 + mi_theap_stat_increase(theap,malloc_requested,size - MI_PADDING_SIZE); + #endif + #if MI_DEBUG>3 if (page->free_is_zero && size > sizeof(*block)) { mi_assert_expensive(mi_mem_is_zero(block+1,size - sizeof(*block))); @@ -104,18 +108,6 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // track per-block statistics mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); - #if 0 && (MI_STAT>0) - if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { - mi_theap_stat_increase(theap, malloc_normal, bsize); - #if (MI_STAT>1) - mi_theap_stat_counter_increase(theap, malloc_normal_count, 1); - const size_t bin = _mi_bin(bsize); - mi_theap_stat_increase(theap, malloc_bins[bin], 1); - mi_theap_stat_increase(theap, malloc_requested, size - MI_PADDING_SIZE); - #endif - } - #endif - // in debug mode initialize with 0xD0 #if (MI_DEBUG>0) && !MI_TRACK_ENABLED && !MI_TSAN if mi_likely(!zero && !mi_page_is_huge(page)) { memset(block, MI_DEBUG_UNINIT, bsize); } @@ -954,7 +946,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); #if MI_STAT>1 - // adjust stats to only count the allocated size of the block (and not the guard page) + // adjust request stats to only count the allocated size of the block (and not the guard page) mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size); mi_theap_stat_increase(theap, malloc_requested, size); #endif diff --git a/src/page.c b/src/page.c index 6899908ea..48eb5d7d6 100644 --- a/src/page.c +++ b/src/page.c @@ -159,18 +159,17 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ // allocations if (alloc_count > 0) { mi_theapx_stat_increase(heap, theap, malloc_normal, alloc_count*bsize); - #if MI_STAT>1 mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count); mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); - mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); + #if MI_STATS==1 + // use coarse total requested bytes + mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); #endif } // frees if (free_count > 0) { mi_theapx_stat_decrease(heap, theap, malloc_normal, free_count*bsize); - #if (MI_STAT > 1) - mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count); - #endif + mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count); } } else { @@ -180,8 +179,9 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ if (alloc_count > 0) { mi_theapx_stat_increase(heap, theap, malloc_huge, alloc_count*bsize); mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); - #if MI_STAT>1 - mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); + #if MI_STATS==1 + // use coarse total requested bytes + mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); #endif } // frees @@ -739,10 +739,8 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { size_t page_size; //uint8_t* page_start = mi_page_area(page, &page_size); - #if MI_STAT>0 mi_theap_stat_counter_increase(theap, pages_extended, 1); - #endif - + // calculate the extend count const size_t bsize = mi_page_block_size(page); size_t extend = (size_t)page->reserved - page->capacity; @@ -798,9 +796,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { } // enable the new free list page->capacity += (uint16_t)extend; - #if MI_STAT>0 mi_theap_stat_increase(theap, page_committed, extend * bsize); - #endif mi_assert_expensive(mi_page_is_valid_init(page)); return true; } diff --git a/src/stats.c b/src/stats.c index 5b1080158..7876fd78e 100644 --- a/src/stats.c +++ b/src/stats.c @@ -128,7 +128,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // copy all fields MI_STAT_FIELDS() - #if MI_STAT>1 + #if MI_STAT for (size_t i = 0; i <= MI_BIN_HUGE; i++) { mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]); } @@ -241,7 +241,7 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t mi_stat_print_ex(stat, msg, unit, out, arg, NULL); } -#if MI_STAT>1 +#if MI_STAT static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { _mi_fprintf(out, arg, " %-10s:", msg); _mi_fprintf(out, arg, "%12s", " "); // no peak @@ -275,7 +275,7 @@ static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) { name, "peak ", "total ", "current ", "block ", "total# "); } -#if MI_STAT>1 +#if MI_STAT static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) { bool found = false; char buf[64]; @@ -367,7 +367,7 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou _mi_fprintf(out, arg, "%s %zu\n", name, id); if (stats->malloc_normal.total + stats->malloc_huge.total != 0) { - #if MI_STAT>1 + #if MI_STAT mi_print_header("blocks", out, arg); mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg); #endif @@ -380,6 +380,8 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, ""); #if MI_STAT>1 mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg); + #else + mi_stat_total_print(&stats->malloc_requested, "malloc req~", 1, out, arg); #endif _mi_fprintf(out, arg, "\n"); #endif From 2bb271416e747f34c35a35534fb293fefa22b047 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 07:48:59 -0700 Subject: [PATCH 170/280] nice stat display --- include/mimalloc/types.h | 4 ++-- src/page.c | 4 ++-- src/stats.c | 38 ++++++++++++++++++-------------------- 3 files changed, 22 insertions(+), 24 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 54a52dde8..626ae3413 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -79,12 +79,12 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Statistics (0=only essential, 1=normal, 2=more fine-grained (expensive) tracking) +// Statistics (0=only essential, 1=normal, 2=more expensive tracking of precise requested bytes) #ifndef MI_STAT #if (MI_DEBUG>0) #define MI_STAT 2 #else -#define MI_STAT 2 +#define MI_STAT 1 #endif #endif diff --git a/src/page.c b/src/page.c index 48eb5d7d6..e64c27e8b 100644 --- a/src/page.c +++ b/src/page.c @@ -161,7 +161,7 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ mi_theapx_stat_increase(heap, theap, malloc_normal, alloc_count*bsize); mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count); mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); - #if MI_STATS==1 + #if MI_STAT==1 // use coarse total requested bytes mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); #endif @@ -179,7 +179,7 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ if (alloc_count > 0) { mi_theapx_stat_increase(heap, theap, malloc_huge, alloc_count*bsize); mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); - #if MI_STATS==1 + #if MI_STAT==1 // use coarse total requested bytes mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); #endif diff --git a/src/stats.c b/src/stats.c index 7876fd78e..d458eadf7 100644 --- a/src/stats.c +++ b/src/stats.c @@ -195,7 +195,7 @@ static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* ar } static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { - _mi_fprintf(out, arg," %-10s:", msg); + _mi_fprintf(out, arg," %-12s:", msg); if (unit != 0) { if (unit > 0) { mi_print_amount(stat->peak, unit, out, arg); @@ -205,20 +205,18 @@ static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64 mi_print_amount(unit, 1, out, arg); mi_print_count(stat->total, unit, out, arg); } - else { + else if (unit==-1) { mi_print_amount(stat->peak, -1, out, arg); mi_print_amount(stat->total, -1, out, arg); // mi_print_amount(stat->freed, -1, out, arg); mi_print_amount(stat->current, -1, out, arg); - if (unit == -1) { - _mi_fprintf(out, arg, "%24s", ""); - } - else { - // mi_print_amount(-unit, 1, out, arg); - //mi_print_count((stat->total / -unit), 0, out, arg); - _mi_fprintf(out, arg, "%12s", ""); - mi_print_count(-unit, 0, out, arg); - } + _mi_fprintf(out, arg, "%24s", ""); + } + else { + mi_print_amount(stat->peak, 1, out, arg); + mi_print_amount(stat->total, 1, out, arg); + _mi_fprintf(out, arg, "%12s", ""); + mi_print_count(-unit, 0, out, arg); } if (stat->current != 0) { _mi_fprintf(out, arg, " "); @@ -243,7 +241,7 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t #if MI_STAT static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { - _mi_fprintf(out, arg, " %-10s:", msg); + _mi_fprintf(out, arg, " %-12s:", msg); _mi_fprintf(out, arg, "%12s", " "); // no peak mi_print_amount(stat->total, unit, out, arg); _mi_fprintf(out, arg, "\n"); @@ -251,13 +249,13 @@ static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, in #endif static void mi_stat_counter_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-10s:", msg); + _mi_fprintf(out, arg, " %-12s:", msg); mi_print_amount(stat->total, 0, out, arg); _mi_fprintf(out, arg, "\n"); } static void mi_stat_counter_print_size(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-10s:", msg); + _mi_fprintf(out, arg, " %-12s:", msg); mi_print_amount(stat->total, 1, out, arg); _mi_fprintf(out, arg, "\n"); } @@ -266,12 +264,12 @@ static void mi_stat_average_print(int64_t count, int64_t total, const char* msg, const int64_t avg_tens = (count == 0 ? 0 : (total*10 / count)); const int64_t avg_whole = avg_tens/10; const int64_t avg_frac1 = avg_tens%10; - _mi_fprintf(out, arg, " %-10s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1); + _mi_fprintf(out, arg, " %-12s: %5lld.%lld avg\n", msg, avg_whole, avg_frac1); } static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) { - _mi_fprintf(out, arg, " %-11s %11s %11s %11s %11s %11s\n", + _mi_fprintf(out, arg, " %-13s %11s %11s %11s %11s %11s\n", name, "peak ", "total ", "current ", "block ", "total# "); } @@ -344,8 +342,8 @@ mi_decl_export void mi_process_info_print_out(mi_output_fun* out, void* arg) mi_ size_t peak_commit; size_t page_faults; mi_process_info(&elapsed, &user_time, &sys_time, ¤t_rss, &peak_rss, ¤t_commit, &peak_commit, &page_faults); - _mi_fprintf(out, arg, " %-10s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); - _mi_fprintf(out, arg, " %-10s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", + _mi_fprintf(out, arg, " %-12s: %5zu.%03zu s\n", "elapsed", elapsed/1000, elapsed%1000); + _mi_fprintf(out, arg, " %-12s: user: %zu.%03zu s, system: %zu.%03zu s, faults: %zu, peak rss: ", "process", user_time/1000, user_time%1000, sys_time/1000, sys_time%1000, page_faults); mi_printf_amount((int64_t)peak_rss, 1, out, arg, false); if (peak_commit > 0) { @@ -401,7 +399,7 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stat_counter_print(&stats->pages_unabandon_busy_wait, "waits", out, arg); mi_stat_counter_print(&stats->pages_extended, "extended", out, arg); mi_stat_counter_print(&stats->pages_retire, "retire", out, arg); - mi_stat_counter_print(&stats->pages_stat_updates, "stat upds", out, arg); + mi_stat_counter_print(&stats->pages_stat_updates, "stat updates", out, arg); mi_stat_average_print(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg); mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); _mi_fprintf(out, arg, "\n"); @@ -429,7 +427,7 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_print_header("process", out, arg); mi_stat_print_ex(&stats->threads, "threads", 0, out, arg, ""); - _mi_fprintf(out, arg, " %-10s: %5i\n", "numa nodes", _mi_os_numa_node_count()); + _mi_fprintf(out, arg, " %-12s: %5i\n", "numa nodes", _mi_os_numa_node_count()); mi_process_info_print_out(out, arg); _mi_fprintf(out, arg, "\n"); From 7dbb0775c92b76e2cb7824fd088206f85f36f542 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 08:04:16 -0700 Subject: [PATCH 171/280] add os_abandoned and os_allocated stats --- include/mimalloc-stats.h | 2 ++ src/arena.c | 6 ++++++ src/stats.c | 3 +++ 3 files changed, 11 insertions(+) diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index 9b297f6b7..bf9a88e5e 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -75,6 +75,8 @@ typedef struct mi_stat_counter_s { /* only on v3 */ \ MI_STAT_COUNT(heaps) \ MI_STAT_COUNT(theaps) \ + MI_STAT_COUNT(pages_os_abandoned) /* pages in the os abandoned list (bad) */ \ + MI_STAT_COUNT(pages_os_allocated) /* pages allocated outside arenas (bad) */ \ MI_STAT_COUNTER(pages_reclaim_on_alloc) \ MI_STAT_COUNTER(pages_reclaim_on_free) \ MI_STAT_COUNTER(pages_reabandon_full) \ diff --git a/src/arena.c b/src/arena.c index fba97dd73..abae6c964 100644 --- a/src/arena.c +++ b/src/arena.c @@ -864,6 +864,7 @@ static uint8_t* mi_arenas_page_alloc_fresh_area(mi_theap_t* theap, size_t slice_ start = (uint8_t*)mi_arena_os_alloc_aligned(heap->subproc, alloc_size, page_alignment, 0 /* align offset */, commit, allow_large, req_arena, memid); } #endif + if (start!=NULL) { mi_heap_stat_increase(heap,pages_os_allocated,1); } } if (start == NULL) return NULL; @@ -1288,6 +1289,9 @@ static void mi_arenas_page_free_prim(mi_page_t* page) { mi_assert_internal(mi_bitmap_is_setN(arena->slices_committed, slice_index, slice_count)); } } + else { + mi_heap_stat_decrease(page->heap, pages_os_allocated, 1); + } if (mi_page_meta_is_separated(page)) { page->block_size = 0; } // for assertion checking _mi_arenas_free( mi_page_subproc(page), mi_page_slice_start(page), mi_page_full_size(page), page->memid); } @@ -1366,6 +1370,7 @@ void _mi_arenas_page_abandon(mi_page_t* page, mi_theap_t* current_theapx) { if (page->next != NULL) { page->next->prev = page; } heap->os_abandoned_pages = page; } + mi_theapx_stat_increase(heap, current_theapx, pages_os_abandoned, 1); } mi_theapx_stat_increase(heap, current_theapx, pages_abandoned, 1); mi_abandoned_page_unown(page, current_theapx); @@ -1435,6 +1440,7 @@ void _mi_arenas_page_unabandon(mi_page_t* page, mi_theap_t* current_theapx) { page->next = NULL; page->prev = NULL; } + mi_theapx_stat_decrease(heap, current_theapx, pages_os_abandoned, 1); } } mi_theapx_stat_decrease(heap, current_theapx, pages_abandoned, 1); diff --git a/src/stats.c b/src/stats.c index d458eadf7..fbead92d9 100644 --- a/src/stats.c +++ b/src/stats.c @@ -215,6 +215,7 @@ static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64 else { mi_print_amount(stat->peak, 1, out, arg); mi_print_amount(stat->total, 1, out, arg); + mi_print_amount(stat->current, -1, out, arg); _mi_fprintf(out, arg, "%12s", ""); mi_print_count(-unit, 0, out, arg); } @@ -393,6 +394,8 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou // mi_stat_print(&stats->segments_cache, "-cached", -1, out, arg); mi_stat_print(&stats->pages, "pages", 0, out, arg); mi_stat_print(&stats->pages_abandoned, "abandoned", 0, out, arg); + mi_stat_print(&stats->pages_os_abandoned, "os abandoned", 0, out, arg); + mi_stat_print(&stats->pages_os_allocated, "os allocated", 0, out, arg); mi_stat_counter_print(&stats->pages_reclaim_on_alloc, "reclaima", out, arg); mi_stat_counter_print(&stats->pages_reclaim_on_free, "reclaimf", out, arg); mi_stat_counter_print(&stats->pages_reabandon_full, "reabandon", out, arg); From b4c945e3ba32619e73f8d04c77822fbbc9b1c1f0 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 08:29:00 -0700 Subject: [PATCH 172/280] add comments --- include/mimalloc/internal.h | 2 +- include/mimalloc/types.h | 17 ++++++++++++----- src/arena.c | 2 +- src/page.c | 5 ++--- src/stats.c | 8 +++----- 5 files changed, 19 insertions(+), 15 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index a0168a8f5..88a3f7a40 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -903,7 +903,7 @@ static inline size_t mi_page_used(const mi_page_t* page) { } static inline void mi_page_used_reset(mi_page_t* page) { - page->xused.used_alloc = page->xused.used_alloc & ~0xFFFF; + page->xused = mi_xused_used_reset(page->xused); } static inline size_t mi_page_alloc_count(const mi_page_t* page) { diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 626ae3413..08764788b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -392,8 +392,15 @@ typedef size_t mi_page_flags_t; // This way we can push a block on the thread free list and try to claim ownership atomically in `free.c:mi_free_block_mt`. typedef uintptr_t mi_thread_free_t; +// We store the currently used block count together with the total malloc call count as 16-bit numbers. +// This is done for better codegen `mi_malloc/mi_free` (where we can increment both at once as `used_alloc += 0x10001` for example). +// The `last_used` is the `used` count since the statistics are last updated; on 32-bit platforms this +// is a separate field in `mi_page_t` but on 64-bit we use the upper 32-bits to store it. +// We need the `alloc_count` and `last_used` to efficiently calculate allocation and free statistics even +// in a release build; this way we can update the stats in the slow path (`_mi_page_update_stats`). typedef union mi_used_s { - uintptr_t used_alloc; + uintptr_t used_alloc; // used + alloc_count + // the following struct is unused but nice for debugging struct { uint16_t used_count; uint16_t alloc_count; @@ -401,11 +408,11 @@ typedef union mi_used_s { uint16_t last_used; uint16_t padding; #endif - } debug; + } debug_le; } mi_used_t; -static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); } -static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); } +static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.used_alloc & 0xFFFF); } +static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); } static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; } // A page contains blocks of one specific size (`block_size`). @@ -443,7 +450,7 @@ typedef struct mi_page_s { mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) mi_used_t xused; // number of blocks in use (including blocks in `thread_free`) (and the allocated count for statistics) #if MI_INTPTR_SIZE < 8 - int32_t xlast_used; // for statistics; on 64-bit platforms it is the upper 32-bits of xused. + uint32_t xlast_used; // for statistics; on 64-bit platforms it is the upper 32-bits of xused. #endif mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) diff --git a/src/arena.c b/src/arena.c index abae6c964..24b33b712 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2576,7 +2576,7 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are _mi_page_unguard_all(page); // remove potential interior guard pages #endif // destroy the page - page->xused = mi_xused_used_reset(page->xused); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case + mi_page_used_reset(page); // note: invariant `|local_free| + |free| == reserved - used` does not hold in this case _mi_arenas_page_free(page, theap); } else { diff --git a/src/page.c b/src/page.c index e64c27e8b..fda5c6f21 100644 --- a/src/page.c +++ b/src/page.c @@ -196,9 +196,8 @@ void _mi_page_update_stats(mi_page_t* page) { mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); // get stat counts - const mi_used_t xused = page->xused; - const size_t used = mi_xused_used_count(xused); - const size_t alloc_count = mi_xused_alloc_count(xused); + const size_t used = mi_page_used(page); + const size_t alloc_count = mi_page_alloc_count(page); const size_t last_used = mi_page_last_used(page); mi_assert_internal(last_used + alloc_count >= used); const size_t free_count = last_used + alloc_count - used; diff --git a/src/stats.c b/src/stats.c index fbead92d9..663f5df34 100644 --- a/src/stats.c +++ b/src/stats.c @@ -197,22 +197,20 @@ static void mi_print_count(int64_t n, int64_t unit, mi_output_fun* out, void* ar static void mi_stat_print_ex(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg, const char* notok ) { _mi_fprintf(out, arg," %-12s:", msg); if (unit != 0) { - if (unit > 0) { + if (unit > 0) { // as KiB (power of two) mi_print_amount(stat->peak, unit, out, arg); mi_print_amount(stat->total, unit, out, arg); - // mi_print_amount(stat->freed, unit, out, arg); mi_print_amount(stat->current, unit, out, arg); mi_print_amount(unit, 1, out, arg); mi_print_count(stat->total, unit, out, arg); } - else if (unit==-1) { + else if (unit==-1) { // as K (decimal) mi_print_amount(stat->peak, -1, out, arg); mi_print_amount(stat->total, -1, out, arg); - // mi_print_amount(stat->freed, -1, out, arg); mi_print_amount(stat->current, -1, out, arg); _mi_fprintf(out, arg, "%24s", ""); } - else { + else { // as KiB , unit is final count mi_print_amount(stat->peak, 1, out, arg); mi_print_amount(stat->total, 1, out, arg); mi_print_amount(stat->current, -1, out, arg); From 61cb91ddd9e30414e3079f2e1829ff7f03bedf11 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 08:31:40 -0700 Subject: [PATCH 173/280] disable assertion in mi_free_try_collect_mt for now --- src/free.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/free.c b/src/free.c index e9f153808..66a7ed84b 100644 --- a/src/free.c +++ b/src/free.c @@ -487,7 +487,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mi_assert_internal(mi_page_is_owned(page)); mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mt_free != NULL); - mi_assert_internal(!allow_reclaim || _mi_subproc() == mi_page_subproc(page)); // never reclaim across subprocesses + // mi_assert_internal(!allow_reclaim || _mi_subproc() == mi_page_subproc(page)); // never reclaim across subprocesses // we own the page now, and it is safe to collect the thread atomic free list if (page->block_size <= MI_SMALL_SIZE_MAX) { From bc1ac9777742f651b4571b674c093d31ae27bf4d Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 08:34:32 -0700 Subject: [PATCH 174/280] missing comma on 32-bit --- src/init.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/init.c b/src/init.c index f4d5b9f88..0948046bb 100644 --- a/src/init.c +++ b/src/init.c @@ -21,7 +21,7 @@ static const mi_page_t mi_page_empty = { NULL, // free {0}, // xused #if MI_INTPTR_SIZE < 8 - 0 // xlast_used + 0, // xlast_used #endif NULL, // local_free 0, // block_size From 23a5db4742ad32d9c01d7801b9d71f14f6ece55d Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 08:45:42 -0700 Subject: [PATCH 175/280] prevent mi_page_associated_theap from reading a free'd theap --- src/heap.c | 8 +++++++- src/threadlocal.c | 6 +++--- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/src/heap.c b/src/heap.c index 28e75ba61..29e0a80d0 100644 --- a/src/heap.c +++ b/src/heap.c @@ -182,6 +182,12 @@ static void mi_heap_free_theaps(mi_heap_t* heap) { theap = next; } } + + // set the theap thread local to NULL (so _mi_page_associated_theap does not read from a freed theap) + if (!_mi_is_process_heap_main(heap)) { + _mi_thread_local_free(heap->theap); + heap->theap = 0; + } } // free the heap resources (assuming the pages are already moved/destroyed, and all theaps have been freed) @@ -221,7 +227,7 @@ static void mi_heap_free(mi_heap_t* heap, bool acquire_heaps_lock) { mi_lock_done(&heap->os_abandoned_pages_lock); mi_lock_done(&heap->arena_pages_lock); if (!_mi_is_process_heap_main(heap)) { - _mi_thread_local_free(heap->theap); + // _mi_thread_local_free(heap->theap); _mi_free_subproc_safe(heap); } } diff --git a/src/threadlocal.c b/src/threadlocal.c index 355498cee..6f280d093 100644 --- a/src/threadlocal.c +++ b/src/threadlocal.c @@ -174,7 +174,6 @@ bool _mi_thread_local_set( mi_thread_local_t key, void* val ) { // get a tls slot value static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key ) { - mi_assert_internal(key!=0); const mi_thread_locals_t* const tls = mi_thread_locals_peek(); if mi_unlikely(tls==NULL) { // this can happen if a thread local is accessed after the thread local has been freed @@ -193,12 +192,13 @@ static mi_decl_noinline void* mi_thread_local_get_regular( mi_thread_local_t key // get a thread local value void* _mi_thread_local_get( mi_thread_local_t key ) { - mi_assert_internal(key!=0); if mi_likely(key == mi_thread_local_key_fast) { return mi_slot_fast_get(); } else { - return mi_thread_local_get_regular(key); + void* p = mi_thread_local_get_regular(key); + if (key==0) { mi_assert_internal(p==NULL); } + return p; } } From 27fbcd87f1f6fc24e059a2d834bd6194f9d56d32 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 12:55:24 -0700 Subject: [PATCH 176/280] add option collect_merges_stats to be able disable automatic stat merging --- include/mimalloc.h | 7 ++++--- src/heap.c | 2 +- src/options.c | 6 ++++++ src/theap.c | 4 +++- 4 files changed, 14 insertions(+), 5 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index d38988613..6d88cf248 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -506,18 +506,19 @@ typedef enum mi_option_e { mi_option_guarded_sample_rate, // 1 out of N allocations in the min/max range will be guarded (=1000) mi_option_guarded_sample_seed, // can be set to allow for a (more) deterministic re-execution when a guard page is triggered (=0) mi_option_generic_collect, // collect theaps every N (=10000) generic allocation calls - mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallowr always, 0 allows if the page originated from the current theap, 1 allow always + mi_option_page_reclaim_on_free, // reclaim abandoned pages on a free (=0). -1 disallow always, 0 allows if the page originated from the current theap, 1 allow always mi_option_page_full_retain, // retain N full (small) pages per size class (=2) mi_option_page_max_candidates, // max candidate pages to consider for allocation (=4) mi_option_max_vabits, // max user space virtual address bits to consider (=48) mi_option_pagemap_commit, // commit the full pagemap (to always catch invalid pointer uses) (=0) mi_option_page_commit_on_demand, // commit page memory on-demand mi_option_page_max_reclaim, // don't reclaim pages of the same originating theap if we already own N pages (in that size class) (=-1 (unlimited)) - mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=16) + mi_option_page_cross_thread_max_reclaim, // don't reclaim pages across threads if we already own N pages (in that size class) (=32) mi_option_allow_thp, // allow transparent huge pages? (=1) (on Android =0 by default). Set to 0 to disable THP for the process. mi_option_minimal_purge_size, // set minimal purge size (in KiB) (=0). By default set to either 64 or 2048 if THP is enabled. mi_option_arena_max_object_size, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). - mi_option_arena_is_numa_local, // experimental + mi_option_arena_is_numa_local, // experimental: associate local numa node with an initial arena allocation + mi_option_collect_merges_stats, // on each theap collection, the theap stats are merged automatically with the parent heap _mi_option_last, // legacy option names mi_option_large_os_pages = mi_option_allow_large_os_pages, diff --git a/src/heap.c b/src/heap.c index 29e0a80d0..507d420ec 100644 --- a/src/heap.c +++ b/src/heap.c @@ -183,7 +183,7 @@ static void mi_heap_free_theaps(mi_heap_t* heap) { } } - // set the theap thread local to NULL (so _mi_page_associated_theap does not read from a freed theap) + // set the theap thread local to NULL (so _mi_page_associated_theap does not read from a freed theap (through delete pages -> page_update_stats)) if (!_mi_is_process_heap_main(heap)) { _mi_thread_local_free(heap->theap); heap->theap = 0; diff --git a/src/options.c b/src/options.c index da0d6669c..f1e063202 100644 --- a/src/options.c +++ b/src/options.c @@ -109,6 +109,10 @@ int mi_version(void) { #endif #endif +#ifndef MI_DEFAULT_COLLECT_MERGES_STATS +#define MI_DEFAULT_COLLECT_MERGES_STATS 1 +#endif + // Static options static mi_option_desc_t mi_options[_mi_option_last] = { @@ -176,6 +180,8 @@ static mi_option_desc_t mi_options[_mi_option_last] = { MI_DEFAULT_ARENA_MAX_OBJECT_SIZE, MI_OPTION_UNINIT, MI_OPTION(arena_max_object_size) }, // set maximal object size that can be allocated in an arena (in KiB) (=2GiB on 64-bit). { 0, MI_OPTION_UNINIT, MI_OPTION(arena_is_numa_local) }, // associate local numa node with an initial arena allocation + { MI_DEFAULT_COLLECT_MERGES_STATS, + MI_OPTION_UNINIT, MI_OPTION(collect_merges_stats) }, // on each theap collect, stats are merged with the parent heap }; static void mi_option_init(mi_option_desc_t* desc); diff --git a/src/theap.c b/src/theap.c index 62091083b..470243de7 100644 --- a/src/theap.c +++ b/src/theap.c @@ -145,7 +145,9 @@ static void mi_theap_collect_ex(mi_theap_t* theap, mi_collect_t collect) } // merge statistics - _mi_theap_merge_stats(theap); + if (mi_option_is_enabled(mi_option_collect_merges_stats)) { + _mi_theap_merge_stats(theap); + } } void _mi_theap_collect_abandon(mi_theap_t* theap) { From 0d673d1c82b5fbcbc79c5d14003ed7c3f25bf6b3 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 12:59:21 -0700 Subject: [PATCH 177/280] add mi_theap_stats_merge_to_heap --- include/mimalloc-stats.h | 9 ++++++--- src/stats.c | 5 +++++ 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index bf9a88e5e..e4d753c91 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -144,9 +144,6 @@ mi_decl_export bool mi_heap_stats_get(mi_heap_t* heap, mi_stats_t* stats) mi_ mi_decl_export char* mi_heap_stats_get_json(mi_heap_t* heap, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL mi_decl_export void mi_heap_stats_print_out(mi_heap_t* heap, mi_output_fun* out, void* arg) mi_attr_noexcept; -// stats from a theap -mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept; - // stats from a subprocess and its heaps aggregated mi_decl_export bool mi_subproc_stats_get(mi_subproc_id_t subproc_id, mi_stats_t* stats) mi_attr_noexcept; mi_decl_export char* mi_subproc_stats_get_json(mi_subproc_id_t subproc_id, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL @@ -168,6 +165,12 @@ mi_decl_export bool mi_subproc_stats_get_exclusive(mi_subproc_id_t subproc_id mi_decl_export char* mi_stats_as_json(mi_stats_t* stats, size_t buf_size, char* buf) mi_attr_noexcept; // use mi_free to free the result if the input buf == NULL mi_decl_export size_t mi_stats_get_bin_size(size_t bin) mi_attr_noexcept; +// get theap stats (only thread safe on theaps belonging to the calling thread) +mi_decl_export bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept; + +// add the theap stats to the stats of the parent heap and clear the theap stats +mi_decl_export void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept; + #ifdef __cplusplus } #endif diff --git a/src/stats.c b/src/stats.c index 663f5df34..77c6c4466 100644 --- a/src/stats.c +++ b/src/stats.c @@ -636,6 +636,11 @@ bool mi_theap_stats_get(mi_theap_t* theap, mi_stats_t* stats) mi_attr_noexcept { return mi_stats_copy(stats, mi_theap_get_stats(theap)); } +void mi_theap_stats_merge_to_heap(mi_theap_t* theap) mi_attr_noexcept { + if (theap == NULL) return; + _mi_theap_merge_stats(theap); +} + static bool mi_cdecl mi_heap_aggregate_visitor(mi_heap_t* heap, void* arg) { mi_stats_t* stats = (mi_stats_t*)arg; mi_stats_add_into(stats, mi_heap_get_stats(heap)); From 636d6b4f9a07e5e7d09546e1199af4a259ebf107 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 17:38:22 -0700 Subject: [PATCH 178/280] wip: initial design for profiling integration --- CMakeLists.txt | 5 +++-- include/mimalloc-stats.h | 1 + include/mimalloc/internal.h | 42 ++++++++++++++++++++++++++++++++++--- include/mimalloc/types.h | 14 +++++++++++++ src/alloc-aligned.c | 10 ++++++--- src/alloc.c | 2 ++ src/free.c | 12 +++++++++++ src/heap.c | 1 + src/init.c | 3 ++- src/page.c | 31 ++++++++++++++++++++++----- src/static.c | 1 + src/stats.c | 1 + 12 files changed, 109 insertions(+), 14 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 788040394..131cd6a21 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,6 +111,7 @@ set(mi_sources src/os.c src/page.c src/page-map.c + src/profile.c src/random.c src/stats.c src/subproc.c @@ -840,7 +841,7 @@ endif() # set language for source files now if(MI_USE_CXX) set_source_files_properties(${mi_sources} PROPERTIES LANGUAGE CXX ) - set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c PROPERTIES LANGUAGE CXX ) + set_source_files_properties(src/static.c test/test-api.c test/test-api-fill.c test/test-stress.c test/test-stress-heaps.c test/test-stress-subprocs.c test/test-profile.c PROPERTIES LANGUAGE CXX ) endif() @@ -1057,7 +1058,7 @@ if (MI_BUILD_TESTS) enable_testing() # static link tests - foreach(TEST_NAME api api-fill stress-heaps stress-subprocs stress) + foreach(TEST_NAME api api-fill stress-heaps stress-subprocs stress profile) add_executable(mimalloc-test-${TEST_NAME} test/test-${TEST_NAME}.c) target_compile_definitions(mimalloc-test-${TEST_NAME} PRIVATE ${mi_defines}) target_compile_options(mimalloc-test-${TEST_NAME} PRIVATE ${mi_cflags}) diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index e4d753c91..e1c74494a 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -84,6 +84,7 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNTER(heaps_delete_wait) \ MI_STAT_COUNTER(pages_stat_updates) /* calls to successful page_stat_update */ \ MI_STAT_COUNTER(pages_stat_update_count) /* total free/allocs */ \ + MI_STAT_COUNTER(profile_samples) /* total sampled profiled allocations */ \ // Size bins for chunks typedef enum mi_chunkbin_e { diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 88a3f7a40..64f6521ca 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -332,6 +332,9 @@ size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr bool _mi_page_is_valid(mi_page_t* page); #endif +// "profile.c" +void* _mi_theap_profile_alloc(mi_theap_t* theap, mi_profiler_t* prof, size_t size, bool zero, mi_page_t** ppage); +void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p); // ------------------------------------------------------ // Assertions @@ -1146,14 +1149,15 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) { /* ------------------------------------------------------------------- - Guarded objects + Guarded and profiled objects ------------------------------------------------------------------- */ -#if MI_GUARDED + // we always align guarded pointers in a block at an offset // the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones #define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) +#define MI_BLOCK_TAG_PROFILED ((mi_encoded_t)(1)) #define MI_BLOCK_TAG_GUARDED (~MI_BLOCK_TAG_ALIGNED) -#endif + static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* p) { #if MI_GUARDED @@ -1165,6 +1169,38 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* #endif } +static inline bool mi_block_ptr_is_profiled(const mi_block_t* block, const void* p) { +#if MI_PROFILE + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_PROFILED); +#else + MI_UNUSED(block); MI_UNUSED(p); + return false; +#endif +} + +static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) { + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved1; + return (mi_atomic_load_acquire(penabled) != 0); +} + +static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved1; + return (mi_atomic_exchange_release(penabled, (enable ? 1 : 0)) != 0); +} + + +static inline size_t mi_theap_disable_profiler(mi_theap_t* theap) { + const size_t threshold = theap->profile_threshold; + theap->profile_threshold = SIZE_MAX; + return threshold; +} + +static inline void mi_theap_enable_profiler(mi_theap_t* theap, size_t threshold) { + theap->profile_threshold = threshold; +} + + #if MI_GUARDED static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) { // this code is written to result in fast assembly as it is on the hot path for allocation diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 08764788b..ceb5b86b3 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -24,6 +24,7 @@ terms of the MIT license. A copy of the license can be found in the file #include +#include #include // ptrdiff_t #include // uintptr_t, uint16_t, etc #include // bool @@ -88,6 +89,11 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif +// Enable profiling support +#ifndef MI_PROFILE +#define MI_PROFILE 1 +#endif + // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) #if !defined(MI_GUARDED) && MI_DEBUG && !defined(NDEBUG) && !MI_OPT_FREE_SMALL #define MI_GUARDED 1 @@ -588,6 +594,9 @@ struct mi_theap_s { _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) _Atomic(size_t) refcount; // reference count + + size_t profile_allocated; // allocated since last profiler sample + size_t profile_threshold; // threshold for the next sample unsigned long long heartbeat; // monotonic heartbeat count mi_random_ctx_t random; // random number context used for secure allocation @@ -607,6 +616,7 @@ struct mi_theap_s { bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` + #if MI_GUARDED size_t guarded_size_min; // minimal size for guarded objects size_t guarded_size_max; // maximal size for guarded objects @@ -649,6 +659,8 @@ typedef struct mi_heap_s { mi_theap_t* theaps; // list of all thread-local theaps belonging to this heap (using the `hnext`/`hprev` fields) mi_lock_t theaps_lock; // lock for the theaps list operations + _Atomic(mi_profiler_t*) profiler; + _Atomic(size_t) abandoned_count[MI_BIN_COUNT]; // total count of abandoned pages in this heap mi_page_t* os_abandoned_pages; // list of pages that are OS allocated and not in an arena mi_lock_t os_abandoned_pages_lock; // lock for the os abandoned pages list (this lock protects list operations) @@ -693,6 +705,8 @@ struct mi_subproc_s { _Atomic(size_t) heap_count; // current heaps in this sub-process (== |heaps|) _Atomic(size_t) heap_total_count; // total created heaps in this sub-process + _Atomic(mi_profiler_t*) profiler; + mi_memid_t memid; // provenance of this memory block (meta or static) mi_subproc_t* parent; // subproc in which this one was allocated mi_decl_align(8) // needed on some 32-bit platforms diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 111d63a8b..ef6196ddc 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -117,8 +117,9 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // if (*usable > adjust) { *usable = *usable - adjust; } // mi_assert_internal(*usable >= size); // } - #if MI_GUARDED + #if MI_GUARDED || MI_PROFILE // set tag to aligned so mi_usable_size works with guard pages + // note: we could have gotten a profiled block but that is still ok if (adjust >= sizeof(mi_block_t)) { mi_block_t* const block = (mi_block_t*)p; block->next = MI_BLOCK_TAG_ALIGNED; @@ -135,7 +136,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap #if MI_DEBUG > 1 mi_page_t* const apage = _mi_ptr_page(aligned_p); void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); - mi_assert_internal(p == unalign_p); + mi_assert_internal(p == unalign_p || mi_block_ptr_is_profiled((mi_block_t*)unalign_p, aligned_p)); #endif // now zero the block if needed @@ -149,7 +150,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap if (p != aligned_p) { mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); - #if MI_GUARDED + #if MI_GUARDED || MI_PROFILE mi_track_mem_defined(p, sizeof(mi_block_t)); #endif } @@ -178,7 +179,10 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t } else { // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. + // but it can happen if this allocation just so happens to become a profile sample (which adds a block in front) + #if !MI_PROFILE mi_assert(false); + #endif mi_free(p); } } diff --git a/src/alloc.c b/src/alloc.c index 9f8ed8971..dec4dfcd1 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -932,7 +932,9 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); + const size_t threshold = mi_theap_disable_profiler(theap); mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, ppage); + mi_theap_enable_profiler(theap,threshold); if (block==NULL) return NULL; size_t usable_size = 0; void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); diff --git a/src/free.c b/src/free.c index 66a7ed84b..5cd71ed6a 100644 --- a/src/free.c +++ b/src/free.c @@ -141,11 +141,22 @@ static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, vo } #endif +static inline void mi_block_check_profiled(mi_page_t* page, mi_block_t* block, void* p) { + #if MI_PROFILE + if mi_unlikely(mi_block_ptr_is_profiled(block,p)) { + _mi_page_profile_free(page,block,p); + } + #else + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); + #endif +} + // free a local pointer (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + mi_block_check_profiled(page,block,p); const bool was_guarded = mi_block_check_unguard(page, block, p); mi_free_block_local(page, block, was_guarded, true /* check for a full page */); } @@ -154,6 +165,7 @@ static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_ static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + mi_block_check_profiled(page,block,p); const bool was_guarded = mi_block_check_unguard(page, block, p); mi_free_block_mt(page, block, was_guarded, allow_reclaim); } diff --git a/src/heap.c b/src/heap.c index 507d420ec..c17b18d1f 100644 --- a/src/heap.c +++ b/src/heap.c @@ -107,6 +107,7 @@ void _mi_heap_init(mi_heap_t* heap, mi_thread_local_t theap_slot, mi_subproc_t* heap->heap_seq = mi_atomic_increment_relaxed(&subproc->heap_total_count); heap->exclusive_arena = _mi_arena_from_id(exclusive_arena_id); heap->numa_node = -1; // no initial affinity + heap->profiler = mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler); mi_stats_header_init(&heap->stats); mi_lock_init(&heap->theaps_lock); mi_lock_init(&heap->os_abandoned_pages_lock); diff --git a/src/init.c b/src/init.c index 0948046bb..a08925879 100644 --- a/src/init.c +++ b/src/init.c @@ -126,6 +126,7 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount + 0, 0, // profiler allocated, threshold 0, // heartbeat { {0}, {0}, 0, true }, // random 0, // page count @@ -137,7 +138,7 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { 0, // full page retain false, // allow reclaim true, // allow abandon - true, // is_detached + true, // is_detached #if MI_GUARDED 0, 0, 0, 1, // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) #endif diff --git a/src/page.c b/src/page.c index fda5c6f21..7098bf578 100644 --- a/src/page.c +++ b/src/page.c @@ -154,6 +154,13 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count); const size_t bsize = mi_page_usable_block_size(page); + #if MI_PROFILE + if (alloc_count>0 && theap!=NULL) { + // track profile allocated + theap->profile_allocated += alloc_count*bsize; + } + #endif + if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { const size_t bin = _mi_bin(bsize); // allocations @@ -176,7 +183,7 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ // allocations mi_assert_internal(alloc_count<=1); mi_assert_internal(free_count<=1); - if (alloc_count > 0) { + if (alloc_count > 0) { mi_theapx_stat_increase(heap, theap, malloc_huge, alloc_count*bsize); mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); #if MI_STAT==1 @@ -1104,7 +1111,7 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex Admin ----------------------------------------------------------- */ -static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) +static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap, mi_profiler_t** pprof) { if mi_unlikely(!mi_theap_is_initialized(theap)) { if (theap==&_mi_theap_empty_wrong) { @@ -1121,7 +1128,13 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) if mi_unlikely(theap->generic_count >= 1000) { theap->generic_collect_count += theap->generic_count; theap->generic_count = 0; - + + mi_heap_t* const heap = _mi_theap_heap(theap); + mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); + if (prof!=NULL) { + if (mi_profiler_is_enabled(prof)) { *pprof = prof; } else { prof = NULL; } + } + // do a full theap collect every once in a while (10000 by default) const long generic_collect = mi_option_get_clamp(mi_option_generic_collect, 1, 1000000L); if (theap->generic_collect_count >= generic_collect) { @@ -1132,6 +1145,7 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) // otherwise we do a mini-collect _mi_deferred_free(theap, false); // call potential deferred free routines _mi_theap_collect_retired(theap, false); // free retired pages + if (prof!=NULL) { mi_theap_collect(theap, false /* force? */); } } } return theap; @@ -1144,9 +1158,16 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) { // initialize if necessary - theap = mi_malloc_generic_admin(theap); + mi_profiler_t* prof = NULL; + theap = mi_malloc_generic_admin(theap, &prof); if (theap==NULL) return NULL; - + if mi_unlikely(prof!=NULL && huge_alignment==0) { // we cannot profile huge alignments as we put a block in front + if mi_unlikely(theap->profile_allocated > theap->profile_threshold) { + // sample the allocation + return _mi_theap_profile_alloc(theap,prof,size,zero,ppage); + } + } + // find (or allocate) a page of the right size mi_page_t* page = mi_find_page(theap, size, huge_alignment); if mi_unlikely(page == NULL) { // first time out of memory, try to collect and retry the allocation once more diff --git a/src/static.c b/src/static.c index 3554a4bc6..f35442471 100644 --- a/src/static.c +++ b/src/static.c @@ -39,6 +39,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "threadlocal.c" #include "prim/prim.c" #include "prim/prim-tls.c" +#include "profile.c" #if MI_OSX_ZONE #include "prim/osx/alloc-override-zone.c" #endif diff --git a/src/stats.c b/src/stats.c index 77c6c4466..08172d7a7 100644 --- a/src/stats.c +++ b/src/stats.c @@ -403,6 +403,7 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stat_counter_print(&stats->pages_stat_updates, "stat updates", out, arg); mi_stat_average_print(stats->pages_stat_updates.total, stats->pages_stat_update_count.total, "stat avg", out, arg); mi_stat_average_print(stats->page_searches_count.total, stats->page_searches.total, "searches", out, arg); + mi_stat_counter_print(&stats->profile_samples, "prof samples", out, arg); _mi_fprintf(out, arg, "\n"); } From 6aebb3374bb3f62a343e71bc39bfc32b8f65c1a3 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 17:40:02 -0700 Subject: [PATCH 179/280] wip: initial design for profiling integration --- include/mimalloc-profile.h | 54 ++++++++++++ src/profile.c | 109 +++++++++++++++++++++++ test/test-profile.c | 171 +++++++++++++++++++++++++++++++++++++ 3 files changed, 334 insertions(+) create mode 100644 include/mimalloc-profile.h create mode 100644 src/profile.c create mode 100644 test/test-profile.c diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h new file mode 100644 index 000000000..866852fad --- /dev/null +++ b/include/mimalloc-profile.h @@ -0,0 +1,54 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2024-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#pragma once +#ifndef MIMALLOC_PROFILE_H +#define MIMALLOC_PROFILE_H + +#include +#include // bool + +typedef struct mi_profiler_data_s { + size_t usable_size; + size_t requested_size; + void* user_data[14]; +} mi_profiler_data_t; + +typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_data_t* profiler_data, void* ptr, size_t threshold, size_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg); +typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_data_t* profiler_data, void* ptr, size_t old_requested_size, const mi_heap_t* heap, void* profiler_arg); +typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_data_t* profiler_data, void* ptr, const mi_heap_t* heap, void* profiler_arg); + +// A profiler +// All fields are considered immutable such that they can be copied and accessed concurrently. +// All fields can be NULL/0. +typedef struct mi_profiler_s { + void* reserved1; // opaque -- internal use for mimalloc + void* reserved2; + void* reserved3; + void* profiler_arg; // opaque profiler state pointer -- passed to each callback + size_t profiler_data_size; // size of required profiler data for each sampled allocation + mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently) + mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently) + mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // called on in-place reallocation of a previous sampled allocation (may be called concurrently) +} mi_profiler_t; + +// Exported definitions +#ifdef __cplusplus +extern "C" { +#endif + +mi_decl_export bool mi_heap_profile(mi_heap_t* heap, const mi_profiler_t* profiler); +mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, const mi_profiler_t* profiler); +mi_decl_export bool mi_profile(const mi_profiler_t* profiler); + +mi_decl_export bool mi_profiler_start(const mi_profiler_t* profiler); +mi_decl_export bool mi_profiler_stop(const mi_profiler_t* profiler); + +#ifdef __cplusplus +} +#endif + +#endif // MIMALLOC_PROFILE_H diff --git a/src/profile.c b/src/profile.c new file mode 100644 index 000000000..0508be1dc --- /dev/null +++ b/src/profile.c @@ -0,0 +1,109 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" + +static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) { + return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); +} + + + +/* ---------------------------------------------------------------------------- + Profile an allocation and free +-----------------------------------------------------------------------------*/ +void* _mi_theap_profile_alloc(mi_theap_t* theap, mi_profiler_t* prof, size_t size, bool zero, mi_page_t** ppage) +{ + mi_assert_internal(theap!=NULL && prof!=NULL); + mi_assert_internal(theap->profile_allocated > theap->profile_threshold); + const size_t allocated = theap->profile_allocated; + theap->profile_allocated = 0; + + // Overallocate a larger block to store the profiler data + // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] + const size_t zero_huge_alignment = (zero ? 1 : 0); + const size_t profiler_data_offset = sizeof(mi_block_t); + size_t profiler_data_size = sizeof(mi_profiler_data_t); + if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > 512 ? 512 : prof->profiler_data_size); }; + const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); + const size_t oversize = profiler_user_offset + size; + mi_page_t* page = NULL; + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap,oversize,zero_huge_alignment,&page); // cannot recurse as the profile_allocated is now zero + if (block==NULL) return NULL; + mi_assert_internal(page!=NULL); + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(!mi_block_ptr_is_guarded(_mi_page_ptr_unalign(page,block),block)); + + // Set up the profiled block + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_PROFILED; + const size_t usable_size = _mi_page_usable_size(page,block) - profiler_user_offset; + void* const p = (uint8_t*)block + profiler_user_offset; + mi_profiler_data_t* profiler_data = (mi_profiler_data_t*)((uint8_t*)block + profiler_data_offset); + profiler_data->requested_size = size - MI_PADDING_SIZE; + profiler_data->usable_size = usable_size; + + // and call the profiler on_alloc + if (prof->on_alloc!=NULL) { + const size_t new_threshold = (*prof->on_alloc)(profiler_data, p, theap->profile_threshold, allocated, _mi_theap_heap(theap), prof->profiler_arg); + if (new_threshold!=0) { + theap->profile_threshold = new_threshold; + } + mi_theap_stat_counter_increase(theap,profile_samples,1); + } + return p; +} + +void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { + mi_assert_internal(mi_block_ptr_is_profiled(block,p)); + + // get the heap and profiler + mi_heap_t* const heap = mi_page_heap(page); + if (heap==NULL) return; + mi_profiler_t* prof = mi_heap_profiler(heap); + if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; + + // call the on_free callback + mi_profiler_data_t* const profiler_data = (mi_profiler_data_t*)((uint8_t*)block + sizeof(mi_block_t)); + prof->on_free(profiler_data, p, heap, prof->profiler_arg); +} + + +/* ---------------------------------------------------------------------------- + API +-----------------------------------------------------------------------------*/ + +bool mi_heap_profile(mi_heap_t* heap, const mi_profiler_t* profiler) { + // if (mi_heap_profiler(heap)!=NULL) return false; // always overwrite? + mi_atomic_store_ptr_release(mi_profiler_t,&heap->profiler,(mi_profiler_t*)profiler); + return true; +} + +bool mi_subproc_profile(mi_subproc_id_t subproc_id, const mi_profiler_t* profiler) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return false; + // if (mi_subproc_profiler(subproc)!=NULL) return false; // always overwrite? + mi_atomic_store_ptr_release(mi_profiler_t,&subproc->profiler, (mi_profiler_t*)profiler); + mi_lock(&subproc->heaps_lock) { + for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) { + mi_heap_profile(heap,profiler); + } + } + return true; +} + +bool mi_profile(const mi_profiler_t* profiler) { + return mi_subproc_profile(mi_subproc_main(),profiler); +} + +bool mi_profiler_start(const mi_profiler_t* profiler ) { + return mi_profiler_set_enabled((mi_profiler_t*)profiler,true); +} + +bool mi_profiler_stop(const mi_profiler_t* profiler) { + return mi_profiler_set_enabled((mi_profiler_t*)profiler,false); +} diff --git a/test/test-profile.c b/test/test-profile.c new file mode 100644 index 000000000..e13c35492 --- /dev/null +++ b/test/test-profile.c @@ -0,0 +1,171 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2026-2026, Microsoft Research, Daniel Schwartz-Narbonne, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ + +// Tests for the mimalloc heap profiler (src/profile.c). + +#include +#include +#include +#include +#include + +#include "mimalloc.h" +#include "mimalloc-profile.h" +#include "mimalloc/internal.h" +#include "testhelper.h" + +// --------------------------------------------------------------------------- +// Shared callback state (not thread safe!) +// --------------------------------------------------------------------------- + +typedef struct { + int64_t alloc_count; + int64_t free_count; + size_t last_size; + size_t last_upscaled; + void* last_ptr; +} profile_state_t; + +// We store ptr in user_data so on_free can verify the round-trip. + +static profile_state_t g_state; + +#define TEST_THRESHOLD (16 * 1024) + +static size_t mi_cdecl on_alloc(mi_profiler_data_t* data, void* ptr, size_t threshold, size_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg) { + MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(profiler_arg); + assert(profiler_arg==&g_state); + g_state.alloc_count++; + g_state.last_ptr = ptr; + g_state.last_size = data->requested_size; + g_state.last_upscaled = bytes_since_last_sample; + // store ptr to verify round-trip + data->user_data[0] = ptr; + return TEST_THRESHOLD; +} + +static void mi_cdecl on_free(mi_profiler_data_t* data, void* ptr, const mi_heap_t* heap, void* profiler_arg) { + MI_UNUSED(heap); MI_UNUSED_RELEASE(profiler_arg); MI_UNUSED_RELEASE(data); + assert(profiler_arg==&g_state); + g_state.free_count++; + // verify the user_data round-trip + assert(data->user_data[0] == ptr); + if (g_state.last_ptr == ptr) { assert(data->requested_size == g_state.last_size); } +} + +mi_profiler_t my_profiler = { + NULL, NULL, NULL, // reserved + &g_state, // profiler_arg + 3*sizeof(void*), // needed data size + &on_alloc, + &on_free, + NULL +}; + +// --------------------------------------------------------------------------- +// Helpers +// --------------------------------------------------------------------------- + +// Force at least one sample by allocating well over the threshold. +static void allocate_past_threshold(void) { + size_t total = 0; + while (total < TEST_THRESHOLD * 1000) { + void* p = mi_malloc(4096); + mi_free(p); + total += 4096; + // mi_collect() + } +} + +// --------------------------------------------------------------------------- +// Tests +// --------------------------------------------------------------------------- + +bool test_profiler_samples(void) { + CHECK_BODY("profiler: on_alloc called after threshold") { + int before = g_state.alloc_count; + allocate_past_threshold(); + result = (g_state.alloc_count > before); + } + return true; +} + +bool test_profiler_record_fields(void) { + CHECK_BODY("profiler: record ptr and size are non-zero") { + int before = g_state.alloc_count; + while (g_state.alloc_count == before) { + void* p = mi_malloc(1024); + mi_free(p); + } + result = (g_state.last_ptr != NULL && g_state.last_size > 0 && g_state.last_upscaled > 0); + } + return true; +} + +bool test_profiler_on_free_called(void) { + CHECK_BODY("profiler: on_free called for sampled allocation") { + int alloc_before = g_state.alloc_count; + int free_before = g_state.free_count; + + // Keep the pointer live until we confirm a sample was taken, then free it. + void* sampled = NULL; + while (g_state.alloc_count == alloc_before) { + if (sampled) { mi_free(sampled); } + sampled = mi_malloc(1024); + } + // At this point g_state.last_ptr is the sampled pointer. + // Free it and check on_free fires. + void* expected = g_state.last_ptr; + mi_free(expected); + sampled = NULL; + + result = (g_state.free_count > free_before); + } + return true; +} + +bool test_profiler_upscaled_at_least_size(void) { + CHECK_BODY("profiler: upscaled_size >= size") { + int before = g_state.alloc_count; + while (g_state.alloc_count == before) { + void* p = mi_malloc(256); + mi_free(p); + } + result = (g_state.last_upscaled >= g_state.last_size); + } + return true; +} + +bool test_profiler_free_count_le_alloc_count(void) { + CHECK_BODY("profiler: on_free never called more times than on_alloc") { + // Free can only fire for sampled allocations, so free_count <= alloc_count + // must hold at all times. + allocate_past_threshold(); + result = (g_state.free_count <= g_state.alloc_count); + } + return true; +} + + +// --------------------------------------------------------------------------- +// main +// --------------------------------------------------------------------------- + +int main(void) { + mi_profile(&my_profiler); + mi_profiler_start(&my_profiler); + + test_profiler_samples(); + test_profiler_record_fields(); + test_profiler_on_free_called(); + test_profiler_upscaled_at_least_size(); + test_profiler_free_count_le_alloc_count(); + + mi_profiler_stop(&my_profiler); + + return print_test_summary(); +} From 4d1a2293bdeec3a621156d4df691794683c1cf4d Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 21:00:26 -0700 Subject: [PATCH 180/280] increase page stat updates when a profiler is active --- CMakeLists.txt | 2 +- include/mimalloc/internal.h | 2 +- src/page.c | 66 ++++++++++++++++++++----------------- src/profile.c | 31 ++++++++++++++--- test/test-profile.c | 2 +- 5 files changed, 65 insertions(+), 38 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 131cd6a21..a91f3e72d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1070,7 +1070,7 @@ if (MI_BUILD_TESTS) else() message(STATUS "cannot build TSAN tests without MI_BUILD_SHARED being enabled") endif() - if(MI_GUARDED) + if(MI_GUARDED AND NOT (TEST_NAME STREQUAL "profile")) add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) elseif(TEST_NAME STREQUAL "stress-heaps") add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=0 $) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 64f6521ca..dac967d33 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -333,7 +333,7 @@ bool _mi_page_is_valid(mi_page_t* page); #endif // "profile.c" -void* _mi_theap_profile_alloc(mi_theap_t* theap, mi_profiler_t* prof, size_t size, bool zero, mi_page_t** ppage); +void* _mi_theap_profile_alloc(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage); void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p); // ------------------------------------------------------ diff --git a/src/page.c b/src/page.c index 7098bf578..84bb6a903 100644 --- a/src/page.c +++ b/src/page.c @@ -155,9 +155,9 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ const size_t bsize = mi_page_usable_block_size(page); #if MI_PROFILE - if (alloc_count>0 && theap!=NULL) { + if (alloc_count>0 && theap!=NULL && theap->profile_threshold!=0) { // track profile allocated - theap->profile_allocated += alloc_count*bsize; + theap->profile_allocated += alloc_count*bsize; } #endif @@ -295,10 +295,8 @@ static void mi_page_thread_free_collect(mi_page_t* page) // potentially keep growing by repeated allocation/free in the same page. // When it gets too large where it could overflow the 16 bits, we update // the stats to reset the allocated counter. -static inline void mi_page_maybe_update_stats(mi_page_t* page) { - if mi_unlikely(mi_xused_alloc_count(page->xused) > 0x7FFF) { - _mi_page_update_stats(page); - } +static inline bool mi_page_has_high_alloc_count(const mi_page_t* page) { + return mi_xused_alloc_count(page->xused) > 0x7FFF; } // returns `true` if after collection `mi_page_immediate_available` is true. @@ -309,7 +307,8 @@ static inline bool mi_page_free_quick_collect(mi_page_t* page) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - mi_page_maybe_update_stats(page); + mi_assert_internal(page->theap != NULL); + if (page->theap->profile_threshold!=0 || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } return true; } @@ -339,7 +338,7 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { page->local_free = NULL; page->free_is_zero = false; } - mi_page_maybe_update_stats(page); + if ((page->theap != NULL && page->theap->profile_threshold!=0) || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } } mi_assert_internal(!force || page->local_free == NULL); } @@ -360,7 +359,7 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - mi_page_maybe_update_stats(page); + if (mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } } } if (mi_page_used(page) == 1) { @@ -1111,7 +1110,7 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex Admin ----------------------------------------------------------- */ -static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap, mi_profiler_t** pprof) +static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) { if mi_unlikely(!mi_theap_is_initialized(theap)) { if (theap==&_mi_theap_empty_wrong) { @@ -1131,8 +1130,12 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap, mi_profiler_t** pp mi_heap_t* const heap = _mi_theap_heap(theap); mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); - if (prof!=NULL) { - if (mi_profiler_is_enabled(prof)) { *pprof = prof; } else { prof = NULL; } + const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof)); + if (theap->profile_threshold==0 && prof_enabled) { + theap->profile_threshold = 1; // start profiling + } + else if (theap->profile_threshold>0 && !prof_enabled) { + theap->profile_threshold = 0; // stop profiling } // do a full theap collect every once in a while (10000 by default) @@ -1145,7 +1148,7 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap, mi_profiler_t** pp // otherwise we do a mini-collect _mi_deferred_free(theap, false); // call potential deferred free routines _mi_theap_collect_retired(theap, false); // free retired pages - if (prof!=NULL) { mi_theap_collect(theap, false /* force? */); } + if (prof_enabled) { mi_theap_collect(theap, false /* force? */); } // update stats } } return theap; @@ -1158,14 +1161,14 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap, mi_profiler_t** pp static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) { // initialize if necessary - mi_profiler_t* prof = NULL; - theap = mi_malloc_generic_admin(theap, &prof); + theap = mi_malloc_generic_admin(theap); if (theap==NULL) return NULL; - if mi_unlikely(prof!=NULL && huge_alignment==0) { // we cannot profile huge alignments as we put a block in front - if mi_unlikely(theap->profile_allocated > theap->profile_threshold) { - // sample the allocation - return _mi_theap_profile_alloc(theap,prof,size,zero,ppage); - } + + if mi_unlikely(huge_alignment==0 && theap->profile_allocated > theap->profile_threshold) { + // profile the allocation + void* const p = _mi_theap_profile_alloc(theap,size,zero,ppage); + if (p!=NULL) { return p; } + // else fall through } // find (or allocate) a page of the right size @@ -1218,16 +1221,19 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm // fast path objects that fit in a small page if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if (req_size < MI_SMALL_MAX_OBJ_SIZE) { - mi_page_queue_t* pq = mi_page_queue(theap, size); - mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); - page = mi_page_queue_find_free(theap,pq); - // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); - if (page!=NULL) { - if (ppage!=NULL) { *ppage = page; } - mi_assert_internal(mi_page_immediate_available(page)); - return _mi_page_malloc_zero(theap,page,size,zero); + // fast profile check + if (theap->profile_allocated <= theap->profile_threshold) { + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (req_size < MI_SMALL_MAX_OBJ_SIZE) { + mi_page_queue_t* pq = mi_page_queue(theap, size); + mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); + page = mi_page_queue_find_free(theap,pq); + // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + if (page!=NULL) { + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(mi_page_immediate_available(page)); + return _mi_page_malloc_zero(theap,page,size,zero); + } } } } diff --git a/src/profile.c b/src/profile.c index 0508be1dc..695adf2fd 100644 --- a/src/profile.c +++ b/src/profile.c @@ -6,20 +6,32 @@ terms of the MIT license. A copy of the license can be found in the file -----------------------------------------------------------------------------*/ #include "mimalloc.h" #include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) { return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); } - +static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { + mi_heap_t* const heap = _mi_theap_heap(theap); + mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); + if (prof!=NULL && mi_profiler_is_enabled(prof)) { + return prof; + } + else { + return NULL; + } +} /* ---------------------------------------------------------------------------- Profile an allocation and free -----------------------------------------------------------------------------*/ -void* _mi_theap_profile_alloc(mi_theap_t* theap, mi_profiler_t* prof, size_t size, bool zero, mi_page_t** ppage) +void* _mi_theap_profile_alloc(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - mi_assert_internal(theap!=NULL && prof!=NULL); + mi_assert_internal(theap!=NULL); mi_assert_internal(theap->profile_allocated > theap->profile_threshold); + mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); + if (prof == NULL) { return NULL; } const size_t allocated = theap->profile_allocated; theap->profile_allocated = 0; @@ -79,7 +91,7 @@ void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { bool mi_heap_profile(mi_heap_t* heap, const mi_profiler_t* profiler) { // if (mi_heap_profiler(heap)!=NULL) return false; // always overwrite? - mi_atomic_store_ptr_release(mi_profiler_t,&heap->profiler,(mi_profiler_t*)profiler); + mi_atomic_store_ptr_release(mi_profiler_t,&heap->profiler,(mi_profiler_t*)profiler); return true; } @@ -101,7 +113,16 @@ bool mi_profile(const mi_profiler_t* profiler) { } bool mi_profiler_start(const mi_profiler_t* profiler ) { - return mi_profiler_set_enabled((mi_profiler_t*)profiler,true); + const bool was_running = mi_profiler_set_enabled((mi_profiler_t*)profiler,true); + if (was_running) return true; + mi_heap_t* heap = mi_heap_main(); + if (mi_heap_profiler(heap)==profiler) { + mi_theap_t* theap = _mi_heap_theap_peek(heap); + if (theap!=NULL && theap->profile_threshold == 0) { + theap->profile_threshold = 1; // get started more quickly + } + } + return false; } bool mi_profiler_stop(const mi_profiler_t* profiler) { diff --git a/test/test-profile.c b/test/test-profile.c index e13c35492..077fc6b10 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -73,7 +73,7 @@ mi_profiler_t my_profiler = { // Force at least one sample by allocating well over the threshold. static void allocate_past_threshold(void) { size_t total = 0; - while (total < TEST_THRESHOLD * 1000) { + while (total < TEST_THRESHOLD * 3) { void* p = mi_malloc(4096); mi_free(p); total += 4096; From 70af0eb5058d2b4ed284e2874dbd1d49c542de15 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 21:55:37 -0700 Subject: [PATCH 181/280] clean up guarded and profiled block handling; allow aligned blocks inside guarded or profiled blocks --- include/mimalloc/internal.h | 10 ++++++ src/alloc-aligned.c | 45 +++++++++--------------- src/free.c | 68 +++++++++++++++++++++---------------- 3 files changed, 64 insertions(+), 59 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index dac967d33..c8ee934c8 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1179,6 +1179,16 @@ static inline bool mi_block_ptr_is_profiled(const mi_block_t* block, const void* #endif } +static inline bool mi_block_ptr_is_profiled_or_guarded(const mi_block_t* block, const void* p) { +#if MI_GUARDED || MI_PROFILE + const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; + return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next != MI_BLOCK_TAG_ALIGNED); +#else + MI_UNUSED(block); MI_UNUSED(p); + return false; +#endif +} + static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) { _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved1; return (mi_atomic_load_acquire(penabled) != 0); diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index ef6196ddc..d5c347aee 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -49,14 +49,14 @@ static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(m } static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - #if MI_THEAP_INITASNULL - if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } - #endif - const size_t rate = theap->guarded_sample_rate; + // #if MI_THEAP_INITASNULL + // if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } + // #endif + // const size_t rate = theap->guarded_sample_rate; // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty` - if (rate != 0) { theap->guarded_sample_rate = 0; } + // if (rate != 0) { theap->guarded_sample_rate = 0; } void* p = _mi_theap_malloc_zero(theap, size, zero, ppage); - if (rate != 0) { theap->guarded_sample_rate = rate; } + // if (rate != 0) { theap->guarded_sample_rate = rate; } return p; } #else @@ -112,14 +112,11 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap // (we can access the page though since the just allocated pointer keeps it alive) if (aligned_p != p) { mi_page_set_has_interior_pointers(page, true); - // if (usable!=NULL) { - // mi_assert_internal(*usable > adjust); - // if (*usable > adjust) { *usable = *usable - adjust; } - // mi_assert_internal(*usable >= size); - // } + #if MI_GUARDED || MI_PROFILE // set tag to aligned so mi_usable_size works with guard pages - // note: we could have gotten a profiled block but that is still ok + // note: we could have gotten a profiled or guarded block but that is still ok + // and in such case the MI_BLOCK_TAG_ALIGNED is just not at the start of the real heap block if (adjust >= sizeof(mi_block_t)) { mi_block_t* const block = (mi_block_t*)p; block->next = MI_BLOCK_TAG_ALIGNED; @@ -135,19 +132,10 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap mi_assert_internal(mi_usable_size(p) == mi_usable_size(aligned_p)+adjust); #if MI_DEBUG > 1 mi_page_t* const apage = _mi_ptr_page(aligned_p); - void* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); - mi_assert_internal(p == unalign_p || mi_block_ptr_is_profiled((mi_block_t*)unalign_p, aligned_p)); + mi_block_t* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); + mi_assert_internal(p == (void*)unalign_p || mi_block_ptr_is_profiled_or_guarded(unalign_p, aligned_p)); #endif - // now zero the block if needed - //if (alignment > MI_PAGE_MAX_OVERALLOC_ALIGN) { - // // for the tracker, on huge aligned allocations only from the start of the large block is defined - // mi_track_mem_undefined(aligned_p, size); - // if (zero) { - // _mi_memzero_(aligned_p, mi_usable_size(aligned_p)); - // } - //} - if (p != aligned_p) { mi_track_align(p,aligned_p,adjust,mi_usable_size(aligned_p)); #if MI_GUARDED || MI_PROFILE @@ -171,18 +159,17 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, ppage); - mi_assert_internal(p == NULL || ((uintptr_t)p % alignment) == 0); + mi_page_t* page = NULL; + void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, &page); + if (ppage!=NULL) { *ppage = page; } const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; if mi_likely(is_aligned_or_null) { return p; } else { // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. - // but it can happen if this allocation just so happens to become a profile sample (which adds a block in front) - #if !MI_PROFILE - mi_assert(false); - #endif + // but it can happen if this allocation just so happens to become a profile or guarded allocation (which adds a block in front) + mi_assert_internal(p==NULL || mi_block_ptr_is_profiled_or_guarded(_mi_page_ptr_unalign(page,p),p)); mi_free(p); } } diff --git a/src/free.c b/src/free.c index 5cd71ed6a..f347e9c20 100644 --- a/src/free.c +++ b/src/free.c @@ -124,49 +124,57 @@ static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, con // forward declaration for a MI_GUARDED build #if MI_GUARDED -static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration -static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { - if (mi_block_ptr_is_guarded(block, p)) { - mi_block_unguard(page, block, p); - return true; - } - else { - return false; - } -} -#else -static inline bool mi_block_check_unguard(mi_page_t* page, mi_block_t* block, void* p) { - MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); - return false; -} +static void mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration #endif -static inline void mi_block_check_profiled(mi_page_t* page, mi_block_t* block, void* p) { - #if MI_PROFILE - if mi_unlikely(mi_block_ptr_is_profiled(block,p)) { - _mi_page_profile_free(page,block,p); +static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool* was_guarded) mi_attr_noexcept { + if mi_likely(!mi_page_has_interior_pointers(page)) { + return mi_validate_block_from_ptr(page,p); + } + else { + mi_block_t* const block = _mi_page_ptr_unalign(page,p); + #if MI_GUARDED || MI_PROFILE + const size_t offset = (uint8_t*)p - (uint8_t*)block; + if (offset >= sizeof(mi_block_t)) { + #if MI_PROFILE + if (block->next == MI_BLOCK_TAG_PROFILED) { + _mi_page_profile_free(page,block,p); + } + else + #endif + #if MI_GUARDED + if (block->next == MI_BLOCK_TAG_GUARDED) { + mi_page_block_unguard(page, block, p); + *was_guarded = true; + } + #else + { MI_UNUSED(was_guarded); } + #endif + } + #endif + return block; } - #else - MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); - #endif } - // free a local pointer (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_local(mi_page_t* page, void* p) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); - mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); - mi_block_check_profiled(page,block,p); - const bool was_guarded = mi_block_check_unguard(page, block, p); + bool was_guarded = false; + mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded); + // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + // mi_block_check_profiled(page,block,p); + // const bool was_guarded = mi_block_check_unguard(page, block, p); mi_free_block_local(page, block, was_guarded, true /* check for a full page */); } // free a pointer owned by another thread (page parameter comes first for better codegen) static void mi_decl_noinline mi_free_generic_mt(mi_page_t* page, void* p, bool allow_reclaim) mi_attr_noexcept { mi_assert_internal(p!=NULL && page != NULL); - mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); - mi_block_check_profiled(page,block,p); - const bool was_guarded = mi_block_check_unguard(page, block, p); + bool was_guarded = false; + mi_block_t* block = mi_page_ptr_block_check(page,p,&was_guarded); + // mi_block_t* const block = (mi_page_has_interior_pointers(page) ? _mi_page_ptr_unalign(page, p) : mi_validate_block_from_ptr(page,p)); + // mi_block_check_profiled(page,block,p); + // const bool was_guarded = mi_block_check_unguard(page, block, p); mi_free_block_mt(page, block, was_guarded, allow_reclaim); } @@ -804,7 +812,7 @@ void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { // Remove guard page when building with MI_GUARDED #if MI_GUARDED -static void mi_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { +static void mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { MI_UNUSED(p); mi_assert_internal(mi_block_ptr_is_guarded(block, p)); mi_assert_internal(mi_page_has_interior_pointers(page)); From 3d3c55bf974232a35064bc393fd17b46a73e38f7 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 21:57:19 -0700 Subject: [PATCH 182/280] increase test threshold for profile --- test/test-profile.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/test-profile.c b/test/test-profile.c index 077fc6b10..fce76c954 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -73,7 +73,7 @@ mi_profiler_t my_profiler = { // Force at least one sample by allocating well over the threshold. static void allocate_past_threshold(void) { size_t total = 0; - while (total < TEST_THRESHOLD * 3) { + while (total < TEST_THRESHOLD * 10) { void* p = mi_malloc(4096); mi_free(p); total += 4096; From 56b1a37a9eebdf9d7e23286fa9c1630d8ee2f0c9 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 2 Sep 2026 22:09:01 -0700 Subject: [PATCH 183/280] fix access to stale page->theap --- src/page.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/page.c b/src/page.c index 84bb6a903..7864bcc0d 100644 --- a/src/page.c +++ b/src/page.c @@ -338,7 +338,8 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { page->local_free = NULL; page->free_is_zero = false; } - if ((page->theap != NULL && page->theap->profile_threshold!=0) || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } + mi_theap_t* const theap = _mi_page_associated_theap_peek(page); + if ((theap != NULL && theap->profile_threshold!=0) || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } } mi_assert_internal(!force || page->local_free == NULL); } From cb30f644916bb43d911b6af3abd0765c9c5f999c Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 3 Sep 2026 10:04:07 -0700 Subject: [PATCH 184/280] add default for MI_OPT_FREE_SMALL option --- CMakeLists.txt | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c47a3d05a..bef2ec884 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,7 +72,6 @@ set_property(CACHE MI_TLS_MODEL PROPERTY STRINGS "LOCAL" "PTHREADS" "LOCAL_DYNAM # deprecated options option(MI_USE_LIBATOMIC "Deprecated: Explicitly link with -latomic (on older systems) (deprecated and detected automatically)" OFF) -option(MI_OPT_FREE_SMALL "Deprecated: Use aligned `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)" OFF) option(MI_DEBUG_INTERNAL "Deprecated: Enable assertion and internal invariant checks (use MI_DEBUG=INTERNAL instead)" OFF) option(MI_DEBUG_FULL "Deprecated: Enable assertion checks and expensive internal heap invariant checking (use MI_DEBUG=FULL instead)" OFF) option(MI_CHECK_FULL "Deprecated: Use full internal invariant checking in DEBUG mode (use MI_DEBUG=FULL instead)" OFF) @@ -89,6 +88,8 @@ option(MI_TRACK_VALGRIND "Deprecated: Compile with Valgrind support (adds a option(MI_TRACK_ASAN "Deprecated: Compile with address sanitizer support (adds a small overhead) (use MI_TRACK=ASAN instead)" OFF) option(MI_TRACK_ETW "Deprecated: Compile with Windows event tracing (ETW) support (adds a small overhead) (use MI_TRACK=ETW instead)" OFF) +set(MI_OPT_FREE_SMALL "DEFAULT" CACHE STRING "Deprecated: Use aligned `mi_free_small` without pagemap lookup (can only be used if SECURE or GUARDED mode are off)") +set_property(CACHE MI_OPT_FREE_SMALL PROPERTY STRINGS "OFF" "ON" "DEFAULT") include(CheckLinkerFlag) # requires cmake 3.18 include(CheckIncludeFiles) @@ -483,13 +484,18 @@ if(MI_FREE_USE_PAGEMAP) endif() # deprecated -if(MI_OPT_FREE_SMALL) +if(MI_OPT_FREE_SMALL STREQUAL "DEFAULT") + # do nothing +elseif(MI_OPT_FREE_SMALL) if(MI_GUARDED OR MI_SECURE) message(STATUS "Cannot enable MI_OPT_FREE_SMALL in secure or guarded mode (MI_OPT_FREE_SMALL=OFF)") else() message(STATUS "Use aligned `mi_free_small` (MI_OPT_FREE_SMALL=ON)") list(APPEND mi_defines MI_OPT_FREE_SMALL=1) endif() +else() + message(STATUS "Disable aligned `mi_free_small` (MI_OPT_FREE_SMALL=OFF)") + list(APPEND mi_defines MI_OPT_FREE_SMALL=0) endif() if(MI_XMALLOC) From 6725ca9396cbb6dcbe5f5d8f08f9f714a950ee5b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ang=C3=A9lica=20Moreira?= Date: Thu, 3 Sep 2026 16:50:55 -0700 Subject: [PATCH 185/280] avoid re-visiting full pages in mi_page_queue_find_free_ex Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/free.c | 2 +- src/page-queue.c | 9 +++++++++ src/page.c | 11 ++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/free.c b/src/free.c index 5d033aab4..8260b31c4 100644 --- a/src/free.c +++ b/src/free.c @@ -489,7 +489,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mt_free != NULL); // mi_assert_internal(_mi_subproc() == mi_page_subproc(page)); // never collect across subprocesses - + // we own the page now, and it is safe to collect the thread atomic free list if (page->block_size <= MI_SMALL_SIZE_MAX) { // use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list diff --git a/src/page-queue.c b/src/page-queue.c index c10c09257..ac9a2e20a 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -341,6 +341,15 @@ static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queu mi_assert_internal(queue->first == page); } +static void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(mi_page_queue_contains(queue, page)); + if (queue->last == page) return; + mi_page_queue_remove(queue, page); + mi_page_queue_push_at_end(theap, queue, page); + mi_assert_internal(queue->last == page); +} + static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_internal(from->count >= 1); diff --git a/src/page.c b/src/page.c index 02ca72044..b9027a61a 100644 --- a/src/page.c +++ b/src/page.c @@ -771,8 +771,9 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages mi_page_t* page_candidate = NULL; // a page with free space mi_page_t* page = pq->first; + mi_page_t* const last = pq->last; - while (page != NULL) + while (page!=NULL) { mi_page_t* next = page->next; // remember next (as this page can move to another queue) count++; @@ -796,6 +797,10 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); } + else if (page!=last && pq->last!=page) { + // avoid revisiting this page for a while + mi_page_queue_move_to_back(theap, pq, page); + } } else { // the page has free space, make it a candidate @@ -831,8 +836,8 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); #endif - - page = next; + if (page==last) { page=NULL; } // don't revisit earlier pages that moved to the back + else { page = next; } } // for each page mi_theap_stat_counter_increase(theap, page_searches, count); From 7e7769c4d5fa20e346d626ed6cde43baa2ead00f Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 3 Sep 2026 18:47:30 -0700 Subject: [PATCH 186/280] return new expected value from collect_partly --- include/mimalloc/internal.h | 2 +- src/free.c | 2 +- src/page.c | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index fdd5c78be..f6bc9092c 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -282,7 +282,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free t void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... void _mi_deferred_free(mi_theap_t* theap, bool force); void _mi_page_free_collect(mi_page_t* page, bool force); -void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq); diff --git a/src/free.c b/src/free.c index 8260b31c4..a84b1cf04 100644 --- a/src/free.c +++ b/src/free.c @@ -496,7 +496,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted). // however, if the page became completely free, the used count is guaranteed to be 0.) mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used. - _mi_page_free_collect_partly(page, mt_free); + mt_free = _mi_page_free_collect_partly(page, mt_free); } else { // for larger blocks we use the regular collect diff --git a/src/page.c b/src/page.c index b9027a61a..9374f57f9 100644 --- a/src/page.c +++ b/src/page.c @@ -244,12 +244,13 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // Collect elements in the thread-free list starting at `head`. This is an optimized // version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`. +// returns a possibly updated expected value for the thread_free pointer. // // `head` must be in the `xthread_free` list. It will not collect `head` itself // so the `used` count is not fully updated in general. However, if the `head` is // the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true). -void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { - if (head == NULL) return; +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { + if (head == NULL) return NULL; mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops) if (next != NULL) { mi_block_set_next(page, head, NULL); @@ -265,6 +266,10 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head); mi_assert_internal(mi_block_next(page,head) == NULL); _mi_page_free_collect(page, false); // collect the final element + return NULL; + } + else { + return head; } } From 3257a7f2e090510f330cc57a85b71bae4130abfc Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 3 Sep 2026 21:43:31 -0700 Subject: [PATCH 187/280] wip: rework profile and guarded as sampled allocations --- include/mimalloc/internal.h | 85 +++++++++++++++++++---------------- include/mimalloc/types.h | 29 +++++++----- src/alloc-aligned.c | 88 +++++++++++++++++++++---------------- src/alloc.c | 65 +++++++++++++++++++-------- src/init.c | 10 ++--- src/page.c | 70 +++++++++++++++++++++-------- src/profile.c | 22 ++++------ src/theap.c | 12 +++-- 8 files changed, 237 insertions(+), 144 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 92d1ac3b0..b9a18918f 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -266,6 +266,7 @@ void _mi_page_map_unsafe_destroy(void); // "page.c" void* _mi_malloc_generic(mi_theap_t* heap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic_no_sample(mi_theap_t* heap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); @@ -323,6 +324,9 @@ void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); +mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; + // "free.c" void _mi_free_subproc_safe(void* p) mi_attr_noexcept; void _mi_page_unguard_all(mi_page_t* page); @@ -333,7 +337,7 @@ bool _mi_page_is_valid(mi_page_t* page); #endif // "profile.c" -void* _mi_theap_profile_alloc(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage); +mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p); // ------------------------------------------------------ @@ -1152,6 +1156,36 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) { Guarded and profiled objects ------------------------------------------------------------------- */ +#define MI_SAMPLE_RATE_MAX (2*MI_GiB) + +static mi_ssize_t mi_samples_from_size(size_t size) { + const mi_ssize_t samples = (size > 1024 ? 1024 : (mi_ssize_t)size); + mi_assert_internal(samples <= INT16_MAX); + return samples; +} + +#if MI_SAMPLE==2 // fine grained +static inline bool mi_theap_sample(mi_theap_t* theap, size_t size) { + // a bit convoluted but we need to avoid writing to the empty theap + // theap->sample_countdown -= mi_sample_from_size(size); + // return (theap->sample_countdown < 0); + const mi_ssize_t sample_countdown = theap->sample_countdown - mi_samples_from_size(size); + if mi_likely(sample_countdown >= 0) { + theap->sample_countdown = sample_countdown; + return false; + } + else { + if mi_unlikely(theap->sample_rate == 0) { + return false; + } + else { + theap->sample_countdown = sample_countdown; + return true; + } + } +} +#endif + // we always align guarded pointers in a block at an offset // the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones #define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) @@ -1201,46 +1235,21 @@ static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { static inline size_t mi_theap_disable_profiler(mi_theap_t* theap) { - const size_t threshold = theap->profile_threshold; - theap->profile_threshold = SIZE_MAX; - return threshold; -} - -static inline void mi_theap_enable_profiler(mi_theap_t* theap, size_t threshold) { - theap->profile_threshold = threshold; -} - - -#if MI_GUARDED -static inline bool mi_theap_malloc_use_guarded(mi_theap_t* theap, size_t size) { - // this code is written to result in fast assembly as it is on the hot path for allocation - const size_t count = theap->guarded_sample_count - 1; // if the rate was 0, this will underflow and count for a long time.. - if mi_likely(count != 0) { - // no sample - theap->guarded_sample_count = count; - return false; - } - else { - // count == 0 - const size_t rate = theap->guarded_sample_rate; - if (rate == 0) { - return false; // don't write to an empty theap - } - else if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { - // use guarded allocation - theap->guarded_sample_count = rate; // reset - return true; - } - else { - // failed size criteria, rewind count - theap->guarded_sample_count = 1; - return false; - } + const mi_ssize_t sample_rate = theap->profile_sample_rate; + theap->profile_sample_rate = 0; + theap->profile_sample_countdown = 0; + theap->sample_rate = theap->guarded_sample_rate; + return (sample_rate < 0 ? 0 : (size_t)sample_rate); +} + +static inline void mi_theap_enable_profiler(mi_theap_t* theap, size_t sample_rate) { + theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : (mi_ssize_t)sample_rate); + theap->profile_sample_countdown = theap->profile_sample_rate; + if (theap->sample_rate==0 || theap->sample_rate > theap->profile_sample_rate) { + theap->sample_rate = theap->profile_sample_rate; } } -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -#endif /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index ceb5b86b3..36cf90ff7 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -91,7 +91,7 @@ terms of the MIT license. A copy of the license can be found in the file // Enable profiling support #ifndef MI_PROFILE -#define MI_PROFILE 1 +#define MI_PROFILE 2 #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) @@ -99,6 +99,14 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_GUARDED 1 #endif +#if MI_PROFILE || MI_GUARDED +#if MI_PROFILE>1 || MI_GUARDED>1 +#define MI_SAMPLE 2 +#else +#define MI_SAMPLE 1 +#endif +#endif + // Reserve extra padding at the end of each block to be more resilient against theap block overflows. // The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. #if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) @@ -588,15 +596,22 @@ typedef struct mi_padding_s { // A thread-local heap ("theap") owns a set of thread-local pages. struct mi_theap_s { // put in front for fast small allocations + mi_ssize_t sample_countdown; mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_tld_t* tld; // thread-local data _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) _Atomic(size_t) refcount; // reference count - - size_t profile_allocated; // allocated since last profiler sample - size_t profile_threshold; // threshold for the next sample + + mi_ssize_t sample_rate; + mi_ssize_t profile_sample_rate; + mi_ssize_t profile_sample_countdown; + mi_ssize_t guarded_sample_rate; + mi_ssize_t guarded_sample_countdown; + size_t guarded_size_min; // minimal size for guarded objects + size_t guarded_size_max; // maximal size for guarded objects + unsigned long long heartbeat; // monotonic heartbeat count mi_random_ctx_t random; // random number context used for secure allocation @@ -617,12 +632,6 @@ struct mi_theap_s { bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` - #if MI_GUARDED - size_t guarded_size_min; // minimal size for guarded objects - size_t guarded_size_max; // maximal size for guarded objects - size_t guarded_sample_rate; // sample rate (set to 0 to disable guarded pages) - size_t guarded_sample_count; // current sample count (counting down to 0) - #endif mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin") mi_memid_t memid; // provenance of the theap struct itself (meta or os) mi_stats_t stats; // thread-local statistics diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index d5c347aee..9339602b0 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -27,7 +27,7 @@ static bool mi_malloc_is_naturally_aligned( size_t size, size_t alignment ) { return ok; } -#if MI_GUARDED +#if 0 static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(mi_theap_t* theap, size_t size, size_t alignment, bool zero, mi_page_t** ppage) mi_attr_noexcept { // use over allocation for guarded blocksl #if MI_THEAP_INITASNULL @@ -47,7 +47,8 @@ static mi_decl_noinline mi_decl_restrict void* mi_theap_malloc_guarded_aligned(m mi_assert_internal(_mi_is_aligned(p, alignment)); return p; } - +#endif +#if MI_GUARDED static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { // #if MI_THEAP_INITASNULL // if mi_unlikely(theap==NULL) { theap = _mi_theap_empty_get(); } @@ -158,19 +159,30 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t // use regular allocation if it is guaranteed to fit the alignment constraints. // this is important to try as the fast path in `mi_theap_malloc_zero_aligned` only works when there exist // a page with the right block size, and if we always use the over-alloc fallback that would never happen. - if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { - mi_page_t* page = NULL; - void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, &page); - if (ppage!=NULL) { *ppage = page; } - const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; - if mi_likely(is_aligned_or_null) { - return p; - } - else { - // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. - // but it can happen if this allocation just so happens to become a profile or guarded allocation (which adds a block in front) - mi_assert_internal(p==NULL || mi_block_ptr_is_profiled_or_guarded(_mi_page_ptr_unalign(page,p),p)); - mi_free(p); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL) + #endif + { + #if MI_SAMPLE // only try if we would not take a sample + const mi_ssize_t samples = mi_samples_from_size(size); + if (theap->sample_countdown >= samples) + #endif + { + if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { + mi_page_t* page = NULL; + void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, &page); + if (ppage!=NULL) { *ppage = page; } + const bool is_aligned_or_null = (((uintptr_t)p) & (alignment-1))==0; + if mi_likely(is_aligned_or_null) { + return p; + } + else { + // this should never happen if the `mi_malloc_is_naturally_aligned` check is correct.. + // (but it can happen if this allocation just so happens to be sampled but we checked for that) + mi_assert_internal(false); // p==NULL || mi_block_ptr_is_profiled_or_guarded(_mi_page_ptr_unalign(page,p),p)); + mi_free(p); + } + } } } @@ -192,15 +204,6 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con return mi_error_bad_alignment(size, alignment, offset); } - #if MI_GUARDED - #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL) - #endif - if (offset==0 && alignment < MI_PAGE_MAX_OVERALLOC_ALIGN && mi_theap_malloc_use_guarded(theap,size)) { - return mi_theap_malloc_guarded_aligned(theap, size, alignment, zero, ppage); - } - #endif - // try first if there happens to be a small block available with just the right alignment // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already // naturally aligned this can be often the case. @@ -208,20 +211,29 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con if mi_likely(theap!=NULL) #endif { - if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { - const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` - const size_t padsize = size + MI_PADDING_SIZE; - mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize); - if mi_likely(page->free != NULL) { - const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; - if mi_likely(is_aligned) - { - if (ppage!=NULL) { *ppage = page; } - void* p = _mi_page_malloc_zero(theap, page, padsize, zero); - mi_assert_internal(p != NULL); - mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); - mi_track_malloc(p, size, zero); - return p; + #if MI_SAMPLE // check if we shouldn't take a sample + const mi_ssize_t samples = mi_samples_from_size(size); + if (theap->sample_countdown >= samples) + #endif + { + if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { + const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` + const size_t padsize = size + MI_PADDING_SIZE; + mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize); + if mi_likely(page->free != NULL) { + const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; + if mi_likely(is_aligned) + { + #if MI_SAMPLE==2 // fine grained needs to update the sample countdown + theap->sample_countdown -= samples; + #endif + if (ppage!=NULL) { *ppage = page; } + void* p = _mi_page_malloc_zero(theap, page, padsize, zero); + mi_assert_internal(p != NULL); + mi_assert_internal(((uintptr_t)p + offset) % alignment == 0); + mi_track_malloc(p, size, zero); + return p; + } } } } diff --git a/src/alloc.c b/src/alloc.c index dec4dfcd1..05264269e 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -138,6 +138,7 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz } // main allocation primitives for small and generic allocation +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; // internal small size allocation static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept @@ -150,14 +151,16 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non #if (MI_PADDING || MI_GUARDED) if mi_unlikely(size == 0) { size = sizeof(void*); } #endif - #if MI_GUARDED - if mi_unlikely(mi_theap_malloc_use_guarded(theap,size)) { - return _mi_theap_malloc_guarded(theap, size, zero, ppage); - } + + // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic) + #if MI_SAMPLE==2 + if mi_unlikely(mi_theap_sample(theap,size)) { return _mi_theap_malloc_sample(theap,size,zero,ppage); } #endif - - // get page in constant time, and allocate from it + + // get page in constant time mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE); + + // and allocate void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, ppage); mi_track_malloc(p,size,zero); @@ -172,14 +175,6 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non // internal generic allocation static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { - #if MI_GUARDED - #if MI_THEAP_INITASNULL - if (theap!=NULL) - #endif - if (huge_alignment==0 && mi_theap_malloc_use_guarded(theap, size)) { - return _mi_theap_malloc_guarded(theap, size, zero, ppage); - } - #endif #if !MI_THEAP_INITASNULL mi_assert(theap!=NULL); #endif @@ -870,11 +865,10 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { } } -#if MI_GUARDED + // We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). // We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) // and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` - static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { // todo: we can still make padding work by moving it out of the guard page area mi_page_t* const page = _mi_ptr_page(block); @@ -933,7 +927,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); const size_t threshold = mi_theap_disable_profiler(theap); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap, req_size, 0 /* don't zero */, ppage); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); mi_theap_enable_profiler(theap,threshold); if (block==NULL) return NULL; size_t usable_size = 0; @@ -959,7 +953,42 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, #endif return p; } -#endif + +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + mi_assert_internal(theap->sample_countdown < 0); + mi_assert_internal(!_mi_is_empty_theap(theap)); + + // update countdown + const mi_ssize_t samples = theap->sample_rate - theap->sample_countdown; + mi_assert_internal(samples > 0); + theap->sample_countdown = theap->sample_rate; // reset sampling + + // update derived countdowns + mi_assert_internal(theap->profile_sample_rate!=0 || theap->profile_sample_countdown==0); + mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0); + if (theap->profile_sample_rate!=0) { theap->profile_sample_countdown -= samples; } + if (theap->guarded_sample_rate!=0) { theap->guarded_sample_countdown -= samples; } + if (theap->profile_sample_countdown < 0) { + theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown + if (theap->guarded_sample_countdown < 0) { theap->guarded_sample_countdown = -1; }; // prevent underflow if profile sampling is usually taken + return _mi_theap_malloc_profiled(theap,size,zero,ppage); + } + else if (theap->guarded_sample_countdown < 0) { + if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { + // use guarded allocation + theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown + return _mi_theap_malloc_guarded(theap,size,zero,ppage); + } + else { + // failed size criteria, rewind the sample countdown so we sample on the next allocation + theap->sample_countdown = -1; + } + } + + // take generic path + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); +} + // ------------------------------------------------------ // ensure explicit external inline definitions are emitted! diff --git a/src/init.c b/src/init.c index a08925879..b1226a5d9 100644 --- a/src/init.c +++ b/src/init.c @@ -121,12 +121,15 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = { }; mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { + -1, // sample countdown: -1 (with a sample rate of 0, so we won't write to the empty theap) MI_SMALL_PAGES_EMPTY, // direct small pages &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount - 0, 0, // profiler allocated, threshold + 0, // sample rate + 0, 0, // profile rate, countdown + 0, 0, 0, 0, // guarded rate, countdown, min, max 0, // heartbeat { {0}, {0}, 0, true }, // random 0, // page count @@ -138,10 +141,7 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { 0, // full page retain false, // allow reclaim true, // allow abandon - true, // is_detached - #if MI_GUARDED - 0, 0, 0, 1, // rate is 0 and count is 1 so we never write to it (see `internal.h:mi_heap_malloc_use_guarded`) - #endif + true, // is_detached MI_PAGE_QUEUES_EMPTY, MI_MEMID_STATIC, MI_STATS_NULL, // stats diff --git a/src/page.c b/src/page.c index b287461b0..8c2c73fc3 100644 --- a/src/page.c +++ b/src/page.c @@ -153,14 +153,22 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1); mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count); + // adjust sample countdown const size_t bsize = mi_page_usable_block_size(page); - #if MI_PROFILE - if (alloc_count>0 && theap!=NULL && theap->profile_threshold!=0) { - // track profile allocated - theap->profile_allocated += alloc_count*bsize; + #if MI_SAMPLE==1 + if (alloc_count>0 && theap!=NULL && theap->sample_rate!=0) { + // adjust countdown + const mi_ssize_t samples = alloc_count*mi_sample_from_size(bsize); + if (theap->sample_countdown > samples) { + theap->sample_countdown -= samples; + } + else { + theap->sample_countdown = -1; + } } #endif + // adjust stats if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { const size_t bin = _mi_bin(bsize); // allocations @@ -308,7 +316,7 @@ static inline bool mi_page_free_quick_collect(mi_page_t* page) { page->local_free = NULL; page->free_is_zero = false; mi_assert_internal(page->theap != NULL); - if (page->theap->profile_threshold!=0 || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } + if (page->theap->sample_rate!=0 || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } return true; } @@ -339,7 +347,7 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { page->free_is_zero = false; } mi_theap_t* const theap = _mi_page_associated_theap_peek(page); - if ((theap != NULL && theap->profile_threshold!=0) || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } + if ((theap != NULL && theap->sample_rate!=0) || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } } mi_assert_internal(!force || page->local_free == NULL); } @@ -1139,14 +1147,17 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) theap->generic_collect_count += theap->generic_count; theap->generic_count = 0; + // check if the profiler is enabled mi_heap_t* const heap = _mi_theap_heap(theap); mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof)); - if (theap->profile_threshold==0 && prof_enabled) { - theap->profile_threshold = 1; // start profiling + if (prof_enabled && theap->sample_rate==0) { + theap->sample_rate = 1; // start profiling + theap->profile_sample_rate = 1; } - else if (theap->profile_threshold>0 && !prof_enabled) { - theap->profile_threshold = 0; // stop profiling + else if (!prof_enabled && theap->sample_rate!=0) { + theap->sample_rate = theap->guarded_sample_rate; // stop profiling + theap->profile_sample_rate = 0; } // do a full theap collect every once in a while (10000 by default) @@ -1159,7 +1170,9 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) // otherwise we do a mini-collect _mi_deferred_free(theap, false); // call potential deferred free routines _mi_theap_collect_retired(theap, false); // free retired pages - if (prof_enabled) { mi_theap_collect(theap, false /* force? */); } // update stats + if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling + mi_theap_collect(theap, false /* force? */); + } } } return theap; @@ -1175,11 +1188,16 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size theap = mi_malloc_generic_admin(theap); if (theap==NULL) return NULL; - if mi_unlikely(huge_alignment==0 && theap->profile_allocated > theap->profile_threshold) { - // profile the allocation - void* const p = _mi_theap_profile_alloc(theap,size,zero,ppage); - if (p!=NULL) { return p; } - // else fall through + // take a sample? + if mi_unlikely(huge_alignment==0 && theap->sample_rate!=0) { + #if MI_SAMPLE==2 // fine grained + if (mi_theap_sample(theap,size)) + #elif MI_SAMPLE==1 + if (theap->sample_countdown < 0) + #endif + { + return _mi_theap_malloc_sample(theap,size,zero,ppage); + } } // find (or allocate) a page of the right size @@ -1232,8 +1250,14 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm // fast path objects that fit in a small page if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { - // fast profile check - if (theap->profile_allocated <= theap->profile_threshold) { + #if MI_SAMPLE + const mi_ssize_t samples = mi_samples_from_size(size); + if (theap->sample_countdown >= samples) + #endif + { + #if MI_SAMPLE==2 // fine grained needs to update the countdown + theap->sample_countdown -= samples; + #endif const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` if (req_size < MI_SMALL_MAX_OBJ_SIZE) { mi_page_queue_t* pq = mi_page_queue(theap, size); @@ -1249,5 +1273,13 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm } } // otherwise fallback - return mi_malloc_generic_fallback(theap,size,zero, huge_alignment,ppage); + return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage); } + +void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + const mi_ssize_t sample_rate = theap->sample_rate; + theap->sample_rate = 0; // prevent a recursive call to _mi_theap_malloc_sample from _mi_malloc_generic + void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); + theap->sample_rate = sample_rate; + return p; +} \ No newline at end of file diff --git a/src/profile.c b/src/profile.c index 695adf2fd..bbfd02b97 100644 --- a/src/profile.c +++ b/src/profile.c @@ -26,25 +26,21 @@ static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { /* ---------------------------------------------------------------------------- Profile an allocation and free -----------------------------------------------------------------------------*/ -void* _mi_theap_profile_alloc(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - mi_assert_internal(theap!=NULL); - mi_assert_internal(theap->profile_allocated > theap->profile_threshold); + mi_assert_internal(theap!=NULL); mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); - if (prof == NULL) { return NULL; } - const size_t allocated = theap->profile_allocated; - theap->profile_allocated = 0; + if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } // Overallocate a larger block to store the profiler data // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] - const size_t zero_huge_alignment = (zero ? 1 : 0); const size_t profiler_data_offset = sizeof(mi_block_t); size_t profiler_data_size = sizeof(mi_profiler_data_t); if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > 512 ? 512 : prof->profiler_data_size); }; const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); const size_t oversize = profiler_user_offset + size; mi_page_t* page = NULL; - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic(theap,oversize,zero_huge_alignment,&page); // cannot recurse as the profile_allocated is now zero + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); if (block==NULL) return NULL; mi_assert_internal(page!=NULL); if (ppage!=NULL) { *ppage = page; } @@ -61,9 +57,9 @@ void* _mi_theap_profile_alloc(mi_theap_t* theap, size_t size, bool zero, mi_page // and call the profiler on_alloc if (prof->on_alloc!=NULL) { - const size_t new_threshold = (*prof->on_alloc)(profiler_data, p, theap->profile_threshold, allocated, _mi_theap_heap(theap), prof->profiler_arg); - if (new_threshold!=0) { - theap->profile_threshold = new_threshold; + const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, (size_t)theap->profile_sample_rate, size /* TODO: bytes since last sample */, _mi_theap_heap(theap), prof->profiler_arg); + if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { + mi_theap_enable_profiler(theap,new_sample_rate); } mi_theap_stat_counter_increase(theap,profile_samples,1); } @@ -118,8 +114,8 @@ bool mi_profiler_start(const mi_profiler_t* profiler ) { mi_heap_t* heap = mi_heap_main(); if (mi_heap_profiler(heap)==profiler) { mi_theap_t* theap = _mi_heap_theap_peek(heap); - if (theap!=NULL && theap->profile_threshold == 0) { - theap->profile_threshold = 1; // get started more quickly + if (theap!=NULL) { + mi_theap_enable_profiler(theap,1); } } return false; diff --git a/src/theap.c b/src/theap.c index 470243de7..c1cc3b09e 100644 --- a/src/theap.c +++ b/src/theap.c @@ -192,13 +192,19 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { #if MI_GUARDED mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - theap->guarded_sample_rate = sample_rate; - theap->guarded_sample_count = sample_rate; // count down samples + theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : (mi_ssize_t)sample_rate); + theap->guarded_sample_countdown = theap->guarded_sample_rate; + if (theap->sample_rate==0 || theap->sample_rate > theap->guarded_sample_rate) { + theap->sample_rate = theap->guarded_sample_rate; + } if (theap->guarded_sample_rate > 1) { if (seed == 0) { seed = _mi_theap_random_next(theap); } - theap->guarded_sample_count = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` + theap->guarded_sample_countdown = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` + } + if (theap->sample_countdown > theap->guarded_sample_countdown) { + theap->sample_countdown = theap->guarded_sample_countdown; } } From 64955e241f99527c3b4db837ff1a7fc713e49f34 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 4 Sep 2026 11:56:35 -0700 Subject: [PATCH 188/280] wip: improve profile and guarded as sampled allocations --- include/mimalloc/internal.h | 38 ++++++++-------------- include/mimalloc/types.h | 13 ++++---- src/alloc-aligned.c | 28 ++++++++-------- src/alloc.c | 65 +++++++++++++++++++++++++------------ src/init.c | 4 +-- src/page.c | 51 +++++++++++++++++------------ src/profile.c | 5 +-- test/test-profile.c | 5 +-- 8 files changed, 117 insertions(+), 92 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index b9a18918f..f14d3adf2 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -324,8 +324,6 @@ void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); -mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; // "free.c" void _mi_free_subproc_safe(void* p) mi_attr_noexcept; @@ -337,7 +335,9 @@ bool _mi_page_is_valid(mi_page_t* page); #endif // "profile.c" -mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, size_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p); // ------------------------------------------------------ @@ -1156,32 +1156,19 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) { Guarded and profiled objects ------------------------------------------------------------------- */ -#define MI_SAMPLE_RATE_MAX (2*MI_GiB) - -static mi_ssize_t mi_samples_from_size(size_t size) { - const mi_ssize_t samples = (size > 1024 ? 1024 : (mi_ssize_t)size); - mi_assert_internal(samples <= INT16_MAX); - return samples; -} +#define MI_SAMPLE_RATE_MAX (SIZE_MAX/4) +#define MI_SAMPLE_COUNTDOWN_MAX (SIZE_MAX/4) #if MI_SAMPLE==2 // fine grained -static inline bool mi_theap_sample(mi_theap_t* theap, size_t size) { - // a bit convoluted but we need to avoid writing to the empty theap - // theap->sample_countdown -= mi_sample_from_size(size); - // return (theap->sample_countdown < 0); - const mi_ssize_t sample_countdown = theap->sample_countdown - mi_samples_from_size(size); - if mi_likely(sample_countdown >= 0) { +static inline bool mi_theap_sample_small(mi_theap_t* theap, size_t req_size) { + mi_assert_internal(req_size <= SIZE_MAX/2); + const size_t sample_countdown = theap->sample_countdown - req_size; + if mi_likely((mi_ssize_t)sample_countdown >= 0) { theap->sample_countdown = sample_countdown; return false; } else { - if mi_unlikely(theap->sample_rate == 0) { - return false; - } - else { - theap->sample_countdown = sample_countdown; - return true; - } + return true; } } #endif @@ -1237,16 +1224,17 @@ static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { static inline size_t mi_theap_disable_profiler(mi_theap_t* theap) { const mi_ssize_t sample_rate = theap->profile_sample_rate; theap->profile_sample_rate = 0; - theap->profile_sample_countdown = 0; + // theap->profile_sample_countdown = 0; theap->sample_rate = theap->guarded_sample_rate; return (sample_rate < 0 ? 0 : (size_t)sample_rate); } static inline void mi_theap_enable_profiler(mi_theap_t* theap, size_t sample_rate) { theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : (mi_ssize_t)sample_rate); - theap->profile_sample_countdown = theap->profile_sample_rate; + if (theap->profile_sample_countdown==0) { theap->profile_sample_countdown = theap->profile_sample_rate; } if (theap->sample_rate==0 || theap->sample_rate > theap->profile_sample_rate) { theap->sample_rate = theap->profile_sample_rate; + theap->sample_countdown = theap->profile_sample_countdown; // TODO: track any previous allocation countdown into sample_requested } } diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 36cf90ff7..3ca6af61e 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -596,7 +596,7 @@ typedef struct mi_padding_s { // A thread-local heap ("theap") owns a set of thread-local pages. struct mi_theap_s { // put in front for fast small allocations - mi_ssize_t sample_countdown; + size_t sample_countdown; mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. mi_tld_t* tld; // thread-local data @@ -604,11 +604,12 @@ struct mi_theap_s { _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) _Atomic(size_t) refcount; // reference count - mi_ssize_t sample_rate; - mi_ssize_t profile_sample_rate; - mi_ssize_t profile_sample_countdown; - mi_ssize_t guarded_sample_rate; - mi_ssize_t guarded_sample_countdown; + size_t sample_rate; + size_t sample_requested; + size_t profile_sample_rate; + size_t profile_sample_countdown; + size_t guarded_sample_rate; + size_t guarded_sample_countdown; size_t guarded_size_min; // minimal size for guarded objects size_t guarded_size_max; // maximal size for guarded objects diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 9339602b0..10636262e 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -163,12 +163,12 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t if mi_likely(theap!=NULL) #endif { - #if MI_SAMPLE // only try if we would not take a sample - const mi_ssize_t samples = mi_samples_from_size(size); - if (theap->sample_countdown >= samples) - #endif - { - if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { + if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) + { + #if MI_SAMPLE // only try if we would not take a sample + if (theap->sample_countdown >= size) + #endif + { mi_page_t* page = NULL; void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, &page); if (ppage!=NULL) { *ppage = page; } @@ -207,16 +207,16 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con // try first if there happens to be a small block available with just the right alignment // since most small power-of-2 blocks (under MI_PAGE_MAX_BLOCK_START_ALIGN2) are already // naturally aligned this can be often the case. - #if MI_THEAP_INITASNULL - if mi_likely(theap!=NULL) - #endif + if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { - #if MI_SAMPLE // check if we shouldn't take a sample - const mi_ssize_t samples = mi_samples_from_size(size); - if (theap->sample_countdown >= samples) + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL) #endif { - if mi_likely(size <= MI_SMALL_SIZE_MAX && alignment <= size) { + #if MI_SAMPLE // check if we shouldn't take a sample + if (theap->sample_countdown >= size) + #endif + { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` const size_t padsize = size + MI_PADDING_SIZE; mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize); @@ -225,7 +225,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con if mi_likely(is_aligned) { #if MI_SAMPLE==2 // fine grained needs to update the sample countdown - theap->sample_countdown -= samples; + theap->sample_countdown -= size; #endif if (ppage!=NULL) { *ppage = page; } void* p = _mi_page_malloc_zero(theap, page, padsize, zero); diff --git a/src/alloc.c b/src/alloc.c index 05264269e..c46645178 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -148,13 +148,13 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non #if MI_DEBUG mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local #endif - #if (MI_PADDING || MI_GUARDED) + #if (MI_PADDING) // || MI_GUARDED) if mi_unlikely(size == 0) { size = sizeof(void*); } #endif // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic) #if MI_SAMPLE==2 - if mi_unlikely(mi_theap_sample(theap,size)) { return _mi_theap_malloc_sample(theap,size,zero,ppage); } + if mi_unlikely(mi_theap_sample_small(theap,size)) { return _mi_theap_malloc_sample(theap,size,zero,ppage); } #endif // get page in constant time @@ -926,9 +926,9 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); - const size_t threshold = mi_theap_disable_profiler(theap); + // const size_t threshold = mi_theap_disable_profiler(theap); mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); - mi_theap_enable_profiler(theap,threshold); + // mi_theap_enable_profiler(theap,threshold); if (block==NULL) return NULL; size_t usable_size = 0; void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); @@ -954,37 +954,62 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, return p; } -mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - mi_assert_internal(theap->sample_countdown < 0); - mi_assert_internal(!_mi_is_empty_theap(theap)); +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE) + mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE); + mi_assert_internal(theap->sample_countdown < req_size); + const size_t size = req_size + MI_PADDING_SIZE; + + // handle empty theap and disabled profiling + if (theap->sample_rate==0) { + #if MI_SAMPLE==2 // fine-grained; reset the countdown to avoid the slow sample path + if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; + } + #endif + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); + } // update countdown - const mi_ssize_t samples = theap->sample_rate - theap->sample_countdown; - mi_assert_internal(samples > 0); + mi_assert_internal(!_mi_is_empty_theap(theap)); + mi_assert_internal(req_size <= SIZE_MAX/2); + mi_assert_internal(theap->sample_rate > 0); + mi_assert_internal(theap->sample_rate <= SIZE_MAX/2); + mi_assert_internal(theap->sample_rate >= theap->sample_countdown); + const size_t requested = theap->sample_rate + (req_size - theap->sample_countdown) + theap->sample_requested; + mi_assert_internal(requested > 0); + mi_assert_internal(requested >= size); + if (requested < size) { + _mi_error_message(EFAULT,"wrong requested/size"); + } + theap->sample_requested = 0; // reset extra count theap->sample_countdown = theap->sample_rate; // reset sampling // update derived countdowns mi_assert_internal(theap->profile_sample_rate!=0 || theap->profile_sample_countdown==0); mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0); - if (theap->profile_sample_rate!=0) { theap->profile_sample_countdown -= samples; } - if (theap->guarded_sample_rate!=0) { theap->guarded_sample_countdown -= samples; } - if (theap->profile_sample_countdown < 0) { + bool sample_profile = false; + bool sample_guarded = false; + if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= requested; } else { sample_profile = true; } + if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= requested; } else { sample_guarded = true; } + + // invoke callback? + if (sample_profile) { theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown - if (theap->guarded_sample_countdown < 0) { theap->guarded_sample_countdown = -1; }; // prevent underflow if profile sampling is usually taken - return _mi_theap_malloc_profiled(theap,size,zero,ppage); + if (sample_guarded) { theap->guarded_sample_countdown = 0; }; // ensure it will get sampled at some point + return _mi_theap_malloc_profiled(theap,size,requested,zero,ppage); } - else if (theap->guarded_sample_countdown < 0) { - if (size >= theap->guarded_size_min && size <= theap->guarded_size_max) { + else if (sample_guarded) { + if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) { // use guarded allocation theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown return _mi_theap_malloc_guarded(theap,size,zero,ppage); } else { - // failed size criteria, rewind the sample countdown so we sample on the next allocation - theap->sample_countdown = -1; - } + // failed size criteria, rewind the sample countdown so we sample asap again + theap->sample_countdown = 0; + } } - // take generic path return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } diff --git a/src/init.c b/src/init.c index b1226a5d9..2ef71b214 100644 --- a/src/init.c +++ b/src/init.c @@ -121,13 +121,13 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = { }; mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { - -1, // sample countdown: -1 (with a sample rate of 0, so we won't write to the empty theap) + ~0, // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) MI_SMALL_PAGES_EMPTY, // direct small pages &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount - 0, // sample rate + 0, 0, // sample rate, requested 0, 0, // profile rate, countdown 0, 0, 0, 0, // guarded rate, countdown, min, max 0, // heartbeat diff --git a/src/page.c b/src/page.c index 8c2c73fc3..5ef7e4015 100644 --- a/src/page.c +++ b/src/page.c @@ -158,12 +158,16 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ #if MI_SAMPLE==1 if (alloc_count>0 && theap!=NULL && theap->sample_rate!=0) { // adjust countdown - const mi_ssize_t samples = alloc_count*mi_sample_from_size(bsize); - if (theap->sample_countdown > samples) { - theap->sample_countdown -= samples; + size_t requested = 0; + if (mi_mul_overflow(alloc_count,bsize,&requested)) { + mi_assert(false); + } + if (theap->sample_countdown >= requested) { + theap->sample_countdown -= requested; } else { - theap->sample_countdown = -1; + theap->sample_countdown = 0; + theap->sample_requested += (requested - theap->sample_countdown); // TODO: check that overflow never happens } } #endif @@ -1070,7 +1074,7 @@ static mi_page_t* mi_huge_page_alloc(mi_theap_t* theap, size_t size, size_t page // Note: in debug mode the size includes MI_PADDING_SIZE and might have overflowed. static mi_page_t* mi_find_page(mi_theap_t* theap, size_t size, size_t huge_alignment) mi_attr_noexcept { const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE) { // TODO: remove as we check in generic_fallback ? _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); return NULL; } @@ -1188,16 +1192,21 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size theap = mi_malloc_generic_admin(theap); if (theap==NULL) return NULL; - // take a sample? + // check allocation size + const size_t req_size = size - MI_PADDING_SIZE; + if mi_unlikely(req_size > MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { + _mi_error_message(EOVERFLOW, "allocation request is too large (%zu bytes)\n", req_size); + return NULL; + } + + // take a sample? if mi_unlikely(huge_alignment==0 && theap->sample_rate!=0) { - #if MI_SAMPLE==2 // fine grained - if (mi_theap_sample(theap,size)) - #elif MI_SAMPLE==1 - if (theap->sample_countdown < 0) - #endif - { - return _mi_theap_malloc_sample(theap,size,zero,ppage); + if mi_unlikely(theap->sample_countdown < req_size) { + return _mi_theap_malloc_sample(theap,req_size,zero,ppage); } + #if MI_SAMPLE==2 + else { theap->sample_countdown -= req_size; } // fine grained updates directly + #endif } // find (or allocate) a page of the right size @@ -1250,21 +1259,21 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm // fast path objects that fit in a small page if mi_likely(mi_theap_is_initialized(theap) && ++theap->generic_count < 1000 && huge_alignment==0) { - #if MI_SAMPLE - const mi_ssize_t samples = mi_samples_from_size(size); - if (theap->sample_countdown >= samples) - #endif + const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (req_size < MI_SMALL_MAX_OBJ_SIZE) { - #if MI_SAMPLE==2 // fine grained needs to update the countdown - theap->sample_countdown -= samples; + #if MI_SAMPLE + if (theap->sample_countdown >= req_size) // ensure we don't need to take a sample #endif - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` - if (req_size < MI_SMALL_MAX_OBJ_SIZE) { + { mi_page_queue_t* pq = mi_page_queue(theap, size); mi_assert_internal(pq!=NULL && !mi_page_queue_is_huge(pq)); page = mi_page_queue_find_free(theap,pq); // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); if (page!=NULL) { + #if MI_SAMPLE==2 // fine grained sampling needs to update the countdown + theap->sample_countdown -= req_size; + #endif if (ppage!=NULL) { *ppage = page; } mi_assert_internal(mi_page_immediate_available(page)); return _mi_page_malloc_zero(theap,page,size,zero); diff --git a/src/profile.c b/src/profile.c index bbfd02b97..1e76b453c 100644 --- a/src/profile.c +++ b/src/profile.c @@ -26,9 +26,10 @@ static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { /* ---------------------------------------------------------------------------- Profile an allocation and free -----------------------------------------------------------------------------*/ -mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, size_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(theap!=NULL); + mi_assert_internal(size<=requested_since_last_sample); mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } @@ -57,7 +58,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th // and call the profiler on_alloc if (prof->on_alloc!=NULL) { - const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, (size_t)theap->profile_sample_rate, size /* TODO: bytes since last sample */, _mi_theap_heap(theap), prof->profiler_arg); + const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, (size_t)theap->profile_sample_rate, requested_since_last_sample /* TODO: bytes since last sample */, _mi_theap_heap(theap), prof->profiler_arg); if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { mi_theap_enable_profiler(theap,new_sample_rate); } diff --git a/test/test-profile.c b/test/test-profile.c index fce76c954..632eca7e6 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -39,10 +39,11 @@ static profile_state_t g_state; static size_t mi_cdecl on_alloc(mi_profiler_data_t* data, void* ptr, size_t threshold, size_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg) { MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(profiler_arg); assert(profiler_arg==&g_state); + assert(bytes_since_last_sample >= data->requested_size); g_state.alloc_count++; g_state.last_ptr = ptr; g_state.last_size = data->requested_size; - g_state.last_upscaled = bytes_since_last_sample; + g_state.last_upscaled = bytes_since_last_sample; // store ptr to verify round-trip data->user_data[0] = ptr; return TEST_THRESHOLD; @@ -159,10 +160,10 @@ int main(void) { mi_profile(&my_profiler); mi_profiler_start(&my_profiler); + test_profiler_upscaled_at_least_size(); test_profiler_samples(); test_profiler_record_fields(); test_profiler_on_free_called(); - test_profiler_upscaled_at_least_size(); test_profiler_free_count_le_alloc_count(); mi_profiler_stop(&my_profiler); From e80f3f9cd3f72947377edfe006b51fcc8b03b78a Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 4 Sep 2026 12:40:57 -0700 Subject: [PATCH 189/280] fix warnings and should sample check --- include/mimalloc/internal.h | 12 +++++++++--- include/mimalloc/types.h | 2 +- src/alloc-aligned.c | 4 ++-- src/init.c | 2 +- src/page.c | 6 +++--- src/theap.c | 2 +- 6 files changed, 17 insertions(+), 11 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index f14d3adf2..ef8243858 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1159,6 +1159,12 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) { #define MI_SAMPLE_RATE_MAX (SIZE_MAX/4) #define MI_SAMPLE_COUNTDOWN_MAX (SIZE_MAX/4) +static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) { + mi_assert_internal(req_size <= SIZE_MAX/2); + const size_t sample_countdown = theap->sample_countdown - req_size; + return ((mi_ssize_t)sample_countdown < 0); +} + #if MI_SAMPLE==2 // fine grained static inline bool mi_theap_sample_small(mi_theap_t* theap, size_t req_size) { mi_assert_internal(req_size <= SIZE_MAX/2); @@ -1222,15 +1228,15 @@ static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { static inline size_t mi_theap_disable_profiler(mi_theap_t* theap) { - const mi_ssize_t sample_rate = theap->profile_sample_rate; + const size_t sample_rate = theap->profile_sample_rate; theap->profile_sample_rate = 0; // theap->profile_sample_countdown = 0; theap->sample_rate = theap->guarded_sample_rate; - return (sample_rate < 0 ? 0 : (size_t)sample_rate); + return sample_rate; } static inline void mi_theap_enable_profiler(mi_theap_t* theap, size_t sample_rate) { - theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : (mi_ssize_t)sample_rate); + theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); if (theap->profile_sample_countdown==0) { theap->profile_sample_countdown = theap->profile_sample_rate; } if (theap->sample_rate==0 || theap->sample_rate > theap->profile_sample_rate) { theap->sample_rate = theap->profile_sample_rate; diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 3ca6af61e..b86776bac 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -91,7 +91,7 @@ terms of the MIT license. A copy of the license can be found in the file // Enable profiling support #ifndef MI_PROFILE -#define MI_PROFILE 2 +#define MI_PROFILE 1 #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 10636262e..b1e08e6af 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -166,7 +166,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { #if MI_SAMPLE // only try if we would not take a sample - if (theap->sample_countdown >= size) + if mi_likely(!mi_theap_should_sample(theap,size)) #endif { mi_page_t* page = NULL; @@ -214,7 +214,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con #endif { #if MI_SAMPLE // check if we shouldn't take a sample - if (theap->sample_countdown >= size) + if mi_likely(!mi_theap_should_sample(theap,size)) #endif { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` diff --git a/src/init.c b/src/init.c index 2ef71b214..c89b4df31 100644 --- a/src/init.c +++ b/src/init.c @@ -121,7 +121,7 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = { }; mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { - ~0, // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) + ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) MI_SMALL_PAGES_EMPTY, // direct small pages &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap diff --git a/src/page.c b/src/page.c index 5ef7e4015..811b3129a 100644 --- a/src/page.c +++ b/src/page.c @@ -1201,7 +1201,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size // take a sample? if mi_unlikely(huge_alignment==0 && theap->sample_rate!=0) { - if mi_unlikely(theap->sample_countdown < req_size) { + if mi_unlikely(mi_theap_should_sample(theap,req_size)) { return _mi_theap_malloc_sample(theap,req_size,zero,ppage); } #if MI_SAMPLE==2 @@ -1263,7 +1263,7 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm if (req_size < MI_SMALL_MAX_OBJ_SIZE) { #if MI_SAMPLE - if (theap->sample_countdown >= req_size) // ensure we don't need to take a sample + if mi_likely(!mi_theap_should_sample(theap,req_size)) // ensure we don't need to take a sample #endif { mi_page_queue_t* pq = mi_page_queue(theap, size); @@ -1286,7 +1286,7 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm } void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - const mi_ssize_t sample_rate = theap->sample_rate; + const size_t sample_rate = theap->sample_rate; theap->sample_rate = 0; // prevent a recursive call to _mi_theap_malloc_sample from _mi_malloc_generic void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); theap->sample_rate = sample_rate; diff --git a/src/theap.c b/src/theap.c index c1cc3b09e..f8d5b2b5e 100644 --- a/src/theap.c +++ b/src/theap.c @@ -192,7 +192,7 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { #if MI_GUARDED mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : (mi_ssize_t)sample_rate); + theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); theap->guarded_sample_countdown = theap->guarded_sample_rate; if (theap->sample_rate==0 || theap->sample_rate > theap->guarded_sample_rate) { theap->sample_rate = theap->guarded_sample_rate; From 542a96660b697f5c48afea3a96a7abff726f3c0c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Ang=C3=A9lica=20Moreira?= Date: Thu, 3 Sep 2026 16:50:55 -0700 Subject: [PATCH 190/280] avoid re-visiting full pages in mi_page_queue_find_free_ex Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com> --- src/free.c | 2 +- src/page-queue.c | 9 +++++++++ src/page.c | 11 ++++++++--- 3 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/free.c b/src/free.c index 5d033aab4..8260b31c4 100644 --- a/src/free.c +++ b/src/free.c @@ -489,7 +489,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* mi_assert_internal(mi_page_is_abandoned(page)); mi_assert_internal(mt_free != NULL); // mi_assert_internal(_mi_subproc() == mi_page_subproc(page)); // never collect across subprocesses - + // we own the page now, and it is safe to collect the thread atomic free list if (page->block_size <= MI_SMALL_SIZE_MAX) { // use the `_partly` version to avoid atomic operations since we already have the `mt_free` pointing into the thread free list diff --git a/src/page-queue.c b/src/page-queue.c index c10c09257..ac9a2e20a 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -341,6 +341,15 @@ static void mi_page_queue_move_to_front(mi_theap_t* theap, mi_page_queue_t* queu mi_assert_internal(queue->first == page); } +static void mi_page_queue_move_to_back(mi_theap_t* theap, mi_page_queue_t* queue, mi_page_t* page) { + mi_assert_internal(mi_page_theap(page) == theap); + mi_assert_internal(mi_page_queue_contains(queue, page)); + if (queue->last == page) return; + mi_page_queue_remove(queue, page); + mi_page_queue_push_at_end(theap, queue, page); + mi_assert_internal(queue->last == page); +} + static void mi_page_queue_enqueue_from_ex(mi_page_queue_t* to, mi_page_queue_t* from, bool enqueue_at_end, mi_page_t* page) { mi_assert_internal(page != NULL); mi_assert_internal(from->count >= 1); diff --git a/src/page.c b/src/page.c index 02ca72044..b9027a61a 100644 --- a/src/page.c +++ b/src/page.c @@ -771,8 +771,9 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, long page_full_retain = (pq->block_size > MI_SMALL_MAX_OBJ_SIZE ? 0 : theap->page_full_retain); // only retain small pages mi_page_t* page_candidate = NULL; // a page with free space mi_page_t* page = pq->first; + mi_page_t* const last = pq->last; - while (page != NULL) + while (page!=NULL) { mi_page_t* next = page->next; // remember next (as this page can move to another queue) count++; @@ -796,6 +797,10 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); } + else if (page!=last && pq->last!=page) { + // avoid revisiting this page for a while + mi_page_queue_move_to_back(theap, pq, page); + } } else { // the page has free space, make it a candidate @@ -831,8 +836,8 @@ static mi_decl_noinline mi_page_t* mi_page_queue_find_free_ex(mi_theap_t* theap, mi_assert_internal(!mi_page_is_in_full(page) && !mi_page_immediate_available(page)); mi_page_to_full(page, pq); #endif - - page = next; + if (page==last) { page=NULL; } // don't revisit earlier pages that moved to the back + else { page = next; } } // for each page mi_theap_stat_counter_increase(theap, page_searches, count); From 866c8116925565c1fe3f16a669fc32612c18796c Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 3 Sep 2026 18:47:30 -0700 Subject: [PATCH 191/280] return new expected value from collect_partly --- include/mimalloc/internal.h | 2 +- src/free.c | 2 +- src/page.c | 9 +++++++-- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index fdd5c78be..f6bc9092c 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -282,7 +282,7 @@ void _mi_page_free(mi_page_t* page, mi_page_queue_t* pq); // free t void _mi_page_abandon(mi_page_t* page, mi_page_queue_t* pq); // abandon the page, to be picked up by another thread... void _mi_deferred_free(mi_theap_t* theap, bool force); void _mi_page_free_collect(mi_page_t* page, bool force); -void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head); mi_decl_nodiscard bool _mi_page_init(mi_theap_t* theap, mi_page_t* page); bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq); diff --git a/src/free.c b/src/free.c index 8260b31c4..a84b1cf04 100644 --- a/src/free.c +++ b/src/free.c @@ -496,7 +496,7 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // (after this the `used` count might be too high (as some blocks may have been concurrently added to the thread free list and are yet uncounted). // however, if the page became completely free, the used count is guaranteed to be 0.) mi_assert_internal(page->reserved>=16); // below this even one freed block goes from full to no longer mostly used. - _mi_page_free_collect_partly(page, mt_free); + mt_free = _mi_page_free_collect_partly(page, mt_free); } else { // for larger blocks we use the regular collect diff --git a/src/page.c b/src/page.c index b9027a61a..9374f57f9 100644 --- a/src/page.c +++ b/src/page.c @@ -244,12 +244,13 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { // Collect elements in the thread-free list starting at `head`. This is an optimized // version of `_mi_page_free_collect` to be used from `free.c:_mi_free_collect_mt` that avoids atomic access to `xthread_free`. +// returns a possibly updated expected value for the thread_free pointer. // // `head` must be in the `xthread_free` list. It will not collect `head` itself // so the `used` count is not fully updated in general. However, if the `head` is // the last remaining element, it will be collected and the used count will become `0` (so `mi_page_all_free` becomes true). -void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { - if (head == NULL) return; +mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { + if (head == NULL) return NULL; mi_block_t* next = mi_block_next(page,head); // we cannot collect the head element itself as `page->thread_free` may point to it (and we want to avoid atomic ops) if (next != NULL) { mi_block_set_next(page, head, NULL); @@ -265,6 +266,10 @@ void _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { mi_assert_internal(mi_tf_block(mi_atomic_load_relaxed(&page->xthread_free)) == head); mi_assert_internal(mi_block_next(page,head) == NULL); _mi_page_free_collect(page, false); // collect the final element + return NULL; + } + else { + return head; } } From f3dbaf11287700d6b96702ec3d124a71e4f5a005 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 07:47:50 -0700 Subject: [PATCH 192/280] add constructor priority for gcc on non-macOS (see also #1391) --- src/prim/prim.c | 5 ++--- src/subproc.c | 2 +- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/src/prim/prim.c b/src/prim/prim.c index 7f31c994d..3d3d0c2e8 100644 --- a/src/prim/prim.c +++ b/src/prim/prim.c @@ -29,9 +29,8 @@ terms of the MIT license. A copy of the license can be found in the file // Generic process initialization #if !defined(MI_PRIM_HAS_PROCESS_ATTACH) #if defined(__GNUC__) || defined(__clang__) - // gcc,clang: use the constructor/destructor attribute - // which for both seem to run before regular constructors/destructors - #if defined(__clang__) + // gcc,clang: use the constructor/destructor attribute with the highest user priority (<=100 is reserved for the compiler itself) + #if defined(__clang__) || (__GNUC__ >= 5 && !defined(__APPLE__)) // gcc has support since 4.2 but not on macOS (https://gcc.gnu.org/bugzilla/show_bug.cgi?id=119435) #define mi_attr_constructor __attribute__((constructor(101))) #define mi_attr_destructor __attribute__((destructor(101))) #else diff --git a/src/subproc.c b/src/subproc.c index c8c3202fe..c248fca22 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -102,7 +102,7 @@ bool _mi_subproc_is_main(mi_subproc_t* subproc) { mi_subproc_t* _mi_subproc(void) { mi_theap_t* theap = _mi_theap_default(); - if (theap == NULL || theap->tld == NULL || theap->tld->subproc == NULL) { // see issue #1289 + if (theap == NULL || theap->tld == NULL || theap->tld->subproc == NULL) { // see issue #1289 and #1391 return _mi_subproc_main(); } else { From ed0d501795c09732c78486821a9659d03b41201a Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 09:06:40 -0700 Subject: [PATCH 193/280] slightly better codegen for profile==2 --- include/mimalloc/internal.h | 8 ++++++++ include/mimalloc/types.h | 6 +++--- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 41b9499dd..83eb83ee0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -654,9 +654,17 @@ static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) { static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) { mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + #if MI_INTPTR_SIZE==MI_SIZE_SIZE + // the sample_countdown field comes before the pages_direct; the following generates better code in general + // (where we treat the countdown as a first -1 entry of the pages_free_direct array) + const size_t idx1 = (size + 2*MI_INTPTR_SIZE - 1)/MI_INTPTR_SIZE; + mi_assert_internal(idx1 <= MI_PAGES_DIRECT); + return ((mi_page_t**)theap)[idx1]; + #else const size_t idx = _mi_wsize_from_size(size); mi_assert_internal(idx < MI_PAGES_DIRECT); return theap->pages_free_direct[idx]; + #endif } static inline bool mi_theap_is_detached(mi_theap_t* theap) { diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index b86776bac..21f4b9b0e 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -91,7 +91,7 @@ terms of the MIT license. A copy of the license can be found in the file // Enable profiling support #ifndef MI_PROFILE -#define MI_PROFILE 1 +#define MI_PROFILE 2 #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) @@ -596,9 +596,10 @@ typedef struct mi_padding_s { // A thread-local heap ("theap") owns a set of thread-local pages. struct mi_theap_s { // put in front for fast small allocations - size_t sample_countdown; + size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:_mi_theap_get_free_small_page`) mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. + // less frequently accessed fields mi_tld_t* tld; // thread-local data _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) @@ -613,7 +614,6 @@ struct mi_theap_s { size_t guarded_size_min; // minimal size for guarded objects size_t guarded_size_max; // maximal size for guarded objects - unsigned long long heartbeat; // monotonic heartbeat count mi_random_ctx_t random; // random number context used for secure allocation size_t page_count; // total number of pages in the `pages` queues. From e349fc314ac926cea951761f41cd6dddee87ad31 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 09:13:41 -0700 Subject: [PATCH 194/280] comments --- src/alloc.c | 8 ++++---- src/free.c | 2 +- src/page.c | 6 +++--- src/stats.c | 2 +- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index b45bc2624..07155bffe 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -95,7 +95,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ theap->sample_countdown -= req_size; #endif - #if MI_STAT==2 + #if MI_STAT>=2 mi_theap_stat_increase(theap,malloc_requested,size - MI_PADDING_SIZE); #endif @@ -947,7 +947,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, mi_track_malloc(p, usable_size, zero); if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); - #if MI_STAT>1 + #if MI_STAT // adjust request stats to only count the allocated size of the block (and not the guard page) mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size); mi_theap_stat_increase(theap, malloc_requested, size); @@ -968,8 +968,8 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* thea // handle empty theap and disabled profiling if (theap->sample_rate==0) { - if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap - theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time + if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time } return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } diff --git a/src/free.c b/src/free.c index 2be4968d8..b9d9edc48 100644 --- a/src/free.c +++ b/src/free.c @@ -784,7 +784,7 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { } const size_t bsize = mi_page_usable_block_size(page); - // #if (MI_STAT>1) + // #if (MI_STAT>=2) // const size_t usize = mi_page_usable_size_of(page, block); // mi_theap_stat_decrease(theap, malloc_requested, usize); // #endif diff --git a/src/page.c b/src/page.c index adf3b59e9..c54ddac46 100644 --- a/src/page.c +++ b/src/page.c @@ -169,7 +169,7 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ theap->sample_countdown = 0; theap->sample_requested += (requested - theap->sample_countdown); // TODO: check that overflow never happens } - } + } #endif // adjust stats @@ -1201,13 +1201,13 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size // take a sample? if mi_unlikely(huge_alignment==0 && mi_theap_should_sample(theap,req_size)) { - if (theap->sample_rate!=0) { + if (theap->sample_rate!=0) { // must check to avoid recursion with `_mi_malloc_generic_nosample` return _mi_theap_malloc_sample(theap,req_size,zero,ppage); } else { mi_assert_internal(!_mi_is_empty_theap(theap)); mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX); - theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter to avoid sampling for a while } } diff --git a/src/stats.c b/src/stats.c index 868fef8d0..9fca930a4 100644 --- a/src/stats.c +++ b/src/stats.c @@ -366,7 +366,7 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stat_count_add_mt(&total, &stats->malloc_normal); mi_stat_count_add_mt(&total, &stats->malloc_huge); mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, ""); - #if MI_STAT>1 + #if MI_STAT>=2 mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg); #else mi_stat_total_print(&stats->malloc_requested, "malloc req~", 1, out, arg); From 525f2c91069cb310c79eb58ecb84a153ff954f85 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 09:54:30 -0700 Subject: [PATCH 195/280] fix countdown counter assertion --- src/page.c | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/src/page.c b/src/page.c index c54ddac46..098d672fe 100644 --- a/src/page.c +++ b/src/page.c @@ -1200,15 +1200,21 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size } // take a sample? - if mi_unlikely(huge_alignment==0 && mi_theap_should_sample(theap,req_size)) { - if (theap->sample_rate!=0) { // must check to avoid recursion with `_mi_malloc_generic_nosample` + if mi_unlikely(mi_theap_should_sample(theap,req_size)) { + if (huge_alignment==0 && theap->sample_rate!=0) { return _mi_theap_malloc_sample(theap,req_size,zero,ppage); + } + mi_assert_internal(!_mi_is_empty_theap(theap)); // cannot write to the empty theap + mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX); + if (theap->sample_rate==0) { + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter } + #if MI_SAMPLE==2 else { - mi_assert_internal(!_mi_is_empty_theap(theap)); - mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX); - theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // reset the countdown counter to avoid sampling for a while + mi_assert_internal(huge_alignment!=0); + theap->sample_countdown += req_size; // adjust such that after `_mi_page_malloc_zero` the countdown is correct again } + #endif } // find (or allocate) a page of the right size From 2682e06c07f28166fb59f89fe426e7d6ed866ac1 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 09:59:44 -0700 Subject: [PATCH 196/280] readjust sample count in case of an error --- src/page.c | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/src/page.c b/src/page.c index 098d672fe..10f1ad83a 100644 --- a/src/page.c +++ b/src/page.c @@ -1200,6 +1200,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size } // take a sample? + bool sample_countdown_is_adjusted = false; if mi_unlikely(mi_theap_should_sample(theap,req_size)) { if (huge_alignment==0 && theap->sample_rate!=0) { return _mi_theap_malloc_sample(theap,req_size,zero,ppage); @@ -1212,6 +1213,8 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size #if MI_SAMPLE==2 else { mi_assert_internal(huge_alignment!=0); + mi_assert_internal(theap->sample_countdown <= SIZE_MAX - req_size); + sample_countdown_is_adjusted = true; theap->sample_countdown += req_size; // adjust such that after `_mi_page_malloc_zero` the countdown is correct again } #endif @@ -1225,9 +1228,9 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size } if mi_unlikely(page == NULL) { // out of memory - const size_t req_size = size - MI_PADDING_SIZE; // correct for padding_size in case of an overflow on `size` + if (sample_countdown_is_adjusted) { theap->sample_countdown -= req_size; } // unadjust again if we failed to find a page _mi_error_message(ENOMEM, "unable to allocate memory (%zu bytes)\n", req_size); - return NULL; + return NULL; } mi_assert_internal(mi_page_immediate_available(page)); @@ -1235,7 +1238,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size mi_assert_internal(_mi_is_aligned(mi_page_slice_start(page), MI_PAGE_ALIGN)); mi_assert_internal(_mi_ptr_page(mi_page_start(page))==page); - // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc) + // and try again, this time succeeding! (i.e. this should never recurse through _mi_page_malloc_zero) if (ppage!=NULL) { *ppage = page; } void* const p = _mi_page_malloc_zero(theap,page,size,zero); mi_assert_internal(p != NULL); From 929a7be0532027007e3bc766cfab06a066d548e6 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 10:08:04 -0700 Subject: [PATCH 197/280] set default MI_PROFILE to 1 --- include/mimalloc/types.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 21f4b9b0e..1a6cfdda6 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -91,7 +91,7 @@ terms of the MIT license. A copy of the license can be found in the file // Enable profiling support #ifndef MI_PROFILE -#define MI_PROFILE 2 +#define MI_PROFILE 1 #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) From 26db822850140d1979cc4ccc610d32c28c75e934 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 10:15:11 -0700 Subject: [PATCH 198/280] possible fix for fine-grained proile crash --- include/mimalloc/types.h | 2 +- src/alloc-aligned.c | 8 ++++---- src/page.c | 27 +++++++++++++++++---------- 3 files changed, 22 insertions(+), 15 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 1a6cfdda6..c5b467405 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -91,7 +91,7 @@ terms of the MIT license. A copy of the license can be found in the file // Enable profiling support #ifndef MI_PROFILE -#define MI_PROFILE 1 +#define MI_PROFILE 2 #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 97466db00..17da719ee 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -163,11 +163,11 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_generic(mi_theap_t if mi_likely(theap!=NULL) #endif { - if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) + #if MI_SAMPLE // only try if we would not take a sample + if mi_likely(!mi_theap_should_sample(theap,size)) + #endif { - #if MI_SAMPLE // only try if we would not take a sample - if mi_likely(!mi_theap_should_sample(theap,size)) - #endif + if (offset == 0 && mi_malloc_is_naturally_aligned(size,alignment)) { mi_page_t* page = NULL; void* p = mi_theap_malloc_zero_no_guarded(theap, size, zero, &page); diff --git a/src/page.c b/src/page.c index 10f1ad83a..85790cc38 100644 --- a/src/page.c +++ b/src/page.c @@ -1132,18 +1132,23 @@ void mi_register_deferred_free(mi_deferred_free_fun* fn, void* arg) mi_attr_noex /* ----------------------------------------------------------- Admin ----------------------------------------------------------- */ +static mi_theap_t* mi_theap_init(mi_theap_t* theap) { + if mi_likely(mi_theap_is_initialized(theap)) return theap; + if (theap==&_mi_theap_empty_wrong) { + // we were unable to allocate a theap for a first-class heap + return NULL; + } + // otherwise we initialize the thread and its default theap + theap = _mi_thread_init(); + if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } + mi_assert_internal(mi_theap_is_initialized(theap)); + return theap; +} static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) { - if mi_unlikely(!mi_theap_is_initialized(theap)) { - if (theap==&_mi_theap_empty_wrong) { - // we were unable to allocate a theap for a first-class heap - return NULL; - } - // otherwise we initialize the thread and its default theap - theap = _mi_thread_init(); - if mi_unlikely(!mi_theap_is_initialized(theap)) { return NULL; } - } + theap = mi_theap_init(theap); + if (theap==NULL) return NULL; mi_assert_internal(mi_theap_is_initialized(theap)); // do administrative tasks every N generic mallocs @@ -1294,11 +1299,13 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm } void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + theap = mi_theap_init(theap); + if (theap==NULL) return NULL; const size_t sample_rate = theap->sample_rate; const size_t sample_countdown = theap->sample_countdown; theap->sample_rate = 0; // prevent a recursive call to _mi_theap_malloc_sample from _mi_malloc_generic theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; - void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); + void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); theap->sample_rate = sample_rate; theap->sample_countdown = sample_countdown; return p; From bde3e6afa76f816a8c2c23bbd0777916a8f886e8 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 10:29:41 -0700 Subject: [PATCH 199/280] fix assertion in mi_malloc_sample --- src/alloc.c | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 07155bffe..980472e50 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -963,7 +963,8 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept { // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE) mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE); - mi_assert_internal(theap->sample_countdown < req_size); + mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) || + (theap->sample_countdown <= MI_SAMPLE_COUNTDOWN_MAX && theap->sample_countdown < req_size)); const size_t size = req_size + MI_PADDING_SIZE; // handle empty theap and disabled profiling From 7dec07acb857200192b68f95b091d87ed3d1248d Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 10:51:29 -0700 Subject: [PATCH 200/280] don't sample the meta theap --- src/init.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/init.c b/src/init.c index c89b4df31..f8f90f80b 100644 --- a/src/init.c +++ b/src/init.c @@ -205,6 +205,8 @@ static void mi_heap_main_init_once(void) { _mi_theap_init(&mi_process_theap_meta,&mi_process_heap_main,&mi_tld_detached); mi_process_theap_meta.allow_page_abandon = false; // for security, don't share with other threads mi_process_theap_meta.page_full_retain = 2; + mi_process_theap_meta.sample_rate = 0; // no sampling for meta data + mi_process_theap_meta.sample_countdown = 0; subproc_main->theap_meta = &mi_process_theap_meta; // mi_heap_theap_set(&mi_process_heap_main,&mi_process_theap_main); // set in `mi_thread_init(_theap_default)` From 1e73e166dbc3ae08a85aee237999e278caa4dca9 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 11:09:50 -0700 Subject: [PATCH 201/280] improve sample countdown check for msvc --- include/mimalloc/internal.h | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 83eb83ee0..ef808ed25 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1175,8 +1175,9 @@ static inline bool mi_page_claim_ownership(mi_page_t* page) { static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) { // note: this should return `true` on an empty theap so we initialize it's countdown to `-1`. mi_assert_internal(req_size <= SIZE_MAX/2); - const size_t sample_countdown = theap->sample_countdown - req_size; - return ((mi_ssize_t)sample_countdown < 0); + // const size_t sample_countdown = theap->sample_countdown - req_size; + // return ((mi_ssize_t)sample_countdown < 0); + return ((mi_ssize_t)theap->sample_countdown < (mi_ssize_t)req_size); } #if MI_SAMPLE==2 // fine grained From ab404608cc4f1dada6efbcfdfde94b4af2c5a305 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 11:39:41 -0700 Subject: [PATCH 202/280] improve codegen for profile==2 on arm64 --- src/alloc.c | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 980472e50..cfc5da22b 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -56,7 +56,7 @@ static mi_decl_maybe_unused mi_decl_noinline void* mi_block_zero(mi_block_t* blo // Fast allocation in a page: just pop from the free list. // Fall back to generic allocation only if the list is empty. // Note: in release mode the (inlined) routine is about 7 instructions with a single test. -static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, size_t sample_countdown, bool zero, mi_page_t** ppage) mi_attr_noexcept { if (page->block_size != 0) { // not the empty theap mi_assert_internal(mi_page_block_size(page) >= size); @@ -70,7 +70,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ mi_used_t xused = page->xused; xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count #if defined(__GNUC__) - __asm("" : : : "memory"); // always load the `used` field before the test + __asm("" : : : "memory" ); // always load the `used` field before the test #endif if (block == NULL) { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); @@ -81,9 +81,11 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // pop from the free list mi_block_t* next = mi_block_next(page,block); mi_track_mem_undefined(block,sizeof(*block)); + #if MI_SECURE if (!zero) block->next = 0; // don't leak internal data #endif + page->free = next; page->xused = xused; mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); @@ -91,8 +93,11 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ #if MI_SAMPLE==2 const size_t req_size = size - MI_PADDING_SIZE; + mi_assert_internal(theap->sample_countdown == sample_countdown); // we pass it to improve codegen mi_assert_internal(theap->sample_countdown >= req_size); - theap->sample_countdown -= req_size; + theap->sample_countdown = sample_countdown - req_size; + #else + MI_UNUSED(sample_countdown); #endif #if MI_STAT>=2 @@ -140,7 +145,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // extra entries for improved efficiency in `alloc-aligned.c` (and in `page.c:mi_malloc_generic`. extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept { - return mi_page_malloc_zero(theap, page, size, zero, NULL); + return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, NULL); } // main allocation primitives for small and generic allocation @@ -167,7 +172,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE); // and allocate - void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, zero, ppage); + void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage); mi_track_malloc(p,size,zero); #if MI_DEBUG>3 From aa1e6a24a28577cbbcfe76b1f4ae0cdc51964154 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 12:54:27 -0700 Subject: [PATCH 203/280] improve codegen for mi_free_small on gcc --- src/free.c | 31 ++++++++++++++++++------------- 1 file changed, 18 insertions(+), 13 deletions(-) diff --git a/src/free.c b/src/free.c index b9d9edc48..512bdcc0e 100644 --- a/src/free.c +++ b/src/free.c @@ -199,21 +199,26 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha mi_page_t* page; #if MI_PAGE_META_SMALL_IS_ALIGNED - if (free_small) { page = (mi_page_t*)_mi_align_down_ptr(p,MI_SMALL_PAGE_SIZE); } + if (free_small) { + const uintptr_t up = _mi_align_down((uintptr_t)p,MI_SMALL_PAGE_SIZE); // like this for codegen on gcc + if mi_unlikely(up==0 && check_p_for_null) { return false; } + page = (mi_page_t*)up; + } else #endif - #if MI_PAGE_META_IS_ALIGNED - { page = _mi_aligned_ptr_page0(p); } - #else - { page = _mi_ptr_page(p); } - #endif - - if mi_unlikely(check_p_for_null && page==NULL) { - #if MI_DEBUG - if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } - #endif - return false; - } + { + #if MI_PAGE_META_IS_ALIGNED + { page = _mi_aligned_ptr_page0(p); } + #else + { page = _mi_ptr_page(p); } + #endif + if mi_unlikely(page==NULL && check_p_for_null) { + #if MI_DEBUG + if (p!=NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } + #endif + return false; + } + } #if MI_DEBUG mi_page_t* const cpage = _mi_checked_ptr_page(p); if mi_unlikely(cpage==NULL) { _mi_error_message(EINVAL, "%s: invalid pointer: %p\n", msg, p); } From 2520d4f0de948a935d244737c428a0f5ef5736ec Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 5 Sep 2026 16:57:11 -0700 Subject: [PATCH 204/280] use 64-bit for total requested --- include/mimalloc-profile.h | 5 +++-- include/mimalloc-stats.h | 2 +- include/mimalloc/internal.h | 35 ++++++++++++++++++----------- include/mimalloc/types.h | 4 ++-- src/alloc.c | 13 ++++++----- src/page.c | 44 ++++++++++++++++++++----------------- src/profile.c | 6 ++--- src/stats.c | 32 +++++++++++++++++---------- test/test-profile.c | 22 +++++++++---------- 9 files changed, 93 insertions(+), 70 deletions(-) diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h index 866852fad..254caba78 100644 --- a/include/mimalloc-profile.h +++ b/include/mimalloc-profile.h @@ -10,14 +10,15 @@ terms of the MIT license. A copy of the license can be found in the file #include #include // bool +#include // uint64_t typedef struct mi_profiler_data_s { size_t usable_size; size_t requested_size; - void* user_data[14]; + void* user_data[6]; // default, but can be less or more (up to 1KiB), depending on `profiler_data_size` } mi_profiler_data_t; -typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_data_t* profiler_data, void* ptr, size_t threshold, size_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg); +typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_data_t* profiler_data, void* ptr, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg); typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_data_t* profiler_data, void* ptr, size_t old_requested_size, const mi_heap_t* heap, void* profiler_arg); typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_data_t* profiler_data, void* ptr, const mi_heap_t* heap, void* profiler_arg); diff --git a/include/mimalloc-stats.h b/include/mimalloc-stats.h index e1c74494a..ab45a0715 100644 --- a/include/mimalloc-stats.h +++ b/include/mimalloc-stats.h @@ -49,7 +49,7 @@ typedef struct mi_stat_counter_s { MI_STAT_COUNT(threads) /* number of threads */ \ MI_STAT_COUNT(malloc_normal) /* allocated bytes <= MI_LARGE_OBJ_SIZE_MAX */ \ MI_STAT_COUNT(malloc_huge) /* allocated bytes in huge pages */ \ - MI_STAT_COUNT(malloc_requested) /* malloc requested bytes */ \ + MI_STAT_COUNTER(malloc_requested) /* malloc requested bytes */ \ \ MI_STAT_COUNTER(mmap_calls) \ MI_STAT_COUNTER(commit_calls) \ diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index ef808ed25..2fa900324 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -337,7 +337,7 @@ bool _mi_page_is_valid(mi_page_t* page); // "profile.c" mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept; mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, size_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; +mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p); // ------------------------------------------------------ @@ -370,24 +370,31 @@ mi_decl_noreturn mi_decl_cold void _mi_assert_fail(const char* assertion, const ----------------------------------------------------------- */ // add to stat keeping track of the peak -void __mi_stat_increase(mi_stat_count_t* stat, size_t amount); -void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount); -void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount); -void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount); +void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount); // adjust stat in special cases to compensate for double counting (and does not adjust peak values and can decrease the total) -void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount); -void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount); +void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount); +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount); // counters can just be increased -static inline void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, size_t amount) { +static inline void __mi_stat_counter_increase_mt(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_atomic_addi64_relaxed(&stat->total, (int64_t)amount); } -static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t amount) { - stat->total += amount; +static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + stat->total += (int64_t)amount; +} + +static inline void __mi_stat_counter_decrease(mi_stat_counter_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); + stat->total -= (int64_t)amount; } #define mi_heap_stat_counter_increase(heap,stat,amount) __mi_stat_counter_increase_mt( &(heap)->stats.stat, amount) @@ -403,15 +410,17 @@ static inline void __mi_stat_counter_increase(mi_stat_counter_t* stat, size_t am #define mi_subproc_stat_adjust_decrease(subproc,stat,amount) __mi_stat_adjust_decrease_mt( &(subproc)->stats.stat, amount) #define mi_theap_stat_counter_increase(theap,stat,amount) __mi_stat_counter_increase( &(theap)->stats.stat, amount) +#define mi_theap_stat_counter_decrease(theap,stat,amount) __mi_stat_counter_decrease( &(theap)->stats.stat, amount) #define mi_theap_stat_increase(theap,stat,amount) __mi_stat_increase( &(theap)->stats.stat, amount) #define mi_theap_stat_decrease(theap,stat,amount) __mi_stat_decrease( &(theap)->stats.stat, amount) #define mi_theap_stat_adjust_increase(theap,stat,amnt) __mi_stat_adjust_increase( &(theap)->stats.stat, amnt) #define mi_theap_stat_adjust_decrease(theap,stat,amnt) __mi_stat_adjust_decrease( &(theap)->stats.stat, amnt) #define mi_theapx_stat_counter_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_counter_increase(theap,stat,amount); } else { mi_heap_stat_counter_increase(heap,stat,amount); } +#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } #define mi_theapx_stat_increase(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_increase(theap,stat,amount); } else { mi_heap_stat_increase(heap,stat,amount); } #define mi_theapx_stat_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_decrease(theap,stat,amount); } else { mi_heap_stat_decrease(heap,stat,amount); } -#define mi_theapx_stat_adjust_decrease(heap,theap,stat,amount) if (theap!=NULL) { mi_theap_stat_adjust_decrease(theap,stat,amount); } else { mi_heap_stat_adjust_decrease(heap,stat,amount); } + /* ----------------------------------------------------------- pthread thread locals diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index c5b467405..a1fff909d 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -91,7 +91,7 @@ terms of the MIT license. A copy of the license can be found in the file // Enable profiling support #ifndef MI_PROFILE -#define MI_PROFILE 2 +#define MI_PROFILE 1 #endif // Enable guard pages behind objects of a certain size (set by the MIMALLOC_GUARDED_MIN/MAX/SAMPLE_RATE options) @@ -606,7 +606,7 @@ struct mi_theap_s { _Atomic(size_t) refcount; // reference count size_t sample_rate; - size_t sample_requested; + uint64_t sample_requested; size_t profile_sample_rate; size_t profile_sample_countdown; size_t guarded_sample_rate; diff --git a/src/alloc.c b/src/alloc.c index cfc5da22b..c93b2da27 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -101,7 +101,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ #endif #if MI_STAT>=2 - mi_theap_stat_increase(theap,malloc_requested,size - MI_PADDING_SIZE); + mi_theap_stat_counter_increase(theap,malloc_requested,size - MI_PADDING_SIZE); #endif #if MI_DEBUG>3 @@ -954,8 +954,8 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); #if MI_STAT // adjust request stats to only count the allocated size of the block (and not the guard page) - mi_theap_stat_adjust_decrease(theap, malloc_requested, req_size); - mi_theap_stat_increase(theap, malloc_requested, size); + mi_theap_stat_counter_decrease(theap, malloc_requested, req_size); + mi_theap_stat_counter_increase(theap, malloc_requested, size); #endif #if MI_DEBUG>3 if (zero) { @@ -986,7 +986,8 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* thea mi_assert_internal(theap->sample_rate > 0); mi_assert_internal(theap->sample_rate <= SIZE_MAX/2); mi_assert_internal(theap->sample_rate >= theap->sample_countdown); - const size_t requested = theap->sample_rate + (req_size - theap->sample_countdown) + theap->sample_requested; + + const uint64_t requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested; mi_assert_internal(requested > 0); mi_assert_internal(requested >= size); if (requested < size) { @@ -1000,8 +1001,8 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* thea mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0); bool sample_profile = false; bool sample_guarded = false; - if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= requested; } else { sample_profile = true; } - if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= requested; } else { sample_guarded = true; } + if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = true; } + if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = true; } // invoke callback? if (sample_profile) { diff --git a/src/page.c b/src/page.c index 85790cc38..63649a2c9 100644 --- a/src/page.c +++ b/src/page.c @@ -146,28 +146,31 @@ bool _mi_page_is_valid(mi_page_t* page) { #if MI_STAT // Merge stats from the page into the corresponding theap or heap. static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_t free_count ) { - // update stats + // get heap and theap (maybe NULL) mi_heap_t* const heap = mi_page_heap(page); mi_theap_t* const theap = _mi_page_associated_theap_peek(page); mi_assert_internal(theap == NULL || (theap->tld != NULL && _mi_thread_id() == theap->tld->thread_id)); mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1); mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count); - // adjust sample countdown + // allocation sizes const size_t bsize = mi_page_usable_block_size(page); + const uint64_t allocated = (uint64_t)alloc_count * (uint64_t)bsize; + const uint64_t requested = allocated - ((uint64_t)alloc_count * MI_PADDING_SIZE); + const uint64_t freed = (uint64_t)free_count * (uint64_t)bsize; + mi_assert_internal(allocated <= INT64_MAX); // safe to cast to int64_t for stats + mi_assert_internal(freed <= INT64_MAX); + + // adjust sample countdown #if MI_SAMPLE==1 if (alloc_count>0 && theap!=NULL && theap->sample_rate!=0) { - // adjust countdown - size_t requested = 0; - if (mi_mul_overflow(alloc_count,bsize,&requested)) { - mi_assert(false); - } - if (theap->sample_countdown >= requested) { - theap->sample_countdown -= requested; + // adjust countdown + if (requested < SIZE_MAX && theap->sample_countdown >= (size_t)requested) { + theap->sample_countdown -= (size_t)requested; } else { theap->sample_countdown = 0; - theap->sample_requested += (requested - theap->sample_countdown); // TODO: check that overflow never happens + theap->sample_requested += (requested - theap->sample_countdown); } } #endif @@ -177,17 +180,17 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ const size_t bin = _mi_bin(bsize); // allocations if (alloc_count > 0) { - mi_theapx_stat_increase(heap, theap, malloc_normal, alloc_count*bsize); mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count); + mi_theapx_stat_increase(heap, theap, malloc_normal, allocated); mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); #if MI_STAT==1 // use coarse total requested bytes - mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); + mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); #endif } // frees if (free_count > 0) { - mi_theapx_stat_decrease(heap, theap, malloc_normal, free_count*bsize); + mi_theapx_stat_decrease(heap, theap, malloc_normal, freed); mi_theapx_stat_decrease(heap, theap, malloc_bins[bin], free_count); } } @@ -196,16 +199,16 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ mi_assert_internal(alloc_count<=1); mi_assert_internal(free_count<=1); if (alloc_count > 0) { - mi_theapx_stat_increase(heap, theap, malloc_huge, alloc_count*bsize); mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); + mi_theapx_stat_increase(heap, theap, malloc_huge, allocated); #if MI_STAT==1 // use coarse total requested bytes - mi_theapx_stat_increase(heap, theap, malloc_requested, alloc_count*(bsize - MI_PADDING_SIZE)); + mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); #endif } // frees if (free_count > 0) { - mi_theapx_stat_decrease(heap, theap, malloc_huge, free_count*bsize); + mi_theapx_stat_decrease(heap, theap, malloc_huge, freed); } } } @@ -1175,13 +1178,13 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) theap->generic_collect_count = 0; mi_theap_collect(theap, false /* force? */); } + else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling + mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ? + } else { // otherwise we do a mini-collect _mi_deferred_free(theap, false); // call potential deferred free routines - _mi_theap_collect_retired(theap, false); // free retired pages - if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling - mi_theap_collect(theap, false /* force? */); - } + _mi_theap_collect_retired(theap, false); // free retired pages } } return theap; @@ -1249,6 +1252,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size mi_assert_internal(p != NULL); // move full pages to the full queue + // this will also call _mi_page_update_stats for huge pages if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) { if (mi_page_is_full(page)) { mi_page_to_full(page, mi_page_queue_of(page)); diff --git a/src/profile.c b/src/profile.c index 1e76b453c..f41b349e2 100644 --- a/src/profile.c +++ b/src/profile.c @@ -26,7 +26,7 @@ static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { /* ---------------------------------------------------------------------------- Profile an allocation and free -----------------------------------------------------------------------------*/ -mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, size_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(theap!=NULL); mi_assert_internal(size<=requested_since_last_sample); @@ -37,7 +37,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] const size_t profiler_data_offset = sizeof(mi_block_t); size_t profiler_data_size = sizeof(mi_profiler_data_t); - if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > 512 ? 512 : prof->profiler_data_size); }; + if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > 1024 ? 1024 : prof->profiler_data_size); }; const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); const size_t oversize = profiler_user_offset + size; mi_page_t* page = NULL; @@ -58,7 +58,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th // and call the profiler on_alloc if (prof->on_alloc!=NULL) { - const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, (size_t)theap->profile_sample_rate, requested_since_last_sample /* TODO: bytes since last sample */, _mi_theap_heap(theap), prof->profiler_arg); + const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { mi_theap_enable_profiler(theap,new_sample_rate); } diff --git a/src/stats.c b/src/stats.c index 9fca930a4..8a13a4eeb 100644 --- a/src/stats.c +++ b/src/stats.c @@ -39,17 +39,21 @@ static void mi_stat_update(mi_stat_count_t* stat, int64_t amount) { if (amount > 0) { stat->total += amount; } } -void __mi_stat_increase_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_increase_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update_mt(stat, (int64_t)amount); } -void __mi_stat_increase(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_increase(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update(stat, (int64_t)amount); } -void __mi_stat_decrease_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_decrease_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update_mt(stat, -((int64_t)amount)); } -void __mi_stat_decrease(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_update(stat, -((int64_t)amount)); } @@ -73,16 +77,20 @@ static void mi_stat_adjust(mi_stat_count_t* stat, int64_t amount) { stat->total += amount; } -void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_increase_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust_mt(stat, (int64_t)amount); } -void __mi_stat_adjust_increase(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_increase(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust(stat, (int64_t)amount); } -void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_decrease_mt(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust_mt(stat, -((int64_t)amount)); } -void __mi_stat_adjust_decrease(mi_stat_count_t* stat, size_t amount) { +void __mi_stat_adjust_decrease(mi_stat_count_t* stat, uint64_t amount) { + mi_assert_internal(amount<=INT64_MAX); mi_stat_adjust(stat, -((int64_t)amount)); } @@ -230,10 +238,10 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t } #if MI_STAT -static void mi_stat_total_print(const mi_stat_count_t* stat, const char* msg, int64_t unit, mi_output_fun* out, void* arg) { +static void mi_stat_total_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { _mi_fprintf(out, arg, " %-12s:", msg); _mi_fprintf(out, arg, "%12s", " "); // no peak - mi_print_amount(stat->total, unit, out, arg); + mi_print_amount(stat->total, 1, out, arg); _mi_fprintf(out, arg, "\n"); } #endif @@ -367,9 +375,9 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou mi_stat_count_add_mt(&total, &stats->malloc_huge); mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, ""); #if MI_STAT>=2 - mi_stat_total_print(&stats->malloc_requested, "malloc req", 1, out, arg); + mi_stat_total_print(&stats->malloc_requested, "malloc req", out, arg); #else - mi_stat_total_print(&stats->malloc_requested, "malloc req~", 1, out, arg); + mi_stat_total_print(&stats->malloc_requested, "malloc req~", out, arg); #endif _mi_fprintf(out, arg, "\n"); #endif diff --git a/test/test-profile.c b/test/test-profile.c index eba49c41e..471ab1d4c 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -23,11 +23,11 @@ terms of the MIT license. A copy of the license can be found in the file // --------------------------------------------------------------------------- typedef struct { - int64_t alloc_count; - int64_t free_count; - size_t last_size; - size_t last_upscaled; - void* last_ptr; + uint64_t alloc_count; + uint64_t free_count; + size_t last_size; + uint64_t last_upscaled; + void* last_ptr; } profile_state_t; // We store ptr in user_data so on_free can verify the round-trip. @@ -36,7 +36,7 @@ static profile_state_t g_state; #define TEST_THRESHOLD (16 * 1024) -static size_t mi_cdecl on_alloc(mi_profiler_data_t* data, void* ptr, size_t threshold, size_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg) { +static size_t mi_cdecl on_alloc(mi_profiler_data_t* data, void* ptr, size_t threshold, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg) { MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(profiler_arg); assert(profiler_arg==&g_state); assert(bytes_since_last_sample >= data->requested_size); @@ -88,7 +88,7 @@ static void allocate_past_threshold(void) { bool test_profiler_samples(void) { CHECK_BODY("profiler: on_alloc called after threshold") { - int before = g_state.alloc_count; + uint64_t before = g_state.alloc_count; allocate_past_threshold(); result = (g_state.alloc_count > before); } @@ -99,7 +99,7 @@ bool test_profiler_samples(void) { bool test_profiler_record_fields(void) { CHECK_BODY("profiler: record ptr and size are non-zero") { - int before = g_state.alloc_count; + uint64_t before = g_state.alloc_count; int count; for (count = 0; g_state.alloc_count == before && count < MAXLOOP; count++) { void* p = mi_malloc(1024); @@ -112,8 +112,8 @@ bool test_profiler_record_fields(void) { bool test_profiler_on_free_called(void) { CHECK_BODY("profiler: on_free called for sampled allocation") { - int alloc_before = g_state.alloc_count; - int free_before = g_state.free_count; + uint64_t alloc_before = g_state.alloc_count; + uint64_t free_before = g_state.free_count; // Keep the pointer live until we confirm a sample was taken, then free it. void* sampled = NULL; @@ -135,7 +135,7 @@ bool test_profiler_on_free_called(void) { bool test_profiler_upscaled_at_least_size(void) { CHECK_BODY("profiler: upscaled_size >= size") { - int before = g_state.alloc_count; + uint64_t before = g_state.alloc_count; while (g_state.alloc_count == before) { void* p = mi_malloc(256); mi_free(p); From 5cfedfb66d12a5998e0d74b1577bb8aff860b8a7 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 6 Sep 2026 01:02:48 -0700 Subject: [PATCH 205/280] improve codegen for mi_zalloc --- include/mimalloc/internal.h | 26 +++++++++++++++++++++++++- src/alloc.c | 3 ++- 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 2fa900324..50f4079f6 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1549,7 +1549,7 @@ static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { return memcpy(dst, src, n); } -static inline void* _mi_memset(void* dst, int val, size_t n) { +static inline void* _mi_memset(void* dst, int val, size_t n) { return memset(dst, val, n); } @@ -1599,4 +1599,28 @@ static inline void* _mi_memzero_aligned(void* dst, size_t n) { return _mi_memset_aligned(dst, 0, n); } +// MI_SIZE_SIZE aligned and sized +static inline void* _mi_memzero_alignedw(void* dst, size_t n) { + mi_assert_internal(n%MI_SIZE_SIZE == 0); + mi_assert_internal((uintptr_t)dst % MI_SIZE_SIZE == 0); + #if MI_ARCH_ARM64 && defined(__GNUC__) + if mi_likely(n <= 128) { + __asm__ volatile ( + "tbz %[n], #3, 1f\n\t" + "sub %[n], %[n], #8\n\t" + "str xzr, [%[dst]], #8\n\t" + "1:\n\t" + "cbz %[n], 3f\n\t" + "2:\n\t" + "subs %[n],%[n],#16\n\t" + "stp xzr,xzr,[%[dst]],#16\n\t" + "b.gt 2b\n\t" + "3:" + : [n] "+r" (n), [dst] "+r" (dst) : : "cc"); + return dst; + } + #endif + return _mi_memset_aligned(dst, 0, n); +} + #endif // MI_INTERNAL_H diff --git a/src/alloc.c b/src/alloc.c index c93b2da27..8d3b9cc28 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -127,7 +127,8 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // zero the block? note: we need to zero the full block size (issue #63) if mi_unlikely(zero) { if (!page->free_is_zero) { - _mi_memzero_aligned(block,bsize); + mi_assert_internal(bsize%MI_SIZE_SIZE == 0); + _mi_memzero_alignedw(block,bsize); } else { block->next = 0; From e37102e1f786521dbb08593bbb348baed8019904 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 6 Sep 2026 11:10:09 -0700 Subject: [PATCH 206/280] faster memzero_block --- include/mimalloc/internal.h | 43 +++++++++++++++++++++---------------- src/alloc.c | 2 +- 2 files changed, 26 insertions(+), 19 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 50f4079f6..374769c96 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1600,27 +1600,34 @@ static inline void* _mi_memzero_aligned(void* dst, size_t n) { } // MI_SIZE_SIZE aligned and sized -static inline void* _mi_memzero_alignedw(void* dst, size_t n) { - mi_assert_internal(n%MI_SIZE_SIZE == 0); - mi_assert_internal((uintptr_t)dst % MI_SIZE_SIZE == 0); - #if MI_ARCH_ARM64 && defined(__GNUC__) - if mi_likely(n <= 128) { - __asm__ volatile ( - "tbz %[n], #3, 1f\n\t" - "sub %[n], %[n], #8\n\t" - "str xzr, [%[dst]], #8\n\t" - "1:\n\t" - "cbz %[n], 3f\n\t" - "2:\n\t" - "subs %[n],%[n],#16\n\t" - "stp xzr,xzr,[%[dst]],#16\n\t" - "b.gt 2b\n\t" - "3:" - : [n] "+r" (n), [dst] "+r" (dst) : : "cc"); +static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) { + mi_assert_internal(bsize%MI_SIZE_SIZE == 0); + mi_assert_internal(bsize > 0); + mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); + mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || (uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); + #if MI_ARCH_ARM64 && defined(__SIZEOF_INT128__) // any 64-bit platform with 128-bit stores could benefit (todo: maybe also for x64 instead of using `rep stosb` there?) + // fast memzero based on overlapping writes (and assuming non-zero size_t multiple size, and uintptr_t aligned) + if mi_unlikely(bsize < 16) { + *((uint64_t*)dst) = 0; + return dst; + } + __uint128_t* const start = (__uint128_t*)dst; + __uint128_t* const end = (__uint128_t*)((uint8_t*)dst + bsize); + if mi_likely(bsize < 64) { + const size_t ofs = (bsize>>5)&1; // == (n >= 32) + mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); + __uint128_t* const end0 = end - ofs; + start[0] = 0; start[ofs] = 0; + end0[-1] = 0; end[-1] = 0; + return dst; + } + if mi_likely(bsize <= 128) { + start[0] = 0; start[1] = 0; start[2] = 0; start[3] = 0; + end[-4] = 0; end[-3] = 0; end[-2] = 0; end[-1] = 0; return dst; } #endif - return _mi_memset_aligned(dst, 0, n); + return _mi_memset_aligned(dst, 0, bsize); } #endif // MI_INTERNAL_H diff --git a/src/alloc.c b/src/alloc.c index 8d3b9cc28..58b313f95 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -128,7 +128,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ if mi_unlikely(zero) { if (!page->free_is_zero) { mi_assert_internal(bsize%MI_SIZE_SIZE == 0); - _mi_memzero_alignedw(block,bsize); + _mi_memzero_block(block,bsize); } else { block->next = 0; From 44fd9ab001750e234963161062937306f410d5e1 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 6 Sep 2026 13:57:46 -0700 Subject: [PATCH 207/280] update vs2022 projects --- ide/vs2022/mimalloc-lib.vcxproj | 2 ++ ide/vs2022/mimalloc-lib.vcxproj.filters | 6 ++++++ ide/vs2022/mimalloc-override-dll.vcxproj | 2 ++ ide/vs2022/mimalloc-override-dll.vcxproj.filters | 6 ++++++ 4 files changed, 16 insertions(+) diff --git a/ide/vs2022/mimalloc-lib.vcxproj b/ide/vs2022/mimalloc-lib.vcxproj index cba6f88f0..29ebec431 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj +++ b/ide/vs2022/mimalloc-lib.vcxproj @@ -451,6 +451,7 @@ + @@ -488,6 +489,7 @@ + diff --git a/ide/vs2022/mimalloc-lib.vcxproj.filters b/ide/vs2022/mimalloc-lib.vcxproj.filters index 0c23b2834..49b5c3e7e 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj.filters +++ b/ide/vs2022/mimalloc-lib.vcxproj.filters @@ -70,6 +70,9 @@ Sources + + Sources + @@ -108,6 +111,9 @@ Headers + + Headers + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj b/ide/vs2022/mimalloc-override-dll.vcxproj index 25269e3d8..cde7d3d6f 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj +++ b/ide/vs2022/mimalloc-override-dll.vcxproj @@ -433,6 +433,7 @@ + @@ -480,6 +481,7 @@ + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj.filters b/ide/vs2022/mimalloc-override-dll.vcxproj.filters index f57933c17..2f9beb4ce 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj.filters +++ b/ide/vs2022/mimalloc-override-dll.vcxproj.filters @@ -70,6 +70,9 @@ Sources + + Sources + @@ -108,6 +111,9 @@ Headers + + Headers + From 35a7b47d82b551e7e2b0740c65a44b1cc32b3592 Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 6 Sep 2026 14:27:20 -0700 Subject: [PATCH 208/280] use fast memzero on x64 as well --- include/mimalloc/internal.h | 25 +++++++++++++++---------- 1 file changed, 15 insertions(+), 10 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 374769c96..b1834402a 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1502,7 +1502,7 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) { // Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows. // --------------------------------------------------------------------------------- -#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__)) +#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__)) && !defined(__SIZEOF_INT128__) extern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c extern mi_decl_hidden size_t _mi_cpu_stosb_max; @@ -1605,29 +1605,34 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize mi_assert_internal(bsize > 0); mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || (uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); - #if MI_ARCH_ARM64 && defined(__SIZEOF_INT128__) // any 64-bit platform with 128-bit stores could benefit (todo: maybe also for x64 instead of using `rep stosb` there?) + #if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) + void* const adst = __builtin_assume_aligned(dst,MI_INTPTR_SIZE); + #else + void* const adst = dst; + #endif + #if (MI_ARCH_ARM64 || MI_ARCH_X64) && defined(__SIZEOF_INT128__) // any 64-bit platform with 128-bit stores can benefit. // fast memzero based on overlapping writes (and assuming non-zero size_t multiple size, and uintptr_t aligned) if mi_unlikely(bsize < 16) { - *((uint64_t*)dst) = 0; - return dst; + *((uint64_t*)adst) = 0; + return adst; } - __uint128_t* const start = (__uint128_t*)dst; - __uint128_t* const end = (__uint128_t*)((uint8_t*)dst + bsize); + __int128_t* const start = (__int128_t*)adst; + __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); if mi_likely(bsize < 64) { const size_t ofs = (bsize>>5)&1; // == (n >= 32) mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); - __uint128_t* const end0 = end - ofs; + __int128_t* const end0 = end - ofs; start[0] = 0; start[ofs] = 0; end0[-1] = 0; end[-1] = 0; - return dst; + return adst; } if mi_likely(bsize <= 128) { start[0] = 0; start[1] = 0; start[2] = 0; start[3] = 0; end[-4] = 0; end[-3] = 0; end[-2] = 0; end[-1] = 0; - return dst; + return adst; } #endif - return _mi_memset_aligned(dst, 0, bsize); + return _mi_memset_aligned(adst, 0, bsize); } #endif // MI_INTERNAL_H From 282d188a1f19bc0523e0f786c06f150794f1532b Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 6 Sep 2026 17:23:52 -0700 Subject: [PATCH 209/280] cleanup mi_memzero_block --- include/mimalloc/internal.h | 87 ++++++++++++++++--------------------- 1 file changed, 38 insertions(+), 49 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index b1834402a..2f652d4aa 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -40,6 +40,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #if !MI_TRACK_ASAN #define mi_decl_forceinline __attribute__((always_inline)) inline @@ -55,10 +56,12 @@ terms of the MIT license. A copy of the license can be found in the file #else #define mi_decl_hidden __attribute__((visibility("hidden"))) #endif -#if (__GNUC__ >= 4) || defined(__clang__) +#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) #define mi_decl_cold __attribute__((cold)) +#define mi_assume_aligned(p,sz) __builtin_assume_aligned(p,sz) #else #define mi_decl_cold +#define mi_assume_aligned(p,sz) #endif #elif __cplusplus >= 201103L // c++11 #define mi_decl_forceinline inline @@ -68,6 +71,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) #else #define mi_decl_forceinline inline #define mi_decl_noinline @@ -76,6 +80,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold +#define mi_assume_aligned(p,sz) #endif #if defined(__GNUC__) || defined(__clang__) @@ -89,7 +94,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_likely(x) (x) #endif -#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) // includes clang and icc +#if (defined(__GNUC__) && (__GNUC__ >= 7)) || defined(__clang__) #define mi_decl_maybe_unused __attribute__((unused)) #elif __cplusplus >= 201703L // c++17 #define mi_decl_maybe_unused [[maybe_unused]] @@ -1560,37 +1565,20 @@ static inline void* _mi_memset(void* dst, int val, size_t n) { // This is used for example in `mi_realloc`. // ------------------------------------------------------------------------------- -#if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) - // On GCC/CLang we provide a hint that the pointers are word aligned. static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); - const void* asrc = __builtin_assume_aligned(src, MI_INTPTR_SIZE); + void* adst = mi_assume_aligned(dst, MI_INTPTR_SIZE); + const void* asrc = mi_assume_aligned(src, MI_INTPTR_SIZE); return _mi_memcpy(adst, asrc, n); } static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - void* adst = __builtin_assume_aligned(dst, MI_INTPTR_SIZE); + void* adst = mi_assume_aligned(dst, MI_INTPTR_SIZE); return _mi_memset(adst, val, n); } -#else - -// Default fallback on `_mi_memcpy` -static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { - mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - return _mi_memcpy(dst, src, n); -} - -static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { - mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - return _mi_memset(dst, val, n); -} - -#endif - static inline void* _mi_memzero(void* dst, size_t n) { return _mi_memset(dst, 0, n); } @@ -1599,40 +1587,41 @@ static inline void* _mi_memzero_aligned(void* dst, size_t n) { return _mi_memset_aligned(dst, 0, n); } -// MI_SIZE_SIZE aligned and sized + +// Zero a block: blocks are always aligned with a positive bsize in machine-word bytes. static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) { mi_assert_internal(bsize%MI_SIZE_SIZE == 0); mi_assert_internal(bsize > 0); mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || (uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); - #if (defined(__GNUC__) && (__GNUC__ >= 4)) || defined(__clang__) - void* const adst = __builtin_assume_aligned(dst,MI_INTPTR_SIZE); - #else - void* const adst = dst; - #endif + #if (MI_ARCH_ARM64 || MI_ARCH_X64) && defined(__SIZEOF_INT128__) // any 64-bit platform with 128-bit stores can benefit. - // fast memzero based on overlapping writes (and assuming non-zero size_t multiple size, and uintptr_t aligned) - if mi_unlikely(bsize < 16) { - *((uint64_t*)adst) = 0; - return adst; - } - __int128_t* const start = (__int128_t*)adst; - __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); - if mi_likely(bsize < 64) { - const size_t ofs = (bsize>>5)&1; // == (n >= 32) - mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); - __int128_t* const end0 = end - ofs; - start[0] = 0; start[ofs] = 0; - end0[-1] = 0; end[-1] = 0; - return adst; - } - if mi_likely(bsize <= 128) { - start[0] = 0; start[1] = 0; start[2] = 0; start[3] = 0; - end[-4] = 0; end[-3] = 0; end[-2] = 0; end[-1] = 0; - return adst; - } + // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and uintptr_t aligned) + if mi_unlikely(bsize < 16) { + *((uint64_t*)dst) = 0; + return dst; + } + mi_assert_internal((uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); + void* const adst = mi_assume_aligned(dst,MI_MAX_ALIGN_SIZE); + __int128_t* const start = (__int128_t*)adst; + __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); + if mi_likely(bsize < 64) { + const size_t ofs = (bsize>>5)&1; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); + __int128_t* const end0 = end - ofs; + start[0] = 0; start[ofs] = 0; + end0[-1] = 0; end[-1] = 0; + return adst; + } + if mi_likely(bsize <= 128) { + start[0] = 0; start[1] = 0; start[2] = 0; start[3] = 0; + end[-4] = 0; end[-3] = 0; end[-2] = 0; end[-1] = 0; + return adst; + } #endif - return _mi_memset_aligned(adst, 0, bsize); + + // regular memset + void* const wdst = mi_assume_aligned(dst,MI_INTPTR_SIZE); + return _mi_memset_aligned(wdst, 0, bsize); } #endif // MI_INTERNAL_H From 5ed8e01ce0feb7212a61722e487f6dc043d4936d Mon Sep 17 00:00:00 2001 From: Daan Date: Sun, 6 Sep 2026 17:25:14 -0700 Subject: [PATCH 210/280] remove old block_zero --- src/alloc.c | 6 ------ 1 file changed, 6 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 58b313f95..3a78bf0d3 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -45,12 +45,6 @@ static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_blo } #endif } -#else -static mi_decl_maybe_unused mi_decl_noinline void* mi_block_zero(mi_block_t* block, size_t bsize) mi_attr_noexcept { - // const size_t bsize = mi_page_usable_block_size(page); - _mi_memzero_aligned(block, bsize); - return block; -} #endif // Fast allocation in a page: just pop from the free list. From eaaed425687ba8fabd215b350f7e24f6d7cf6317 Mon Sep 17 00:00:00 2001 From: daanx Date: Sun, 6 Sep 2026 21:29:32 -0700 Subject: [PATCH 211/280] fix mi_memzero_block for msvc x64 --- include/mimalloc/internal.h | 49 +++++++++++++++++++++++-------------- 1 file changed, 30 insertions(+), 19 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 2f652d4aa..762b7cfdb 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -40,7 +40,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold -#define mi_assume_aligned(p,sz) +#define mi_assume_aligned(p,sz) (p) #elif (defined(__GNUC__) && (__GNUC__ >= 3)) || defined(__clang__) // includes clang and icc #if !MI_TRACK_ASAN #define mi_decl_forceinline __attribute__((always_inline)) inline @@ -61,7 +61,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_assume_aligned(p,sz) __builtin_assume_aligned(p,sz) #else #define mi_decl_cold -#define mi_assume_aligned(p,sz) +#define mi_assume_aligned(p,sz) (p) #endif #elif __cplusplus >= 201103L // c++11 #define mi_decl_forceinline inline @@ -71,7 +71,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold -#define mi_assume_aligned(p,sz) +#define mi_assume_aligned(p,sz) (p) #else #define mi_decl_forceinline inline #define mi_decl_noinline @@ -80,7 +80,7 @@ terms of the MIT license. A copy of the license can be found in the file #define mi_decl_weak #define mi_decl_hidden #define mi_decl_cold -#define mi_assume_aligned(p,sz) +#define mi_assume_aligned(p,sz) (p) #endif #if defined(__GNUC__) || defined(__clang__) @@ -1507,7 +1507,11 @@ static inline uintptr_t _mi_random_shuffle(uintptr_t x) { // Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows. // --------------------------------------------------------------------------------- -#if !MI_TRACK_ENABLED && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__)) && !defined(__SIZEOF_INT128__) +#if !MI_TRACK_ENABLED && (MI_ARCH_ARM64 || MI_ARCH_X64) && (defined(__SIZEOF_INT128__) || (defined(_MSC_VER) && defined(__AVX2__))) // any 64-bit platform with 128-bit stores can benefit. +#define MI_USE_MEMZERO128 1 +#endif + +#if !MI_TRACK_ENABLED && !MI_USE_MEMZERO128 && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__)) extern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c extern mi_decl_hidden size_t _mi_cpu_stosb_max; @@ -1594,27 +1598,34 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize mi_assert_internal(bsize > 0); mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || (uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); - - #if (MI_ARCH_ARM64 || MI_ARCH_X64) && defined(__SIZEOF_INT128__) // any 64-bit platform with 128-bit stores can benefit. + + #if MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and uintptr_t aligned) + #if defined(_MSC_VER) && defined(__AVX2__) + typedef __m128i __int128_t; + const __int128_t zero = _mm_setzero_si128(); + #else + const __int128_t zero = 0; + #endif if mi_unlikely(bsize < 16) { - *((uint64_t*)dst) = 0; + *((uint64_t*)dst) = 0; return dst; } - mi_assert_internal((uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); - void* const adst = mi_assume_aligned(dst,MI_MAX_ALIGN_SIZE); - __int128_t* const start = (__int128_t*)adst; - __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); - if mi_likely(bsize < 64) { - const size_t ofs = (bsize>>5)&1; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); + mi_assert_internal((uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); + mi_assert_internal(bsize % 16 == 0); + void* const adst = mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); + __int128_t* const start = (__int128_t*)adst; + __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); + if mi_likely(bsize < 64) { + const size_t ofs = (bsize>>5)&1; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); __int128_t* const end0 = end - ofs; - start[0] = 0; start[ofs] = 0; - end0[-1] = 0; end[-1] = 0; + start[0] = zero; start[ofs] = zero; + end0[-1] = zero; end[-1] = zero; return adst; } - if mi_likely(bsize <= 128) { - start[0] = 0; start[1] = 0; start[2] = 0; start[3] = 0; - end[-4] = 0; end[-3] = 0; end[-2] = 0; end[-1] = 0; + if mi_likely(bsize <= 128) { + start[0] = zero; start[1] = zero; start[2] = zero; start[3] = zero; + end[-4] = zero; end[-3] = zero; end[-2] = zero; end[-1] = zero; return adst; } #endif From f95e0e29c1407254751362c94a0508bd6fbbd3e7 Mon Sep 17 00:00:00 2001 From: David Carlier Date: Mon, 7 Sep 2026 20:47:50 +0100 Subject: [PATCH 212/280] guard the reset range in mi_segment_huge_page_alloc decommit_size is a ptrdiff_t passed as a size_t, so it wraps if the aligned block starts within sizeof(mi_block_t) of the page start. --- src/segment.c | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/src/segment.c b/src/segment.c index b2af61c9f..0bae59f91 100644 --- a/src/segment.c +++ b/src/segment.c @@ -1272,10 +1272,13 @@ static mi_page_t* mi_segment_huge_page_alloc(size_t size, size_t page_alignment, if (page_alignment > 0 && segment->allow_decommit && page->is_committed) { uint8_t* aligned_p = (uint8_t*)_mi_align_up((uintptr_t)start, page_alignment); mi_assert_internal(_mi_is_aligned(aligned_p, page_alignment)); + mi_assert_internal(aligned_p >= start); mi_assert_internal(psize - (aligned_p - start) >= size); uint8_t* decommit_start = start + sizeof(mi_block_t); // for the free list - ptrdiff_t decommit_size = aligned_p - decommit_start; - _mi_os_reset(decommit_start, decommit_size); // do not decommit as it may be in a region + if (aligned_p > decommit_start) { // any room for alignment ? + size_t decommit_size = (size_t)(aligned_p - decommit_start); + _mi_os_reset(decommit_start, decommit_size); // do not decommit as it may be in a region + } } return page; From f73b46312a7afb85ff606120328d070468e4f6aa Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 8 Sep 2026 20:08:24 -0700 Subject: [PATCH 213/280] adjust sampling countdown more aggressively --- include/mimalloc/internal.h | 12 ++- include/mimalloc/types.h | 14 ++-- src/page.c | 148 +++++++++++++++++++++++++----------- 3 files changed, 124 insertions(+), 50 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 762b7cfdb..99c1ab550 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -945,13 +945,21 @@ static inline size_t mi_page_alloc_count(const mi_page_t* page) { } static inline size_t mi_page_last_used(const mi_page_t* page) { - #if MI_INTPTR_SIZE >= 8 + #if MI_SIZE_SIZE >= 8 return (page->xused.used_alloc >> 32) & 0xFFFF; #else return page->xlast_used; #endif } +static inline size_t mi_page_last_alloc(const mi_page_t* page) { + #if MI_SIZE_SIZE >= 8 + return (page->xused.used_alloc >> 48) & 0xFFFF; + #else + return page->xlast_alloc; + #endif +} + // are all blocks in a page freed? // note: needs up-to-date used count, (as the `xthread_free` list may not be empty). see `_mi_page_collect_free`. @@ -1600,7 +1608,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || (uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); #if MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) - // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and uintptr_t aligned) + // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) #if defined(_MSC_VER) && defined(__AVX2__) typedef __m128i __int128_t; const __int128_t zero = _mm_setzero_si128(); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index a1fff909d..c018bee7b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -39,6 +39,9 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_MAX_ALIGN_SIZE 16 // sizeof(max_align_t) #endif +#if MI_MAX_ALIGN_SIZE < MI_INTPTR_SIZE +#error MI_MAX_ALIGN_SIZE must be at least MI_INTPTR_SIZE +#endif // ------------------------------------------------------ // Variants @@ -413,14 +416,14 @@ typedef uintptr_t mi_thread_free_t; // We need the `alloc_count` and `last_used` to efficiently calculate allocation and free statistics even // in a release build; this way we can update the stats in the slow path (`_mi_page_update_stats`). typedef union mi_used_s { - uintptr_t used_alloc; // used + alloc_count + size_t used_alloc; // used + alloc_count // the following struct is unused but nice for debugging struct { uint16_t used_count; uint16_t alloc_count; - #if MI_INTPTR_SIZE >= 8 + #if MI_SIZE_SIZE >= 8 uint16_t last_used; - uint16_t padding; + uint16_t last_alloc; #endif } debug_le; } mi_used_t; @@ -463,8 +466,9 @@ typedef struct mi_page_s { _Atomic(mi_threadid_t) xthread_id; // thread this page belongs to. (= `theap->thread_id (or 0 or 4 if abandoned) | page_flags`) mi_block_t* free; // list of available free blocks (`malloc` allocates from this list) mi_used_t xused; // number of blocks in use (including blocks in `thread_free`) (and the allocated count for statistics) - #if MI_INTPTR_SIZE < 8 - uint32_t xlast_used; // for statistics; on 64-bit platforms it is the upper 32-bits of xused. + #if MI_SIZE_SIZE < 8 + uint16_t xlast_used; // for statistics; on 64-bit platforms it is in bits 32..47 of xused. + uint16_t xlast_alloc; // for sampling; on 64-bit platforms it is in bits 48..63 of xused. #endif mi_block_t* local_free; // list of deferred free blocks by this thread (migrates to `free`) diff --git a/src/page.c b/src/page.c index 63649a2c9..31a3b8ae1 100644 --- a/src/page.c +++ b/src/page.c @@ -139,16 +139,55 @@ bool _mi_page_is_valid(mi_page_t* page) { // mi_assert_internal(mi_theap_contains_queue(mi_page_theap(page),pq)); } } + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); return true; } #endif +#if MI_STAT || MI_SAMPLE +// Gets the theap belonging to a page. +static mi_theap_t* mi_theap_of_page(mi_page_t* page) { + mi_theap_t* theap = page->theap; + if mi_unlikely(mi_page_thread_id(page) != _mi_prim_thread_id()) { + theap = _mi_page_associated_theap_peek(page); + } + return theap; +} +#endif + +#if MI_SAMPLE==1 /* for ==2, the countdown is already done at every `alloc.c:mi_page_alloc_zero` */ +static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count) +{ + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(theap!=NULL); + if (theap->sample_rate==0) return; + + const size_t last_alloc = mi_page_last_alloc(page); + if (alloc_count <= last_alloc) return; + + // update countdown + const size_t bsize = mi_page_usable_block_size(page); mi_assert_internal(bsize >= MI_PADDING_SIZE); + const uint64_t requested = (uint64_t)(alloc_count - last_alloc) * (uint64_t)(bsize - MI_PADDING_SIZE); + if (requested <= SIZE_MAX && theap->sample_countdown >= (size_t)requested) { + theap->sample_countdown -= requested; + } + else { + theap->sample_requested += (requested - theap->sample_countdown); + theap->sample_countdown = 0; + } +} +#else +static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count) { + MI_UNUSED(theap); MI_UNUSED(page); MI_UNUSED(alloc_count); +} +#endif + #if MI_STAT // Merge stats from the page into the corresponding theap or heap. -static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_t free_count ) { - // get heap and theap (maybe NULL) +static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, size_t alloc_count, size_t free_count ) { + // get heap (as the theap might be NULL) mi_heap_t* const heap = mi_page_heap(page); - mi_theap_t* const theap = _mi_page_associated_theap_peek(page); mi_assert_internal(theap == NULL || (theap->tld != NULL && _mi_thread_id() == theap->tld->thread_id)); mi_theapx_stat_counter_increase(heap,theap,pages_stat_updates,1); mi_theapx_stat_counter_increase(heap,theap,pages_stat_update_count, alloc_count + free_count); @@ -156,25 +195,13 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ // allocation sizes const size_t bsize = mi_page_usable_block_size(page); const uint64_t allocated = (uint64_t)alloc_count * (uint64_t)bsize; - const uint64_t requested = allocated - ((uint64_t)alloc_count * MI_PADDING_SIZE); const uint64_t freed = (uint64_t)free_count * (uint64_t)bsize; + #if MI_STAT==1 + const uint64_t requested = allocated - ((uint64_t)alloc_count * MI_PADDING_SIZE); + #endif mi_assert_internal(allocated <= INT64_MAX); // safe to cast to int64_t for stats mi_assert_internal(freed <= INT64_MAX); - // adjust sample countdown - #if MI_SAMPLE==1 - if (alloc_count>0 && theap!=NULL && theap->sample_rate!=0) { - // adjust countdown - if (requested < SIZE_MAX && theap->sample_countdown >= (size_t)requested) { - theap->sample_countdown -= (size_t)requested; - } - else { - theap->sample_countdown = 0; - theap->sample_requested += (requested - theap->sample_countdown); - } - } - #endif - // adjust stats if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { const size_t bin = _mi_bin(bsize); @@ -214,8 +241,9 @@ static void mi_page_merge_stats(const mi_page_t* page, size_t alloc_count, size_ } // Update stats for a page -void _mi_page_update_stats(mi_page_t* page) { +static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) { mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); // get stat counts const size_t used = mi_page_used(page); @@ -223,30 +251,73 @@ void _mi_page_update_stats(mi_page_t* page) { const size_t last_used = mi_page_last_used(page); mi_assert_internal(last_used + alloc_count >= used); const size_t free_count = last_used + alloc_count - used; + + #if MI_SAMPLE==1 // for ==2 it is already counted in every `alloc.c:mi_page_alloc_zero` + if (theap!=NULL) { mi_theap_adjust_sample_countdown(theap,page,alloc_count); } + #endif + + // update stats? if (alloc_count + free_count == 0) { mi_assert_internal(last_used == used); return; } - // reset the alloc_count and set last_used to used - #if MI_INTPTR_SIZE >= 8 + // reset the `alloc_count` (and `last_alloc`), and set `last_used` to `used` + #if MI_SIZE_SIZE >= 8 page->xused.used_alloc = (used << 32) | used; #else page->xused.used_alloc = used; page->xlast_used = used; + page->xlast_alloc = 0; #endif mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); - mi_page_merge_stats(page, alloc_count, free_count); + mi_theap_page_merge_stats(theap, page, alloc_count, free_count); } +void _mi_page_update_stats(mi_page_t* page) { // called on abandoned pages etc. + mi_theap_page_update_stats(mi_theap_of_page(page),page); +} #else +static void mi_page_update_stats(mi_page_t* page) { + MI_UNUSED(page); +} void _mi_page_update_stats(mi_page_t* page) { MI_UNUSED(page); } #endif +static void mi_page_update_sample_countdown(mi_page_t* page) +{ + // adjust count down + const size_t alloc_count = mi_page_alloc_count(page); + if (alloc_count==0) { + return; + } + else if mi_unlikely(alloc_count>=0x8000) { // if the count could overflow, update stats so the counter is reset + _mi_page_update_stats(page); + } + #if MI_SAMPLE==1 // if ==2 the countdown is already always counted in `alloc.c:mi_page_alloc_zero_ex` + else { + mi_theap_t* theap = mi_theap_of_page(page); + if (theap==NULL) return; + mi_theap_adjust_sample_countdown(theap,page,alloc_count); + // update last_alloc to alloc_count + #if MI_SIZE_SIZE >= 8 + mi_assert_internal(alloc_count <= UINT16_MAX); + page->xused.used_alloc = (alloc_count << 48) | (page->xused.used_alloc & (~MI_ZU(0) >> 16)); + #else + page->xlast_alloc = alloc_count; + #endif + mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); + mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); + + } + #endif +} + /* ----------------------------------------------------------- Page collect the `local_free` and `thread_free` lists @@ -306,14 +377,6 @@ static void mi_page_thread_free_collect(mi_page_t* page) } -// When we collect the local free list, the `mi_page_alloc_count` can -// potentially keep growing by repeated allocation/free in the same page. -// When it gets too large where it could overflow the 16 bits, we update -// the stats to reset the allocated counter. -static inline bool mi_page_has_high_alloc_count(const mi_page_t* page) { - return mi_xused_alloc_count(page->xused) > 0x7FFF; -} - // returns `true` if after collection `mi_page_immediate_available` is true. static inline bool mi_page_free_quick_collect(mi_page_t* page) { if mi_likely(page->free != NULL) return true; @@ -322,8 +385,7 @@ static inline bool mi_page_free_quick_collect(mi_page_t* page) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - mi_assert_internal(page->theap != NULL); - if (page->theap->sample_rate!=0 || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } + mi_page_update_sample_countdown(page); return true; } @@ -352,9 +414,8 @@ void _mi_page_free_collect(mi_page_t* page, bool force) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - } - mi_theap_t* const theap = _mi_page_associated_theap_peek(page); - if ((theap != NULL && theap->sample_rate!=0) || mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } + } + mi_page_update_sample_countdown(page); } mi_assert_internal(!force || page->local_free == NULL); } @@ -376,8 +437,8 @@ mi_block_t* _mi_page_free_collect_partly(mi_page_t* page, mi_block_t* head) { page->free = page->local_free; page->local_free = NULL; page->free_is_zero = false; - if (mi_page_has_high_alloc_count(page)) { _mi_page_update_stats(page); } - } + mi_page_update_sample_countdown(page); + } } if (mi_page_used(page) == 1) { // all elements are free'd since we skipped the `head` element itself @@ -574,7 +635,6 @@ void _mi_page_retire(mi_page_t* page) mi_attr_noexcept { if (index < theap->page_retired_min) theap->page_retired_min = index; if (index > theap->page_retired_max) theap->page_retired_max = index; mi_assert_internal(mi_page_all_free(page)); - // _mi_page_update_stats(page); return; // don't free after all } } @@ -1178,9 +1238,9 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) theap->generic_collect_count = 0; mi_theap_collect(theap, false /* force? */); } - else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling - mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ? - } + // else if (theap->sample_rate != 0) { // update stats more aggressively if we are sampling + // mi_theap_collect(theap, false /* force? */); // TODO: make specialized mi_theap_collect_update_stats ? + // } else { // otherwise we do a mini-collect _mi_deferred_free(theap, false); // call potential deferred free routines @@ -1220,6 +1280,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size } #if MI_SAMPLE==2 else { + // huge_alignment!=0 so we need to adjust the countdown mi_assert_internal(huge_alignment!=0); mi_assert_internal(theap->sample_countdown <= SIZE_MAX - req_size); sample_countdown_is_adjusted = true; @@ -1250,9 +1311,10 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size if (ppage!=NULL) { *ppage = page; } void* const p = _mi_page_malloc_zero(theap,page,size,zero); mi_assert_internal(p != NULL); - + mi_page_update_sample_countdown(page); + // move full pages to the full queue - // this will also call _mi_page_update_stats for huge pages + // this will also call _mi_page_update_stats for huge pages if (mi_page_block_size(page) > MI_SMALL_MAX_OBJ_SIZE) { if (mi_page_is_full(page)) { mi_page_to_full(page, mi_page_queue_of(page)); From b614a944c9801467c15eef653d09ddfb0b053010 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 8 Sep 2026 20:56:10 -0700 Subject: [PATCH 214/280] better cmake options for MI_PROFILE, and MI_GUARDED --- CMakeLists.txt | 58 ++++++++++++++++++++++++++++++++-------- include/mimalloc/types.h | 5 ++-- src/free.c | 5 ++-- src/options.c | 26 ++++++++++-------- src/page.c | 7 ----- 5 files changed, 68 insertions(+), 33 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index f7d9a4a5f..8f33edbd0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -10,7 +10,9 @@ set_property(CACHE MI_SECURE PROPERTY STRINGS "OFF" "ON" "FULL") option(MI_OVERRIDE "Override the standard malloc interface (i.e. define entry points for 'malloc', 'free', etc)" ON) option(MI_XMALLOC "Enable abort() call on memory allocation failure by default" OFF) option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) -option(MI_GUARDED "Build with guard pages behind certain object allocations (enabled by default in a debug build)" OFF) + +set(MI_GUARDED "DEFAULT" CACHE STRING "Build with ability to put guard pages behind certain object allocations") +set_property(CACHE MI_STAT PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") set(MI_USE_CXX "DEFAULT" CACHE STRING "Use the C++ compiler to compile the library (instead of the C compiler)") set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") @@ -18,11 +20,6 @@ set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") set(MI_OPT_ARCH "DEFAULT" CACHE STRING "Only for optimized builds: turn on architecture specific optimizations (for x64: '-march=haswell;-mavx2' (2013), for arm64: '-march=armv8.1-a' (2016), for riscV: '-march=rv64gcb_zacas' (2023))") set_property(CACHE MI_OPT_ARCH PROPERTY STRINGS "OFF" "ON" "avx2" "armv8.1" "armv8.3" "rva22" "rva23" "rv64gcb_zacas" "rv64gcb_zacas_zalasr" "DEFAULT") -option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) - -set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") -set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "OFF" "SHARED" "ON" "DEFAULT") - set(MI_DEBUG "DEFAULT" CACHE STRING "Enable assertion checks: OFF, ON, INTERNAL, FULL, DEFAULT. INTERNAL adds extra internal invariant checks, while FULL adds expensive heap invariant checks") set_property(CACHE MI_DEBUG PROPERTY STRINGS "OFF" "ON" "INTERNAL" "FULL" "DEFAULT") @@ -32,9 +29,17 @@ set_property(CACHE MI_TRACK PROPERTY STRINGS "OFF" "ASAN" "VALGRIND" "ETW") set(MI_STAT "DEFAULT" CACHE STRING "Enable statistics.") set_property(CACHE MI_STAT PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") +set(MI_PROFILE "DEFAULT" CACHE STRING "Enable profiling.") +set_property(CACHE MI_PROFILE PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") + +set(MI_FREE_IS_CHECKED "DEFAULT" CACHE STRING "Check for invalid pointers in 'mi_free': OFF, SHARED, ON, DEFAULT. 'SHARED' enables it only for the dynamic/shared library when MI_OVERRIDE is ON") +set_property(CACHE MI_FREE_IS_CHECKED PROPERTY STRINGS "OFF" "SHARED" "ON" "DEFAULT") + set(MI_ALLOW_THP "DEFAULT" CACHE STRING "Enable transparent huge pages on Linux: OFF, ON, FULL, DEFAULT. 'FULL' enables THP and also avoids splitting up huge pages during purging.") set_property(CACHE MI_ALLOW_THP PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") +option(MI_OPT_SIMD "Use SIMD instructions (requires MI_OPT_ARCH to be enabled)" OFF) + option(MI_DEBUG_TSAN "Build with thread sanitizer (needs clang)" OFF) option(MI_DEBUG_UBSAN "Build with undefined-behavior sanitizer (needs clang++)" OFF) @@ -437,13 +442,23 @@ elseif(MI_DEBUG) list(APPEND mi_defines MI_DEBUG=1) # assertion checking (mi_assert) endif() -if(MI_DEBUG AND !MI_GUARDED) - message(STATUS "Enable MI_GUARDED (since MI_DEBUG is enabled)") - set(MI_GUARDED=ON) +if(MI_GUARDED STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_GUARDED "FULL") + else() + set(MI_GUARDED "ON") + endif() endif() -if(MI_GUARDED) - message(STATUS "Compile guard pages behind certain object allocations (MI_GUARDED=ON)") + +if(MI_GUARDED STREQUAL "FULL") + message(STATUS "Enable fine-grained sampling for guard pages behind certain object allocations (MI_GUARDED=FULL)") + list(APPEND mi_defines MI_GUARDED=2) +elseif(MI_GUARDED) + message(STATUS "Enable sampled guard pages behind certain object allocations (MI_GUARDED=${MI_GUARDED})") list(APPEND mi_defines MI_GUARDED=1) +else() + # message(STATUS "Disable guard pages (MI_GUARDED=${MI_GUARDED})") + list(APPEND mi_defines MI_GUARDED=0) endif() if(MI_NO_PADDING) @@ -497,6 +512,25 @@ else() list(APPEND mi_defines MI_STAT=0) endif() +if(MI_PROFILE STREQUAL "DEFAULT") + if(MI_DEBUG) + set(MI_PROFILE "FULL") + else() + set(MI_PROFILE "ON") + endif() +endif() + +if(MI_PROFILE STREQUAL "FULL") + message(STATUS "Enable fine-grained sampled profiling (MI_PROFILE=FULL)") + list(APPEND mi_defines MI_PROFILE=2) +elseif(MI_PROFILE) + message(STATUS "Enable sampled profiling (MI_PROFILE=${MI_PROFILE})") + list(APPEND mi_defines MI_PROFILE=1) +else() + message(STATUS "Disable profiling (MI_PROFILE=${MI_PROFILE})") + list(APPEND mi_defines MI_PROFILE=0) +endif() + if(CMAKE_SYSTEM_NAME STREQUAL "WASI") message(STATUS "Enabling MI_FREE_USE_PAGEMAP since the target (WASI) does not support large OS aligned allocations well") set(MI_FREE_USE_PAGEMAP ON) @@ -1080,6 +1114,8 @@ if (MI_BUILD_TESTS) add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) elseif(TEST_NAME STREQUAL "stress-heaps") add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=0 $) + elseif(NOT MI_PROFILE AND (TEST_NAME STREQUAL "profile")) + # don't add as a test else() add_test(NAME test-${TEST_NAME} COMMAND mimalloc-test-${TEST_NAME}) endif() diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index c018bee7b..3b63185a5 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -83,7 +83,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Statistics (0=only essential, 1=normal, 2=more expensive tracking of precise requested bytes) +// Statistics (0=only essential, 1=detailed (fast, can be enabled always), 2=more expensive tracking of precise requested bytes) #ifndef MI_STAT #if (MI_DEBUG>0) #define MI_STAT 2 @@ -92,7 +92,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Enable profiling support +// Enable profiling support (0=off, 1=fast, can be enabled always, 2=allow fine-grained sample rates, more expensive(<64 KiB)) #ifndef MI_PROFILE #define MI_PROFILE 1 #endif @@ -102,6 +102,7 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_GUARDED 1 #endif +// For profiling or guarded pages we need to sample every once in a while. (1=fast, 2=allow fine grained sample rates, more expensive (<64KiB)) #if MI_PROFILE || MI_GUARDED #if MI_PROFILE>1 || MI_GUARDED>1 #define MI_SAMPLE 2 diff --git a/src/free.c b/src/free.c index 512bdcc0e..3d908a747 100644 --- a/src/free.c +++ b/src/free.c @@ -128,6 +128,7 @@ static void mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p); #endif static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool* was_guarded) mi_attr_noexcept { + MI_UNUSED(was_guarded); if mi_likely(!mi_page_has_interior_pointers(page)) { return mi_validate_block_from_ptr(page,p); } @@ -147,8 +148,8 @@ static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool mi_page_block_unguard(page, block, p); *was_guarded = true; } - #else - { MI_UNUSED(was_guarded); } + #else + { } #endif } #endif diff --git a/src/options.c b/src/options.c index f1e063202..32ba894c9 100644 --- a/src/options.c +++ b/src/options.c @@ -77,10 +77,10 @@ int mi_version(void) { #endif #ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE -#if MI_GUARDED && !MI_DEBUG -#define MI_DEFAULT_GUARDED_SAMPLE_RATE 4000 +#if MI_GUARDED && MI_DEBUG +#define MI_DEFAULT_GUARDED_SAMPLE_RATE MI_MiB #else -#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 #endif #endif @@ -251,24 +251,28 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); #if MI_GUARDED - _mi_fprintf(out, arg, "guarded build: %s\n", mi_option_get(mi_option_guarded_sample_rate) != 0 ? "enabled" : "disabled"); - #endif + _mi_fprintf(out, arg, "guarded mode: %s, rate=%ld\n", MI_SAMPLE==2 ? "fine-grained" : "enabled", mi_option_get(mi_option_guarded_sample_rate)); + #endif #if MI_TSAN _mi_fprintf(out, arg, "thread santizer enabled\n"); #endif + #if MI_PROFILE + _mi_fprintf(out, arg, "profiling : %s\n", MI_SAMPLE==2 ? "fine-grained" : "enabled"); + #endif + #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED - _mi_fprintf(out, arg, "free: (small) aligned, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : (small) aligned, page size: %zu\n", sizeof(mi_page_t)); #elif MI_PAGE_META_IS_ALIGNED - _mi_fprintf(out, arg, "free: aligned, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : aligned, page size: %zu\n", sizeof(mi_page_t)); #elif MI_PAGE_META_SMALL_IS_ALIGNED - _mi_fprintf(out, arg, "free: small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : small aligned + pagemap, page size: %zu\n", sizeof(mi_page_t)); #elif MI_FREE_IS_CHECKED - _mi_fprintf(out, arg, "free: checked, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : checked, page size: %zu\n", sizeof(mi_page_t)); #else - _mi_fprintf(out, arg, "free: pagemap, page size: %zu\n", sizeof(mi_page_t)); + _mi_fprintf(out, arg, "free mode : pagemap, page size: %zu\n", sizeof(mi_page_t)); #endif #if MI_ENCODE_FREELIST - _mi_fprintf(out, arg, "free lists: encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); + _mi_fprintf(out, arg, "free lists : encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); #endif } diff --git a/src/page.c b/src/page.c index 31a3b8ae1..57018abd6 100644 --- a/src/page.c +++ b/src/page.c @@ -177,10 +177,6 @@ static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, theap->sample_countdown = 0; } } -#else -static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, size_t alloc_count) { - MI_UNUSED(theap); MI_UNUSED(page); MI_UNUSED(alloc_count); -} #endif #if MI_STAT @@ -281,9 +277,6 @@ void _mi_page_update_stats(mi_page_t* page) { // called on abandoned pag } #else -static void mi_page_update_stats(mi_page_t* page) { - MI_UNUSED(page); -} void _mi_page_update_stats(mi_page_t* page) { MI_UNUSED(page); } From d58665096c78ca5679024bcb371cb9f0f322dfc4 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 8 Sep 2026 20:58:19 -0700 Subject: [PATCH 215/280] rename MI_STAT to MI_STATS --- CMakeLists.txt | 28 ++++++++++++++-------------- include/mimalloc/types.h | 10 +++++----- src/alloc.c | 4 ++-- src/free.c | 6 +++--- src/page.c | 10 +++++----- src/stats.c | 12 ++++++------ 6 files changed, 35 insertions(+), 35 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8f33edbd0..c7289186b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -12,7 +12,7 @@ option(MI_XMALLOC "Enable abort() call on memory allocation failure by option(MI_SHOW_ERRORS "Show error and warning messages by default (only enabled by default in DEBUG mode)" OFF) set(MI_GUARDED "DEFAULT" CACHE STRING "Build with ability to put guard pages behind certain object allocations") -set_property(CACHE MI_STAT PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") +set_property(CACHE MI_GUARDED PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") set(MI_USE_CXX "DEFAULT" CACHE STRING "Use the C++ compiler to compile the library (instead of the C compiler)") set_property(CACHE MI_USE_CXX PROPERTY STRINGS "OFF" "ON" "DEFAULT") @@ -26,8 +26,8 @@ set_property(CACHE MI_DEBUG PROPERTY STRINGS "OFF" "ON" "INTERNAL" "FULL" "DEFAU set(MI_TRACK "OFF" CACHE STRING "Enable tracking: OFF, ASAN (address sanitizer), VALGRIND, ETW (Windows event tracing). Adds a small overhead.") set_property(CACHE MI_TRACK PROPERTY STRINGS "OFF" "ASAN" "VALGRIND" "ETW") -set(MI_STAT "DEFAULT" CACHE STRING "Enable statistics.") -set_property(CACHE MI_STAT PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") +set(MI_STATS "DEFAULT" CACHE STRING "Enable statistics.") +set_property(CACHE MI_STATS PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") set(MI_PROFILE "DEFAULT" CACHE STRING "Enable profiling.") set_property(CACHE MI_PROFILE PROPERTY STRINGS "OFF" "ON" "FULL" "DEFAULT") @@ -493,23 +493,23 @@ else() message(STATUS "Invalid value for MI_FREE_IS_CHECKED: ${MI_FREE_IS_CHECKED} (expecting: DEFAULT, OFF, SHARED, or ON)") endif() -if(MI_STAT STREQUAL "DEFAULT") +if(MI_STATS STREQUAL "DEFAULT") if(MI_DEBUG) - set(MI_STAT "FULL") + set(MI_STATS "FULL") else() - set(MI_STAT "ON") + set(MI_STATS "ON") endif() endif() -if(MI_STAT STREQUAL "FULL") - message(STATUS "Enable detailed statistics including precise requested bytes (expensive) (MI_STAT=FULL)") - list(APPEND mi_defines MI_STAT=2) -elseif(MI_STAT) - message(STATUS "Enable detailed statistics (MI_STAT=${MI_STAT})") - list(APPEND mi_defines MI_STAT=1) +if(MI_STATS STREQUAL "FULL") + message(STATUS "Enable detailed statistics including precise requested bytes (expensive) (MI_STATS=FULL)") + list(APPEND mi_defines MI_STATS=2) +elseif(MI_STATS) + message(STATUS "Enable detailed statistics (MI_STATS=${MI_STATS})") + list(APPEND mi_defines MI_STATS=1) else() - message(STATUS "Disable detailed statistics (MI_STAT=${MI_STAT})") - list(APPEND mi_defines MI_STAT=0) + message(STATUS "Disable detailed statistics (MI_STATS=${MI_STATS})") + list(APPEND mi_defines MI_STATS=0) endif() if(MI_PROFILE STREQUAL "DEFAULT") diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 3b63185a5..5e7cfb466 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -55,8 +55,8 @@ terms of the MIT license. A copy of the license can be found in the file // #define MI_TRACK_ASAN 1 // #define MI_TRACK_ETW 1 -// Define MI_STAT as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). -// #define MI_STAT 1 +// Define MI_STATS as 1 to maintain statistics; set it to 2 to have detailed statistics (but costs some performance). +// #define MI_STATS 1 // Define MI_SECURE to enable security mitigations // #define MI_SECURE 1 // check invalid pointer free, guard pages around meta data, randomize arena allocation addresses (like ASLR), abort on detected meta data corruption @@ -84,11 +84,11 @@ terms of the MIT license. A copy of the license can be found in the file #endif // Statistics (0=only essential, 1=detailed (fast, can be enabled always), 2=more expensive tracking of precise requested bytes) -#ifndef MI_STAT +#ifndef MI_STATS #if (MI_DEBUG>0) -#define MI_STAT 2 +#define MI_STATS 2 #else -#define MI_STAT 1 +#define MI_STATS 1 #endif #endif diff --git a/src/alloc.c b/src/alloc.c index 3a78bf0d3..84f4a2a96 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -94,7 +94,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ MI_UNUSED(sample_countdown); #endif - #if MI_STAT>=2 + #if MI_STATS>=2 mi_theap_stat_counter_increase(theap,malloc_requested,size - MI_PADDING_SIZE); #endif @@ -947,7 +947,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, mi_track_malloc(p, usable_size, zero); if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); - #if MI_STAT + #if MI_STATS // adjust request stats to only count the allocated size of the block (and not the guard page) mi_theap_stat_counter_decrease(theap, malloc_requested, req_size); mi_theap_stat_counter_increase(theap, malloc_requested, size); diff --git a/src/free.c b/src/free.c index 3d908a747..1b562a64d 100644 --- a/src/free.c +++ b/src/free.c @@ -774,7 +774,7 @@ mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, co #endif // only maintain stats for smaller objects if requested -#if 0 && (MI_STAT>0) +#if 0 && (MI_STATS>0) static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { MI_UNUSED(block); mi_theap_t* theap = _mi_theap_default(); @@ -790,13 +790,13 @@ static void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { } const size_t bsize = mi_page_usable_block_size(page); - // #if (MI_STAT>=2) + // #if (MI_STATS>=2) // const size_t usize = mi_page_usable_size_of(page, block); // mi_theap_stat_decrease(theap, malloc_requested, usize); // #endif if (bsize <= MI_LARGE_MAX_OBJ_SIZE) { mi_theap_stat_decrease(theap, malloc_normal, bsize); - #if (MI_STAT > 1) + #if (MI_STATS > 1) mi_theap_stat_decrease(theap, malloc_bins[_mi_bin(bsize)], 1); #endif } diff --git a/src/page.c b/src/page.c index 57018abd6..110e41e3b 100644 --- a/src/page.c +++ b/src/page.c @@ -145,7 +145,7 @@ bool _mi_page_is_valid(mi_page_t* page) { } #endif -#if MI_STAT || MI_SAMPLE +#if MI_STATS || MI_SAMPLE // Gets the theap belonging to a page. static mi_theap_t* mi_theap_of_page(mi_page_t* page) { mi_theap_t* theap = page->theap; @@ -179,7 +179,7 @@ static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, } #endif -#if MI_STAT +#if MI_STATS // Merge stats from the page into the corresponding theap or heap. static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, size_t alloc_count, size_t free_count ) { // get heap (as the theap might be NULL) @@ -192,7 +192,7 @@ static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, const size_t bsize = mi_page_usable_block_size(page); const uint64_t allocated = (uint64_t)alloc_count * (uint64_t)bsize; const uint64_t freed = (uint64_t)free_count * (uint64_t)bsize; - #if MI_STAT==1 + #if MI_STATS==1 const uint64_t requested = allocated - ((uint64_t)alloc_count * MI_PADDING_SIZE); #endif mi_assert_internal(allocated <= INT64_MAX); // safe to cast to int64_t for stats @@ -206,7 +206,7 @@ static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, mi_theapx_stat_counter_increase(heap, theap, malloc_normal_count, alloc_count); mi_theapx_stat_increase(heap, theap, malloc_normal, allocated); mi_theapx_stat_increase(heap, theap, malloc_bins[bin], alloc_count); - #if MI_STAT==1 + #if MI_STATS==1 // use coarse total requested bytes mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); #endif @@ -224,7 +224,7 @@ static void mi_theap_page_merge_stats(mi_theap_t* theap, const mi_page_t* page, if (alloc_count > 0) { mi_theapx_stat_counter_increase(heap, theap, malloc_huge_count, alloc_count); mi_theapx_stat_increase(heap, theap, malloc_huge, allocated); - #if MI_STAT==1 + #if MI_STATS==1 // use coarse total requested bytes mi_theapx_stat_counter_increase(heap, theap, malloc_requested, requested); #endif diff --git a/src/stats.c b/src/stats.c index 8a13a4eeb..854282b07 100644 --- a/src/stats.c +++ b/src/stats.c @@ -127,7 +127,7 @@ static void mi_stats_add(mi_stats_t* stats, const mi_stats_t* src) { // copy all fields MI_STAT_FIELDS() - #if MI_STAT + #if MI_STATS for (size_t i = 0; i <= MI_BIN_HUGE; i++) { mi_stat_count_add_mt(&stats->malloc_bins[i], &src->malloc_bins[i]); } @@ -237,7 +237,7 @@ static void mi_stat_print(const mi_stat_count_t* stat, const char* msg, int64_t mi_stat_print_ex(stat, msg, unit, out, arg, NULL); } -#if MI_STAT +#if MI_STATS static void mi_stat_total_print(const mi_stat_counter_t* stat, const char* msg, mi_output_fun* out, void* arg) { _mi_fprintf(out, arg, " %-12s:", msg); _mi_fprintf(out, arg, "%12s", " "); // no peak @@ -271,7 +271,7 @@ static void mi_print_header(const char* name,mi_output_fun* out, void* arg ) { name, "peak ", "total ", "current ", "block ", "total# "); } -#if MI_STAT +#if MI_STATS static bool mi_stats_print_bins(const mi_stat_count_t* bins, size_t max, mi_output_fun* out, void* arg) { bool found = false; char buf[64]; @@ -363,18 +363,18 @@ void _mi_stats_print(const char* name, size_t id, const mi_stats_t* stats, mi_ou _mi_fprintf(out, arg, "%s %zu\n", name, id); if (stats->malloc_normal.total + stats->malloc_huge.total != 0) { - #if MI_STAT + #if MI_STATS mi_print_header("blocks", out, arg); mi_stats_print_bins(stats->malloc_bins, MI_BIN_HUGE, out, arg); #endif - #if MI_STAT + #if MI_STATS mi_stat_print(&stats->malloc_normal, "binned", -stats->malloc_normal_count.total, out, arg); mi_stat_print(&stats->malloc_huge, "huge", -stats->malloc_huge_count.total, out, arg); mi_stat_count_t total = { 0,0,0 }; mi_stat_count_add_mt(&total, &stats->malloc_normal); mi_stat_count_add_mt(&total, &stats->malloc_huge); mi_stat_print_ex(&total, "total", -(stats->malloc_normal_count.total + stats->malloc_huge_count.total), out, arg, ""); - #if MI_STAT>=2 + #if MI_STATS>=2 mi_stat_total_print(&stats->malloc_requested, "malloc req", out, arg); #else mi_stat_total_print(&stats->malloc_requested, "malloc req~", out, arg); From 60dee63ff04cadb252a1794608bb7da448d5a6f6 Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 8 Sep 2026 21:53:50 -0700 Subject: [PATCH 216/280] fix guarded sampling --- .github/workflows/test.yaml | 8 ++++---- CMakeLists.txt | 2 +- include/mimalloc/internal.h | 4 ++-- include/mimalloc/types.h | 14 ++++++++------ src/alloc.c | 4 ++-- src/profile.c | 4 ++-- src/theap.c | 2 +- 7 files changed, 20 insertions(+), 18 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index d95821599..e10ffb874 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -238,20 +238,20 @@ jobs: - name: Debug, Guarded if: matrix.tests == 'extra' && runner.os != 'macOS' run: | - cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=ON + cmake . -B out/debug-guarded -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG=FULL -DMI_GUARDED=FULL cmake --build out/debug-guarded --parallel 8 --config Debug ctest --test-dir out/debug-guarded --verbose --timeout 240 -C Debug env: - MIMALLOC_GUARDED_SAMPLE_RATE: 10 + MIMALLOC_GUARDED_SAMPLE_RATE: 1024 - name: Release, Guarded if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') run: | - cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=ON + cmake . -B out/release-guarded -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_GUARDED=FULL cmake --build out/release-guarded --parallel 8 --config Release ctest --test-dir out/release-guarded --verbose --timeout 240 -C Release env: - MIMALLOC_GUARDED_SAMPLE_RATE: 10 + MIMALLOC_GUARDED_SAMPLE_RATE: 128 - name: Debug, Use pagemap if: matrix.tests == 'extra' && runner.os == 'Linux' && startsWith(github.ref_name,'dev3') diff --git a/CMakeLists.txt b/CMakeLists.txt index c7289186b..0ca742a97 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1111,7 +1111,7 @@ if (MI_BUILD_TESTS) message(STATUS "cannot build TSAN tests without MI_BUILD_SHARED being enabled") endif() if(MI_GUARDED AND NOT (TEST_NAME STREQUAL "profile")) - add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1 $) + add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_GUARDED_SAMPLE_RATE=1024 $) elseif(TEST_NAME STREQUAL "stress-heaps") add_test(NAME test-${TEST_NAME} COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=0 $) elseif(NOT MI_PROFILE AND (TEST_NAME STREQUAL "profile")) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 99c1ab550..0aca5d75d 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1264,7 +1264,7 @@ static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { } -static inline size_t mi_theap_disable_profiler(mi_theap_t* theap) { +static inline size_t mi_theap_disable_profiling(mi_theap_t* theap) { const size_t sample_rate = theap->profile_sample_rate; theap->profile_sample_rate = 0; // theap->profile_sample_countdown = 0; @@ -1272,7 +1272,7 @@ static inline size_t mi_theap_disable_profiler(mi_theap_t* theap) { return sample_rate; } -static inline void mi_theap_enable_profiler(mi_theap_t* theap, size_t sample_rate) { +static inline void mi_theap_enable_profiling(mi_theap_t* theap, size_t sample_rate) { theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); if (theap->profile_sample_countdown==0) { theap->profile_sample_countdown = theap->profile_sample_rate; } if (theap->sample_rate==0 || theap->sample_rate > theap->profile_sample_rate) { diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 5e7cfb466..cac057e68 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -83,7 +83,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Statistics (0=only essential, 1=detailed (fast, can be enabled always), 2=more expensive tracking of precise requested bytes) +// Statistics (0=only essential, 1=detailed (fast, can be enabled always), 2=slightly more expensive tracking of precise requested bytes) #ifndef MI_STATS #if (MI_DEBUG>0) #define MI_STATS 2 @@ -92,7 +92,7 @@ terms of the MIT license. A copy of the license can be found in the file #endif #endif -// Enable profiling support (0=off, 1=fast, can be enabled always, 2=allow fine-grained sample rates, more expensive(<64 KiB)) +// Enable profiling support (0=off, 1=fast, can be enabled always, 2=allow fine-grained sample rates(<64 KiB), a tad more expensive) #ifndef MI_PROFILE #define MI_PROFILE 1 #endif @@ -102,12 +102,14 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_GUARDED 1 #endif -// For profiling or guarded pages we need to sample every once in a while. (1=fast, 2=allow fine grained sample rates, more expensive (<64KiB)) -#if MI_PROFILE || MI_GUARDED +// For profiling or guarded pages we need to sample every once in a while. (0=no sampling, 1=fast, 2=allow fine grained sample rates(<64 KiB), a tad more expensive) +#ifndef MI_SAMPLE #if MI_PROFILE>1 || MI_GUARDED>1 -#define MI_SAMPLE 2 + #define MI_SAMPLE 2 +#elif MI_PROFILE || MI_GUARDED + #define MI_SAMPLE 1 #else -#define MI_SAMPLE 1 + #define MI_SAMPLE 0 #endif #endif diff --git a/src/alloc.c b/src/alloc.c index 84f4a2a96..43ee1e8c2 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -996,8 +996,8 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* thea mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0); bool sample_profile = false; bool sample_guarded = false; - if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = true; } - if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = true; } + if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = (theap->profile_sample_rate!=0); } + if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = (theap->guarded_sample_rate!=0); } // invoke callback? if (sample_profile) { diff --git a/src/profile.c b/src/profile.c index f41b349e2..82178e585 100644 --- a/src/profile.c +++ b/src/profile.c @@ -60,7 +60,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th if (prof->on_alloc!=NULL) { const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { - mi_theap_enable_profiler(theap,new_sample_rate); + mi_theap_enable_profiling(theap,new_sample_rate); } mi_theap_stat_counter_increase(theap,profile_samples,1); } @@ -116,7 +116,7 @@ bool mi_profiler_start(const mi_profiler_t* profiler ) { if (mi_heap_profiler(heap)==profiler) { mi_theap_t* theap = _mi_heap_theap_peek(heap); if (theap!=NULL) { - mi_theap_enable_profiler(theap,1); + mi_theap_enable_profiling(theap,1); } } return false; diff --git a/src/theap.c b/src/theap.c index f8d5b2b5e..abf874339 100644 --- a/src/theap.c +++ b/src/theap.c @@ -215,7 +215,7 @@ mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t mi static void mi_theap_guarded_init(mi_theap_t* theap) { mi_theap_guarded_set_sample_rate(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, MI_SAMPLE_RATE_MAX), (size_t)mi_option_get(mi_option_guarded_sample_seed)); mi_theap_guarded_set_size_bound(theap, (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), From fc4665d690d4cacc8bc3b501a0793b924e06367e Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 09:43:41 -0700 Subject: [PATCH 217/280] reorganize sampling code --- CMakeLists.txt | 11 +- ide/vs2022/mimalloc-lib.vcxproj | 3 +- ide/vs2022/mimalloc-lib.vcxproj.filters | 5 +- ide/vs2022/mimalloc-override-dll.vcxproj | 3 +- .../mimalloc-override-dll.vcxproj.filters | 5 +- include/mimalloc/internal.h | 63 ++--- src/alloc-aligned.c | 2 +- src/alloc.c | 155 +----------- src/free.c | 38 +-- src/page.c | 4 +- src/profile.c | 127 ---------- src/sample-guarded.c | 162 +++++++++++++ src/sample-profile.c | 223 ++++++++++++++++++ src/static.c | 13 +- src/theap.c | 45 +--- test/test-profile.c | 9 +- 16 files changed, 440 insertions(+), 428 deletions(-) delete mode 100644 src/profile.c create mode 100644 src/sample-guarded.c create mode 100644 src/sample-profile.c diff --git a/CMakeLists.txt b/CMakeLists.txt index 0ca742a97..7fb3b31ac 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -116,15 +116,16 @@ set(mi_sources src/options.c src/os.c src/page.c - src/page-map.c - src/profile.c + src/page-map.c + src/prim/prim.c + src/prim/prim-tls.c src/random.c + src/sample-guarded.c + src/sample-profile.c src/stats.c src/subproc.c src/theap.c - src/threadlocal.c - src/prim/prim.c - src/prim/prim-tls.c) + src/threadlocal.c) set(mi_cflags "") set(mi_cflags_static "") # extra flags for a static library build diff --git a/ide/vs2022/mimalloc-lib.vcxproj b/ide/vs2022/mimalloc-lib.vcxproj index 29ebec431..9a17c0ae3 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj +++ b/ide/vs2022/mimalloc-lib.vcxproj @@ -451,7 +451,8 @@ - + + diff --git a/ide/vs2022/mimalloc-lib.vcxproj.filters b/ide/vs2022/mimalloc-lib.vcxproj.filters index 49b5c3e7e..c140f17cf 100644 --- a/ide/vs2022/mimalloc-lib.vcxproj.filters +++ b/ide/vs2022/mimalloc-lib.vcxproj.filters @@ -70,7 +70,10 @@ Sources - + + Sources + + Sources diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj b/ide/vs2022/mimalloc-override-dll.vcxproj index cde7d3d6f..c2338bb2e 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj +++ b/ide/vs2022/mimalloc-override-dll.vcxproj @@ -481,7 +481,8 @@ - + + diff --git a/ide/vs2022/mimalloc-override-dll.vcxproj.filters b/ide/vs2022/mimalloc-override-dll.vcxproj.filters index 2f9beb4ce..8bced90ae 100644 --- a/ide/vs2022/mimalloc-override-dll.vcxproj.filters +++ b/ide/vs2022/mimalloc-override-dll.vcxproj.filters @@ -70,7 +70,10 @@ Sources - + + Sources + + Sources diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 0aca5d75d..da07174e5 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -339,11 +339,20 @@ size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr bool _mi_page_is_valid(mi_page_t* page); #endif -// "profile.c" -mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +// "sample-guarded.c" mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p); +void _mi_page_unguard_all(mi_page_t* page); +void _mi_theap_guarded_init(mi_theap_t* theap); + +// "sample-profile.c" +mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept; +size_t _mi_theap_update_sample_rate(mi_theap_t* theap); + mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p); +size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate); + // ------------------------------------------------------ // Assertions @@ -514,6 +523,12 @@ typedef struct mi_option_desc_s { // initialize a local variable to zero; use memset as compilers optimize constant sized memset's #define _mi_memzero_var(x) memset(&x,0,sizeof(x)) +// minimum +static inline size_t mi_min(size_t x, size_t y) { return (x <= y ? x : y); } + +// maximum +static inline size_t mi_max(size_t x, size_t y) { return (x >= y ? x : y); } + // Is `x` a power of two? (0 is considered a power of two) static inline bool _mi_is_power_of_two(uintptr_t x) { return ((x & (x - 1)) == 0); @@ -1202,20 +1217,6 @@ static inline bool mi_theap_should_sample(mi_theap_t* theap, size_t req_size) { return ((mi_ssize_t)theap->sample_countdown < (mi_ssize_t)req_size); } -#if MI_SAMPLE==2 // fine grained -static inline bool mi_theap_sample_small(mi_theap_t* theap, size_t req_size) { - mi_assert_internal(req_size <= SIZE_MAX/2); - const size_t sample_countdown = theap->sample_countdown - req_size; - if mi_likely((mi_ssize_t)sample_countdown >= 0) { - theap->sample_countdown = sample_countdown; - return false; - } - else { - return true; - } -} -#endif - // we always align guarded pointers in a block at an offset // the block `next` field is then used as a tag to distinguish regular offset aligned blocks from guarded ones #define MI_BLOCK_TAG_ALIGNED ((mi_encoded_t)(0)) @@ -1233,17 +1234,7 @@ static inline bool mi_block_ptr_is_guarded(const mi_block_t* block, const void* #endif } -static inline bool mi_block_ptr_is_profiled(const mi_block_t* block, const void* p) { -#if MI_PROFILE - const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; - return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next == MI_BLOCK_TAG_PROFILED); -#else - MI_UNUSED(block); MI_UNUSED(p); - return false; -#endif -} - -static inline bool mi_block_ptr_is_profiled_or_guarded(const mi_block_t* block, const void* p) { +static inline bool mi_block_ptr_is_sampled(const mi_block_t* block, const void* p) { #if MI_GUARDED || MI_PROFILE const ptrdiff_t offset = (uint8_t*)p - (uint8_t*)block; return (offset >= (ptrdiff_t)(sizeof(mi_block_t)) && block->next != MI_BLOCK_TAG_ALIGNED); @@ -1264,24 +1255,6 @@ static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { } -static inline size_t mi_theap_disable_profiling(mi_theap_t* theap) { - const size_t sample_rate = theap->profile_sample_rate; - theap->profile_sample_rate = 0; - // theap->profile_sample_countdown = 0; - theap->sample_rate = theap->guarded_sample_rate; - return sample_rate; -} - -static inline void mi_theap_enable_profiling(mi_theap_t* theap, size_t sample_rate) { - theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); - if (theap->profile_sample_countdown==0) { theap->profile_sample_countdown = theap->profile_sample_rate; } - if (theap->sample_rate==0 || theap->sample_rate > theap->profile_sample_rate) { - theap->sample_rate = theap->profile_sample_rate; - theap->sample_countdown = theap->profile_sample_countdown; // TODO: track any previous allocation countdown into sample_requested - } -} - - /* ------------------------------------------------------------------- Encoding/Decoding the free list next pointers diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 17da719ee..967571ced 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -134,7 +134,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap #if MI_DEBUG > 1 mi_page_t* const apage = _mi_ptr_page(aligned_p); mi_block_t* unalign_p = _mi_page_ptr_unalign(apage, aligned_p); - mi_assert_internal(p == (void*)unalign_p || mi_block_ptr_is_profiled_or_guarded(unalign_p, aligned_p)); + mi_assert_internal(p == (void*)unalign_p || mi_block_ptr_is_sampled(unalign_p, aligned_p)); #endif if (p != aligned_p) { diff --git a/src/alloc.c b/src/alloc.c index 43ee1e8c2..561865f9e 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -143,9 +143,6 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz return mi_page_malloc_zero(theap, page, size, theap->sample_countdown, zero, NULL); } -// main allocation primitives for small and generic allocation -mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; - // internal small size allocation static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { @@ -160,7 +157,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic) #if MI_SAMPLE==2 - if mi_unlikely(mi_theap_should_sample(theap,size)) { return _mi_theap_malloc_sample(theap,size,zero,ppage); } + if mi_unlikely(mi_theap_should_sample(theap,size)) { return _mi_theap_malloc_sampled(theap,size,zero,ppage); } #endif // get page in constant time @@ -871,156 +868,6 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { } } - -// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). -// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) -// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` -static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { - // todo: we can still make padding work by moving it out of the guard page area - mi_page_t* const page = _mi_ptr_page(block); - mi_page_set_has_interior_pointers(page, true); - block->next = MI_BLOCK_TAG_GUARDED; - - // set guard page at the end of the block - const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` - const size_t os_page_size = _mi_os_page_size(); - mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); - if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { - // should never happen - mi_free(block); - return NULL; - } - uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; - // note: the alignment of the guard page relies on blocks being os_page_size aligned which - // is ensured in `mi_arena_page_alloc_fresh`. - mi_assert_internal(_mi_is_aligned(block, os_page_size)); - mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); - if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) { - const bool ok = _mi_os_protect(guard_page, os_page_size); - if mi_unlikely(!ok) { - _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); - } - } - else { - _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); - } - - // align pointer just in front of the guard page - size_t offset = block_size - os_page_size - obj_size; - mi_assert_internal(offset > sizeof(mi_block_t)); - if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) { - // give up to place it right in front of the guard page if the offset is too large for unalignment - offset = MI_PAGE_MAX_OVERALLOC_ALIGN; - } - uint8_t* const p = (uint8_t*)block + offset; - mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN); - if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); } - mi_track_align(block, p, offset, obj_size); - mi_track_mem_defined(block, sizeof(mi_block_t)); - return p; -} - -mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept -{ - // allocate multiple of page size ending in a guard page - // ensure minimal alignment requirement? - if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow - _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size); - return NULL; - } - const size_t os_page_size = _mi_os_page_size(); - const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); - const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); - const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); - // const size_t threshold = mi_theap_disable_profiler(theap); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); - // mi_theap_enable_profiler(theap,threshold); - if (block==NULL) return NULL; - size_t usable_size = 0; - void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); - if (p == NULL) return NULL; - if (zero) { - _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) - } - - // stats - mi_track_malloc(p, usable_size, zero); - if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } - mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); - #if MI_STATS - // adjust request stats to only count the allocated size of the block (and not the guard page) - mi_theap_stat_counter_decrease(theap, malloc_requested, req_size); - mi_theap_stat_counter_increase(theap, malloc_requested, size); - #endif - #if MI_DEBUG>3 - if (zero) { - mi_assert_expensive(mi_mem_is_zero(p, size)); - } - #endif - return p; -} - -mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sample(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE) - mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE); - mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) || - (theap->sample_countdown <= MI_SAMPLE_COUNTDOWN_MAX && theap->sample_countdown < req_size)); - const size_t size = req_size + MI_PADDING_SIZE; - - // handle empty theap and disabled profiling - if (theap->sample_rate==0) { - if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap - theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time - } - return _mi_malloc_generic_no_sample(theap,size,zero,ppage); - } - - // update countdown - mi_assert_internal(!_mi_is_empty_theap(theap)); - mi_assert_internal(req_size <= SIZE_MAX/2); - mi_assert_internal(theap->sample_rate > 0); - mi_assert_internal(theap->sample_rate <= SIZE_MAX/2); - mi_assert_internal(theap->sample_rate >= theap->sample_countdown); - - const uint64_t requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested; - mi_assert_internal(requested > 0); - mi_assert_internal(requested >= size); - if (requested < size) { - _mi_error_message(EFAULT,"wrong requested/size"); - } - theap->sample_requested = 0; // reset extra count - theap->sample_countdown = theap->sample_rate; // reset sampling - - // update derived countdowns - mi_assert_internal(theap->profile_sample_rate!=0 || theap->profile_sample_countdown==0); - mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0); - bool sample_profile = false; - bool sample_guarded = false; - if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = (theap->profile_sample_rate!=0); } - if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = (theap->guarded_sample_rate!=0); } - - // invoke callback? - if (sample_profile) { - theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown - if (sample_guarded) { theap->guarded_sample_countdown = 0; }; // ensure it will get sampled at some point - return _mi_theap_malloc_profiled(theap,size,requested,zero,ppage); - } - else if (sample_guarded) { - if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) { - // use guarded allocation - theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown - return _mi_theap_malloc_guarded(theap,size,zero,ppage); - } - else { - // failed size criteria, rewind the sample countdown so we sample asap again - theap->sample_countdown = 0; - } - } - // take generic path - return _mi_malloc_generic_no_sample(theap,size,zero,ppage); -} - - // ------------------------------------------------------ // ensure explicit external inline definitions are emitted! // ------------------------------------------------------ diff --git a/src/free.c b/src/free.c index 1b562a64d..b92176f96 100644 --- a/src/free.c +++ b/src/free.c @@ -122,11 +122,6 @@ static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, con #endif } -// forward declaration for a MI_GUARDED build -#if MI_GUARDED -static void mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p); // forward declaration -#endif - static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool* was_guarded) mi_attr_noexcept { MI_UNUSED(was_guarded); if mi_likely(!mi_page_has_interior_pointers(page)) { @@ -145,7 +140,7 @@ static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool #endif #if MI_GUARDED if (block->next == MI_BLOCK_TAG_GUARDED) { - mi_page_block_unguard(page, block, p); + _mi_page_block_unguard(page, block, p); *was_guarded = true; } #else @@ -816,34 +811,3 @@ void mi_stat_free(const mi_page_t* page, const mi_block_t* block) { #endif -// Remove guard page when building with MI_GUARDED -#if MI_GUARDED -static void mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { - MI_UNUSED(p); - mi_assert_internal(mi_block_ptr_is_guarded(block, p)); - mi_assert_internal(mi_page_has_interior_pointers(page)); - mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); - mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); - - const size_t bsize = mi_page_block_size(page); - const size_t psize = _mi_os_page_size(); - mi_assert_internal(bsize > psize); - mi_assert_internal(!page->memid.is_pinned); - void* gpage = (uint8_t*)block + bsize - psize; - mi_assert_internal(_mi_is_aligned(gpage, psize)); - _mi_os_unprotect(gpage, psize); -} - -// unguard a whole page (called from `mi_heap_destroy`) -void _mi_page_unguard_all(mi_page_t* page) { - if mi_likely(!mi_page_has_interior_pointers(page)) return; - uint8_t* const start = mi_page_start(page); - const size_t psize = mi_page_committed(page); - _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded -} -#else -void _mi_page_unguard_all(mi_page_t* page) { - MI_UNUSED(page); - // nothing to do -} -#endif diff --git a/src/page.c b/src/page.c index 110e41e3b..93c025f3e 100644 --- a/src/page.c +++ b/src/page.c @@ -1264,7 +1264,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size bool sample_countdown_is_adjusted = false; if mi_unlikely(mi_theap_should_sample(theap,req_size)) { if (huge_alignment==0 && theap->sample_rate!=0) { - return _mi_theap_malloc_sample(theap,req_size,zero,ppage); + return _mi_theap_malloc_sampled(theap,req_size,zero,ppage); } mi_assert_internal(!_mi_is_empty_theap(theap)); // cannot write to the empty theap mi_assert_internal(req_size <= MI_SAMPLE_COUNTDOWN_MAX); @@ -1362,7 +1362,7 @@ void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi if (theap==NULL) return NULL; const size_t sample_rate = theap->sample_rate; const size_t sample_countdown = theap->sample_countdown; - theap->sample_rate = 0; // prevent a recursive call to _mi_theap_malloc_sample from _mi_malloc_generic + theap->sample_rate = 0; // prevent a recursive call to mi_theap_malloc_sampled from _mi_malloc_generic theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); theap->sample_rate = sample_rate; diff --git a/src/profile.c b/src/profile.c deleted file mode 100644 index 82178e585..000000000 --- a/src/profile.c +++ /dev/null @@ -1,127 +0,0 @@ -/* ---------------------------------------------------------------------------- -Copyright (c) 2019-2026, Microsoft Research, Daan Leijen -This is free software; you can redistribute it and/or modify it under the -terms of the MIT license. A copy of the license can be found in the file -"LICENSE" at the root of this distribution. ------------------------------------------------------------------------------*/ -#include "mimalloc.h" -#include "mimalloc/internal.h" -#include "mimalloc/prim-tls.h" - -static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) { - return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); -} - -static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { - mi_heap_t* const heap = _mi_theap_heap(theap); - mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); - if (prof!=NULL && mi_profiler_is_enabled(prof)) { - return prof; - } - else { - return NULL; - } -} - -/* ---------------------------------------------------------------------------- - Profile an allocation and free ------------------------------------------------------------------------------*/ -mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept -{ - mi_assert_internal(theap!=NULL); - mi_assert_internal(size<=requested_since_last_sample); - mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); - if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } - - // Overallocate a larger block to store the profiler data - // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] - const size_t profiler_data_offset = sizeof(mi_block_t); - size_t profiler_data_size = sizeof(mi_profiler_data_t); - if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > 1024 ? 1024 : prof->profiler_data_size); }; - const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); - const size_t oversize = profiler_user_offset + size; - mi_page_t* page = NULL; - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); - if (block==NULL) return NULL; - mi_assert_internal(page!=NULL); - if (ppage!=NULL) { *ppage = page; } - mi_assert_internal(!mi_block_ptr_is_guarded(_mi_page_ptr_unalign(page,block),block)); - - // Set up the profiled block - mi_page_set_has_interior_pointers(page, true); - block->next = MI_BLOCK_TAG_PROFILED; - const size_t usable_size = _mi_page_usable_size(page,block) - profiler_user_offset; - void* const p = (uint8_t*)block + profiler_user_offset; - mi_profiler_data_t* profiler_data = (mi_profiler_data_t*)((uint8_t*)block + profiler_data_offset); - profiler_data->requested_size = size - MI_PADDING_SIZE; - profiler_data->usable_size = usable_size; - - // and call the profiler on_alloc - if (prof->on_alloc!=NULL) { - const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); - if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { - mi_theap_enable_profiling(theap,new_sample_rate); - } - mi_theap_stat_counter_increase(theap,profile_samples,1); - } - return p; -} - -void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { - mi_assert_internal(mi_block_ptr_is_profiled(block,p)); - - // get the heap and profiler - mi_heap_t* const heap = mi_page_heap(page); - if (heap==NULL) return; - mi_profiler_t* prof = mi_heap_profiler(heap); - if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; - - // call the on_free callback - mi_profiler_data_t* const profiler_data = (mi_profiler_data_t*)((uint8_t*)block + sizeof(mi_block_t)); - prof->on_free(profiler_data, p, heap, prof->profiler_arg); -} - - -/* ---------------------------------------------------------------------------- - API ------------------------------------------------------------------------------*/ - -bool mi_heap_profile(mi_heap_t* heap, const mi_profiler_t* profiler) { - // if (mi_heap_profiler(heap)!=NULL) return false; // always overwrite? - mi_atomic_store_ptr_release(mi_profiler_t,&heap->profiler,(mi_profiler_t*)profiler); - return true; -} - -bool mi_subproc_profile(mi_subproc_id_t subproc_id, const mi_profiler_t* profiler) { - mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - if (subproc==NULL) return false; - // if (mi_subproc_profiler(subproc)!=NULL) return false; // always overwrite? - mi_atomic_store_ptr_release(mi_profiler_t,&subproc->profiler, (mi_profiler_t*)profiler); - mi_lock(&subproc->heaps_lock) { - for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) { - mi_heap_profile(heap,profiler); - } - } - return true; -} - -bool mi_profile(const mi_profiler_t* profiler) { - return mi_subproc_profile(mi_subproc_main(),profiler); -} - -bool mi_profiler_start(const mi_profiler_t* profiler ) { - const bool was_running = mi_profiler_set_enabled((mi_profiler_t*)profiler,true); - if (was_running) return true; - mi_heap_t* heap = mi_heap_main(); - if (mi_heap_profiler(heap)==profiler) { - mi_theap_t* theap = _mi_heap_theap_peek(heap); - if (theap!=NULL) { - mi_theap_enable_profiling(theap,1); - } - } - return false; -} - -bool mi_profiler_stop(const mi_profiler_t* profiler) { - return mi_profiler_set_enabled((mi_profiler_t*)profiler,false); -} diff --git a/src/sample-guarded.c b/src/sample-guarded.c new file mode 100644 index 000000000..99c99c82a --- /dev/null +++ b/src/sample-guarded.c @@ -0,0 +1,162 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { + theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); + if (theap->guarded_sample_rate > 1) { + if (seed == 0) { + seed = _mi_theap_random_next(theap); + } + theap->guarded_sample_countdown = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` + } + else if (theap->guarded_sample_countdown > theap->guarded_sample_rate) { + theap->guarded_sample_countdown = theap->guarded_sample_rate; + } + _mi_theap_update_sample_rate(theap); +} + +mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { + theap->guarded_size_min = min; + theap->guarded_size_max = (min > max ? min : max); +} + +void _mi_theap_guarded_init(mi_theap_t* theap) { + mi_theap_guarded_set_sample_rate(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), + (size_t)mi_option_get(mi_option_guarded_sample_seed)); + mi_theap_guarded_set_size_bound(theap, + (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), + (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); +} + + +// We always allocate a guarded allocation at an offset (`mi_page_has_interior_pointers` will be true). +// We then set the first word of the block to `0` for regular offset aligned allocations (in `alloc-aligned.c`) +// and the first word to `~0` for guarded allocations to have a correct `mi_usable_size` +static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t* usable_size) { + // todo: we can still make padding work by moving it out of the guard page area + mi_page_t* const page = _mi_ptr_page(block); + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_GUARDED; + + // set guard page at the end of the block + const size_t block_size = mi_page_block_size(page); // must use `block_size` to match `mi_free_local` + const size_t os_page_size = _mi_os_page_size(); + mi_assert_internal(block_size >= obj_size + os_page_size + sizeof(mi_block_t)); + if (block_size < obj_size + os_page_size + sizeof(mi_block_t)) { + // should never happen + mi_free(block); + return NULL; + } + uint8_t* guard_page = (uint8_t*)block + block_size - os_page_size; + // note: the alignment of the guard page relies on blocks being os_page_size aligned which + // is ensured in `mi_arena_page_alloc_fresh`. + mi_assert_internal(_mi_is_aligned(block, os_page_size)); + mi_assert_internal(_mi_is_aligned(guard_page, os_page_size)); + if (!page->memid.is_pinned && _mi_is_aligned(guard_page, os_page_size)) { + const bool ok = _mi_os_protect(guard_page, os_page_size); + if mi_unlikely(!ok) { + _mi_warning_message("failed to set a guard page behind an object (object %p of size %zu)\n", block, block_size); + } + } + else { + _mi_warning_message("unable to set a guard page behind an object due to pinned memory (large OS pages?) (object %p of size %zu)\n", block, block_size); + } + + // align pointer just in front of the guard page + size_t offset = block_size - os_page_size - obj_size; + mi_assert_internal(offset > sizeof(mi_block_t)); + if (offset > MI_PAGE_MAX_OVERALLOC_ALIGN) { + // give up to place it right in front of the guard page if the offset is too large for unalignment + offset = MI_PAGE_MAX_OVERALLOC_ALIGN; + } + uint8_t* const p = (uint8_t*)block + offset; + mi_assert_internal(p == guard_page - obj_size || offset >= MI_PAGE_MAX_OVERALLOC_ALIGN); + if (usable_size != NULL) { *usable_size = (guard_page - p); mi_assert_internal(mi_usable_size(p)==*usable_size); } + mi_track_align(block, p, offset, obj_size); + mi_track_mem_defined(block, sizeof(mi_block_t)); + return p; +} + +// Allocate a block with a guard page behind it. +mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // allocate multiple of page size ending in a guard page + // ensure minimal alignment requirement? + if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow + _mi_error_message(EOVERFLOW, "(guarded) allocation request is too large (%zu bytes)\n", size); + return NULL; + } + const size_t os_page_size = _mi_os_page_size(); + const size_t obj_size = (mi_option_is_enabled(mi_option_guarded_precise) ? size : _mi_align_up(size, MI_MAX_ALIGN_SIZE)); + const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); + const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); + // const size_t threshold = mi_theap_disable_profiler(theap); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); + // mi_theap_enable_profiler(theap,threshold); + if (block==NULL) return NULL; + size_t usable_size = 0; + void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); + if (p == NULL) return NULL; + if (zero) { + _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) + } + + // stats + mi_track_malloc(p, usable_size, zero); + if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } + mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); + #if MI_STATS + // adjust request stats to only count the allocated size of the block (and not the guard page) + mi_theap_stat_counter_decrease(theap, malloc_requested, req_size); + mi_theap_stat_counter_increase(theap, malloc_requested, size); + #endif + #if MI_DEBUG>3 + if (zero) { + mi_assert_expensive(mi_mem_is_zero(p, size)); + } + #endif + return p; +} + + +// Remove guard page when building with MI_GUARDED +#if MI_GUARDED +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + if (!mi_block_ptr_is_guarded(block,p)) return; + mi_assert_internal(mi_block_ptr_is_guarded(block, p)); + mi_assert_internal(mi_page_has_interior_pointers(page)); + mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); + mi_assert_internal(block->next == MI_BLOCK_TAG_GUARDED); + + const size_t bsize = mi_page_block_size(page); + const size_t psize = _mi_os_page_size(); + mi_assert_internal(bsize > psize); + mi_assert_internal(!page->memid.is_pinned); + void* gpage = (uint8_t*)block + bsize - psize; + mi_assert_internal(_mi_is_aligned(gpage, psize)); + _mi_os_unprotect(gpage, psize); +} + +// unguard a whole page (called from `mi_heap_destroy`) +void _mi_page_unguard_all(mi_page_t* page) { + if mi_likely(!mi_page_has_interior_pointers(page)) return; + uint8_t* const start = mi_page_start(page); + const size_t psize = mi_page_committed(page); + _mi_os_unprotect(start,psize); // unprotect all at once as we cannot know which blocks are guarded +} +#else +void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED(page); MI_UNUSED(block); MI_UNUSED(p); +} +void _mi_page_unguard_all(mi_page_t* page) { + MI_UNUSED(page); +} +#endif diff --git a/src/sample-profile.c b/src/sample-profile.c new file mode 100644 index 000000000..f5fcdafd3 --- /dev/null +++ b/src/sample-profile.c @@ -0,0 +1,223 @@ +/* ---------------------------------------------------------------------------- +Copyright (c) 2019-2026, Microsoft Research, Daan Leijen +This is free software; you can redistribute it and/or modify it under the +terms of the MIT license. A copy of the license can be found in the file +"LICENSE" at the root of this distribution. +-----------------------------------------------------------------------------*/ +#include "mimalloc.h" +#include "mimalloc/internal.h" +#include "mimalloc/prim-tls.h" + +//---------------------------------------------------------------------------- +// General sampled allocation (called from `page.c:mi_malloc_generic_fallback`) +// This is for both guarded and profiled sampling +//---------------------------------------------------------------------------- + +size_t _mi_theap_update_sample_rate(mi_theap_t* theap) { + const size_t old_sample_rate = theap->sample_rate; + theap->sample_rate = theap->profile_sample_rate; + if (theap->sample_rate == 0 || (theap->guarded_sample_rate!=0 && theap->sample_rate > theap->guarded_sample_rate)) { + theap->sample_rate = theap->guarded_sample_rate; + } + if (theap->sample_countdown > theap->sample_rate) { + theap->sample_countdown = theap->sample_rate; // todo: adjust difference? + } + return old_sample_rate; +} + +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE) + mi_assert_internal(req_size <= MI_MAX_ALLOC_SIZE); + mi_assert_internal((theap->sample_countdown==SIZE_MAX && _mi_is_empty_theap(theap)) || + (theap->sample_countdown <= MI_SAMPLE_COUNTDOWN_MAX && theap->sample_countdown < req_size)); + const size_t size = req_size + MI_PADDING_SIZE; + + // handle empty theap and disabled sampling + if (theap->sample_rate==0) { + if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap + theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time + } + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); + } + + // update countdown and total accummulated requested bytes since the last sample + mi_assert_internal(!_mi_is_empty_theap(theap)); + mi_assert_internal(req_size <= SIZE_MAX/2); + mi_assert_internal(theap->sample_rate > 0); + mi_assert_internal(theap->sample_rate <= SIZE_MAX/2); + mi_assert_internal(theap->sample_rate >= theap->sample_countdown); + + const uint64_t requested = theap->sample_requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested; + mi_assert_internal(requested > 0); + mi_assert_internal(requested >= size); + theap->sample_countdown = theap->sample_rate; // reset sampling + + // update derived countdowns + mi_assert_internal(theap->profile_sample_rate!=0 || theap->profile_sample_countdown==0); + mi_assert_internal(theap->guarded_sample_rate!=0 || theap->guarded_sample_countdown==0); + bool sample_profile = false; + bool sample_guarded = false; + if (theap->profile_sample_rate!=0) { + if (theap->profile_sample_countdown >= requested) { theap->profile_sample_countdown -= (size_t)requested; } else { sample_profile = true; } + } + if (theap->guarded_sample_rate!=0) { + if (theap->guarded_sample_countdown >= requested) { theap->guarded_sample_countdown -= (size_t)requested; } else { sample_guarded = true; } + } + + // invoke callback? + if (sample_profile) { + theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown + theap->sample_requested = 0; // reset requested as we pass it to malloc_profiled + return _mi_theap_malloc_profiled(theap,size,requested,zero,ppage); + } + else if (sample_guarded) { + if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) { + // use guarded allocation + theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown + return _mi_theap_malloc_guarded(theap,size,zero,ppage); + } + else { + // failed size criteria, rewind the sample countdown so we sample asap again + // todo: can we do better here as this will cause many samples until it fits the size.. + theap->sample_countdown = 0; + } + } + // take generic path + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); +} + + + +//---------------------------------------------------------------------------- +// Util +//---------------------------------------------------------------------------- + +static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) { + return mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); +} + +static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { + mi_heap_t* const heap = _mi_theap_heap(theap); + mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); + if (prof!=NULL && mi_profiler_is_enabled(prof)) { + return prof; + } + else { + return NULL; + } +} + +//---------------------------------------------------------------------------- +// Profile an allocation +//----------------------------------------------------------------------------- + +mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept +{ + mi_assert_internal(theap!=NULL); + mi_assert_internal(size<=requested_since_last_sample); + mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); + if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } + + // Overallocate a larger block to store the profiler data + // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] + const size_t profiler_data_offset = sizeof(mi_block_t); + size_t profiler_data_size = sizeof(mi_profiler_data_t); + if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > 1024 ? 1024 : prof->profiler_data_size); }; + const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); + const size_t oversize = profiler_user_offset + size; + mi_page_t* page = NULL; + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); + if (block==NULL) return NULL; + mi_assert_internal(page!=NULL); + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(!mi_block_ptr_is_guarded(_mi_page_ptr_unalign(page,block),block)); + + // Set up the profiled block + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_PROFILED; + const size_t usable_size = _mi_page_usable_size(page,block) - profiler_user_offset; + void* const p = (uint8_t*)block + profiler_user_offset; + mi_profiler_data_t* profiler_data = (mi_profiler_data_t*)((uint8_t*)block + profiler_data_offset); + profiler_data->requested_size = size - MI_PADDING_SIZE; + profiler_data->usable_size = usable_size; + + // and call the profiler on_alloc + if (prof->on_alloc!=NULL) { + const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); + if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { + _mi_theap_set_profile_sample_rate(theap,new_sample_rate); + } + mi_theap_stat_counter_increase(theap,profile_samples,1); + } + return p; +} + +void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { + mi_assert_internal(mi_block_ptr_is_sampled(block,p)); + + // get the heap and profiler + mi_heap_t* const heap = mi_page_heap(page); + if (heap==NULL) return; + mi_profiler_t* prof = mi_heap_profiler(heap); + if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; + + // call the on_free callback + mi_profiler_data_t* const profiler_data = (mi_profiler_data_t*)((uint8_t*)block + sizeof(mi_block_t)); + prof->on_free(profiler_data, p, heap, prof->profiler_arg); +} + + +//---------------------------------------------------------------------------- +// Profiling API +//-----------------------------------------------------------------------------* + +size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) { + const size_t old_sample_rate = theap->profile_sample_rate; + theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); + if (theap->profile_sample_countdown > theap->profile_sample_rate) { + theap->profile_sample_countdown = theap->profile_sample_rate; // todo: adjust difference? + } + _mi_theap_update_sample_rate(theap); + return old_sample_rate; +} + +bool mi_heap_profile(mi_heap_t* heap, const mi_profiler_t* profiler) { + // if (mi_heap_profiler(heap)!=NULL) return false; // always overwrite? + mi_atomic_store_ptr_release(mi_profiler_t,&heap->profiler,(mi_profiler_t*)profiler); + return true; +} + +bool mi_subproc_profile(mi_subproc_id_t subproc_id, const mi_profiler_t* profiler) { + mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); + if (subproc==NULL) return false; + // if (mi_subproc_profiler(subproc)!=NULL) return false; // always overwrite? + mi_atomic_store_ptr_release(mi_profiler_t,&subproc->profiler, (mi_profiler_t*)profiler); + mi_lock(&subproc->heaps_lock) { + for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) { + mi_heap_profile(heap,profiler); + } + } + return true; +} + +bool mi_profile(const mi_profiler_t* profiler) { + return mi_subproc_profile(mi_subproc_main(),profiler); +} + +bool mi_profiler_start(const mi_profiler_t* profiler ) { + const bool was_running = mi_profiler_set_enabled((mi_profiler_t*)profiler,true); + if (was_running) return true; + mi_heap_t* heap = mi_heap_main(); + if (mi_heap_profiler(heap)==profiler) { + mi_theap_t* theap = _mi_heap_theap_peek(heap); + if (theap!=NULL) { + _mi_theap_set_profile_sample_rate(theap,1); + } + } + return false; +} + +bool mi_profiler_stop(const mi_profiler_t* profiler) { + return mi_profiler_set_enabled((mi_profiler_t*)profiler,false); +} diff --git a/src/static.c b/src/static.c index f35442471..8a4f8c3d5 100644 --- a/src/static.c +++ b/src/static.c @@ -30,16 +30,17 @@ terms of the MIT license. A copy of the license can be found in the file #include "libc.c" #include "options.c" #include "os.c" +#include "prim/prim.c" // includes platform specific prim//prim.c +#include "prim/prim-tls.c" +#if MI_OSX_ZONE +#include "prim/osx/alloc-override-zone.c" +#endif #include "page.c" // includes page-queue.c #include "page-map.c" #include "random.c" +#include "sample-guarded.c" +#include "sample-profile.c" #include "stats.c" #include "subproc.c" #include "theap.c" #include "threadlocal.c" -#include "prim/prim.c" -#include "prim/prim-tls.c" -#include "profile.c" -#if MI_OSX_ZONE -#include "prim/osx/alloc-override-zone.c" -#endif diff --git a/src/theap.c b/src/theap.c index abf874339..1731830a7 100644 --- a/src/theap.c +++ b/src/theap.c @@ -190,49 +190,6 @@ mi_theap_t* mi_theap_set_default(mi_theap_t* theap) { return previous; } -#if MI_GUARDED -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - theap->guarded_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); - theap->guarded_sample_countdown = theap->guarded_sample_rate; - if (theap->sample_rate==0 || theap->sample_rate > theap->guarded_sample_rate) { - theap->sample_rate = theap->guarded_sample_rate; - } - if (theap->guarded_sample_rate > 1) { - if (seed == 0) { - seed = _mi_theap_random_next(theap); - } - theap->guarded_sample_countdown = (seed % theap->guarded_sample_rate) + 1; // start at random count between 1 and `sample_rate` - } - if (theap->sample_countdown > theap->guarded_sample_countdown) { - theap->sample_countdown = theap->guarded_sample_countdown; - } -} - -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - theap->guarded_size_min = min; - theap->guarded_size_max = (min > max ? min : max); -} - -static void mi_theap_guarded_init(mi_theap_t* theap) { - mi_theap_guarded_set_sample_rate(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, MI_SAMPLE_RATE_MAX), - (size_t)mi_option_get(mi_option_guarded_sample_seed)); - mi_theap_guarded_set_size_bound(theap, - (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), - (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); -} -#else -mi_decl_export void mi_theap_guarded_set_sample_rate(mi_theap_t* theap, size_t sample_rate, size_t seed) { - MI_UNUSED(theap); MI_UNUSED(sample_rate); MI_UNUSED(seed); -} - -mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t min, size_t max) { - MI_UNUSED(theap); MI_UNUSED(min); MI_UNUSED(max); -} -static void mi_theap_guarded_init(mi_theap_t* theap) { - MI_UNUSED(theap); -} -#endif static void mi_theap_options_init(mi_theap_t* theap) { theap->allow_page_reclaim = (mi_option_get(mi_option_page_reclaim_on_free) >= 0); @@ -296,7 +253,7 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) _mi_random_split(&head_random, &theap->random); // &theap->random is used as nonce so it is ok if threads capture the same head->random } // theap->cookie = _mi_theap_random_next(theap) | 1; - mi_theap_guarded_init(theap); // needs theap->random + _mi_theap_guarded_init(theap); // needs theap->random if (!theap->is_detached) { mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem } diff --git a/test/test-profile.c b/test/test-profile.c index 471ab1d4c..8e6743c93 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -105,6 +105,7 @@ bool test_profiler_record_fields(void) { void* p = mi_malloc(1024); mi_free(p); } + assert(count!=MAXLOOP); result = (g_state.last_ptr != NULL && g_state.last_size > 0 && g_state.last_upscaled > 0 && count!=MAXLOOP); } return true; @@ -127,7 +128,7 @@ bool test_profiler_on_free_called(void) { void* expected = g_state.last_ptr; mi_free(expected); sampled = NULL; - + assert(count!=MAXLOOP); result = (g_state.free_count > free_before && count!=MAXLOOP); } return true; @@ -136,11 +137,13 @@ bool test_profiler_on_free_called(void) { bool test_profiler_upscaled_at_least_size(void) { CHECK_BODY("profiler: upscaled_size >= size") { uint64_t before = g_state.alloc_count; - while (g_state.alloc_count == before) { + int count; + for (count = 0; g_state.alloc_count == before && count < MAXLOOP; count++) { void* p = mi_malloc(256); mi_free(p); } - result = (g_state.last_upscaled >= g_state.last_size); + assert(count!=MAXLOOP); + result = (g_state.last_upscaled >= g_state.last_size && count!=MAXLOOP); } return true; } From d94818a1c0f2a9b0182dd12c6e22911c2b7425c9 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 10:12:13 -0700 Subject: [PATCH 218/280] fix initialization on 32-bit --- include/mimalloc-profile.h | 20 +++++++++++++------- include/mimalloc/types.h | 29 +++++++++++++++++------------ src/init.c | 9 +++++---- src/page.c | 5 +++-- 4 files changed, 38 insertions(+), 25 deletions(-) diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h index 254caba78..a91030e2c 100644 --- a/include/mimalloc-profile.h +++ b/include/mimalloc-profile.h @@ -12,24 +12,30 @@ terms of the MIT license. A copy of the license can be found in the file #include // bool #include // uint64_t +// Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.) typedef struct mi_profiler_data_s { - size_t usable_size; - size_t requested_size; - void* user_data[6]; // default, but can be less or more (up to 1KiB), depending on `profiler_data_size` + size_t profiler_data_size; // size of the custom profile data (should be the `mi_profiler_t.profiler_data_size`) + size_t usable_size; // usable size in the allocated block + size_t requested_size; // user requested size (i.e. the size passed to `mi_malloc`) + void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `profiler_data_size` } mi_profiler_data_t; +// Profiling callback invoked on each sampled allocation. typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_data_t* profiler_data, void* ptr, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg); -typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_data_t* profiler_data, void* ptr, size_t old_requested_size, const mi_heap_t* heap, void* profiler_arg); + +// Profiling callback invoked on each sampled in-place re-allocation. +typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap, void* profiler_arg); + +// Profiling callback invoked on a previously sampled allocation. typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_data_t* profiler_data, void* ptr, const mi_heap_t* heap, void* profiler_arg); // A profiler -// All fields are considered immutable such that they can be copied and accessed concurrently. -// All fields can be NULL/0. +// All fields are considered immutable such that they can be copied and accessed concurrently. All fields can be NULL/0. typedef struct mi_profiler_s { void* reserved1; // opaque -- internal use for mimalloc void* reserved2; void* reserved3; - void* profiler_arg; // opaque profiler state pointer -- passed to each callback + void* profiler_arg; // opaque user profiler state pointer -- passed to each callback size_t profiler_data_size; // size of required profiler data for each sampled allocation mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently) mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index cac057e68..3cf34a2de 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -610,17 +610,25 @@ struct mi_theap_s { mi_tld_t* tld; // thread-local data _Atomic(mi_heap_t*) heap; // the heap this theap belongs to. _Atomic(mi_subproc_t*)subproc; // subproc this belongs too (always `subproc == heap->subproc` but needed for safe destruction) - _Atomic(size_t) refcount; // reference count + _Atomic(size_t) refcount; // reference count (needed for safe heap destroy) - size_t sample_rate; - uint64_t sample_requested; - size_t profile_sample_rate; - size_t profile_sample_countdown; - size_t guarded_sample_rate; - size_t guarded_sample_countdown; + // config + long page_full_retain; // how many full pages can be retained per queue (before abandoning them) + bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages + bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint + bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` + + // sampling + size_t sample_rate; // current sampling rate in requested bytes (or 0 to disable) (for profiling and guarded mode) + uint64_t sample_requested; // total allocated/requested bytes since the last sample + size_t profile_sample_rate; // sampling rate in requested bytes for profiling + size_t profile_sample_countdown; // countdown in requested bytes for profiling + size_t guarded_sample_rate; // sampling rate in requested bytes for guarded objects + size_t guarded_sample_countdown; // countdown in requested bytes for guarded objects size_t guarded_size_min; // minimal size for guarded objects size_t guarded_size_max; // maximal size for guarded objects + // stats unsigned long long heartbeat; // monotonic heartbeat count mi_random_ctx_t random; // random number context used for secure allocation size_t page_count; // total number of pages in the `pages` queues. @@ -630,16 +638,13 @@ struct mi_theap_s { long generic_count; // how often is `_mi_malloc_generic` called? long generic_collect_count; // how often is `_mi_malloc_generic` called without collecting? + // theaps belong to heaps and threads mi_theap_t* tnext; // list of theaps in this thread mi_theap_t* tprev; mi_theap_t* hnext; // list of theaps of the owning `heap` mi_theap_t* hprev; - long page_full_retain; // how many full pages can be retained per queue (before abandoning them) - bool allow_page_reclaim; // `true` if this theap can reclaim abandoned pages - bool allow_page_abandon; // `true` if this theap can abandon pages to reduce memory footprint - bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` - + // page queues mi_page_queue_t pages[MI_BIN_COUNT]; // queue of pages for each size class (or "bin") mi_memid_t memid; // provenance of the theap struct itself (meta or os) mi_stats_t stats; // thread-local statistics diff --git a/src/init.c b/src/init.c index f8f90f80b..55289fd11 100644 --- a/src/init.c +++ b/src/init.c @@ -22,6 +22,7 @@ static const mi_page_t mi_page_empty = { {0}, // xused #if MI_INTPTR_SIZE < 8 0, // xlast_used + 0, // xlast_alloc #endif NULL, // local_free 0, // block_size @@ -127,6 +128,10 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { MI_ATOMIC_VAR_INIT(NULL), // heap MI_ATOMIC_VAR_INIT(NULL), // subproc MI_ATOMIC_VAR_INIT(1), // refcount + 0, // full page retain + false, // allow reclaim + true, // allow abandon + true, // is_detached 0, 0, // sample rate, requested 0, 0, // profile rate, countdown 0, 0, 0, 0, // guarded rate, countdown, min, max @@ -138,10 +143,6 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { 0, 0, // generic count NULL, NULL, // tnext, tprev NULL, NULL, // hnext, hprev - 0, // full page retain - false, // allow reclaim - true, // allow abandon - true, // is_detached MI_PAGE_QUEUES_EMPTY, MI_MEMID_STATIC, MI_STATS_NULL, // stats diff --git a/src/page.c b/src/page.c index 93c025f3e..762ec6592 100644 --- a/src/page.c +++ b/src/page.c @@ -168,12 +168,13 @@ static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, // update countdown const size_t bsize = mi_page_usable_block_size(page); mi_assert_internal(bsize >= MI_PADDING_SIZE); - const uint64_t requested = (uint64_t)(alloc_count - last_alloc) * (uint64_t)(bsize - MI_PADDING_SIZE); + const size_t alloc_diff = alloc_count - last_alloc; + const uint64_t requested = (uint64_t)alloc_diff * (uint64_t)(bsize - MI_PADDING_SIZE); if (requested <= SIZE_MAX && theap->sample_countdown >= (size_t)requested) { theap->sample_countdown -= requested; } else { - theap->sample_requested += (requested - theap->sample_countdown); + theap->sample_requested += (requested - theap->sample_countdown); theap->sample_countdown = 0; } } From 01fe31e7e048b86c8d426d5d7db1d4ba00a0c049 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 13:27:32 -0700 Subject: [PATCH 219/280] improve heap_(zm)alloc codegen --- include/mimalloc/internal.h | 4 ++-- include/mimalloc/prim-tls.h | 10 +++++++++ include/mimalloc/types.h | 2 +- src/alloc.c | 43 +++++++++++++++++++++++++++++++------ src/heap.c | 2 +- src/page.c | 6 +++--- src/sample-guarded.c | 4 +++- src/sample-profile.c | 11 ++++++---- 8 files changed, 63 insertions(+), 19 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index da07174e5..bf3db601b 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -270,8 +270,8 @@ mi_page_t* _mi_safe_ptr_page(const void* p); void _mi_page_map_unsafe_destroy(void); // "page.c" -void* _mi_malloc_generic(mi_theap_t* heap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; -void* _mi_malloc_generic_no_sample(mi_theap_t* heap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(size_t size, mi_theap_t* theap, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic_no_sample(size_t size, mi_theap_t* theap, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 9d7eda399..9a08a03bb 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -396,6 +396,16 @@ static inline mi_theap_t* _mi_heap_theap(mi_heap_t* heap) { return _mi_heap_theap_get_or_init(heap); } +static inline mi_theap_t* _mi_heap_theap_cached(mi_heap_t* heap) { + mi_theap_t* theap = _mi_theap_cached(); + #if MI_THEAP_INITASNULL + if mi_likely(theap!=NULL && _mi_theap_heap_peek(theap)==heap) return theap; + #else + if mi_likely(_mi_theap_heap_peek(theap)==heap) return theap; + #endif + return NULL; +} + // Get the theap belonging to a heap without creating it if it is not yet initialized. static inline mi_theap_t* _mi_heap_theap_peek(const mi_heap_t* heap) { mi_theap_t* theap = _mi_theap_cached(); diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 3cf34a2de..11df3f9f8 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -179,7 +179,7 @@ terms of the MIT license. A copy of the license can be found in the file #if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) #if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0 #define MI_PAGE_META_SMALL_IS_ALIGNED 0 -#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // cannot be guarded as that may allocate large blocks for small allocations +#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE #define MI_PAGE_META_SMALL_IS_ALIGNED 1 #else #define MI_PAGE_META_SMALL_IS_ALIGNED 0 diff --git a/src/alloc.c b/src/alloc.c index 561865f9e..e32257737 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -67,7 +67,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ __asm("" : : : "memory" ); // always load the `used` field before the test #endif if (block == NULL) { - return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); + return _mi_malloc_generic(size, theap, (zero ? 1 : 0), ppage); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); if (ppage != NULL) { *ppage = page; }; @@ -122,6 +122,9 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ if mi_unlikely(zero) { if (!page->free_is_zero) { mi_assert_internal(bsize%MI_SIZE_SIZE == 0); + #if !MI_PADDING + return // use tail-call + #endif _mi_memzero_block(block,bsize); } else { @@ -183,7 +186,7 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size #endif mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local mi_assert((huge_alignment & 1)==0); - void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(size + MI_PADDING_SIZE, theap, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p, size, zero); #if MI_DEBUG>3 @@ -218,8 +221,16 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_no return mi_theap_malloc_small(_mi_theap_default(), size); } +static mi_decl_nodiscard mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, NULL); +} + mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, false, NULL); + // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); } + // but the following prevents using a stack frame. We use this to optimize some select functions only. + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, size, false, NULL); } + else { return mi_heap_init_malloc_small(heap, size); } } // The main internal allocation functions @@ -265,8 +276,15 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept return mi_theap_malloc(_mi_theap_default(), size); } +static mi_decl_nodiscard mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); + // return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, false, 0, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_zero_nonnull(theap, size, false, 0, NULL); } + else { return mi_heap_init_malloc(heap, size); } + } @@ -279,11 +297,16 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_ return mi_theap_malloc_small_zero(theap, size, true, NULL); } +static mi_decl_nodiscard mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, NULL); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); + // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, size, true, NULL); } + else { return mi_heap_init_zalloc_small(heap, size); } } - mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { return _mi_theap_malloc_zero(theap, size, true, NULL); } @@ -292,8 +315,14 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL); } +static mi_decl_nodiscard mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); +} mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); + // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); + mi_theap_t* const theap = _mi_heap_theap_cached(heap); + if mi_likely(theap!=NULL) { return mi_theap_malloc_zero_nonnull(theap, size, true, 0, NULL); } + else { return mi_heap_init_zalloc(heap, size); } } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_calloc(mi_theap_t* theap, size_t count, size_t size) mi_attr_noexcept { diff --git a/src/heap.c b/src/heap.c index c17b18d1f..d7f1db132 100644 --- a/src/heap.c +++ b/src/heap.c @@ -87,7 +87,7 @@ static mi_decl_noinline mi_theap_t* mi_heap_init_theap(const mi_heap_t* const_he // get (and possibly create) the theap belonging to a heap -mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) +mi_decl_cold mi_theap_t* _mi_heap_theap_get_or_init(const mi_heap_t* heap) { mi_assert_internal(heap->theap != 0); mi_theap_t* theap = (mi_theap_t*)_mi_thread_local_get(heap->theap); diff --git a/src/page.c b/src/page.c index 762ec6592..944ed4180 100644 --- a/src/page.c +++ b/src/page.c @@ -1324,7 +1324,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size // very large requested alignments in which case we use a huge singleton page. // Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit) // to use 4 parameters which compiles better on msvc for the malloc fast path. -void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept +void* _mi_malloc_generic(size_t size, mi_theap_t* theap, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept { #if !MI_THEAP_INITASNULL mi_assert_internal(theap != NULL); @@ -1358,14 +1358,14 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage); } -void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { +void* _mi_malloc_generic_no_sample(size_t size, mi_theap_t* theap, bool zero, mi_page_t** ppage) mi_attr_noexcept { theap = mi_theap_init(theap); if (theap==NULL) return NULL; const size_t sample_rate = theap->sample_rate; const size_t sample_countdown = theap->sample_countdown; theap->sample_rate = 0; // prevent a recursive call to mi_theap_malloc_sampled from _mi_malloc_generic theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; - void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); + void* p = _mi_malloc_generic(size, theap, (zero ? 1 : 0), ppage); theap->sample_rate = sample_rate; theap->sample_countdown = sample_countdown; return p; diff --git a/src/sample-guarded.c b/src/sample-guarded.c index 99c99c82a..07c7c082e 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -99,7 +99,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); // const size_t threshold = mi_theap_disable_profiler(theap); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(req_size, theap, false /* don't zero */, ppage); // mi_theap_enable_profiler(theap,threshold); if (block==NULL) return NULL; size_t usable_size = 0; @@ -123,6 +123,8 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, mi_assert_expensive(mi_mem_is_zero(p, size)); } #endif + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(_mi_ptr_page(p)) <= MI_SMALL_MAX_OBJ_SIZE); } return p; } diff --git a/src/sample-profile.c b/src/sample-profile.c index f5fcdafd3..d0a32b023 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -38,7 +38,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* the if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time } - return _mi_malloc_generic_no_sample(theap,size,zero,ppage); + return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } // update countdown and total accummulated requested bytes since the last sample @@ -84,7 +84,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* the } } // take generic path - return _mi_malloc_generic_no_sample(theap,size,zero,ppage); + return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } @@ -117,7 +117,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th mi_assert_internal(theap!=NULL); mi_assert_internal(size<=requested_since_last_sample); mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); - if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } + if (prof == NULL) { return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } // Overallocate a larger block to store the profiler data // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] @@ -127,11 +127,14 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); const size_t oversize = profiler_user_offset + size; mi_page_t* page = NULL; - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(oversize,theap,zero,&page); if (block==NULL) return NULL; mi_assert_internal(page!=NULL); if (ppage!=NULL) { *ppage = page; } mi_assert_internal(!mi_block_ptr_is_guarded(_mi_page_ptr_unalign(page,block),block)); + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + // (that is why we need to limit the profiler_data_size as well) + if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } // Set up the profiled block mi_page_set_has_interior_pointers(page, true); From 9dd0d6177b247c3c208ac7edbb58206c59051afc Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 13:38:18 -0700 Subject: [PATCH 220/280] fix attribute order --- src/alloc.c | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index e32257737..2f798c672 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -221,7 +221,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_no return mi_theap_malloc_small(_mi_theap_default(), size); } -static mi_decl_nodiscard mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, NULL); } @@ -276,7 +276,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept return mi_theap_malloc(_mi_theap_default(), size); } -static mi_decl_nodiscard mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -297,7 +297,7 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_ return mi_theap_malloc_small_zero(theap, size, true, NULL); } -static mi_decl_nodiscard mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -315,7 +315,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL); } -static mi_decl_nodiscard mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { From c638b42956050f7cf63d9a1acffa55d9f2a16083 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 16:54:18 -0700 Subject: [PATCH 221/280] provide wsize allocation --- CMakeLists.txt | 6 ++-- include/mimalloc.h | 34 +++++++++++++++----- include/mimalloc/internal.h | 16 +++------- include/mimalloc/types.h | 4 +-- src/alloc-aligned.c | 2 +- src/alloc.c | 63 +++++++++++++++++++++++++++++-------- src/init.c | 2 +- src/page.c | 4 +-- src/sample-guarded.c | 7 ++++- src/sample-profile.c | 3 ++ 10 files changed, 99 insertions(+), 42 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7fb3b31ac..3a6d17928 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -447,15 +447,15 @@ if(MI_GUARDED STREQUAL "DEFAULT") if(MI_DEBUG) set(MI_GUARDED "FULL") else() - set(MI_GUARDED "ON") + set(MI_GUARDED "OFF") endif() endif() if(MI_GUARDED STREQUAL "FULL") - message(STATUS "Enable fine-grained sampling for guard pages behind certain object allocations (MI_GUARDED=FULL)") + message(STATUS "Enable fine-grained sampling for guard pages (MI_GUARDED=FULL)") list(APPEND mi_defines MI_GUARDED=2) elseif(MI_GUARDED) - message(STATUS "Enable sampled guard pages behind certain object allocations (MI_GUARDED=${MI_GUARDED})") + message(STATUS "Enable sampled guard pages (MI_GUARDED=${MI_GUARDED})") list(APPEND mi_defines MI_GUARDED=1) else() # message(STATUS "Disable guard pages (MI_GUARDED=${MI_GUARDED})") diff --git a/include/mimalloc.h b/include/mimalloc.h index 6d88cf248..c355a4095 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -97,6 +97,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // size_t, wchar_t #include // bool +#include // inline assertions #ifdef __cplusplus extern "C" { @@ -120,7 +121,7 @@ mi_decl_nodiscard mi_decl_export mi_decl_restrict char* mi_realpath(const char* // Extended allocation functions // ------------------------------------------------------ #define MI_SMALL_WSIZE_MAX 128 -#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(void*)) +#define MI_SMALL_SIZE_MAX (MI_SMALL_WSIZE_MAX*sizeof(size_t)) // mimalloc considers `sizeof(size_t)` as the machine word size mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); mi_decl_nodiscard mi_decl_export mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept mi_attr_malloc mi_attr_alloc_size(1); @@ -399,23 +400,42 @@ mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_the // Fast constant size allocations. // ------------------------------------------------------ +// `__mi` are internal functions; do not use. +mi_decl_nodiscard mi_decl_restrict void* __mi_wzalloc_small(size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* __mi_wmalloc_small(size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small_nonnull(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small_nonnull(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +static inline size_t mi_wsize_from_size(size_t size) { + return ((size + sizeof(size_t) - 1) / sizeof(size_t)); +} + static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_malloc_small(size); } else { return mi_malloc(size); } + if (size <= MI_SMALL_SIZE_MAX) { return __mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); } } static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_zalloc_small(size); } else { return mi_zalloc(size); } + if (size <= MI_SMALL_SIZE_MAX) { return __mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } } -static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_malloc_small(theap,size); } else { return mi_theap_malloc(theap,size); } +static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } } static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_zalloc_small(theap,size); } else { return mi_theap_zalloc(theap,size); } + if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } +} +static inline mi_decl_restrict void* mi_theap_malloc_csize_nonnull(mi_theap_t* theap, size_t size) mi_attr_noexcept { + assert(theap!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wmalloc_small_nonnull(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } +} +static inline mi_decl_restrict void* mi_theap_zalloc_csize_nonnull(mi_theap_t* theap, size_t size) mi_attr_noexcept { + assert(theap!=NULL); + if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wzalloc_small_nonnull(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } } static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } } static inline void mi_free_csize_nonnull(void* p, size_t size) mi_attr_noexcept { - // assert(p!=NULL); + assert(p!=NULL); if (size <= MI_SMALL_SIZE_MAX) { mi_free_small_nonnull(p); } else { mi_free(p); } } diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index bf3db601b..96d0f7d87 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -270,7 +270,7 @@ mi_page_t* _mi_safe_ptr_page(const void* p); void _mi_page_map_unsafe_destroy(void); // "page.c" -void* _mi_malloc_generic(size_t size, mi_theap_t* theap, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void* _mi_malloc_generic_no_sample(size_t size, mi_theap_t* theap, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks @@ -681,19 +681,11 @@ static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) { return subproc; } -static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t size) { - mi_assert_internal(size <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); - #if MI_INTPTR_SIZE==MI_SIZE_SIZE - // the sample_countdown field comes before the pages_direct; the following generates better code in general - // (where we treat the countdown as a first -1 entry of the pages_free_direct array) - const size_t idx1 = (size + 2*MI_INTPTR_SIZE - 1)/MI_INTPTR_SIZE; - mi_assert_internal(idx1 <= MI_PAGES_DIRECT); - return ((mi_page_t**)theap)[idx1]; - #else - const size_t idx = _mi_wsize_from_size(size); +static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t xsize, bool is_wsize) { + mi_assert_internal(is_wsize ? xsize <= (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE) : xsize <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); + const size_t idx = (is_wsize ? xsize : _mi_wsize_from_size(xsize)); mi_assert_internal(idx < MI_PAGES_DIRECT); return theap->pages_free_direct[idx]; - #endif } static inline bool mi_theap_is_detached(mi_theap_t* theap) { diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 11df3f9f8..6f3c0188b 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -179,7 +179,7 @@ terms of the MIT license. A copy of the license can be found in the file #if !defined(MI_PAGE_META_SMALL_IS_ALIGNED) #if defined(MI_OPT_FREE_SMALL) && MI_OPT_FREE_SMALL==0 #define MI_PAGE_META_SMALL_IS_ALIGNED 0 -#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE +#elif (MI_OPT_FREE_SMALL || MI_PAGE_META_IS_ALIGNED) && !MI_SECURE && !MI_GUARDED // guarded can put small allocations into big blocks in a medium page (which would make `mi_free_small` fail) #define MI_PAGE_META_SMALL_IS_ALIGNED 1 #else #define MI_PAGE_META_SMALL_IS_ALIGNED 0 @@ -603,7 +603,6 @@ typedef struct mi_padding_s { // A thread-local heap ("theap") owns a set of thread-local pages. struct mi_theap_s { // put in front for fast small allocations - size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:_mi_theap_get_free_small_page`) mi_page_t* pages_free_direct[MI_PAGES_DIRECT]; // optimize: array where every entry points a page with possibly free blocks in the corresponding queue for that size. // less frequently accessed fields @@ -619,6 +618,7 @@ struct mi_theap_s { bool is_detached; // `true` if `tld->thread_id == MI_THREADID_DETACHED` // sampling + size_t sample_countdown; // sample countdown in requested bytes (don't change the field order; see `internal.h:_mi_theap_get_free_small_page`) size_t sample_rate; // current sampling rate in requested bytes (or 0 to disable) (for profiling and guarded mode) uint64_t sample_requested; // total allocated/requested bytes since the last sample size_t profile_sample_rate; // sampling rate in requested bytes for profiling diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 967571ced..38600c71f 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -219,7 +219,7 @@ static inline void* mi_theap_malloc_zero_aligned_at(mi_theap_t* const theap, con { const uintptr_t align_mask = alignment-1; // for any x, `(x & align_mask) == (x % alignment)` const size_t padsize = size + MI_PADDING_SIZE; - mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize); + mi_page_t* page = _mi_theap_get_free_small_page(theap, padsize, false); if mi_likely(page->free != NULL) { const bool is_aligned = (((uintptr_t)page->free + offset) & align_mask)==0; if mi_likely(is_aligned) diff --git a/src/alloc.c b/src/alloc.c index 2f798c672..9f0152e5f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -67,7 +67,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ __asm("" : : : "memory" ); // always load the `used` field before the test #endif if (block == NULL) { - return _mi_malloc_generic(size, theap, (zero ? 1 : 0), ppage); + return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); } mi_assert_internal(block != NULL && _mi_ptr_page(block) == page); if (ppage != NULL) { *ppage = page; }; @@ -147,8 +147,9 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz } // internal small size allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { + const size_t size = (is_wsize ? xsize * MI_INTPTR_SIZE : xsize); mi_assert(theap != NULL); mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG @@ -164,7 +165,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non #endif // get page in constant time - mi_page_t* page = _mi_theap_get_free_small_page(theap, size + MI_PADDING_SIZE); + mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? _mi_wsize_from_size(MI_PADDING_SIZE) : MI_PADDING_SIZE), is_wsize); // and allocate void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage); @@ -186,7 +187,7 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size #endif mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local mi_assert((huge_alignment & 1)==0); - void* const p = _mi_malloc_generic(size + MI_PADDING_SIZE, theap, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic + void* const p = _mi_malloc_generic(theap, size + MI_PADDING_SIZE, (zero ? 1 : 0) | huge_alignment, ppage); // note: size can overflow but it is detected in malloc_generic mi_track_malloc(p, size, zero); #if MI_DEBUG>3 @@ -198,10 +199,11 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size } // internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_ex(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { + const size_t size = (is_wsize ? xsize * MI_INTPTR_SIZE : xsize); #if MI_THEAP_INITASNULL if (theap!=NULL) { - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_malloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, 0, ppage); // tailcall @@ -211,25 +213,46 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_ #endif } +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { + return mi_theap_malloc_small_zero_ex(theap,size,false,zero,ppage); +} + +static mi_decl_forceinline mi_decl_restrict void* mi_theap_wmalloc_small_zero(mi_theap_t* theap, size_t wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { + return mi_theap_malloc_small_zero_ex(theap,wsize,true,zero,ppage); +} // allocate a small block mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small_zero(theap, size, false, NULL); } +mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { + return mi_theap_malloc_small_zero_ex(theap, wsize, true, false, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small_nonnull(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_malloc_small_zero_nonnull(theap, wsize, true, false, NULL); +} + + mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { return mi_theap_malloc_small(_mi_theap_default(), size); } +mi_decl_nodiscard mi_decl_restrict void* __mi_wmalloc_small(size_t wsize) mi_attr_noexcept { + return mi_theap_malloc_small_zero_ex(_mi_theap_default(), wsize, true, false, NULL); +} + mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, NULL); + return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); } // but the following prevents using a stack frame. We use this to optimize some select functions only. mi_theap_t* const theap = _mi_heap_theap_cached(heap); - if mi_likely(theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, size, false, NULL); } + if mi_likely(theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, size, false, false, NULL); } else { return mi_heap_init_malloc_small(heap, size); } } @@ -238,7 +261,7 @@ static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_malloc_small_zero_nonnull(theap, size, false, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); @@ -254,7 +277,7 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_malloc_small_zero_nonnull(theap, size, false, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); @@ -293,17 +316,31 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_no return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL); } +mi_decl_nodiscard mi_decl_restrict void* __mi_wzalloc_small(size_t size) mi_attr_noexcept { + return mi_theap_wmalloc_small_zero(_mi_theap_default(), size, true, NULL); +} + mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small_zero(theap, size, true, NULL); } +mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { + return mi_theap_malloc_small_zero_ex(theap, size, true, true, NULL); +} + +mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small_nonnull(mi_theap_t* theap, size_t size) mi_attr_noexcept { + mi_assert(theap!=NULL); + return mi_theap_malloc_small_zero_nonnull(theap, size, true, true, NULL); +} + + mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, NULL); + return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); mi_theap_t* const theap = _mi_heap_theap_cached(heap); - if mi_likely(theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, size, true, NULL); } + if mi_likely(theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, size, false, true, NULL); } else { return mi_heap_init_zalloc_small(heap, size); } } @@ -316,7 +353,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept } mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); + return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, true, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { // optimize: return mi_theap_malloc_zero_nonnull(_mi_heap_theap(heap), size, true, 0, NULL); diff --git a/src/init.c b/src/init.c index 55289fd11..1600a1325 100644 --- a/src/init.c +++ b/src/init.c @@ -122,7 +122,6 @@ static mi_decl_cache_align mi_tld_t mi_tld_detached = { }; mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { - ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) MI_SMALL_PAGES_EMPTY, // direct small pages &mi_tld_detached, // tld MI_ATOMIC_VAR_INIT(NULL), // heap @@ -132,6 +131,7 @@ mi_decl_hidden mi_decl_cache_align const mi_theap_t _mi_theap_empty = { false, // allow reclaim true, // allow abandon true, // is_detached + ~MI_ZU(0), // sample countdown: "-1" (with a sample rate of 0, so we won't write to the empty theap with MI_SAMPLE==2) 0, 0, // sample rate, requested 0, 0, // profile rate, countdown 0, 0, 0, 0, // guarded rate, countdown, min, max diff --git a/src/page.c b/src/page.c index 944ed4180..280b3132f 100644 --- a/src/page.c +++ b/src/page.c @@ -1324,7 +1324,7 @@ static mi_decl_noinline void* mi_malloc_generic_fallback(mi_theap_t* theap, size // very large requested alignments in which case we use a huge singleton page. // Note: we put `bool zero, size_t huge_alignment` into one parameter (with zero in the low bit) // to use 4 parameters which compiles better on msvc for the malloc fast path. -void* _mi_malloc_generic(size_t size, mi_theap_t* theap, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept +void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept { #if !MI_THEAP_INITASNULL mi_assert_internal(theap != NULL); @@ -1365,7 +1365,7 @@ void* _mi_malloc_generic_no_sample(size_t size, mi_theap_t* theap, bool zero, mi const size_t sample_countdown = theap->sample_countdown; theap->sample_rate = 0; // prevent a recursive call to mi_theap_malloc_sampled from _mi_malloc_generic theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; - void* p = _mi_malloc_generic(size, theap, (zero ? 1 : 0), ppage); + void* p = _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); theap->sample_rate = sample_rate; theap->sample_countdown = sample_countdown; return p; diff --git a/src/sample-guarded.c b/src/sample-guarded.c index 07c7c082e..3cfa0007b 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -123,8 +123,13 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, mi_assert_expensive(mi_mem_is_zero(p, size)); } #endif + #if MI_PAGE_META_SMALL_IS_ALIGNED && MI_DEBUG>=2 // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. - if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(_mi_ptr_page(p)) <= MI_SMALL_MAX_OBJ_SIZE); } + if (size <= MI_SMALL_SIZE_MAX) { + mi_page_t* const page = _mi_ptr_page(p); + mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); + } + #endif return p; } diff --git a/src/sample-profile.c b/src/sample-profile.c index d0a32b023..83676df46 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -132,9 +132,12 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th mi_assert_internal(page!=NULL); if (ppage!=NULL) { *ppage = page; } mi_assert_internal(!mi_block_ptr_is_guarded(_mi_page_ptr_unalign(page,block),block)); + #if MI_PAGE_META_SMALL_IS_ALIGNED // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. // (that is why we need to limit the profiler_data_size as well) + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } + #endif // Set up the profiled block mi_page_set_has_interior_pointers(page, true); From 9c8ad51a82005cd1675278bf42a25ba651d91334 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 17:02:46 -0700 Subject: [PATCH 222/280] fix attribute order --- include/mimalloc-profile.h | 9 +++++++-- src/alloc.c | 13 ++++++------- 2 files changed, 13 insertions(+), 9 deletions(-) diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h index a91030e2c..5408964ec 100644 --- a/include/mimalloc-profile.h +++ b/include/mimalloc-profile.h @@ -15,13 +15,13 @@ terms of the MIT license. A copy of the license can be found in the file // Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.) typedef struct mi_profiler_data_s { size_t profiler_data_size; // size of the custom profile data (should be the `mi_profiler_t.profiler_data_size`) - size_t usable_size; // usable size in the allocated block size_t requested_size; // user requested size (i.e. the size passed to `mi_malloc`) void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `profiler_data_size` } mi_profiler_data_t; // Profiling callback invoked on each sampled allocation. -typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_data_t* profiler_data, void* ptr, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg); +// If `profiler_data!=NULL` (i.e. when `on_free` is not NULL), then `profiler_data->requested_size == requested_size`. +typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg); // Profiling callback invoked on each sampled in-place re-allocation. typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap, void* profiler_arg); @@ -47,11 +47,16 @@ typedef struct mi_profiler_s { extern "C" { #endif +// attach a profiler to a heap. mi_decl_export bool mi_heap_profile(mi_heap_t* heap, const mi_profiler_t* profiler); +// attach a profiler to all heaps in a sub-process. mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, const mi_profiler_t* profiler); +// attach a profiler to the main sub-process. mi_decl_export bool mi_profile(const mi_profiler_t* profiler); +// start sampling mi_decl_export bool mi_profiler_start(const mi_profiler_t* profiler); +// end sampling mi_decl_export bool mi_profiler_stop(const mi_profiler_t* profiler); #ifdef __cplusplus diff --git a/src/alloc.c b/src/alloc.c index 9f0152e5f..b57df6774 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -200,16 +200,15 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size // internal small allocation static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_ex(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { - const size_t size = (is_wsize ? xsize * MI_INTPTR_SIZE : xsize); #if MI_THEAP_INITASNULL if (theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); } else { - return mi_theap_malloc_generic(theap, size, zero, 0, ppage); // tailcall + return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall } #else - return mi_theap_malloc_small_zero_nonnull(theap, size, zero, ppage); + return mi_theap_malloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); #endif } @@ -244,7 +243,7 @@ mi_decl_nodiscard mi_decl_restrict void* __mi_wmalloc_small(size_t wsize) mi_att return mi_theap_malloc_small_zero_ex(_mi_theap_default(), wsize, true, false, NULL); } -mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, false, NULL); } @@ -299,7 +298,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept return mi_theap_malloc(_mi_theap_default(), size); } -mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -334,7 +333,7 @@ mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small_nonnull(mi_the } -mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -352,7 +351,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL); } -mi_decl_nodiscard static mi_decl_restrict void* mi_decl_noinline mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, true, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { From d3ebb2b8ac03eaf6f985956f6d46f8d9cb05aa01 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 17:16:13 -0700 Subject: [PATCH 223/280] fix profile for on_free == NULL --- include/mimalloc-profile.h | 2 + src/sample-profile.c | 85 ++++++++++++++++++++++---------------- test/test-profile.c | 9 ++-- 3 files changed, 56 insertions(+), 40 deletions(-) diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h index 5408964ec..6e9e8ad02 100644 --- a/include/mimalloc-profile.h +++ b/include/mimalloc-profile.h @@ -12,6 +12,8 @@ terms of the MIT license. A copy of the license can be found in the file #include // bool #include // uint64_t +#define MI_PROFILE_DATA_MAX_SIZE (1024) + // Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.) typedef struct mi_profiler_data_s { size_t profiler_data_size; // size of the custom profile data (should be the `mi_profiler_t.profiler_data_size`) diff --git a/src/sample-profile.c b/src/sample-profile.c index 83676df46..79dfd9b16 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -116,46 +116,59 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th { mi_assert_internal(theap!=NULL); mi_assert_internal(size<=requested_since_last_sample); + mi_assert_internal(size>=MI_PADDING_SIZE); mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); if (prof == NULL) { return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } - - // Overallocate a larger block to store the profiler data - // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] - const size_t profiler_data_offset = sizeof(mi_block_t); - size_t profiler_data_size = sizeof(mi_profiler_data_t); - if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > 1024 ? 1024 : prof->profiler_data_size); }; - const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); - const size_t oversize = profiler_user_offset + size; - mi_page_t* page = NULL; - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(oversize,theap,zero,&page); - if (block==NULL) return NULL; - mi_assert_internal(page!=NULL); - if (ppage!=NULL) { *ppage = page; } - mi_assert_internal(!mi_block_ptr_is_guarded(_mi_page_ptr_unalign(page,block),block)); - #if MI_PAGE_META_SMALL_IS_ALIGNED - // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. - // (that is why we need to limit the profiler_data_size as well) - // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. - if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } - #endif - - // Set up the profiled block - mi_page_set_has_interior_pointers(page, true); - block->next = MI_BLOCK_TAG_PROFILED; - const size_t usable_size = _mi_page_usable_size(page,block) - profiler_user_offset; - void* const p = (uint8_t*)block + profiler_user_offset; - mi_profiler_data_t* profiler_data = (mi_profiler_data_t*)((uint8_t*)block + profiler_data_offset); - profiler_data->requested_size = size - MI_PADDING_SIZE; - profiler_data->usable_size = usable_size; - - // and call the profiler on_alloc - if (prof->on_alloc!=NULL) { - const size_t new_sample_rate = (*prof->on_alloc)(profiler_data, p, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); - if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { - _mi_theap_set_profile_sample_rate(theap,new_sample_rate); + const size_t req_size = size - MI_PADDING_SIZE; + + void* p = NULL; + size_t new_sample_rate = 0; + if (prof->on_free==NULL) { + // just allocate without profiler data + p = _mi_malloc_generic_no_sample(size,theap,zero,ppage); + if (p==NULL) { return p; } + if (prof->on_alloc!=NULL) { + new_sample_rate = (*prof->on_alloc)(NULL, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); } - mi_theap_stat_counter_increase(theap,profile_samples,1); } + else { + // Overallocate a larger block to store the profiler data + // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] + const size_t profiler_data_offset = sizeof(mi_block_t); + size_t profiler_data_size = sizeof(mi_profiler_data_t); + if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > MI_PROFILE_DATA_MAX_SIZE ? MI_PROFILE_DATA_MAX_SIZE : prof->profiler_data_size); }; + const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); + const size_t oversize = profiler_user_offset + size; + mi_page_t* page = NULL; + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(oversize,theap,zero,&page); + if (block==NULL) return NULL; + mi_assert_internal(page!=NULL); + if (ppage!=NULL) { *ppage = page; } + mi_assert_internal(!mi_block_ptr_is_guarded(_mi_page_ptr_unalign(page,block),block)); + #if MI_PAGE_META_SMALL_IS_ALIGNED + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + // (that is why we need to limit the profiler_data_size as well) + // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. + if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } + #endif + + // Set up the profiled block + mi_page_set_has_interior_pointers(page, true); + block->next = MI_BLOCK_TAG_PROFILED; + p = (uint8_t*)block + profiler_user_offset; + mi_profiler_data_t* profiler_data = (mi_profiler_data_t*)((uint8_t*)block + profiler_data_offset); + profiler_data->profiler_data_size = profiler_data_size; + profiler_data->requested_size = req_size; + + // and call the profiler on_alloc + if (prof->on_alloc!=NULL) { + new_sample_rate = (*prof->on_alloc)(profiler_data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); + } + } + if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { + _mi_theap_set_profile_sample_rate(theap,new_sample_rate); + } + mi_theap_stat_counter_increase(theap,profile_samples,1); return p; } diff --git a/test/test-profile.c b/test/test-profile.c index 8e6743c93..e4ec69f44 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -36,13 +36,14 @@ static profile_state_t g_state; #define TEST_THRESHOLD (16 * 1024) -static size_t mi_cdecl on_alloc(mi_profiler_data_t* data, void* ptr, size_t threshold, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg) { - MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(profiler_arg); +static size_t mi_cdecl on_alloc(mi_profiler_data_t* data, void* ptr, size_t requested_size, size_t threshold, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg) { + MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(profiler_arg); MI_UNUSED(requested_size); assert(profiler_arg==&g_state); - assert(bytes_since_last_sample >= data->requested_size); + assert(bytes_since_last_sample >= requested_size); + assert(requested_size == data->requested_size); g_state.alloc_count++; g_state.last_ptr = ptr; - g_state.last_size = data->requested_size; + g_state.last_size = requested_size; g_state.last_upscaled = bytes_since_last_sample; // store ptr to verify round-trip data->user_data[0] = ptr; From 2a5caea6d5db1b752fad14258586644518ef09b1 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 17:37:24 -0700 Subject: [PATCH 224/280] remove wmalloc_nonnull variants --- include/mimalloc.h | 13 +++---------- include/mimalloc/internal.h | 9 +-------- src/alloc.c | 20 +++----------------- src/page-queue.c | 10 +++++----- 4 files changed, 12 insertions(+), 40 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index c355a4095..a0d569f6f 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -405,8 +405,7 @@ mi_decl_nodiscard mi_decl_restrict void* __mi_wzalloc_small(size_t wsize) mi_att mi_decl_nodiscard mi_decl_restrict void* __mi_wmalloc_small(size_t wsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small_nonnull(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small_nonnull(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; + static inline size_t mi_wsize_from_size(size_t size) { return ((size + sizeof(size_t) - 1) / sizeof(size_t)); } @@ -418,18 +417,12 @@ static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexce if (size <= MI_SMALL_SIZE_MAX) { return __mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } } static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { + assert(theap!=NULL); if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } } static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } -} -static inline mi_decl_restrict void* mi_theap_malloc_csize_nonnull(mi_theap_t* theap, size_t size) mi_attr_noexcept { assert(theap!=NULL); - if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wmalloc_small_nonnull(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } -} -static inline mi_decl_restrict void* mi_theap_zalloc_csize_nonnull(mi_theap_t* theap, size_t size) mi_attr_noexcept { - assert(theap!=NULL); - if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wzalloc_small_nonnull(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } + if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } } static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 96d0f7d87..4372fef17 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -600,13 +600,6 @@ static inline bool mi_mem_is_zero(const void* p, size_t size) { return true; } -// Align a byte size to a size in _machine words_, -// i.e. byte size == `wsize*sizeof(void*)`. -static inline size_t _mi_wsize_from_size(size_t size) { - mi_assert_internal(size <= SIZE_MAX - sizeof(uintptr_t)); - return (size + sizeof(uintptr_t) - 1) / sizeof(uintptr_t); -} - // Overflow detecting multiply #if __has_builtin(__builtin_umul_overflow) || (defined(__GNUC__) && (__GNUC__ >= 5)) #include // UINT_MAX, ULONG_MAX @@ -683,7 +676,7 @@ static inline mi_subproc_t* _mi_theap_subproc(const mi_theap_t* theap) { static inline mi_page_t* _mi_theap_get_free_small_page(mi_theap_t* theap, size_t xsize, bool is_wsize) { mi_assert_internal(is_wsize ? xsize <= (MI_SMALL_WSIZE_MAX + MI_PADDING_WSIZE) : xsize <= (MI_SMALL_SIZE_MAX + MI_PADDING_SIZE)); - const size_t idx = (is_wsize ? xsize : _mi_wsize_from_size(xsize)); + const size_t idx = (is_wsize ? xsize : mi_wsize_from_size(xsize)); mi_assert_internal(idx < MI_PAGES_DIRECT); return theap->pages_free_direct[idx]; } diff --git a/src/alloc.c b/src/alloc.c index b57df6774..cd803b711 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -165,7 +165,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non #endif // get page in constant time - mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? _mi_wsize_from_size(MI_PADDING_SIZE) : MI_PADDING_SIZE), is_wsize); + mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? mi_wsize_from_size(MI_PADDING_SIZE) : MI_PADDING_SIZE), is_wsize); // and allocate void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage); @@ -216,31 +216,22 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_ return mi_theap_malloc_small_zero_ex(theap,size,false,zero,ppage); } -static mi_decl_forceinline mi_decl_restrict void* mi_theap_wmalloc_small_zero(mi_theap_t* theap, size_t wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { - return mi_theap_malloc_small_zero_ex(theap,wsize,true,zero,ppage); -} - // allocate a small block mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small_zero(theap, size, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { - return mi_theap_malloc_small_zero_ex(theap, wsize, true, false, NULL); -} - -mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small_nonnull(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); return mi_theap_malloc_small_zero_nonnull(theap, wsize, true, false, NULL); } - mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { return mi_theap_malloc_small(_mi_theap_default(), size); } mi_decl_nodiscard mi_decl_restrict void* __mi_wmalloc_small(size_t wsize) mi_attr_noexcept { - return mi_theap_malloc_small_zero_ex(_mi_theap_default(), wsize, true, false, NULL); + return mi_theap_malloc_small_zero_ex(_mi_theap_default(), wsize, true, false, NULL ); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -316,7 +307,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_no } mi_decl_nodiscard mi_decl_restrict void* __mi_wzalloc_small(size_t size) mi_attr_noexcept { - return mi_theap_wmalloc_small_zero(_mi_theap_default(), size, true, NULL); + return mi_theap_malloc_small_zero_ex(_mi_theap_default(), size, true, true, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { @@ -324,15 +315,10 @@ mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_ } mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_ex(theap, size, true, true, NULL); -} - -mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small_nonnull(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); return mi_theap_malloc_small_zero_nonnull(theap, size, true, true, NULL); } - mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, true, NULL); } diff --git a/src/page-queue.c b/src/page-queue.c index ac9a2e20a..7ef9609c9 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -62,7 +62,7 @@ static inline size_t mi_page_queue_count(const mi_page_queue_t* pq) { // We use `wsize` for the size in "machine word sizes", // i.e. byte size == `wsize*sizeof(void*)`. static size_t mi_bin(size_t size) { - size_t wsize = _mi_wsize_from_size(size); + size_t wsize = mi_wsize_from_size(size); #if defined(MI_ALIGN4W) if mi_likely(wsize <= 4) { return (wsize <= 1 ? 1 : (wsize+1)&~1); // round to double word sizes @@ -152,10 +152,10 @@ bool _mi_page_queue_is_valid(mi_theap_t* theap, const mi_page_queue_t* pq) { for (mi_page_t* page = pq->first; page != NULL; page = page->next) { mi_assert_internal(page->prev == prev); if (mi_page_is_in_full(page)) { - mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2); + mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 2); } else if (mi_page_is_huge(page)) { - mi_assert_internal(_mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1); + mi_assert_internal(mi_wsize_from_size(pq->block_size) == MI_LARGE_MAX_OBJ_WSIZE + 1); } else { mi_assert_internal(mi_page_block_size(page) == pq->block_size); @@ -215,7 +215,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ if (pq->first == NULL) page = _mi_page_empty_get(); // find index in the right direct page array - const size_t idx = _mi_wsize_from_size(size); + const size_t idx = mi_wsize_from_size(size); mi_page_t** const pages_free = theap->pages_free_direct; if (pages_free[idx] == page) return; // already set @@ -232,7 +232,7 @@ static inline void mi_theap_queue_first_update(mi_theap_t* theap, const mi_page_ while( bin == mi_bin(prev->block_size) && prev > &theap->pages[0]) { prev--; } - start = 1 + _mi_wsize_from_size(prev->block_size); + start = 1 + mi_wsize_from_size(prev->block_size); if (start > idx) start = idx; } From e1bd8031bfe849b5d98b3e0861163fee7760b6a0 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 18:03:14 -0700 Subject: [PATCH 225/280] use sizeof(size_t) = machine word consistently (instead of sizeof(intptr_t)) --- include/mimalloc/internal.h | 34 +++++++++++++++++----------------- include/mimalloc/prim-tls.h | 6 +++--- include/mimalloc/types.h | 8 +++----- src/alloc-override.c | 4 ++-- src/alloc.c | 2 +- src/arena.c | 2 +- src/free.c | 4 ++-- src/init.c | 6 ++---- src/os.c | 8 ++++---- src/page-queue.c | 6 +++--- src/page.c | 8 ++++---- src/prim/prim-tls.c | 4 ++-- src/prim/unix/prim.c | 8 ++++---- src/prim/windows/prim.c | 2 +- src/random.c | 20 ++++++++++---------- test/main-override-static.c | 6 +++--- test/test-api.c | 2 +- 17 files changed, 63 insertions(+), 67 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 4372fef17..246e29a57 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -148,10 +148,10 @@ void _mi_random_init(mi_random_ctx_t* ctx); void _mi_random_init_weak(mi_random_ctx_t* ctx); void _mi_random_reinit_if_weak(mi_random_ctx_t * ctx); void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* new_ctx); -uintptr_t _mi_random_next(mi_random_ctx_t* ctx); -uintptr_t _mi_theap_random_next(mi_theap_t* theap); -uintptr_t _mi_os_random_weak(uintptr_t extra_seed); -static inline uintptr_t _mi_random_shuffle(uintptr_t x); +size_t _mi_random_next(mi_random_ctx_t* ctx); +size_t _mi_theap_random_next(mi_theap_t* theap); +size_t _mi_os_random_weak(size_t extra_seed); +static inline size_t _mi_random_shuffle(size_t x); // prim-tls.c void _mi_tls_slots_init(void); @@ -1443,16 +1443,16 @@ static inline size_t _mi_memid_size(mi_memid_t memid) { // Fast "random" shuffle // ------------------------------------------------------------------- -static inline uintptr_t _mi_random_shuffle(uintptr_t x) { +static inline size_t _mi_random_shuffle(size_t x) { if (x==0) { x = 17; } // ensure we don't get stuck in generating zeros -#if (MI_INTPTR_SIZE>=8) +#if (MI_SIZE_SIZE>=8) // by Sebastiano Vigna, see: x ^= x >> 30; x *= 0xbf58476d1ce4e5b9UL; x ^= x >> 27; x *= 0x94d049bb133111ebUL; x ^= x >> 31; -#elif (MI_INTPTR_SIZE==4) +#elif (MI_SIZE_SIZE==4) // by Chris Wellons, see: x ^= x >> 16; x *= 0x7feb352dUL; @@ -1531,21 +1531,21 @@ static inline void* _mi_memset(void* dst, int val, size_t n) { #endif // ------------------------------------------------------------------------------- -// The `_mi_memcpy_aligned` can be used if the pointers are machine-word aligned +// The `_mi_memcpy_aligned` can be used if the pointers are machine-word (size_t) aligned // This is used for example in `mi_realloc`. // ------------------------------------------------------------------------------- // On GCC/CLang we provide a hint that the pointers are word aligned. static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { - mi_assert_internal(((uintptr_t)dst % MI_INTPTR_SIZE == 0) && ((uintptr_t)src % MI_INTPTR_SIZE == 0)); - void* adst = mi_assume_aligned(dst, MI_INTPTR_SIZE); - const void* asrc = mi_assume_aligned(src, MI_INTPTR_SIZE); + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE) && _mi_is_aligned(src,MI_SIZE_SIZE)); + void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); + const void* asrc = mi_assume_aligned(src, MI_SIZE_SIZE); return _mi_memcpy(adst, asrc, n); } static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { - mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - void* adst = mi_assume_aligned(dst, MI_INTPTR_SIZE); + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); + void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); return _mi_memset(adst, val, n); } @@ -1562,8 +1562,8 @@ static inline void* _mi_memzero_aligned(void* dst, size_t n) { static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) { mi_assert_internal(bsize%MI_SIZE_SIZE == 0); mi_assert_internal(bsize > 0); - mi_assert_internal((uintptr_t)dst % MI_INTPTR_SIZE == 0); - mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || (uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); + mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); + mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); #if MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) @@ -1577,7 +1577,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize *((uint64_t*)dst) = 0; return dst; } - mi_assert_internal((uintptr_t)dst % MI_MAX_ALIGN_SIZE == 0); + mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); mi_assert_internal(bsize % 16 == 0); void* const adst = mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); __int128_t* const start = (__int128_t*)adst; @@ -1597,7 +1597,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize #endif // regular memset - void* const wdst = mi_assume_aligned(dst,MI_INTPTR_SIZE); + void* const wdst = mi_assume_aligned(dst,MI_SIZE_SIZE); return _mi_memset_aligned(wdst, 0, bsize); } diff --git a/include/mimalloc/prim-tls.h b/include/mimalloc/prim-tls.h index 9a08a03bb..da78d0193 100644 --- a/include/mimalloc/prim-tls.h +++ b/include/mimalloc/prim-tls.h @@ -118,7 +118,7 @@ static inline void** mi_prim_thread_pointer(void) { void** tcb; #if defined(__APPLE__) __asm__ ("movq %%gs:0, %0" : "=r" (tcb) : : ); // x86_64 macOSX uses GS - #elif (MI_INTPTR_SIZE==4) + #elif (MI_SIZE_SIZE==4) __asm__ ("movl %%fs:0, %0" : "=r" (tcb) : : ); // x32 ABI #else __asm__ ("movq %%fs:0, %0" : "=r" (tcb) : : ); // x86_64 Linux, BSD uses FS @@ -301,9 +301,9 @@ static inline mi_theap_t* _mi_theap_cached(void) { // We try to use direct slots (64 available), but can also use the expansion slots (upto 1024 extra available) // See for the offsets. #if MI_SIZE_SIZE==4 -#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_INTPTR_SIZE) +#define MI_TLS_EXPANSION_SLOT (0x0F94 / MI_SIZE_SIZE) #else -#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_INTPTR_SIZE) +#define MI_TLS_EXPANSION_SLOT (0x1780 / MI_SIZE_SIZE) #endif extern mi_decl_hidden _Atomic(size_t) _mi_theap_default_slot; diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 6f3c0188b..077637ae9 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -30,7 +30,7 @@ terms of the MIT license. A copy of the license can be found in the file #include // bool #include // SIZE_MAX etc. #include // error codes -#include "bits.h" // size defines (MI_INTPTR_SIZE etc), bit operations +#include "bits.h" // size defines (MI_SIZE_SIZE etc), bit operations #include "atomic.h" // _Atomic primitives // Minimal alignment necessary. On most platforms 16 bytes are needed @@ -167,7 +167,7 @@ terms of the MIT license. A copy of the license can be found in the file #if !MI_FREE_IS_CHECKED && !MI_FREE_USE_PAGEMAP #if MI_PAGE_META_IS_SEPARATED #define MI_PAGE_META_IS_ALIGNED 1 -#define MI_PAGE_META_ALIGNED_CHUNKS MI_INTPTR_SIZE +#define MI_PAGE_META_ALIGNED_CHUNKS MI_SIZE_SIZE #else #warning "cannot optimize free with alignment since the page meta data is not separated (due to MI_PAGE_MAP_FLAT?)" #endif @@ -494,8 +494,6 @@ typedef struct mi_page_s { #if (MI_ENCODE_FREELIST || MI_PADDING) uintptr_t keys[MI_PAGE_KEY_COUNT]; // const: one or two random keys to encode the free lists (see `_mi_block_next`) or padding canary - // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 - // uintptr_t padding[1]; // make it 128 bytes for best codegen in mi_ptr_page_align #endif } mi_page_t; @@ -591,7 +589,7 @@ typedef struct mi_padding_s { uint32_t delta; // padding bytes before the block. (mi_full_usable_size(p) - delta == exact allocated bytes) } mi_padding_t; #define MI_PADDING_SIZE (sizeof(mi_padding_t)) -#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_INTPTR_SIZE - 1) / MI_INTPTR_SIZE) +#define MI_PADDING_WSIZE ((MI_PADDING_SIZE + MI_SIZE_SIZE - 1) / MI_SIZE_SIZE) #else #define MI_PADDING_SIZE 0 #define MI_PADDING_WSIZE 0 diff --git a/src/alloc-override.c b/src/alloc-override.c index 04da5f293..bcb96b52d 100644 --- a/src/alloc-override.c +++ b/src/alloc-override.c @@ -293,7 +293,7 @@ typedef void* mi_nothrow_t; void _ZdlPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete(void*, std::align_val_t, std::nothrow_t const&) void _ZdaPvSt11align_val_tRKSt9nothrow_t(void* p, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); mi_free_aligned(p,al); } // operator delete[](void*, std::align_val_t, std::nothrow_t const&) - #if (MI_INTPTR_SIZE==8) || (MI_INTPTR_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257 + #if (MI_SIZE_SIZE==8) || (MI_SIZE_SIZE==4 && defined(__EMSCRIPTEN__)) // pr #1257 void* _Znwm(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znam(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit void* _ZnwmRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } @@ -302,7 +302,7 @@ typedef void* mi_nothrow_t; void* _ZnamSt11align_val_t(size_t n, size_t al) MI_FORWARD2(mi_new_aligned, n, al) void* _ZnwmSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } void* _ZnamSt11align_val_tRKSt9nothrow_t(size_t n, size_t al, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_aligned_nothrow(n,al); } - #elif (MI_INTPTR_SIZE==4) + #elif (MI_SIZE_SIZE==4) void* _Znwj(size_t n) MI_FORWARD1(mi_new,n) // new 64-bit void* _Znaj(size_t n) MI_FORWARD1(mi_new,n) // new[] 64-bit void* _ZnwjRKSt9nothrow_t(size_t n, mi_nothrow_t tag) { MI_UNUSED(tag); return mi_new_nothrow(n); } diff --git a/src/alloc.c b/src/alloc.c index cd803b711..57ce56494 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -149,7 +149,7 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz // internal small size allocation static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { - const size_t size = (is_wsize ? xsize * MI_INTPTR_SIZE : xsize); + const size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize); mi_assert(theap != NULL); mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG diff --git a/src/arena.c b/src/arena.c index 24b33b712..2bbadf82b 100644 --- a/src/arena.c +++ b/src/arena.c @@ -995,7 +995,7 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou #if !defined(MI_PAGE_BLOCK_START_MAX_OFFSET) #define MI_PAGE_BLOCK_START_MAX_OFFSET (8*MI_INTPTR_BITS) /* 512 */ #endif - if (block_size >= MI_INTPTR_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && + if (block_size >= MI_SIZE_SIZE && block_size <= MI_PAGE_BLOCK_START_MAX_OFFSET && _mi_is_power_of_two(block_size)) { block_start = _mi_align_up(mi_page_info_size(), block_size); // to maintain natural alignment diff --git a/src/free.c b/src/free.c index b92176f96..a0e34f13a 100644 --- a/src/free.c +++ b/src/free.c @@ -187,7 +187,7 @@ static mi_decl_forceinline bool mi_ptr_page_is_valid_ex(const void* p, const cha { MI_UNUSED_RELEASE(msg); MI_UNUSED(free_small); #if MI_DEBUG - if mi_unlikely(((uintptr_t)p & (MI_INTPTR_SIZE - 1)) != 0 && !mi_option_is_enabled(mi_option_guarded_precise)) { + if mi_unlikely(!_mi_is_aligned(p,MI_SIZE_SIZE) && !mi_option_is_enabled(mi_option_guarded_precise)) { _mi_error_message(EINVAL, "%s: invalid (unaligned) pointer: %p\n", msg, p); return false; } @@ -615,7 +615,7 @@ static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const // Used for double free checking to avoid checking free lists too frequently static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { mi_block_t* n = mi_block_nextx(page,block,page->keys); - return (((uintptr_t)n & (MI_INTPTR_SIZE-1))==0 && // quick check: aligned pointer? + return (!_mi_is_aligned(p,MI_SIZE_SIZE) && // quick check: aligned pointer? (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? } diff --git a/src/init.c b/src/init.c index 1600a1325..a725fb704 100644 --- a/src/init.c +++ b/src/init.c @@ -20,7 +20,7 @@ static const mi_page_t mi_page_empty = { MI_ATOMIC_VAR_INIT(0), // xthread_id NULL, // free {0}, // xused - #if MI_INTPTR_SIZE < 8 + #if MI_SIZE_SIZE < 8 0, // xlast_used 0, // xlast_alloc #endif @@ -43,8 +43,6 @@ static const mi_page_t mi_page_empty = { #else { 0 }, // key #endif - // #elif MI_PAGE_META_IS_ALIGNED && MI_INTPTR_SIZE==8 - // { 0 }, // padding #endif }; @@ -58,7 +56,7 @@ static const mi_page_t mi_page_empty = { #error define initializer for direct pages #endif -#if (MI_PADDING>0) && (MI_INTPTR_SIZE >= 8) +#if (MI_PADDING>0) && (MI_SIZE_SIZE >= 8) #define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } #elif (MI_PADDING>0) #define MI_SMALL_PAGES_EMPTY { MI_INIT_PAGES_DIRECT(MI_PAGE_EMPTY), MI_PAGE_EMPTY(), MI_PAGE_EMPTY(), MI_PAGE_EMPTY() } diff --git a/src/os.c b/src/os.c index a727cc844..15111cf94 100644 --- a/src/os.c +++ b/src/os.c @@ -14,7 +14,7 @@ terms of the MIT license. A copy of the license can be found in the file Initialization. ----------------------------------------------------------- */ #ifndef MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB -#if MI_INTPTR_SIZE < 8 +#if MI_SIZE_SIZE < 8 #define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 4*MI_MiB // 4 GiB #else #define MI_DEFAULT_PHYSICAL_MEMORY_IN_KIB 32*MI_MiB // 32 GiB @@ -119,7 +119,7 @@ bool _mi_os_commit(mi_subproc_t* subproc, void* addr, size_t size, bool* is_zero // On systems with enough virtual address bits, we can do efficient aligned allocation by using // the 2TiB to 30TiB area to allocate those. If we have at least 46 bits of virtual address // space (64TiB) we use this technique. (but see issue #939) -#if (MI_INTPTR_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) +#if (MI_SIZE_SIZE >= 8) && !defined(MI_NO_ALIGNED_HINT) // && !defined(WIN32) && !defined(ANDROID) // Return a `try_alignment` aligned address that is probably available. // If this returns NULL, the OS will determine the address but on some OS's that may not be @@ -361,7 +361,7 @@ static void* mi_os_prim_alloc_aligned(mi_subproc_t* subproc, size_t size, size_t if (!(alignment >= _mi_os_page_size() && ((alignment & (alignment - 1)) == 0))) return NULL; size = _mi_align_up(size, _mi_os_page_size()); - #if MI_INTPTR_SIZE >= 8 + #if MI_SIZE_SIZE >= 8 const bool try_direct_alloc = true; #else // try a direct allocation if the alignment is below the default, or less than or equal to 1/4 fraction of the size. @@ -730,7 +730,7 @@ and possibly associated with a specific NUMA node. (use `numa_node>=0`) #define MI_HUGE_OS_PAGE_SIZE (MI_GiB) -#if (MI_INTPTR_SIZE >= 8) +#if (MI_SIZE_SIZE >= 8) // To ensure proper alignment, use our own area for huge OS pages static mi_decl_cache_align _Atomic(uintptr_t) mi_huge_start; // = 0 diff --git a/src/page-queue.c b/src/page-queue.c index 7ef9609c9..c8e9b5425 100644 --- a/src/page-queue.c +++ b/src/page-queue.c @@ -21,11 +21,11 @@ terms of the MIT license. A copy of the license can be found in the file Minimal alignment in machine words (i.e. `sizeof(void*)`) ----------------------------------------------------------- */ -#if (MI_MAX_ALIGN_SIZE > 4*MI_INTPTR_SIZE) +#if (MI_MAX_ALIGN_SIZE > 4*MI_SIZE_SIZE) #error "define alignment for more than 4x word size for this platform" -#elif (MI_MAX_ALIGN_SIZE > 2*MI_INTPTR_SIZE) +#elif (MI_MAX_ALIGN_SIZE > 2*MI_SIZE_SIZE) #define MI_ALIGN4W // 4 machine words minimal alignment -#elif (MI_MAX_ALIGN_SIZE > MI_INTPTR_SIZE) +#elif (MI_MAX_ALIGN_SIZE > MI_SIZE_SIZE) #define MI_ALIGN2W // 2 machine words minimal alignment #else // ok, default alignment is 1 word diff --git a/src/page.c b/src/page.c index 280b3132f..15d237ec6 100644 --- a/src/page.c +++ b/src/page.c @@ -738,15 +738,15 @@ static void mi_page_free_list_extend_secure(mi_theap_t* const theap, mi_page_t* // and initialize the free list by randomly threading through them // set up first element - const uintptr_t r = _mi_theap_random_next(theap); + const size_t r = _mi_theap_random_next(theap); size_t current = r % slice_count; counts[current]--; mi_block_t* const free_start = blocks[current]; // and iterate through the rest; use `random_shuffle` for performance - uintptr_t rnd = _mi_random_shuffle(r|1); // ensure not 0 + size_t rnd = _mi_random_shuffle(r|1); // ensure not 0 for (size_t i = 1; i < extend; i++) { - // call random_shuffle only every INTPTR_SIZE rounds - const size_t round = i%MI_INTPTR_SIZE; + // call random_shuffle only every SIZE_SIZE rounds + const size_t round = i%MI_SIZE_SIZE; if (round == 0) rnd = _mi_random_shuffle(rnd); // select a random next slice index size_t next = ((rnd >> 8*round) & (slice_count-1)); diff --git a/src/prim/prim-tls.c b/src/prim/prim-tls.c index 8c8b0c71f..1b79321a0 100644 --- a/src/prim/prim-tls.c +++ b/src/prim/prim-tls.c @@ -46,9 +46,9 @@ mi_threadid_t _mi_thread_id(void) mi_attr_noexcept { // If we can, we use one of the 64 direct TLS slots (but fall back to expansion slots if needed) // See for the offsets. #if MI_SIZE_SIZE==4 -#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_INTPTR_SIZE) +#define MI_TLS_DIRECT_FIRST (0x0E10 / MI_SIZE_SIZE) #else -#define MI_TLS_DIRECT_FIRST (0x1480 / MI_INTPTR_SIZE) +#define MI_TLS_DIRECT_FIRST (0x1480 / MI_SIZE_SIZE) #endif #define MI_TLS_DIRECT_SLOTS (64) #define MI_TLS_EXPANSION_SLOTS (1024) diff --git a/src/prim/unix/prim.c b/src/prim/unix/prim.c index 1838901d5..25c2adea0 100644 --- a/src/prim/unix/prim.c +++ b/src/prim/unix/prim.c @@ -339,7 +339,7 @@ static void* unix_mmap_prim_aligned(void* addr, size_t size, size_t try_alignmen // fall back to regular mmap } #endif - #if (MI_INTPTR_SIZE >= 8) && !defined(MAP_ALIGNED) + #if (MI_SIZE_SIZE >= 8) && !defined(MAP_ALIGNED) // on 64-bit systems, use the virtual address area after 2TiB for 4MiB aligned allocations if (addr == NULL) { void* hint = _mi_os_get_aligned_hint(try_alignment, size); @@ -612,7 +612,7 @@ int _mi_prim_protect(void* start, size_t size, bool protect) { // Huge page allocation //--------------------------------------------- -#if (MI_INTPTR_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) +#if (MI_SIZE_SIZE >= 8) && !defined(__HAIKU__) && !defined(__CYGWIN__) #ifndef MPOL_PREFERRED #define MPOL_PREFERRED 1 @@ -633,12 +633,12 @@ int _mi_prim_alloc_huge_os_pages(void* hint_addr, size_t size, int numa_node, bo bool is_large = true; *is_zero = true; *addr = unix_mmap(hint_addr, size, MI_ARENA_SLICE_ALIGN, PROT_READ | PROT_WRITE, true, true, &is_large); - if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_INTPTR_SIZE - 1)) { // at most 63 nodes + if (*addr != NULL && numa_node >= 0 && numa_node < (8*MI_SIZE_SIZE - 1)) { // at most 63 nodes unsigned long numa_mask = (1UL << numa_node); // todo: does `mbind` work correctly for huge OS pages? should we // use `set_mempolicy` before calling mmap instead? // see: - long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_INTPTR_SIZE, 0); + long err = mi_prim_mbind(*addr, size, MPOL_PREFERRED, &numa_mask, 8*MI_SIZE_SIZE, 0); if (err != 0) { err = errno; _mi_warning_message("failed to bind huge (1GiB) pages to numa node %d (error: %ld (0x%lx))\n", numa_node, err, err); diff --git a/src/prim/windows/prim.c b/src/prim/windows/prim.c index 0dfc42202..222fa6303 100644 --- a/src/prim/windows/prim.c +++ b/src/prim/windows/prim.c @@ -278,7 +278,7 @@ int _mi_prim_free(void* addr, size_t size ) { //--------------------------------------------- static void* win_virtual_alloc_prim_once(void* addr, size_t size, size_t try_alignment, DWORD flags) { - #if (MI_INTPTR_SIZE >= 8) + #if (MI_SIZE_SIZE >= 8) // on 64-bit systems, try to use the virtual address area after 2TiB for 4MiB aligned allocations if (addr == NULL) { void* hint = _mi_os_get_aligned_hint(try_alignment,size); diff --git a/src/random.c b/src/random.c index 464627acf..3c012adf4 100644 --- a/src/random.c +++ b/src/random.c @@ -139,14 +139,14 @@ void _mi_random_split(mi_random_ctx_t* ctx, mi_random_ctx_t* ctx_new) { chacha_split(ctx, (uintptr_t)ctx_new ^ nonce_rnd /*nonce*/, ctx_new); } -uintptr_t _mi_random_next(mi_random_ctx_t* ctx) { +size_t _mi_random_next(mi_random_ctx_t* ctx) { mi_assert_internal(mi_random_is_initialized(ctx)); - uintptr_t r; + size_t r; do { - #if MI_INTPTR_SIZE <= 4 + #if MI_SIZE_SIZE <= 4 r = chacha_next32(ctx); - #elif MI_INTPTR_SIZE == 8 - r = (((uintptr_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); + #elif MI_SIZE_SIZE == 8 + r = (((size_t)chacha_next32(ctx) << 32) | chacha_next32(ctx)); #else # error "define mi_random_next for this platform" #endif @@ -160,12 +160,12 @@ To initialize a fresh random context. If we cannot get good randomness, we fall back to weak randomness based on a timer and ASLR. -----------------------------------------------------------------------------*/ -uintptr_t _mi_os_random_weak(uintptr_t extra_seed) { - uintptr_t x = (uintptr_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random +size_t _mi_os_random_weak(size_t extra_seed) { + size_t x = (size_t)&_mi_os_random_weak ^ extra_seed; // ASLR makes the address random x ^= _mi_prim_clock_now(); // and do a few randomization steps - uintptr_t max = ((x ^ (x >> 17)) & 0x0F) + 1; - for (uintptr_t i = 0; i < max || x==0; i++, x++) { + size_t max = ((x ^ (x >> 17)) & 0x0F) + 1; + for (size_t i = 0; i < max || x==0; i++, x++) { x = _mi_random_shuffle(x); } mi_assert_internal(x != 0); @@ -180,7 +180,7 @@ static void mi_random_init_ex(mi_random_ctx_t* ctx, bool use_weak) { #if !defined(__wasi__) if (!use_weak) { _mi_warning_message("unable to use secure randomness\n"); } #endif - uintptr_t x = _mi_os_random_weak(0); + size_t x = _mi_os_random_weak(0); for (size_t i = 0; i < 32; i+=4, x++) { x = _mi_random_shuffle(x); key[i] = (uint8_t)(x); diff --git a/test/main-override-static.c b/test/main-override-static.c index c3230e1af..ed8fba65a 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -349,7 +349,7 @@ static void test_large_pages(void) { #include #include -#define MI_LARGE_WSIZE_MAX (4*1024*1024 / MI_INTPTR_SIZE) +#define MI_LARGE_WSIZE_MAX (4*1024*1024 / MI_SIZE_SIZE) #define MI_BIN_HUGE 100 //#define MI_ALIGN2W @@ -390,10 +390,10 @@ static inline uint8_t mi_bsr32(uint32_t x) { // Bit scan reverse: return the index of the highest bit. uint8_t _mi_bsr(uintptr_t x) { if (x == 0) return 0; - #if MI_INTPTR_SIZE==8 + #if MI_SIZE_SIZE==8 uint32_t hi = (x >> 32); return (hi == 0 ? mi_bsr32((uint32_t)x) : 32 + mi_bsr32(hi)); - #elif MI_INTPTR_SIZE==4 + #elif MI_SIZE_SIZE==4 return mi_bsr32(x); #else # error "define bsr for non-32 or 64-bit platforms" diff --git a/test/test-api.c b/test/test-api.c index d9b677e09..f1a9ac1e4 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -476,7 +476,7 @@ int main(void) { } } - #if (MI_INTPTR_SIZE > 4) + #if (MI_SIZE_SIZE > 4) CHECK_BODY("arena_reserve") { result = (0==mi_reserve_os_memory(16*MI_GiB,false,true)); } From 2317ff6669aef98b310dcec9274eba0537ac3018 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 18:03:52 -0700 Subject: [PATCH 226/280] use sizeof(size_t) = machine word consistently (instead of sizeof(intptr_t)) --- src/options.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/options.c b/src/options.c index 32ba894c9..6ef6e0e8e 100644 --- a/src/options.c +++ b/src/options.c @@ -49,7 +49,7 @@ int mi_version(void) { // in KiB #ifndef MI_DEFAULT_ARENA_RESERVE - #if (MI_INTPTR_SIZE>4) + #if (MI_SIZE_SIZE>4) #define MI_DEFAULT_ARENA_RESERVE 1024L*1024L #else #define MI_DEFAULT_ARENA_RESERVE 128L*1024L From 5d2916adc7d1a76edfdea2798bde6607b944f9f6 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 19:40:14 -0700 Subject: [PATCH 227/280] fix assertion --- src/free.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/free.c b/src/free.c index a0e34f13a..eddfb7e74 100644 --- a/src/free.c +++ b/src/free.c @@ -615,7 +615,7 @@ static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const // Used for double free checking to avoid checking free lists too frequently static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { mi_block_t* n = mi_block_nextx(page,block,page->keys); - return (!_mi_is_aligned(p,MI_SIZE_SIZE) && // quick check: aligned pointer? + return (!_mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer? (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? } From 39f4509ecdc15508093721e8fd1d9c843e49cd19 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 20:17:38 -0700 Subject: [PATCH 228/280] theap_malloc functions can assume a non-null theap --- include/mimalloc.h | 19 +++++++------ include/mimalloc/internal.h | 3 +- src/alloc-aligned.c | 6 ++-- src/alloc.c | 55 ++++++++++++++++++------------------- 4 files changed, 40 insertions(+), 43 deletions(-) diff --git a/include/mimalloc.h b/include/mimalloc.h index a0d569f6f..6de3738c0 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -400,29 +400,30 @@ mi_decl_nodiscard mi_decl_export void* mi_theap_rezalloc(mi_the // Fast constant size allocations. // ------------------------------------------------------ -// `__mi` are internal functions; do not use. -mi_decl_nodiscard mi_decl_restrict void* __mi_wzalloc_small(size_t wsize) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_restrict void* __mi_wmalloc_small(size_t wsize) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; -mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +// Machine word size allocation. `wsize` is the allocation size in machine words (`sizeof(size_t)`) +mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept; +// get the machine word size from a byte size. static inline size_t mi_wsize_from_size(size_t size) { return ((size + sizeof(size_t) - 1) / sizeof(size_t)); } static inline mi_decl_restrict void* mi_malloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return __mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_wmalloc_small(mi_wsize_from_size(size)); } else { return mi_malloc(size); } } static inline mi_decl_restrict void* mi_zalloc_csize(size_t size) mi_attr_noexcept { - if (size <= MI_SMALL_SIZE_MAX) { return __mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_wzalloc_small(mi_wsize_from_size(size)); } else { return mi_zalloc(size); } } static inline mi_decl_restrict void* mi_theap_malloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { assert(theap!=NULL); - if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wmalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_malloc(theap,size); } } static inline mi_decl_restrict void* mi_theap_zalloc_csize(mi_theap_t* theap, size_t size) mi_attr_noexcept { assert(theap!=NULL); - if (size <= MI_SMALL_SIZE_MAX) { return __mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } + if (size <= MI_SMALL_SIZE_MAX) { return mi_theap_wzalloc_small(theap,mi_wsize_from_size(size)); } else { return mi_theap_zalloc(theap,size); } } static inline void mi_free_csize(void* p, size_t size) mi_attr_noexcept { if (size <= MI_SMALL_SIZE_MAX) { mi_free_small(p); } else { mi_free(p); } diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 246e29a57..16c279ca4 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -323,8 +323,7 @@ mi_msecs_t _mi_clock_start(void); // "alloc.c" void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t size, bool zero) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` -void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept; -void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` +void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept; // called from `_mi_theap_malloc_aligned` void* _mi_theap_realloc_zero(mi_theap_t* theap, void* p, size_t newsize, bool zero) mi_attr_noexcept; mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p); void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size); diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 38600c71f..752b1d0a7 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -56,13 +56,13 @@ static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, boo // const size_t rate = theap->guarded_sample_rate; // only write if `rate!=0` so we don't write to the constant `_mi_theap_empty` // if (rate != 0) { theap->guarded_sample_rate = 0; } - void* p = _mi_theap_malloc_zero(theap, size, zero, ppage); + void* p = _mi_theap_malloc_zero(theap, size, zero, 0, ppage); // if (rate != 0) { theap->guarded_sample_rate = rate; } return p; } #else static void* mi_theap_malloc_zero_no_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) { - return _mi_theap_malloc_zero(theap, size, zero, ppage); + return _mi_theap_malloc_zero(theap, size, zero, 0, ppage); } #endif @@ -86,7 +86,7 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap } oversize = (size <= MI_SMALL_SIZE_MAX ? MI_SMALL_SIZE_MAX + 1 /* ensure we use generic malloc path */ : size); // note: no guarded as alignment > 0 - p = _mi_theap_malloc_zero_ex(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block + p = _mi_theap_malloc_zero(theap, oversize, zero, alignment, &page); // the page block size should be large enough to align in the single huge page block if (p == NULL) return NULL; } else { diff --git a/src/alloc.c b/src/alloc.c index 57ce56494..21f2e892a 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -199,7 +199,7 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size } // internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_ex(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { +static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { #if MI_THEAP_INITASNULL if (theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); @@ -212,26 +212,23 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_ex( #endif } -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - return mi_theap_malloc_small_zero_ex(theap,size,false,zero,ppage); -} - // allocate a small block mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(theap, size, false, NULL); + mi_assert(theap!=NULL); + return mi_theap_malloc_small_zero_nonnull(theap, size, false, false, NULL); } -mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); return mi_theap_malloc_small_zero_nonnull(theap, wsize, true, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small(_mi_theap_default(), size); + return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, false, NULL); } -mi_decl_nodiscard mi_decl_restrict void* __mi_wmalloc_small(size_t wsize) mi_attr_noexcept { - return mi_theap_malloc_small_zero_ex(_mi_theap_default(), wsize, true, false, NULL ); +mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept { + return mi_theap_malloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -248,6 +245,7 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, s // The main internal allocation functions static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { + mi_assert(theap!=NULL); // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); @@ -258,7 +256,7 @@ static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, } } -extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { +extern mi_decl_forceinline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, size_t huge_alignment, mi_page_t** ppage) mi_attr_noexcept { // fast path for small objects #if MI_THEAP_INITASNULL if mi_likely(theap!=NULL && size <= MI_SMALL_SIZE_MAX) @@ -274,19 +272,17 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero_ex(mi_theap_t* theap, siz } } -extern inline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, zero, 0, ppage); -} // Main allocation functions mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero_ex(theap, size, false, 0, NULL); + mi_assert(theap!=NULL); + return mi_theap_malloc_zero_nonnull(theap, size, false, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc(size_t size) mi_attr_noexcept { - return mi_theap_malloc(_mi_theap_default(), size); + return _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, NULL); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -303,20 +299,21 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t // zero initialized small block mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), size, true, NULL); + return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, true, NULL); } -mi_decl_nodiscard mi_decl_restrict void* __mi_wzalloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_ex(_mi_theap_default(), size, true, true, NULL); +mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept { + return mi_theap_malloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(theap, size, true, NULL); + mi_assert(theap!=NULL); + return mi_theap_malloc_small_zero_nonnull(theap, size, false, true, NULL); } -mi_decl_nodiscard mi_decl_restrict void* __mi_theap_wzalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { +mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_malloc_small_zero_nonnull(theap, size, true, true, NULL); + return mi_theap_malloc_small_zero_nonnull(theap, wsize, true, true, NULL); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -330,11 +327,11 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, s } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc(mi_theap_t* theap, size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(theap, size, true, NULL); + return _mi_theap_malloc_zero(theap, size, true, 0, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_zalloc(size_t size) mi_attr_noexcept { - return _mi_theap_malloc_zero(_mi_theap_default(), size, true, NULL); + return _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, NULL); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc(mi_heap_t* heap, size_t size) mi_attr_noexcept { @@ -374,25 +371,25 @@ static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { // Return usable size mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, &page); + void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, false, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, true, &page); + void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, true, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_umalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, &page); + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, false, 0, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, &page); + void* p = _mi_theap_malloc_zero(_mi_theap_default(), size, true, 0, &page); return mi_ublock_size(p,page,pblock_size); } @@ -475,7 +472,7 @@ static mi_decl_forceinline void* mi_theap_realloc_zero_ex(mi_theap_t* theap, voi } // note: we don't zero allocate upfront so we only zero initialize the expanded part mi_page_t* newpage; // use block_size for zero-ing, issue #763 - void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,&newpage); + void* const newp = _mi_theap_malloc_zero(theap,newsize,false /* no zero */,0,&newpage); if mi_likely(newp != NULL) { if (pblock_size_post!=NULL) { *pblock_size_post = mi_page_block_size(newpage); } const size_t copy_size = (newsize > size ? size : newsize); From 6590d34a5a2473055030f5de7c9d38660d550a88 Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 21:29:45 -0700 Subject: [PATCH 229/280] add profiler heap --- include/mimalloc-profile.h | 41 ++++++++-------- include/mimalloc/internal.h | 4 +- include/mimalloc/types.h | 1 + src/page.c | 10 ++-- src/sample-profile.c | 78 +++++++++++++++++++---------- src/subproc.c | 3 +- test/test-profile.c | 98 ++++++++++++++++++++----------------- 7 files changed, 136 insertions(+), 99 deletions(-) diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h index 6e9e8ad02..3460a7b12 100644 --- a/include/mimalloc-profile.h +++ b/include/mimalloc-profile.h @@ -12,37 +12,38 @@ terms of the MIT license. A copy of the license can be found in the file #include // bool #include // uint64_t -#define MI_PROFILE_DATA_MAX_SIZE (1024) +#define MI_PROFILE_SAMPLE_DATA_MAX_SIZE (1024) // Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.) -typedef struct mi_profiler_data_s { - size_t profiler_data_size; // size of the custom profile data (should be the `mi_profiler_t.profiler_data_size`) - size_t requested_size; // user requested size (i.e. the size passed to `mi_malloc`) - void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `profiler_data_size` -} mi_profiler_data_t; +typedef struct mi_profiler_sample_data_s { + size_t user_data_size; // size of the user_data (should be the `mi_profiler_t.sample_data_size`) + void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `sample_data_size` +} mi_profiler_sample_data_t; + +typedef struct mi_profiler_s mi_profiler_t; // Profiling callback invoked on each sampled allocation. // If `profiler_data!=NULL` (i.e. when `on_free` is not NULL), then `profiler_data->requested_size == requested_size`. -typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg); +typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap); // Profiling callback invoked on each sampled in-place re-allocation. -typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap, void* profiler_arg); +typedef size_t (mi_cdecl mi_profiler_on_realloc_inplace_fun)(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t old_size, const mi_heap_t* heap); // Profiling callback invoked on a previously sampled allocation. -typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_data_t* profiler_data, void* ptr, const mi_heap_t* heap, void* profiler_arg); +typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, const mi_heap_t* heap); // A profiler // All fields are considered immutable such that they can be copied and accessed concurrently. All fields can be NULL/0. -typedef struct mi_profiler_s { - void* reserved1; // opaque -- internal use for mimalloc - void* reserved2; - void* reserved3; - void* profiler_arg; // opaque user profiler state pointer -- passed to each callback - size_t profiler_data_size; // size of required profiler data for each sampled allocation +struct mi_profiler_s { + void* reserved; // opaque; reserved by mimalloc + mi_heap_t* profiler_heap; // if initially NULL, it is filled in by mimalloc when the profiler is attached; this heap will not be sampled. + size_t sample_data_size; // size of required profiler data for each sampled allocation + size_t initial_sample_rate;// initial sample rate (set to at least 1 or higher) (can be adjusted by `on_alloc`) mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently) mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently) - mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // called on in-place reallocation of a previous sampled allocation (may be called concurrently) -} mi_profiler_t; + mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // (as yet unused) called on in-place reallocation of a previous sampled allocation (may be called concurrently) + // ... more user fields allowed +}; // Exported definitions #ifdef __cplusplus @@ -50,11 +51,11 @@ extern "C" { #endif // attach a profiler to a heap. -mi_decl_export bool mi_heap_profile(mi_heap_t* heap, const mi_profiler_t* profiler); +mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler); // attach a profiler to all heaps in a sub-process. -mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, const mi_profiler_t* profiler); +mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler); // attach a profiler to the main sub-process. -mi_decl_export bool mi_profile(const mi_profiler_t* profiler); +mi_decl_export bool mi_profile(mi_profiler_t* profiler); // start sampling mi_decl_export bool mi_profiler_start(const mi_profiler_t* profiler); diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 16c279ca4..060e038a9 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1229,12 +1229,12 @@ static inline bool mi_block_ptr_is_sampled(const mi_block_t* block, const void* } static inline bool mi_profiler_is_enabled(const mi_profiler_t* prof) { - _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved1; + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; return (mi_atomic_load_acquire(penabled) != 0); } static inline bool mi_profiler_set_enabled(mi_profiler_t* prof, bool enable) { - _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved1; + _Atomic(size_t)* penabled = (_Atomic(size_t)*)&prof->reserved; return (mi_atomic_exchange_release(penabled, (enable ? 1 : 0)) != 0); } diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 077637ae9..4aa891a60 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -714,6 +714,7 @@ struct mi_subproc_s { _Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged _Atomic(mi_heap_t*) heap_main; // main heap for this sub process + _Atomic(mi_heap_t*) heap_profiler; // heap for an attached profiler mi_heap_t* heaps; // heaps belonging to this sub-process mi_lock_t heaps_lock; diff --git a/src/page.c b/src/page.c index 15d237ec6..0d2a79a5e 100644 --- a/src/page.c +++ b/src/page.c @@ -1217,13 +1217,11 @@ static mi_theap_t* mi_malloc_generic_admin(mi_theap_t* theap) mi_heap_t* const heap = _mi_theap_heap(theap); mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); const bool prof_enabled = (prof!=NULL && mi_profiler_is_enabled(prof)); - if (prof_enabled && theap->sample_rate==0) { - theap->sample_rate = 1; // start profiling - theap->profile_sample_rate = 1; + if (prof_enabled && theap->profile_sample_rate==0) { + _mi_theap_set_profile_sample_rate(theap,mi_max(1,prof->initial_sample_rate)); // start profiling } - else if (!prof_enabled && theap->sample_rate!=0) { - theap->sample_rate = theap->guarded_sample_rate; // stop profiling - theap->profile_sample_rate = 0; + else if (!prof_enabled && theap->profile_sample_rate!=0) { + _mi_theap_set_profile_sample_rate(theap,0); // stop profiling } // do a full theap collect every once in a while (10000 by default) diff --git a/src/sample-profile.c b/src/sample-profile.c index 79dfd9b16..bc5f395fd 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -118,7 +118,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th mi_assert_internal(size<=requested_since_last_sample); mi_assert_internal(size>=MI_PADDING_SIZE); mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); - if (prof == NULL) { return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } + if (prof == NULL || prof->on_alloc == NULL) { return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } const size_t req_size = size - MI_PADDING_SIZE; void* p = NULL; @@ -128,23 +128,24 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th p = _mi_malloc_generic_no_sample(size,theap,zero,ppage); if (p==NULL) { return p; } if (prof->on_alloc!=NULL) { - new_sample_rate = (*prof->on_alloc)(NULL, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); + // we are just allocation profiling (not heap profiling as on_free == NULL) + new_sample_rate = (*prof->on_alloc)(prof, NULL /* no data */, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); } } else { // Overallocate a larger block to store the profiler data // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] - const size_t profiler_data_offset = sizeof(mi_block_t); - size_t profiler_data_size = sizeof(mi_profiler_data_t); - if (prof->profiler_data_size > 2*sizeof(size_t)) { profiler_data_size = (prof->profiler_data_size > MI_PROFILE_DATA_MAX_SIZE ? MI_PROFILE_DATA_MAX_SIZE : prof->profiler_data_size); }; - const size_t profiler_user_offset = _mi_align_up(profiler_data_offset + profiler_data_size, MI_MAX_ALIGN_SIZE); - const size_t oversize = profiler_user_offset + size; + const size_t sample_data_offset = sizeof(mi_block_t); + size_t sample_user_data_size = 0; + if (prof->sample_data_size > 0) { sample_user_data_size = (prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size); }; + const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; + const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); + const size_t oversize = user_offset + size; mi_page_t* page = NULL; mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(oversize,theap,zero,&page); if (block==NULL) return NULL; mi_assert_internal(page!=NULL); - if (ppage!=NULL) { *ppage = page; } - mi_assert_internal(!mi_block_ptr_is_guarded(_mi_page_ptr_unalign(page,block),block)); + if (ppage!=NULL) { *ppage = page; } #if MI_PAGE_META_SMALL_IS_ALIGNED // we should never allocate something allocated as small in a non-small page or otherwise aligned mi_free_small may fail. // (that is why we need to limit the profiler_data_size as well) @@ -155,14 +156,13 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th // Set up the profiled block mi_page_set_has_interior_pointers(page, true); block->next = MI_BLOCK_TAG_PROFILED; - p = (uint8_t*)block + profiler_user_offset; - mi_profiler_data_t* profiler_data = (mi_profiler_data_t*)((uint8_t*)block + profiler_data_offset); - profiler_data->profiler_data_size = profiler_data_size; - profiler_data->requested_size = req_size; - + p = (uint8_t*)block + user_offset; + mi_profiler_sample_data_t* sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); + sample_data->user_data_size = sample_user_data_size; + // and call the profiler on_alloc if (prof->on_alloc!=NULL) { - new_sample_rate = (*prof->on_alloc)(profiler_data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap), prof->profiler_arg); + new_sample_rate = (*prof->on_alloc)(prof, sample_data, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); } } if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { @@ -182,8 +182,8 @@ void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; // call the on_free callback - mi_profiler_data_t* const profiler_data = (mi_profiler_data_t*)((uint8_t*)block + sizeof(mi_block_t)); - prof->on_free(profiler_data, p, heap, prof->profiler_arg); + mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t)); + prof->on_free(prof, sample_data, p, heap); } @@ -191,6 +191,24 @@ void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { // Profiling API //-----------------------------------------------------------------------------* +static mi_profiler_t mi_nosample_profiler = { MI_ATOMIC_VAR_INIT(NULL), NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL }; + +static mi_heap_t* mi_subproc_get_heap_profiler(mi_subproc_t* subproc) { + mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t, &subproc->heap_profiler); + if (heap!=NULL) return heap; + + heap = mi_heap_new(); + mi_atomic_store_ptr_release(mi_profiler_t,&heap->profiler, &mi_nosample_profiler); + mi_heap_t* prev_heap = NULL; + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_heap_t, &subproc->heap_profiler, &prev_heap, heap)) { + // already set by someone else + mi_heap_delete(heap); + heap = prev_heap; + } + mi_assert_internal(heap!=NULL && heap == mi_atomic_load_ptr_acquire(mi_heap_t, &subproc->heap_profiler)); + return heap; +} + size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) { const size_t old_sample_rate = theap->profile_sample_rate; theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); @@ -201,26 +219,34 @@ size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) return old_sample_rate; } -bool mi_heap_profile(mi_heap_t* heap, const mi_profiler_t* profiler) { - // if (mi_heap_profiler(heap)!=NULL) return false; // always overwrite? - mi_atomic_store_ptr_release(mi_profiler_t,&heap->profiler,(mi_profiler_t*)profiler); +static bool mi_heap_set_profiler(mi_heap_t* heap, mi_profiler_t* profiler) { + mi_profiler_t* previous = NULL; // never overwrite + return mi_atomic_cas_ptr_strong_acq_rel(mi_heap_t, &heap->profiler, &previous, profiler); +} + +bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) { + mi_profiler_stop(profiler); + if (!mi_heap_set_profiler(heap,profiler)) return false; + profiler->profiler_heap = mi_subproc_get_heap_profiler(heap->subproc); return true; } -bool mi_subproc_profile(mi_subproc_id_t subproc_id, const mi_profiler_t* profiler) { +bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) { mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); - if (subproc==NULL) return false; - // if (mi_subproc_profiler(subproc)!=NULL) return false; // always overwrite? - mi_atomic_store_ptr_release(mi_profiler_t,&subproc->profiler, (mi_profiler_t*)profiler); + if (subproc==NULL) return false; + mi_profiler_stop(profiler); + mi_profiler_t* previous = NULL; + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; } // never overwrite + profiler->profiler_heap = mi_subproc_get_heap_profiler(subproc); mi_lock(&subproc->heaps_lock) { for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) { - mi_heap_profile(heap,profiler); + mi_heap_set_profiler(heap,profiler); } } return true; } -bool mi_profile(const mi_profiler_t* profiler) { +bool mi_profile( mi_profiler_t* profiler) { return mi_subproc_profile(mi_subproc_main(),profiler); } diff --git a/src/subproc.c b/src/subproc.c index c248fca22..9e01d4b5c 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -228,7 +228,8 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro } } - subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap + subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap + mi_atomic_store_ptr_relaxed(mi_heap_t,&subproc->heap_profiler,NULL); if (!_mi_subproc_is_main(subproc)) { // merge stats back into the main subproc diff --git a/test/test-profile.c b/test/test-profile.c index e4ec69f44..6ad5d367f 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -23,51 +23,61 @@ terms of the MIT license. A copy of the license can be found in the file // --------------------------------------------------------------------------- typedef struct { + mi_profiler_t profiler; uint64_t alloc_count; uint64_t free_count; size_t last_size; uint64_t last_upscaled; void* last_ptr; -} profile_state_t; +} my_profiler_t; -// We store ptr in user_data so on_free can verify the round-trip. +static inline my_profiler_t* downcast( mi_profiler_t* prof ) { + return (my_profiler_t*)prof; +} -static profile_state_t g_state; +// We store ptr in user_data so on_free can verify the round-trip. #define TEST_THRESHOLD (16 * 1024) -static size_t mi_cdecl on_alloc(mi_profiler_data_t* data, void* ptr, size_t requested_size, size_t threshold, uint64_t bytes_since_last_sample, const mi_heap_t* heap, void* profiler_arg) { - MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(profiler_arg); MI_UNUSED(requested_size); - assert(profiler_arg==&g_state); - assert(bytes_since_last_sample >= requested_size); - assert(requested_size == data->requested_size); - g_state.alloc_count++; - g_state.last_ptr = ptr; - g_state.last_size = requested_size; - g_state.last_upscaled = bytes_since_last_sample; - // store ptr to verify round-trip +static size_t mi_cdecl on_alloc(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, size_t requested_size, size_t threshold, uint64_t bytes_since_last_sample, const mi_heap_t* heap) { + MI_UNUSED(threshold); MI_UNUSED(heap); MI_UNUSED(requested_size); + my_profiler_t* prof = downcast(profiler); + assert(bytes_since_last_sample >= requested_size); + prof->alloc_count++; + prof->last_ptr = ptr; + prof->last_size = requested_size; + prof->last_upscaled = bytes_since_last_sample; + // store ptr to verify round-trip + assert(data->user_data_size >= sizeof(void*)); + assert(data->user_data_size >= prof->profiler.sample_data_size); data->user_data[0] = ptr; return TEST_THRESHOLD; } -static void mi_cdecl on_free(mi_profiler_data_t* data, void* ptr, const mi_heap_t* heap, void* profiler_arg) { - MI_UNUSED(heap); MI_UNUSED_RELEASE(profiler_arg); MI_UNUSED_RELEASE(data); - assert(profiler_arg==&g_state); - g_state.free_count++; +static void mi_cdecl on_free(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, const mi_heap_t* heap) { + MI_UNUSED(heap); MI_UNUSED_RELEASE(data); MI_UNUSED_RELEASE(ptr); + my_profiler_t* prof = downcast(profiler); + prof->free_count++; // verify the user_data round-trip - assert(data->user_data[0] == ptr); - if (g_state.last_ptr == ptr) { assert(data->requested_size == g_state.last_size); } + assert(data->user_data[0] == ptr); } -mi_profiler_t my_profiler = { - NULL, NULL, NULL, // reserved - &g_state, // profiler_arg - 3*sizeof(void*), // needed data size - &on_alloc, - &on_free, - NULL +static my_profiler_t my_profiler = { + { // profiler_t + NULL, // reserved + NULL, // profiler_heap + sizeof(void*), // needed data size (default) + 0, // initial sample rate (default) + &on_alloc, + &on_free, + NULL + }, + 0, 0, 0, 0, NULL }; + + + // --------------------------------------------------------------------------- // Helpers // --------------------------------------------------------------------------- @@ -89,9 +99,9 @@ static void allocate_past_threshold(void) { bool test_profiler_samples(void) { CHECK_BODY("profiler: on_alloc called after threshold") { - uint64_t before = g_state.alloc_count; + uint64_t before = my_profiler.alloc_count; allocate_past_threshold(); - result = (g_state.alloc_count > before); + result = (my_profiler.alloc_count > before); } return true; } @@ -100,51 +110,51 @@ bool test_profiler_samples(void) { bool test_profiler_record_fields(void) { CHECK_BODY("profiler: record ptr and size are non-zero") { - uint64_t before = g_state.alloc_count; + uint64_t before = my_profiler.alloc_count; int count; - for (count = 0; g_state.alloc_count == before && count < MAXLOOP; count++) { + for (count = 0; my_profiler.alloc_count == before && count < MAXLOOP; count++) { void* p = mi_malloc(1024); mi_free(p); } assert(count!=MAXLOOP); - result = (g_state.last_ptr != NULL && g_state.last_size > 0 && g_state.last_upscaled > 0 && count!=MAXLOOP); + result = (my_profiler.last_ptr != NULL && my_profiler.last_size > 0 && my_profiler.last_upscaled > 0 && count!=MAXLOOP); } return true; } bool test_profiler_on_free_called(void) { CHECK_BODY("profiler: on_free called for sampled allocation") { - uint64_t alloc_before = g_state.alloc_count; - uint64_t free_before = g_state.free_count; + uint64_t alloc_before = my_profiler.alloc_count; + uint64_t free_before = my_profiler.free_count; // Keep the pointer live until we confirm a sample was taken, then free it. void* sampled = NULL; int count; - for (count = 0; g_state.alloc_count == alloc_before && count < MAXLOOP; count++) { + for (count = 0; my_profiler.alloc_count == alloc_before && count < MAXLOOP; count++) { if (sampled) { mi_free(sampled); } sampled = mi_malloc(1024); } - // At this point g_state.last_ptr is the sampled pointer. + // At this point my_profiler.last_ptr is the sampled pointer. // Free it and check on_free fires. - void* expected = g_state.last_ptr; + void* expected = my_profiler.last_ptr; mi_free(expected); sampled = NULL; assert(count!=MAXLOOP); - result = (g_state.free_count > free_before && count!=MAXLOOP); + result = (my_profiler.free_count > free_before && count!=MAXLOOP); } return true; } bool test_profiler_upscaled_at_least_size(void) { CHECK_BODY("profiler: upscaled_size >= size") { - uint64_t before = g_state.alloc_count; + uint64_t before = my_profiler.alloc_count; int count; - for (count = 0; g_state.alloc_count == before && count < MAXLOOP; count++) { + for (count = 0; my_profiler.alloc_count == before && count < MAXLOOP; count++) { void* p = mi_malloc(256); mi_free(p); } assert(count!=MAXLOOP); - result = (g_state.last_upscaled >= g_state.last_size && count!=MAXLOOP); + result = (my_profiler.last_upscaled >= my_profiler.last_size && count!=MAXLOOP); } return true; } @@ -154,7 +164,7 @@ bool test_profiler_free_count_le_alloc_count(void) { // Free can only fire for sampled allocations, so free_count <= alloc_count // must hold at all times. allocate_past_threshold(); - result = (g_state.free_count <= g_state.alloc_count); + result = (my_profiler.free_count <= my_profiler.alloc_count); } return true; } @@ -165,8 +175,8 @@ bool test_profiler_free_count_le_alloc_count(void) { // --------------------------------------------------------------------------- int main(void) { - mi_profile(&my_profiler); - mi_profiler_start(&my_profiler); + mi_profile(&my_profiler.profiler); + mi_profiler_start(&my_profiler.profiler); test_profiler_upscaled_at_least_size(); test_profiler_samples(); @@ -174,7 +184,7 @@ int main(void) { test_profiler_on_free_called(); test_profiler_free_count_le_alloc_count(); - mi_profiler_stop(&my_profiler); + mi_profiler_stop(&my_profiler.profiler); return print_test_summary(); } From e1c568d8cf4eb26c073a8b08db0254bbfb76bacd Mon Sep 17 00:00:00 2001 From: Daan Date: Wed, 9 Sep 2026 21:33:38 -0700 Subject: [PATCH 230/280] fix c++ compilation --- src/alloc.c | 1 - src/sample-profile.c | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 21f2e892a..a5f28f97f 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -924,7 +924,6 @@ mi_decl_nodiscard void* mi_new_reallocn(void* p, size_t newcount, size_t size) { void* _mi_externs[] = { (void*)&_mi_page_malloc_zero, (void*)&_mi_theap_malloc_zero, - (void*)&_mi_theap_malloc_zero_ex, (void*)&mi_theap_malloc, (void*)&mi_theap_zalloc, (void*)&mi_theap_malloc_small, diff --git a/src/sample-profile.c b/src/sample-profile.c index bc5f395fd..04159e931 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -191,7 +191,7 @@ void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { // Profiling API //-----------------------------------------------------------------------------* -static mi_profiler_t mi_nosample_profiler = { MI_ATOMIC_VAR_INIT(NULL), NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL }; +static mi_profiler_t mi_nosample_profiler = { NULL, NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL }; static mi_heap_t* mi_subproc_get_heap_profiler(mi_subproc_t* subproc) { mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t, &subproc->heap_profiler); @@ -221,7 +221,7 @@ size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) static bool mi_heap_set_profiler(mi_heap_t* heap, mi_profiler_t* profiler) { mi_profiler_t* previous = NULL; // never overwrite - return mi_atomic_cas_ptr_strong_acq_rel(mi_heap_t, &heap->profiler, &previous, profiler); + return mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t, &heap->profiler, &previous, profiler); } bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) { From 2d53d6ae832111d26fdb00c7bd301914fc6f0788 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 08:36:45 -0700 Subject: [PATCH 231/280] fix double free check without padding --- src/free.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/free.c b/src/free.c index eddfb7e74..63734721d 100644 --- a/src/free.c +++ b/src/free.c @@ -16,8 +16,6 @@ terms of the MIT license. A copy of the license can be found in the file mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, const mi_block_t* block, bool is_guarded, size_t* usable_size); mi_decl_nodiscard static bool mi_check_double_free(const mi_page_t* page, const mi_block_t* block); static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* block, bool was_guarded); -// static void mi_stat_free(const mi_page_t* page, const mi_block_t* block); - // ------------------------------------------------------ // Free @@ -615,7 +613,7 @@ static mi_decl_noinline bool mi_check_double_freex(const mi_page_t* page, const // Used for double free checking to avoid checking free lists too frequently static inline bool mi_block_could_be_double_free(const mi_page_t* page, const mi_block_t* block) { mi_block_t* n = mi_block_nextx(page,block,page->keys); - return (!_mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer? + return (_mi_is_aligned(block,MI_SIZE_SIZE) && // quick check: aligned pointer? (n==NULL || mi_page_contains_address(page,n))); // quick check: in the same page or NULL? } From c31f719ba1f41d300df201e0f2c92390fd4248ee Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 11:49:32 -0700 Subject: [PATCH 232/280] cleanup profiler api --- include/mimalloc-profile.h | 30 ++++++++----- include/mimalloc/types.h | 3 +- src/bitmap.h | 2 +- src/options.c | 27 ++++++------ src/sample-profile.c | 88 +++++++++++++++++--------------------- src/subproc.c | 3 +- test/test-profile.c | 3 +- 7 files changed, 76 insertions(+), 80 deletions(-) diff --git a/include/mimalloc-profile.h b/include/mimalloc-profile.h index 3460a7b12..5a1327ff7 100644 --- a/include/mimalloc-profile.h +++ b/include/mimalloc-profile.h @@ -12,16 +12,21 @@ terms of the MIT license. A copy of the license can be found in the file #include // bool #include // uint64_t +//--------------------------------------------------------------------------- +// Initial support for profiling hooks; still experimental +//--------------------------------------------------------------------------- + #define MI_PROFILE_SAMPLE_DATA_MAX_SIZE (1024) +// User defined profiler; see `test_profile.c` for an example. +typedef struct mi_profiler_s mi_profiler_t; + // Profiler data is stored together with each sampled allocation (unless the `on_free` field in the profiler is NULL.) typedef struct mi_profiler_sample_data_s { size_t user_data_size; // size of the user_data (should be the `mi_profiler_t.sample_data_size`) void* user_data[1]; // default, but can be less or more (up to 1KiB), depending on `sample_data_size` } mi_profiler_sample_data_t; -typedef struct mi_profiler_s mi_profiler_t; - // Profiling callback invoked on each sampled allocation. // If `profiler_data!=NULL` (i.e. when `on_free` is not NULL), then `profiler_data->requested_size == requested_size`. typedef size_t (mi_cdecl mi_profiler_on_alloc_fun )(mi_profiler_t* profiler, mi_profiler_sample_data_t* profiler_data, void* ptr, size_t requested_size, size_t bytes_sample_rate, uint64_t bytes_since_last_sample, const mi_heap_t* heap); @@ -36,9 +41,8 @@ typedef void (mi_cdecl mi_profiler_on_free_fun )(mi_profiler_t* profiler, mi // All fields are considered immutable such that they can be copied and accessed concurrently. All fields can be NULL/0. struct mi_profiler_s { void* reserved; // opaque; reserved by mimalloc - mi_heap_t* profiler_heap; // if initially NULL, it is filled in by mimalloc when the profiler is attached; this heap will not be sampled. - size_t sample_data_size; // size of required profiler data for each sampled allocation - size_t initial_sample_rate;// initial sample rate (set to at least 1 or higher) (can be adjusted by `on_alloc`) + size_t sample_data_size; // size of required profiler data for each sampled allocation (or zero for no data) + size_t initial_sample_rate;// initial sample rate in bytes (set to at least 1 or higher) (can be adjusted by `on_alloc`) mi_profiler_on_alloc_fun* on_alloc; // called on a sampled allocation (may be called concurrently) mi_profiler_on_free_fun* on_free; // called on when previous sampled allocation is freed (may be called concurrently) mi_profiler_on_realloc_inplace_fun* on_realloc_inplace; // (as yet unused) called on in-place reallocation of a previous sampled allocation (may be called concurrently) @@ -50,17 +54,23 @@ struct mi_profiler_s { extern "C" { #endif -// attach a profiler to a heap. +// attach a profiler to a particular heap only. mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler); -// attach a profiler to all heaps in a sub-process. + +// disable profiling for a particular heap; useful for a heap that the profiler uses itself for metadata. +mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap); + +// attach a profiler to any (current and future) heaps in a sub-process (unless those heaps disabled profiling) mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler); -// attach a profiler to the main sub-process. + +// attach a profiler to any heaps in the main sub-process mi_decl_export bool mi_profile(mi_profiler_t* profiler); // start sampling -mi_decl_export bool mi_profiler_start(const mi_profiler_t* profiler); +mi_decl_export bool mi_profiler_start(mi_profiler_t* profiler); + // end sampling -mi_decl_export bool mi_profiler_stop(const mi_profiler_t* profiler); +mi_decl_export bool mi_profiler_stop(mi_profiler_t* profiler); #ifdef __cplusplus } diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 4aa891a60..edd4f84fd 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -713,8 +713,7 @@ struct mi_subproc_s { mi_decl_align(8) // needed on some 32-bit platforms _Atomic(int64_t) purge_expire; // expiration is set if any arenas can be purged - _Atomic(mi_heap_t*) heap_main; // main heap for this sub process - _Atomic(mi_heap_t*) heap_profiler; // heap for an attached profiler + _Atomic(mi_heap_t*) heap_main; // main heap for this sub process mi_heap_t* heaps; // heaps belonging to this sub-process mi_lock_t heaps_lock; diff --git a/src/bitmap.h b/src/bitmap.h index ba8224b8a..79eda77fa 100644 --- a/src/bitmap.h +++ b/src/bitmap.h @@ -71,7 +71,7 @@ typedef size_t mi_bfield_t; #define MI_BCHUNK_FIELDS (MI_BCHUNK_BITS / MI_BFIELD_BITS) // 8 on both 64- and 32-bit -// some compiler (msvc in C mode) cannot have expressions in the alignment attribute +// some compilers (msvc in C mode) cannot have expressions in the alignment attribute #if MI_BCHUNK_SIZE==64 #define mi_decl_bchunk_align mi_decl_align(64) #elif MI_BCHUNK_SIZE==32 diff --git a/src/options.c b/src/options.c index 6ef6e0e8e..6a12ee47d 100644 --- a/src/options.c +++ b/src/options.c @@ -246,20 +246,7 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ _mi_fprintf(out, arg, "option '%s': %ld %s\n", desc->name, desc->value, (mi_option_has_size_in_kib(option) ? "KiB" : "")); } - // show build configuration - _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); - _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); - _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); - #if MI_GUARDED - _mi_fprintf(out, arg, "guarded mode: %s, rate=%ld\n", MI_SAMPLE==2 ? "fine-grained" : "enabled", mi_option_get(mi_option_guarded_sample_rate)); - #endif - #if MI_TSAN - _mi_fprintf(out, arg, "thread santizer enabled\n"); - #endif - #if MI_PROFILE - _mi_fprintf(out, arg, "profiling : %s\n", MI_SAMPLE==2 ? "fine-grained" : "enabled"); - #endif - + // show build configuration #if MI_PAGE_META_IS_ALIGNED && MI_PAGE_META_SMALL_IS_ALIGNED _mi_fprintf(out, arg, "free mode : (small) aligned, page size: %zu\n", sizeof(mi_page_t)); #elif MI_PAGE_META_IS_ALIGNED @@ -274,6 +261,18 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ #if MI_ENCODE_FREELIST _mi_fprintf(out, arg, "free lists : encoded with %d key(s)\n", MI_PAGE_KEY_COUNT); #endif + #if MI_GUARDED + _mi_fprintf(out, arg, "guarded mode: %s, rate=%ld\n", MI_SAMPLE==2 ? "fine-grained" : "enabled", mi_option_get(mi_option_guarded_sample_rate)); + #endif + #if MI_PROFILE + _mi_fprintf(out, arg, "profiling : %s\n", MI_SAMPLE==2 ? "fine-grained" : "enabled"); + #endif + #if MI_TSAN + _mi_fprintf(out, arg, "thread santizer enabled\n"); + #endif + _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); + _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); + _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); } mi_decl_export void mi_options_print(void) mi_attr_noexcept { diff --git a/src/sample-profile.c b/src/sample-profile.c index 04159e931..bff0e0b84 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -25,6 +25,7 @@ size_t _mi_theap_update_sample_rate(mi_theap_t* theap) { return old_sample_rate; } +// Called when the theap sample countdown <= req_size; should call guarded or profiled allocation. mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_size, bool zero, mi_page_t** ppage) mi_attr_noexcept { // the size has not yet been counted against the countdown (and does not include MI_PADDING_SIZE) @@ -100,7 +101,7 @@ static mi_profiler_t* mi_heap_profiler(const mi_heap_t* heap) { static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { mi_heap_t* const heap = _mi_theap_heap(theap); mi_profiler_t* prof = mi_atomic_load_ptr_relaxed(mi_profiler_t, &heap->profiler); - if (prof!=NULL && mi_profiler_is_enabled(prof)) { + if (prof!=NULL && mi_profiler_is_enabled(prof) && prof->on_alloc != NULL) { return prof; } else { @@ -117,30 +118,30 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th mi_assert_internal(theap!=NULL); mi_assert_internal(size<=requested_since_last_sample); mi_assert_internal(size>=MI_PADDING_SIZE); - mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); - if (prof == NULL || prof->on_alloc == NULL) { return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } const size_t req_size = size - MI_PADDING_SIZE; - + mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); + if (prof == NULL) { return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } + mi_assert_internal(prof!=NULL && prof->on_alloc!=NULL && mi_profiler_is_enabled(prof)); + void* p = NULL; size_t new_sample_rate = 0; - if (prof->on_free==NULL) { + if (prof->on_free==NULL || prof->sample_data_size==0) { // just allocate without profiler data p = _mi_malloc_generic_no_sample(size,theap,zero,ppage); if (p==NULL) { return p; } if (prof->on_alloc!=NULL) { - // we are just allocation profiling (not heap profiling as on_free == NULL) - new_sample_rate = (*prof->on_alloc)(prof, NULL /* no data */, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); + // we are just allocation profiling (not heap profiling as on_free == NULL) + new_sample_rate = (*prof->on_alloc)(prof, NULL /* no data */, p, req_size, theap->profile_sample_rate, requested_since_last_sample, _mi_theap_heap(theap) ); } } else { - // Overallocate a larger block to store the profiler data + // overallocate a larger block to store the profiler data // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] - const size_t sample_data_offset = sizeof(mi_block_t); - size_t sample_user_data_size = 0; - if (prof->sample_data_size > 0) { sample_user_data_size = (prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size); }; - const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; - const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); - const size_t oversize = user_offset + size; + const size_t sample_data_offset = sizeof(mi_block_t); + const size_t sample_user_data_size = _mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*)); + const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; // one void* too many just in case + const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); + const size_t oversize = user_offset + size; mi_page_t* page = NULL; mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(oversize,theap,zero,&page); if (block==NULL) return NULL; @@ -153,11 +154,11 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th if (size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); } #endif - // Set up the profiled block + // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free) mi_page_set_has_interior_pointers(page, true); block->next = MI_BLOCK_TAG_PROFILED; p = (uint8_t*)block + user_offset; - mi_profiler_sample_data_t* sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); + mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); sample_data->user_data_size = sample_user_data_size; // and call the profiler on_alloc @@ -183,7 +184,7 @@ void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { // call the on_free callback mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t)); - prof->on_free(prof, sample_data, p, heap); + (*prof->on_free)(prof, sample_data, p, heap); } @@ -191,24 +192,6 @@ void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { // Profiling API //-----------------------------------------------------------------------------* -static mi_profiler_t mi_nosample_profiler = { NULL, NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL }; - -static mi_heap_t* mi_subproc_get_heap_profiler(mi_subproc_t* subproc) { - mi_heap_t* heap = mi_atomic_load_ptr_acquire(mi_heap_t, &subproc->heap_profiler); - if (heap!=NULL) return heap; - - heap = mi_heap_new(); - mi_atomic_store_ptr_release(mi_profiler_t,&heap->profiler, &mi_nosample_profiler); - mi_heap_t* prev_heap = NULL; - if (!mi_atomic_cas_ptr_strong_acq_rel(mi_heap_t, &subproc->heap_profiler, &prev_heap, heap)) { - // already set by someone else - mi_heap_delete(heap); - heap = prev_heap; - } - mi_assert_internal(heap!=NULL && heap == mi_atomic_load_ptr_acquire(mi_heap_t, &subproc->heap_profiler)); - return heap; -} - size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) { const size_t old_sample_rate = theap->profile_sample_rate; theap->profile_sample_rate = (sample_rate > MI_SAMPLE_RATE_MAX ? MI_SAMPLE_RATE_MAX : sample_rate); @@ -220,24 +203,27 @@ size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate) } static bool mi_heap_set_profiler(mi_heap_t* heap, mi_profiler_t* profiler) { - mi_profiler_t* previous = NULL; // never overwrite + mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler) : NULL); // don't overwrite unless it is NULL return mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t, &heap->profiler, &previous, profiler); } -bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) { +mi_decl_export bool mi_heap_profile(mi_heap_t* heap, mi_profiler_t* profiler) { mi_profiler_stop(profiler); - if (!mi_heap_set_profiler(heap,profiler)) return false; - profiler->profiler_heap = mi_subproc_get_heap_profiler(heap->subproc); - return true; + return mi_heap_set_profiler(heap,profiler); +} + +mi_decl_export void mi_heap_profile_disable(mi_heap_t* heap) { + static mi_profiler_t mi_nosample_profiler = { NULL, 0, MI_SAMPLE_RATE_MAX, NULL, NULL, NULL }; + mi_heap_set_profiler(heap,NULL); // first clear any previous one + mi_heap_set_profiler(heap,&mi_nosample_profiler); // and then install the nosample one } -bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) { +mi_decl_export bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) { mi_subproc_t* subproc = _mi_subproc_from_id(subproc_id); if (subproc==NULL) return false; mi_profiler_stop(profiler); - mi_profiler_t* previous = NULL; - if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; } // never overwrite - profiler->profiler_heap = mi_subproc_get_heap_profiler(subproc); + mi_profiler_t* previous = (profiler==NULL ? mi_atomic_load_ptr_acquire(mi_profiler_t,&subproc->profiler) : NULL); // don't overwrite unless it is NULL + if (!mi_atomic_cas_ptr_strong_acq_rel(mi_profiler_t,&subproc->profiler, &previous, profiler)) { return false; } mi_lock(&subproc->heaps_lock) { for (mi_heap_t* heap = subproc->heaps; heap!=NULL; heap = heap->next) { mi_heap_set_profiler(heap,profiler); @@ -246,13 +232,16 @@ bool mi_subproc_profile(mi_subproc_id_t subproc_id, mi_profiler_t* profiler) { return true; } -bool mi_profile( mi_profiler_t* profiler) { +mi_decl_export bool mi_profile( mi_profiler_t* profiler) { return mi_subproc_profile(mi_subproc_main(),profiler); } -bool mi_profiler_start(const mi_profiler_t* profiler ) { - const bool was_running = mi_profiler_set_enabled((mi_profiler_t*)profiler,true); +bool mi_profiler_start(mi_profiler_t* profiler ) { + if (profiler==NULL) return false; + const bool was_running = mi_profiler_set_enabled(profiler,true); if (was_running) return true; + // for the main heap, if this is the profiler, start the theap more aggressively + // otherwise it will be picked up when theaps take the slow generic malloc path. mi_heap_t* heap = mi_heap_main(); if (mi_heap_profiler(heap)==profiler) { mi_theap_t* theap = _mi_heap_theap_peek(heap); @@ -263,6 +252,7 @@ bool mi_profiler_start(const mi_profiler_t* profiler ) { return false; } -bool mi_profiler_stop(const mi_profiler_t* profiler) { - return mi_profiler_set_enabled((mi_profiler_t*)profiler,false); +bool mi_profiler_stop(mi_profiler_t* profiler) { + if (profiler==NULL) return true; + return mi_profiler_set_enabled(profiler,false); } diff --git a/src/subproc.c b/src/subproc.c index 9e01d4b5c..f42d4a11c 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -229,8 +229,7 @@ static void mi_subproc_unsafe_destroy(mi_subproc_t* subproc, bool acquire_subpro } subproc->theap_meta = NULL; // theap meta stats are merged during heap_destroy of the main heap - mi_atomic_store_ptr_relaxed(mi_heap_t,&subproc->heap_profiler,NULL); - + if (!_mi_subproc_is_main(subproc)) { // merge stats back into the main subproc _mi_stats_merge_into(&mi_process_subproc_main.stats, &subproc->stats); diff --git a/test/test-profile.c b/test/test-profile.c index 6ad5d367f..41c636622 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -65,8 +65,7 @@ static void mi_cdecl on_free(mi_profiler_t* profiler, mi_profiler_sample_data_t* static my_profiler_t my_profiler = { { // profiler_t NULL, // reserved - NULL, // profiler_heap - sizeof(void*), // needed data size (default) + sizeof(void*), // needed sample data size 0, // initial sample rate (default) &on_alloc, &on_free, From 8b8be2ff95d5fb4a7eab8bf64f8a314320b6d72e Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 12:32:27 -0700 Subject: [PATCH 233/280] fix MI_DEBUG=OFF option --- CMakeLists.txt | 3 +++ test/main-override-static.c | 2 +- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3a6d17928..2dafb80ed 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -441,6 +441,9 @@ elseif(MI_DEBUG STREQUAL "INTERNAL") elseif(MI_DEBUG) message(STATUS "Set debug level to assertion checking (MI_DEBUG=${MI_DEBUG})") list(APPEND mi_defines MI_DEBUG=1) # assertion checking (mi_assert) +else() + message(STATUS "Disable assertion checks (MI_DEBUG=${MI_DEBUG})") + list(APPEND mi_defines NDEBUG=1) endif() if(MI_GUARDED STREQUAL "DEFAULT") diff --git a/test/main-override-static.c b/test/main-override-static.c index ed8fba65a..368710c3c 100644 --- a/test/main-override-static.c +++ b/test/main-override-static.c @@ -50,11 +50,11 @@ int main() { // double_free1(); // double_free2(); // corrupt_free(); + // invalid_free(); // block_overflow1(); // block_overflow2(); test_canary_leak(); // test_aslr(); - // invalid_free(); // test_reserved(); // negative_stat(); // test_theap_walk(); From 48f505092fc2a2f363ef48aa1a5578c29ae68ea7 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 18:51:19 -0700 Subject: [PATCH 234/280] azure pipeline failure test --- include/mimalloc/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 060e038a9..802def27f 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1472,7 +1472,7 @@ static inline size_t _mi_random_shuffle(size_t x) { // Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows. // --------------------------------------------------------------------------------- -#if !MI_TRACK_ENABLED && (MI_ARCH_ARM64 || MI_ARCH_X64) && (defined(__SIZEOF_INT128__) || (defined(_MSC_VER) && defined(__AVX2__))) // any 64-bit platform with 128-bit stores can benefit. +#if 0 && !MI_TRACK_ENABLED && (MI_ARCH_ARM64 || MI_ARCH_X64) && (defined(__SIZEOF_INT128__) || (defined(_MSC_VER) && defined(__AVX2__))) // any 64-bit platform with 128-bit stores can benefit. #define MI_USE_MEMZERO128 1 #endif From ba3464b169e44f6503ddfec704742d3301a6f964 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 19:33:12 -0700 Subject: [PATCH 235/280] check azure pipeline failure --- azure-pipelines.yml | 405 ++++++++++++++++++------------------ include/mimalloc/internal.h | 2 +- src/options.c | 2 +- src/page.c | 2 +- src/sample-guarded.c | 2 +- src/sample-profile.c | 10 +- 6 files changed, 212 insertions(+), 211 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 5094950e1..6affb5b6c 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,62 +14,62 @@ trigger: - v* jobs: -- job: - displayName: Windows 2022 - pool: - vmImage: - windows-2022 - strategy: - matrix: - Debug: - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - MSBuildConfiguration: Debug - Release: - BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - MSBuildConfiguration: Release - Release SIMD: - BuildType: release-simd - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON -DMI_WIN_USE_FIXED_TLS=ON - MSBuildConfiguration: Release - Secure: - BuildType: secure - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - MSBuildConfiguration: Release - Debug x86: - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -A Win32 - MSBuildConfiguration: Debug - Release x86: - BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -A Win32 - MSBuildConfiguration: Release - Debug Fixed TLS: - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_WIN_USE_FIXED_TLS=ON - MSBuildConfiguration: Debug - Release Fixed TLS: - BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_WIN_USE_FIXED_TLS=ON - MSBuildConfiguration: Release - steps: - - task: CMake@1 - inputs: - workingDirectory: $(BuildType) - cmakeArgs: .. $(cmakeExtraArgs) - - task: MSBuild@1 - inputs: - solution: $(BuildType)/libmimalloc.sln - configuration: '$(MSBuildConfiguration)' - msbuildArguments: -m - - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) - workingDirectory: $(BuildType) - displayName: CTest - #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress - # displayName: TestStress - #- upload: $(Build.SourcesDirectory)/$(BuildType) - # artifact: mimalloc-windows-$(BuildType) +# - job: +# displayName: Windows 2022 +# pool: +# vmImage: +# windows-2022 +# strategy: +# matrix: +# Debug: +# BuildType: debug +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON +# MSBuildConfiguration: Debug +# Release: +# BuildType: release +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release +# MSBuildConfiguration: Release +# Release SIMD: +# BuildType: release-simd +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON -DMI_WIN_USE_FIXED_TLS=ON +# MSBuildConfiguration: Release +# Secure: +# BuildType: secure +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON +# MSBuildConfiguration: Release +# Debug x86: +# BuildType: debug +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -A Win32 +# MSBuildConfiguration: Debug +# Release x86: +# BuildType: release +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -A Win32 +# MSBuildConfiguration: Release +# Debug Fixed TLS: +# BuildType: debug +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_WIN_USE_FIXED_TLS=ON +# MSBuildConfiguration: Debug +# Release Fixed TLS: +# BuildType: release +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_WIN_USE_FIXED_TLS=ON +# MSBuildConfiguration: Release +# steps: +# - task: CMake@1 +# inputs: +# workingDirectory: $(BuildType) +# cmakeArgs: .. $(cmakeExtraArgs) +# - task: MSBuild@1 +# inputs: +# solution: $(BuildType)/libmimalloc.sln +# configuration: '$(MSBuildConfiguration)' +# msbuildArguments: -m +# - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) +# workingDirectory: $(BuildType) +# displayName: CTest +# #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress +# # displayName: TestStress +# #- upload: $(Build.SourcesDirectory)/$(BuildType) +# # artifact: mimalloc-windows-$(BuildType) - job: displayName: Ubuntu 22.04 @@ -78,71 +78,71 @@ jobs: ubuntu-22.04 strategy: matrix: - Debug: - CC: gcc - CXX: g++ - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + # Debug: + # CC: gcc + # CXX: g++ + # BuildType: debug + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON Release: CC: gcc CXX: g++ BuildType: release cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - Secure: - CC: gcc - CXX: g++ - BuildType: secure - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - Debug++: - CC: gcc - CXX: g++ - BuildType: debug-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + # Secure: + # CC: gcc + # CXX: g++ + # BuildType: secure + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + # Debug++: + # CC: gcc + # CXX: g++ + # BuildType: debug-cxx + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON Debug Clang: CC: clang CXX: clang++ BuildType: debug-clang cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Release Clang: - CC: clang - CXX: clang++ - BuildType: release-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - Release SIMD Clang: - CC: clang - CXX: clang++ - BuildType: release-simd-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON - Secure Clang: - CC: clang - CXX: clang++ - BuildType: secure-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - Debug++ Clang: - CC: clang - CXX: clang++ - BuildType: debug-clang-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - Debug ASAN Clang: - CC: clang - CXX: clang++ - BuildType: debug-asan-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON - Debug UBSAN Clang: - CC: clang - CXX: clang++ - BuildType: debug-ubsan-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON + # Release Clang: + # CC: clang + # CXX: clang++ + # BuildType: release-clang + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + # Release SIMD Clang: + # CC: clang + # CXX: clang++ + # BuildType: release-simd-clang + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON + # Secure Clang: + # CC: clang + # CXX: clang++ + # BuildType: secure-clang + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + # Debug++ Clang: + # CC: clang + # CXX: clang++ + # BuildType: debug-clang-cxx + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + # Debug ASAN Clang: + # CC: clang + # CXX: clang++ + # BuildType: debug-asan-clang + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON + # Debug UBSAN Clang: + # CC: clang + # CXX: clang++ + # BuildType: debug-ubsan-clang + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON Debug TSAN Clang++: CC: clang CXX: clang++ BuildType: debug-tsan-clang-cxx cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON - Debug Guarded Clang: - CC: clang - CXX: clang - BuildType: debug-guarded-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_DEBUG_FULL=ON -DMI_GUARDED=ON + # Debug Guarded Clang: + # CC: clang + # CXX: clang + # BuildType: debug-guarded-clang + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_DEBUG_FULL=ON -DMI_GUARDED=ON steps: - task: CMake@1 @@ -159,105 +159,106 @@ jobs: # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-ubuntu-$(BuildType) -- job: - displayName: macOS 14 (Sonoma) - pool: - vmImage: - macOS-14 - strategy: - matrix: - Debug: - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Release: - BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - Release SIMD: - BuildType: release-simd - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON - Secure: - BuildType: secure - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - steps: - - task: CMake@1 - inputs: - workingDirectory: $(BuildType) - cmakeArgs: .. $(cmakeExtraArgs) - - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) - displayName: Make - - script: ctest --verbose --timeout 240 - workingDirectory: $(BuildType) - displayName: CTest -# - upload: $(Build.SourcesDirectory)/$(BuildType) -# artifact: mimalloc-macos-$(BuildType) +# - job: +# displayName: macOS 14 (Sonoma) +# pool: +# vmImage: +# macOS-14 +# strategy: +# matrix: +# Debug: +# BuildType: debug +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON +# Release: +# BuildType: release +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release +# Release SIMD: +# BuildType: release-simd +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON +# Secure: +# BuildType: secure +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON +# steps: +# - task: CMake@1 +# inputs: +# workingDirectory: $(BuildType) +# cmakeArgs: .. $(cmakeExtraArgs) +# - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) +# displayName: Make +# - script: ctest --verbose --timeout 240 +# workingDirectory: $(BuildType) +# displayName: CTest + +# # - upload: $(Build.SourcesDirectory)/$(BuildType) +# # artifact: mimalloc-macos-$(BuildType) # ---------------------------------------------------------- # Other OS versions (just debug mode) # ---------------------------------------------------------- -- job: - displayName: Ubuntu 24.04 - pool: - vmImage: - ubuntu-24.04 - strategy: - matrix: - Debug: - CC: gcc - CXX: g++ - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Debug++: - CC: gcc - CXX: g++ - BuildType: debug-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - Debug Clang: - CC: clang - CXX: clang++ - BuildType: debug-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Debug++ Clang: - CC: clang - CXX: clang++ - BuildType: debug-clang-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - Release Clang: - CC: clang - CXX: clang++ - BuildType: release-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - steps: - - task: CMake@1 - inputs: - workingDirectory: $(BuildType) - cmakeArgs: .. $(cmakeExtraArgs) - - script: make -j$(nproc) -C $(BuildType) - displayName: Make - - script: ctest --verbose --timeout 240 - workingDirectory: $(BuildType) - displayName: CTest +# - job: +# displayName: Ubuntu 24.04 +# pool: +# vmImage: +# ubuntu-24.04 +# strategy: +# matrix: +# Debug: +# CC: gcc +# CXX: g++ +# BuildType: debug +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON +# Debug++: +# CC: gcc +# CXX: g++ +# BuildType: debug-cxx +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON +# Debug Clang: +# CC: clang +# CXX: clang++ +# BuildType: debug-clang +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON +# Debug++ Clang: +# CC: clang +# CXX: clang++ +# BuildType: debug-clang-cxx +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON +# Release Clang: +# CC: clang +# CXX: clang++ +# BuildType: release-clang +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release +# steps: +# - task: CMake@1 +# inputs: +# workingDirectory: $(BuildType) +# cmakeArgs: .. $(cmakeExtraArgs) +# - script: make -j$(nproc) -C $(BuildType) +# displayName: Make +# - script: ctest --verbose --timeout 240 +# workingDirectory: $(BuildType) +# displayName: CTest -- job: - displayName: macOS 15 (Sequoia) - pool: - vmImage: - macOS-15 - strategy: - matrix: - Debug: - BuildType: debug - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - # Release: - # BuildType: release - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - steps: - - task: CMake@1 - inputs: - workingDirectory: $(BuildType) - cmakeArgs: .. $(cmakeExtraArgs) - - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) - displayName: Make - - script: ctest --verbose --timeout 240 - workingDirectory: $(BuildType) - displayName: CTest +# - job: +# displayName: macOS 15 (Sequoia) +# pool: +# vmImage: +# macOS-15 +# strategy: +# matrix: +# Debug: +# BuildType: debug +# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON +# # Release: +# # BuildType: release +# # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release +# steps: +# - task: CMake@1 +# inputs: +# workingDirectory: $(BuildType) +# cmakeArgs: .. $(cmakeExtraArgs) +# - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) +# displayName: Make +# - script: ctest --verbose --timeout 240 +# workingDirectory: $(BuildType) +# displayName: CTest diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 802def27f..c69c45efe 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -271,7 +271,7 @@ void _mi_page_map_unsafe_destroy(void); // "page.c" void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignment, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; -void* _mi_malloc_generic_no_sample(size_t size, mi_theap_t* theap, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; +void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept mi_attr_malloc; void _mi_page_retire(mi_page_t* page) mi_attr_noexcept; // free the page if there are no other pages with many free blocks void _mi_page_unfull(mi_page_t* page); diff --git a/src/options.c b/src/options.c index 6a12ee47d..fcc160332 100644 --- a/src/options.c +++ b/src/options.c @@ -78,7 +78,7 @@ int mi_version(void) { #ifndef MI_DEFAULT_GUARDED_SAMPLE_RATE #if MI_GUARDED && MI_DEBUG -#define MI_DEFAULT_GUARDED_SAMPLE_RATE MI_MiB +#define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 /* MI_MiB */ #else #define MI_DEFAULT_GUARDED_SAMPLE_RATE 0 #endif diff --git a/src/page.c b/src/page.c index 0d2a79a5e..0cf6648f2 100644 --- a/src/page.c +++ b/src/page.c @@ -1356,7 +1356,7 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm return mi_malloc_generic_fallback(theap,size,zero,huge_alignment,ppage); } -void* _mi_malloc_generic_no_sample(size_t size, mi_theap_t* theap, bool zero, mi_page_t** ppage) mi_attr_noexcept { +void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept { theap = mi_theap_init(theap); if (theap==NULL) return NULL; const size_t sample_rate = theap->sample_rate; diff --git a/src/sample-guarded.c b/src/sample-guarded.c index 3cfa0007b..dff5a3304 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -99,7 +99,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, const size_t bsize = _mi_align_up(_mi_align_up(obj_size, MI_MAX_ALIGN_SIZE) + sizeof(mi_block_t), MI_MAX_ALIGN_SIZE); const size_t req_size = _mi_align_up(bsize + os_page_size, os_page_size); // const size_t threshold = mi_theap_disable_profiler(theap); - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(req_size, theap, false /* don't zero */, ppage); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap, req_size, false /* don't zero */, ppage); // mi_theap_enable_profiler(theap,threshold); if (block==NULL) return NULL; size_t usable_size = 0; diff --git a/src/sample-profile.c b/src/sample-profile.c index bff0e0b84..4f9cd8f73 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -39,7 +39,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* the if (!_mi_is_empty_theap(theap)) { // avoid writing to the initial empty theap theap->sample_countdown = MI_SAMPLE_COUNTDOWN_MAX; // avoid the sampling path for a long time } - return _mi_malloc_generic_no_sample(size,theap,zero,ppage); + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } // update countdown and total accummulated requested bytes since the last sample @@ -85,7 +85,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* the } } // take generic path - return _mi_malloc_generic_no_sample(size,theap,zero,ppage); + return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } @@ -120,14 +120,14 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th mi_assert_internal(size>=MI_PADDING_SIZE); const size_t req_size = size - MI_PADDING_SIZE; mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); - if (prof == NULL) { return _mi_malloc_generic_no_sample(size,theap,zero,ppage); } + if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } mi_assert_internal(prof!=NULL && prof->on_alloc!=NULL && mi_profiler_is_enabled(prof)); void* p = NULL; size_t new_sample_rate = 0; if (prof->on_free==NULL || prof->sample_data_size==0) { // just allocate without profiler data - p = _mi_malloc_generic_no_sample(size,theap,zero,ppage); + p = _mi_malloc_generic_no_sample(theap,size,zero,ppage); if (p==NULL) { return p; } if (prof->on_alloc!=NULL) { // we are just allocation profiling (not heap profiling as on_free == NULL) @@ -143,7 +143,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); const size_t oversize = user_offset + size; mi_page_t* page = NULL; - mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(oversize,theap,zero,&page); + mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); if (block==NULL) return NULL; mi_assert_internal(page!=NULL); if (ppage!=NULL) { *ppage = page; } From c449187870dc2ee8763ee93a6c155a5344be4b2b Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 20:55:22 -0700 Subject: [PATCH 236/280] fix padding assertions --- include/mimalloc/track.h | 9 -------- include/mimalloc/types.h | 2 -- src/alloc-aligned.c | 3 ++- src/alloc.c | 46 ++++++++++++++++++++-------------------- src/free.c | 1 + src/sample-profile.c | 4 ++-- 6 files changed, 28 insertions(+), 37 deletions(-) diff --git a/include/mimalloc/track.h b/include/mimalloc/track.h index 753363366..d5e6128ac 100644 --- a/include/mimalloc/track.h +++ b/include/mimalloc/track.h @@ -132,19 +132,10 @@ defined, undefined, or not accessible at all: #define mi_track_mem_noaccess(p,size) #endif - -#if MI_PADDING -#define mi_track_malloc(p,reqsize,zero) \ - do { if ((p)!=NULL) { \ - mi_assert_internal(mi_usable_size(p)==(reqsize)); \ - mi_track_malloc_size(p,reqsize,reqsize,zero); \ - } } while(0) -#else #define mi_track_malloc(p,reqsize,zero) \ do { if ((p)!=NULL) { \ mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ } } while(0) -#endif #endif // MI_TRACK_H diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index edd4f84fd..890623d74 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -69,8 +69,6 @@ terms of the MIT license. A copy of the license can be found in the file #define MI_SECURE 0 #endif -#define MI_PADDING 0 - // Define MI_DEBUG for assertion and invariant checking // #define MI_DEBUG 1 // basic assertion checks and statistics, check double free, corrupted free list, and invalid pointer free. (cmake -DMI_DEBUG=ON) // #define MI_DEBUG 2 // + internal assertion checks (cmake -DMI_DEBUG_INTERNAL=ON) diff --git a/src/alloc-aligned.c b/src/alloc-aligned.c index 752b1d0a7..3a4383e0b 100644 --- a/src/alloc-aligned.c +++ b/src/alloc-aligned.c @@ -123,7 +123,8 @@ static mi_decl_noinline void* mi_theap_malloc_zero_aligned_at_overalloc(mi_theap block->next = MI_BLOCK_TAG_ALIGNED; } #endif - _mi_padding_shrink(page, (mi_block_t*)p, adjust + size); + mi_block_t* const base = _mi_page_ptr_unalign(page, p); + _mi_padding_shrink(page, base, adjust + size); } // todo: expand padding if overallocated ? diff --git a/src/alloc.c b/src/alloc.c index a5f28f97f..1c718ffd3 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -147,16 +147,16 @@ extern void* _mi_page_malloc_zero(mi_theap_t* theap, mi_page_t* page, size_t siz } // internal small size allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_nonnull(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept +static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero_nonnull(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { - const size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize); + size_t size = (is_wsize ? xsize * MI_SIZE_SIZE : xsize); mi_assert(theap != NULL); mi_assert(size <= MI_SMALL_SIZE_MAX); #if MI_DEBUG mi_assert(mi_theap_matches_thread(theap)); // theaps are thread local #endif #if (MI_PADDING) // || MI_GUARDED) - if mi_unlikely(size == 0) { size = sizeof(void*); } + if mi_unlikely(xsize == 0) { xsize = (is_wsize ? 1 : MI_SIZE_SIZE); size += MI_SIZE_SIZE; } #endif // we only sample if fine-grained sampling is enabled (otherwise we sample in mi_malloc_generic) @@ -165,7 +165,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero_non #endif // get page in constant time - mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? mi_wsize_from_size(MI_PADDING_SIZE) : MI_PADDING_SIZE), is_wsize); + mi_page_t* page = _mi_theap_get_free_small_page(theap, xsize + (is_wsize ? MI_PADDING_WSIZE : MI_PADDING_SIZE), is_wsize); // and allocate void* const p = mi_page_malloc_zero(theap, page, size + MI_PADDING_SIZE, theap->sample_countdown, zero, ppage); @@ -199,47 +199,47 @@ static mi_decl_forceinline void* mi_theap_malloc_generic(mi_theap_t* theap, size } // internal small allocation -static mi_decl_forceinline mi_decl_restrict void* mi_theap_malloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { +static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi_theap_t* theap, size_t xsize, bool is_wsize, bool zero, mi_page_t** ppage) mi_attr_noexcept { #if MI_THEAP_INITASNULL if (theap!=NULL) { - return mi_theap_malloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); } else { return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall } #else - return mi_theap_malloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); + return mi_theap_malloc_xsmall_zero_nonnull(theap, xsize, is_wsize, zero, ppage); #endif } // allocate a small block mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_malloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_malloc_small_zero_nonnull(theap, size, false, false, NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_theap_wmalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_malloc_small_zero_nonnull(theap, wsize, true, false, NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, wsize, true, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_malloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, false, NULL); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_wmalloc_small(size_t wsize) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, false, NULL ); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, false, NULL); + return mi_theap_xmalloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, false, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { // we could also use: mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(theap), size, false, NULL); } // but the following prevents using a stack frame. We use this to optimize some select functions only. mi_theap_t* const theap = _mi_heap_theap_cached(heap); - if mi_likely(theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, size, false, false, NULL); } + if mi_likely(theap!=NULL) { return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, false, NULL); } else { return mi_heap_init_malloc_small(heap, size); } } @@ -249,7 +249,7 @@ static mi_decl_forceinline void* mi_theap_malloc_zero_nonnull(mi_theap_t* theap, // fast path for small objects if mi_likely(size <= MI_SMALL_SIZE_MAX) { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, false, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); @@ -265,7 +265,7 @@ extern mi_decl_forceinline void* _mi_theap_malloc_zero(mi_theap_t* theap, size_t #endif { mi_assert_internal(huge_alignment == 0); - return mi_theap_malloc_small_zero_nonnull(theap, size, false, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, zero, ppage); } else { return mi_theap_malloc_generic(theap, size, zero, huge_alignment, ppage); @@ -299,30 +299,30 @@ mi_decl_nodiscard mi_decl_restrict void* mi_heap_malloc(mi_heap_t* heap, size_t // zero initialized small block mi_decl_nodiscard mi_decl_restrict void* mi_zalloc_small(size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), size, false, true, NULL); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_wzalloc_small(size_t wsize) mi_attr_noexcept { - return mi_theap_malloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); + return mi_theap_xmalloc_small_zero(_mi_theap_default(), wsize, true, true, NULL); } mi_decl_nodiscard extern inline mi_decl_restrict void* mi_theap_zalloc_small(mi_theap_t* theap, size_t size) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_malloc_small_zero_nonnull(theap, size, false, true, NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_theap_wzalloc_small(mi_theap_t* theap, size_t wsize) mi_attr_noexcept { mi_assert(theap!=NULL); - return mi_theap_malloc_small_zero_nonnull(theap, wsize, true, true, NULL); + return mi_theap_xmalloc_small_zero_nonnull(theap, wsize, true, true, NULL); } mi_decl_nodiscard static mi_decl_noinline mi_decl_restrict void* mi_heap_init_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { - return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, true, NULL); + return mi_theap_xmalloc_small_zero_nonnull(_mi_heap_theap_get_or_init(heap), size, false, true, NULL); } mi_decl_nodiscard mi_decl_restrict void* mi_heap_zalloc_small(mi_heap_t* heap, size_t size) mi_attr_noexcept { // optimize: return mi_theap_malloc_small_zero_nonnull(_mi_heap_theap(heap), size, true, NULL); mi_theap_t* const theap = _mi_heap_theap_cached(heap); - if mi_likely(theap!=NULL) { return mi_theap_malloc_small_zero_nonnull(theap, size, false, true, NULL); } + if mi_likely(theap!=NULL) { return mi_theap_xmalloc_small_zero_nonnull(theap, size, false, true, NULL); } else { return mi_heap_init_zalloc_small(heap, size); } } @@ -371,13 +371,13 @@ static void* mi_ublock_size( void* p, mi_page_t* page, size_t* pblock_size ) { // Return usable size mi_decl_nodiscard mi_decl_restrict void* mi_umalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, false, &page); + void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, false, &page); return mi_ublock_size(p,page,pblock_size); } mi_decl_nodiscard mi_decl_restrict void* mi_uzalloc_small(size_t size, size_t* pblock_size) mi_attr_noexcept { mi_page_t* page; - void* p = mi_theap_malloc_small_zero(_mi_theap_default(), size, false, true, &page); + void* p = mi_theap_xmalloc_small_zero(_mi_theap_default(), size, false, true, &page); return mi_ublock_size(p,page,pblock_size); } diff --git a/src/free.c b/src/free.c index 63734721d..eff462b9f 100644 --- a/src/free.c +++ b/src/free.c @@ -676,6 +676,7 @@ static size_t mi_page_usable_size_of(const mi_page_t* page, const mi_block_t* bl // contain the pointer for the delayed list, then shrink the padding (by decreasing delta) // so it will later not trigger an overflow error in `mi_free_block`. void _mi_padding_shrink(const mi_page_t* page, const mi_block_t* block, const size_t min_size) { + if (mi_block_ptr_is_guarded(block,block + 1)) return; size_t bsize; size_t delta; bool ok = mi_page_decode_padding(page, block, &delta, &bsize, NULL); diff --git a/src/sample-profile.c b/src/sample-profile.c index 4f9cd8f73..6b93f3f4b 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -51,7 +51,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* the const uint64_t requested = theap->sample_requested = (uint64_t)theap->sample_rate + (uint64_t)(req_size - theap->sample_countdown) + theap->sample_requested; mi_assert_internal(requested > 0); - mi_assert_internal(requested >= size); + mi_assert_internal(requested >= req_size); theap->sample_countdown = theap->sample_rate; // reset sampling // update derived countdowns @@ -116,9 +116,9 @@ static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept { mi_assert_internal(theap!=NULL); - mi_assert_internal(size<=requested_since_last_sample); mi_assert_internal(size>=MI_PADDING_SIZE); const size_t req_size = size - MI_PADDING_SIZE; + mi_assert_internal(req_size<=requested_since_last_sample); mi_profiler_t* const prof = mi_theap_get_enabled_profiler(theap); if (prof == NULL) { return _mi_malloc_generic_no_sample(theap,size,zero,ppage); } mi_assert_internal(prof!=NULL && prof->on_alloc!=NULL && mi_profiler_is_enabled(prof)); From e2a957dcadcfb42ec2db2a149bcad8e912debcab Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 20:56:45 -0700 Subject: [PATCH 237/280] fix linux compilation --- src/alloc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/alloc.c b/src/alloc.c index 1c718ffd3..15184c35e 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -208,7 +208,7 @@ static mi_decl_forceinline mi_decl_restrict void* mi_theap_xmalloc_small_zero(mi return mi_theap_malloc_generic(theap, (is_wsize ? xsize * MI_SIZE_SIZE : xsize), zero, 0, ppage); // tailcall } #else - return mi_theap_malloc_xsmall_zero_nonnull(theap, xsize, is_wsize, zero, ppage); + return mi_theap_xmalloc_small_zero_nonnull(theap, xsize, is_wsize, zero, ppage); #endif } From 3110f3f2cb3260877ea0561fede9447b674b9b0f Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 21:03:10 -0700 Subject: [PATCH 238/280] test azure pipeline failure --- azure-pipelines.yml | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 6affb5b6c..cee96321b 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,7 +87,7 @@ jobs: CC: gcc CXX: g++ BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release # Secure: # CC: gcc # CXX: g++ @@ -103,6 +103,11 @@ jobs: CXX: clang++ BuildType: debug-clang cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Release Clang with Assertions: + CC: clang + CXX: clang++ + BuildType: release-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_DEBUG=ON -DMI_OPT_ARCH=ON -DMI_OPT_FREE_SMALL=ON # Release Clang: # CC: clang # CXX: clang++ From 8a8b4761569b5500b382070eadd897561d2b1353 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 21:11:01 -0700 Subject: [PATCH 239/280] test azure pipeline failure --- src/arena.c | 1 + src/sample-guarded.c | 1 + 2 files changed, 2 insertions(+) diff --git a/src/arena.c b/src/arena.c index 2bbadf82b..dbb614ccd 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1918,6 +1918,7 @@ static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool com mi_memid_t memid; void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); if (start == NULL) return ENOMEM; + _mi_raw_message("os alloc aligned: %p, size %zu\n", start, size); if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { _mi_os_free_ex(subproc, start, size, commit, memid); _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); diff --git a/src/sample-guarded.c b/src/sample-guarded.c index dff5a3304..566b5b1c8 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -106,6 +106,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); if (p == NULL) return NULL; if (zero) { + _mi_raw_message("guarded memzero: %p, size: %zu, block=%p, usable_size:%zu\n", p, obj_size, block, usable_size); _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) } From 84f86e70f5a1e6618bd0eb7fb1824fc9a109f192 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 21:16:46 -0700 Subject: [PATCH 240/280] test azure pipeline failure --- azure-pipelines.yml | 23 ++++++++++++++--------- 1 file changed, 14 insertions(+), 9 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index cee96321b..ba51dd8d1 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -88,6 +88,11 @@ jobs: CXX: g++ BuildType: release cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + Release opt arch: + CC: gcc + CXX: g++ + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON # Secure: # CC: gcc # CXX: g++ @@ -98,15 +103,15 @@ jobs: # CXX: g++ # BuildType: debug-cxx # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - Debug Clang: - CC: clang - CXX: clang++ - BuildType: debug-clang - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Release Clang with Assertions: - CC: clang - CXX: clang++ - BuildType: release-clang + # Debug Clang: + # CC: clang + # CXX: clang++ + # BuildType: debug-clang + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Release GCC with Assertions: + CC: gcc + CXX: g++ + BuildType: release-gcc cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_DEBUG=ON -DMI_OPT_ARCH=ON -DMI_OPT_FREE_SMALL=ON # Release Clang: # CC: clang From ec2f2f26b96355a1c00d7038e2e0a472d83a483e Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 21:23:22 -0700 Subject: [PATCH 241/280] test azure pipeline failure --- azure-pipelines.yml | 24 ++++++++++++------------ src/sample-guarded.c | 3 ++- 2 files changed, 14 insertions(+), 13 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ba51dd8d1..e30e34c35 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -88,11 +88,11 @@ jobs: CXX: g++ BuildType: release cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - Release opt arch: + Release opt arch debug: CC: gcc CXX: g++ BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_DEBUG=ON # Secure: # CC: gcc # CXX: g++ @@ -133,21 +133,21 @@ jobs: # CXX: clang++ # BuildType: debug-clang-cxx # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - # Debug ASAN Clang: - # CC: clang - # CXX: clang++ - # BuildType: debug-asan-clang - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_TRACK_ASAN=ON + Debug ASAN Clang: + CC: clang + CXX: clang++ + BuildType: debug-asan-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_TRACK_ASAN=ON # Debug UBSAN Clang: # CC: clang # CXX: clang++ # BuildType: debug-ubsan-clang # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON - Debug TSAN Clang++: - CC: clang - CXX: clang++ - BuildType: debug-tsan-clang-cxx - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON + # Debug TSAN Clang++: + # CC: clang + # CXX: clang++ + # BuildType: debug-tsan-clang-cxx + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON # Debug Guarded Clang: # CC: clang # CXX: clang diff --git a/src/sample-guarded.c b/src/sample-guarded.c index 566b5b1c8..dda1fcdb7 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -106,7 +106,8 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); if (p == NULL) return NULL; if (zero) { - _mi_raw_message("guarded memzero: %p, size: %zu, block=%p, usable_size:%zu\n", p, obj_size, block, usable_size); + mi_page_t* page = _mi_ptr_page(p); + _mi_raw_message("guarded memzero: %p, size: %zu, block=%p, usable_size:%zu, block_size:%zu\n", p, obj_size, block, usable_size, page->block_size); _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) } From 0fef88609b188fa037631256d57767c62181dc8c Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 21:39:34 -0700 Subject: [PATCH 242/280] test azure pipeline failure --- src/free.c | 2 +- src/sample-guarded.c | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/free.c b/src/free.c index eff462b9f..d0061463f 100644 --- a/src/free.c +++ b/src/free.c @@ -650,7 +650,7 @@ static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_ const bool ok = (mi_ptr_encode_canary(page,block,page->keys) == canary && *delta <= *bsize); if (double_free!=NULL) { if mi_unlikely(!ok) { *double_free = mi_ptr_decode_canary_is_freed(canary); } // double free? - else { padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed + else { *double_free = false; padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed } mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); return ok; diff --git a/src/sample-guarded.c b/src/sample-guarded.c index dda1fcdb7..4897295d9 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -112,7 +112,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, } // stats - mi_track_malloc(p, usable_size, zero); + // mi_track_malloc(p, usable_size, zero); if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); #if MI_STATS From 4f9608f4d4957291dcc2f01231612db4456b7f26 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 21:47:28 -0700 Subject: [PATCH 243/280] test azure pipeline failure --- azure-pipelines.yml | 3 ++- src/os.c | 2 +- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index e30e34c35..10dddd698 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -92,7 +92,7 @@ jobs: CC: gcc CXX: g++ BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_DEBUG=ON + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_OPT_ARCH=ON # Secure: # CC: gcc # CXX: g++ @@ -166,6 +166,7 @@ jobs: displayName: CTest env: MIMALLOC_GUARDED_SAMPLE_RATE: 1000 + MIMALLOC_VERBOSE: 2 # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-ubuntu-$(BuildType) diff --git a/src/os.c b/src/os.c index 15111cf94..94d618827 100644 --- a/src/os.c +++ b/src/os.c @@ -150,7 +150,7 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; - #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode + #if 0 && (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point const uintptr_t r = _mi_theap_random_next(theap); From 2c2038bc96d883334c95bf29645f2c9ec03465ad Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 21:53:33 -0700 Subject: [PATCH 244/280] test azure pipeline failure --- azure-pipelines.yml | 2 +- src/sample-profile.c | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 10dddd698..d518a92d7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -87,7 +87,7 @@ jobs: CC: gcc CXX: g++ BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_GUARDED=ON Release opt arch debug: CC: gcc CXX: g++ diff --git a/src/sample-profile.c b/src/sample-profile.c index 6b93f3f4b..0f3c37474 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -70,19 +70,25 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* the if (sample_profile) { theap->profile_sample_countdown = theap->profile_sample_rate; // reset countdown theap->sample_requested = 0; // reset requested as we pass it to malloc_profiled + #if MI_PROFILE return _mi_theap_malloc_profiled(theap,size,requested,zero,ppage); + #endif } else if (sample_guarded) { if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) { // use guarded allocation theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown + // #if MI_GUARDED return _mi_theap_malloc_guarded(theap,size,zero,ppage); + // #endif } + // #if MI_GUARDED else { // failed size criteria, rewind the sample countdown so we sample asap again // todo: can we do better here as this will cause many samples until it fits the size.. theap->sample_countdown = 0; } + // #endif } // take generic path return _mi_malloc_generic_no_sample(theap,size,zero,ppage); From e3c2a9c10c03150ae277744ef061cfa4522cb8cb Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 22:03:43 -0700 Subject: [PATCH 245/280] fix non-guarded build still guarding objects when MIMALLOC_GUARDED_RATE was set --- src/sample-guarded.c | 6 +++++- src/sample-profile.c | 8 ++++---- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/src/sample-guarded.c b/src/sample-guarded.c index 4897295d9..f424729e9 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -28,12 +28,16 @@ mi_decl_export void mi_theap_guarded_set_size_bound(mi_theap_t* theap, size_t mi } void _mi_theap_guarded_init(mi_theap_t* theap) { + #if MI_GUARDED mi_theap_guarded_set_sample_rate(theap, (size_t)mi_option_get_clamp(mi_option_guarded_sample_rate, 0, LONG_MAX), (size_t)mi_option_get(mi_option_guarded_sample_seed)); mi_theap_guarded_set_size_bound(theap, (size_t)mi_option_get_clamp(mi_option_guarded_min, 0, LONG_MAX), (size_t)mi_option_get_clamp(mi_option_guarded_max, 0, LONG_MAX) ); + #else + mi_theap_guarded_set_sample_rate(theap,0,0); + #endif } @@ -87,7 +91,7 @@ static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t // Allocate a block with a guard page behind it. mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept -{ +{ // allocate multiple of page size ending in a guard page // ensure minimal alignment requirement? if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow diff --git a/src/sample-profile.c b/src/sample-profile.c index 0f3c37474..084c8f5d1 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -78,17 +78,17 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* the if (req_size >= theap->guarded_size_min && req_size <= theap->guarded_size_max) { // use guarded allocation theap->guarded_sample_countdown = theap->guarded_sample_rate; // reset countdown - // #if MI_GUARDED + #if MI_GUARDED return _mi_theap_malloc_guarded(theap,size,zero,ppage); - // #endif + #endif } - // #if MI_GUARDED + #if MI_GUARDED else { // failed size criteria, rewind the sample countdown so we sample asap again // todo: can we do better here as this will cause many samples until it fits the size.. theap->sample_countdown = 0; } - // #endif + #endif } // take generic path return _mi_malloc_generic_no_sample(theap,size,zero,ppage); From 1148e5c2b619ab8b2fefb28b12ff8c6f3547301a Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 22:23:17 -0700 Subject: [PATCH 246/280] roll back azure pipeline test code --- azure-pipelines.yml | 418 ++++++++++++++++++------------------ include/mimalloc/internal.h | 3 +- include/mimalloc/track.h | 15 +- src/arena.c | 3 +- src/free.c | 2 +- src/os.c | 2 +- src/sample-guarded.c | 6 +- 7 files changed, 223 insertions(+), 226 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index d518a92d7..84ce387f9 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -14,62 +14,62 @@ trigger: - v* jobs: -# - job: -# displayName: Windows 2022 -# pool: -# vmImage: -# windows-2022 -# strategy: -# matrix: -# Debug: -# BuildType: debug -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -# MSBuildConfiguration: Debug -# Release: -# BuildType: release -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -# MSBuildConfiguration: Release -# Release SIMD: -# BuildType: release-simd -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON -DMI_WIN_USE_FIXED_TLS=ON -# MSBuildConfiguration: Release -# Secure: -# BuildType: secure -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON -# MSBuildConfiguration: Release -# Debug x86: -# BuildType: debug -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -A Win32 -# MSBuildConfiguration: Debug -# Release x86: -# BuildType: release -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -A Win32 -# MSBuildConfiguration: Release -# Debug Fixed TLS: -# BuildType: debug -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_WIN_USE_FIXED_TLS=ON -# MSBuildConfiguration: Debug -# Release Fixed TLS: -# BuildType: release -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_WIN_USE_FIXED_TLS=ON -# MSBuildConfiguration: Release -# steps: -# - task: CMake@1 -# inputs: -# workingDirectory: $(BuildType) -# cmakeArgs: .. $(cmakeExtraArgs) -# - task: MSBuild@1 -# inputs: -# solution: $(BuildType)/libmimalloc.sln -# configuration: '$(MSBuildConfiguration)' -# msbuildArguments: -m -# - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) -# workingDirectory: $(BuildType) -# displayName: CTest -# #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress -# # displayName: TestStress -# #- upload: $(Build.SourcesDirectory)/$(BuildType) -# # artifact: mimalloc-windows-$(BuildType) +- job: + displayName: Windows 2022 + pool: + vmImage: + windows-2022 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + MSBuildConfiguration: Debug + Release: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + MSBuildConfiguration: Release + Release SIMD: + BuildType: release-simd + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON -DMI_WIN_USE_FIXED_TLS=ON + MSBuildConfiguration: Release + Secure: + BuildType: secure + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + MSBuildConfiguration: Release + Debug x86: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -A Win32 + MSBuildConfiguration: Debug + Release x86: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -A Win32 + MSBuildConfiguration: Release + Debug Fixed TLS: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_WIN_USE_FIXED_TLS=ON + MSBuildConfiguration: Debug + Release Fixed TLS: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_WIN_USE_FIXED_TLS=ON + MSBuildConfiguration: Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - task: MSBuild@1 + inputs: + solution: $(BuildType)/libmimalloc.sln + configuration: '$(MSBuildConfiguration)' + msbuildArguments: -m + - script: ctest --verbose --timeout 240 -C $(MSBuildConfiguration) + workingDirectory: $(BuildType) + displayName: CTest + #- script: $(BuildType)\$(BuildType)\mimalloc-test-stress + # displayName: TestStress + #- upload: $(Build.SourcesDirectory)/$(BuildType) + # artifact: mimalloc-windows-$(BuildType) - job: displayName: Ubuntu 22.04 @@ -78,81 +78,71 @@ jobs: ubuntu-22.04 strategy: matrix: - # Debug: - # CC: gcc - # CXX: g++ - # BuildType: debug - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug: + CC: gcc + CXX: g++ + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON Release: CC: gcc CXX: g++ BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_GUARDED=ON - Release opt arch debug: + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + Secure: CC: gcc CXX: g++ - BuildType: release - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_OPT_ARCH=ON - # Secure: - # CC: gcc - # CXX: g++ - # BuildType: secure - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - # Debug++: - # CC: gcc - # CXX: g++ - # BuildType: debug-cxx - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON - # Debug Clang: - # CC: clang - # CXX: clang++ - # BuildType: debug-clang - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON - Release GCC with Assertions: + BuildType: secure + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + Debug++: CC: gcc CXX: g++ - BuildType: release-gcc - cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_DEBUG=ON -DMI_OPT_ARCH=ON -DMI_OPT_FREE_SMALL=ON - # Release Clang: - # CC: clang - # CXX: clang++ - # BuildType: release-clang - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release - # Release SIMD Clang: - # CC: clang - # CXX: clang++ - # BuildType: release-simd-clang - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON - # Secure Clang: - # CC: clang - # CXX: clang++ - # BuildType: secure-clang - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON - # Debug++ Clang: - # CC: clang - # CXX: clang++ - # BuildType: debug-clang-cxx - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + BuildType: debug-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Debug Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Release Clang: + CC: clang + CXX: clang++ + BuildType: release-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + Release SIMD Clang: + CC: clang + CXX: clang++ + BuildType: release-simd-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON + Secure Clang: + CC: clang + CXX: clang++ + BuildType: secure-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + Debug++ Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON Debug ASAN Clang: CC: clang CXX: clang++ BuildType: debug-asan-clang cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_TRACK_ASAN=ON - # Debug UBSAN Clang: - # CC: clang - # CXX: clang++ - # BuildType: debug-ubsan-clang - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON - # Debug TSAN Clang++: - # CC: clang - # CXX: clang++ - # BuildType: debug-tsan-clang-cxx - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON - # Debug Guarded Clang: - # CC: clang - # CXX: clang - # BuildType: debug-guarded-clang - # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_DEBUG_FULL=ON -DMI_GUARDED=ON + Debug UBSAN Clang: + CC: clang + CXX: clang++ + BuildType: debug-ubsan-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_DEBUG_UBSAN=ON + Debug TSAN Clang++: + CC: clang + CXX: clang++ + BuildType: debug-tsan-clang-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_USE_CXX=ON -DMI_DEBUG_TSAN=ON + Debug Guarded Clang: + CC: clang + CXX: clang + BuildType: debug-guarded-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=RelWithDebInfo -DMI_DEBUG_FULL=ON -DMI_GUARDED=ON steps: - task: CMake@1 @@ -170,106 +160,106 @@ jobs: # - upload: $(Build.SourcesDirectory)/$(BuildType) # artifact: mimalloc-ubuntu-$(BuildType) -# - job: -# displayName: macOS 14 (Sonoma) -# pool: -# vmImage: -# macOS-14 -# strategy: -# matrix: -# Debug: -# BuildType: debug -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -# Release: -# BuildType: release -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -# Release SIMD: -# BuildType: release-simd -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON -# Secure: -# BuildType: secure -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON -# steps: -# - task: CMake@1 -# inputs: -# workingDirectory: $(BuildType) -# cmakeArgs: .. $(cmakeExtraArgs) -# - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) -# displayName: Make -# - script: ctest --verbose --timeout 240 -# workingDirectory: $(BuildType) -# displayName: CTest +- job: + displayName: macOS 14 (Sonoma) + pool: + vmImage: + macOS-14 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Release: + BuildType: release + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + Release SIMD: + BuildType: release-simd + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_OPT_SIMD=ON + Secure: + BuildType: secure + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -DMI_SECURE=ON + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 240 + workingDirectory: $(BuildType) + displayName: CTest -# # - upload: $(Build.SourcesDirectory)/$(BuildType) -# # artifact: mimalloc-macos-$(BuildType) +# - upload: $(Build.SourcesDirectory)/$(BuildType) +# artifact: mimalloc-macos-$(BuildType) # ---------------------------------------------------------- # Other OS versions (just debug mode) # ---------------------------------------------------------- -# - job: -# displayName: Ubuntu 24.04 -# pool: -# vmImage: -# ubuntu-24.04 -# strategy: -# matrix: -# Debug: -# CC: gcc -# CXX: g++ -# BuildType: debug -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -# Debug++: -# CC: gcc -# CXX: g++ -# BuildType: debug-cxx -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON -# Debug Clang: -# CC: clang -# CXX: clang++ -# BuildType: debug-clang -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -# Debug++ Clang: -# CC: clang -# CXX: clang++ -# BuildType: debug-clang-cxx -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON -# Release Clang: -# CC: clang -# CXX: clang++ -# BuildType: release-clang -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -# steps: -# - task: CMake@1 -# inputs: -# workingDirectory: $(BuildType) -# cmakeArgs: .. $(cmakeExtraArgs) -# - script: make -j$(nproc) -C $(BuildType) -# displayName: Make -# - script: ctest --verbose --timeout 240 -# workingDirectory: $(BuildType) -# displayName: CTest +- job: + displayName: Ubuntu 24.04 + pool: + vmImage: + ubuntu-24.04 + strategy: + matrix: + Debug: + CC: gcc + CXX: g++ + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug++: + CC: gcc + CXX: g++ + BuildType: debug-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Debug Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + Debug++ Clang: + CC: clang + CXX: clang++ + BuildType: debug-clang-cxx + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -DMI_USE_CXX=ON + Release Clang: + CC: clang + CXX: clang++ + BuildType: release-clang + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(nproc) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 240 + workingDirectory: $(BuildType) + displayName: CTest -# - job: -# displayName: macOS 15 (Sequoia) -# pool: -# vmImage: -# macOS-15 -# strategy: -# matrix: -# Debug: -# BuildType: debug -# cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON -# # Release: -# # BuildType: release -# # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release -# steps: -# - task: CMake@1 -# inputs: -# workingDirectory: $(BuildType) -# cmakeArgs: .. $(cmakeExtraArgs) -# - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) -# displayName: Make -# - script: ctest --verbose --timeout 240 -# workingDirectory: $(BuildType) -# displayName: CTest +- job: + displayName: macOS 15 (Sequoia) + pool: + vmImage: + macOS-15 + strategy: + matrix: + Debug: + BuildType: debug + cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Debug -DMI_DEBUG_FULL=ON + # Release: + # BuildType: release + # cmakeExtraArgs: -DCMAKE_BUILD_TYPE=Release + steps: + - task: CMake@1 + inputs: + workingDirectory: $(BuildType) + cmakeArgs: .. $(cmakeExtraArgs) + - script: make -j$(sysctl -n hw.ncpu) -C $(BuildType) + displayName: Make + - script: ctest --verbose --timeout 240 + workingDirectory: $(BuildType) + displayName: CTest diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index c69c45efe..347f93c05 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1472,7 +1472,7 @@ static inline size_t _mi_random_shuffle(size_t x) { // Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows. // --------------------------------------------------------------------------------- -#if 0 && !MI_TRACK_ENABLED && (MI_ARCH_ARM64 || MI_ARCH_X64) && (defined(__SIZEOF_INT128__) || (defined(_MSC_VER) && defined(__AVX2__))) // any 64-bit platform with 128-bit stores can benefit. +#if !MI_TRACK_ENABLED && (MI_ARCH_ARM64 || MI_ARCH_X64) && (defined(__SIZEOF_INT128__) || (defined(_MSC_VER) && defined(__AVX2__))) // any 64-bit platform with 128-bit stores can benefit. #define MI_USE_MEMZERO128 1 #endif @@ -1577,7 +1577,6 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize return dst; } mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); - mi_assert_internal(bsize % 16 == 0); void* const adst = mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); __int128_t* const start = (__int128_t*)adst; __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); diff --git a/include/mimalloc/track.h b/include/mimalloc/track.h index d5e6128ac..b391d8902 100644 --- a/include/mimalloc/track.h +++ b/include/mimalloc/track.h @@ -132,10 +132,21 @@ defined, undefined, or not accessible at all: #define mi_track_mem_noaccess(p,size) #endif + +#if MI_PADDING_CHECK_BYTES && !MI_GUARDED +#define mi_track_malloc(p,reqsize,zero) \ + do { if ((p)!=NULL) { \ + const size_t __usize = mi_usable_size(p); MI_UNUSED(__usize); \ + mi_assert_internal(__usize==(reqsize)); \ + mi_track_malloc_size(p,reqsize,reqsize,zero); \ + } } while(0) +#else #define mi_track_malloc(p,reqsize,zero) \ do { if ((p)!=NULL) { \ - mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ - mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ + const size_t __usize = mi_usable_size(p); MI_UNUSED(__usize); \ + mi_assert_internal(__usize>=(reqsize)); \ + mi_track_malloc_size(p,reqsize,__usize,zero); \ } } while(0) +#endif #endif // MI_TRACK_H diff --git a/src/arena.c b/src/arena.c index dbb614ccd..fd894c828 100644 --- a/src/arena.c +++ b/src/arena.c @@ -1917,8 +1917,7 @@ static int mi_reserve_os_memory_ex2(mi_subproc_t* subproc, size_t size, bool com } mi_memid_t memid; void* start = _mi_os_alloc_aligned(subproc, size, MI_ARENA_ALIGNMENT, commit, allow_large, &memid); - if (start == NULL) return ENOMEM; - _mi_raw_message("os alloc aligned: %p, size %zu\n", start, size); + if (start == NULL) return ENOMEM; if (!mi_manage_os_memory_ex2(subproc, start, size, -1 /* numa node */, exclusive, memid, NULL, NULL, arena_id)) { _mi_os_free_ex(subproc, start, size, commit, memid); _mi_verbose_message("failed to reserve %zu KiB memory\n", _mi_divide_up(size, 1024)); diff --git a/src/free.c b/src/free.c index d0061463f..eff462b9f 100644 --- a/src/free.c +++ b/src/free.c @@ -650,7 +650,7 @@ static inline bool mi_page_decode_padding(const mi_page_t* page, const mi_block_ const bool ok = (mi_ptr_encode_canary(page,block,page->keys) == canary && *delta <= *bsize); if (double_free!=NULL) { if mi_unlikely(!ok) { *double_free = mi_ptr_decode_canary_is_freed(canary); } // double free? - else { *double_free = false; padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed + else { padding->canary = mi_ptr_encode_canary_freed(); } // mark as freed } mi_track_mem_noaccess(padding,sizeof(mi_padding_t)); return ok; diff --git a/src/os.c b/src/os.c index 94d618827..15111cf94 100644 --- a/src/os.c +++ b/src/os.c @@ -150,7 +150,7 @@ void* _mi_os_get_aligned_hint(size_t try_alignment, size_t sze) uintptr_t hint = mi_atomic_add_acq_rel(&aligned_base, req_size); if (hint == 0 || hint > MI_HINT_MAX) { // wrap or initialize uintptr_t init = MI_HINT_BASE; - #if 0 && (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode + #if (MI_SECURE>=1 || !MI_DEBUG) // security: randomize start of aligned allocations unless in debug mode mi_theap_t* const theap = _mi_theap_default(); // don't use `mi_theap_get_default()` as that can cause allocation recursively (issue #1267) if (!mi_theap_is_initialized(theap)) return NULL; // no hint as we lack randomness at this point const uintptr_t r = _mi_theap_random_next(theap); diff --git a/src/sample-guarded.c b/src/sample-guarded.c index f424729e9..424dbff6d 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -91,7 +91,7 @@ static void* mi_block_ptr_set_guarded(mi_block_t* block, size_t obj_size, size_t // Allocate a block with a guard page behind it. mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, bool zero, mi_page_t** ppage) mi_attr_noexcept -{ +{ // allocate multiple of page size ending in a guard page // ensure minimal alignment requirement? if mi_unlikely(size >= MI_MAX_ALLOC_SIZE - MI_PADDING_SIZE) { // check up front so the `req_size` won't overflow @@ -110,13 +110,11 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, void* const p = mi_block_ptr_set_guarded(block, obj_size, &usable_size); if (p == NULL) return NULL; if (zero) { - mi_page_t* page = _mi_ptr_page(p); - _mi_raw_message("guarded memzero: %p, size: %zu, block=%p, usable_size:%zu, block_size:%zu\n", p, obj_size, block, usable_size, page->block_size); _mi_memzero(p,obj_size); // we have to zero afterwards as padding might have written inside the block (if the `blocksize > reqsize + os_page_size`) } // stats - // mi_track_malloc(p, usable_size, zero); + mi_track_malloc(p, usable_size, zero); if (!mi_theap_is_initialized(theap)) { theap = _mi_theap_default(); } mi_theap_stat_counter_increase(theap, malloc_guarded_count, 1); #if MI_STATS From 68d4a8cd99a1a6540a8327c08c208efa31236d78 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 22:37:00 -0700 Subject: [PATCH 247/280] disable memzero128 --- include/mimalloc/internal.h | 2 +- src/options.c | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 347f93c05..93379eb7c 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1564,7 +1564,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); - #if MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) + #if 0 && MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) #if defined(_MSC_VER) && defined(__AVX2__) typedef __m128i __int128_t; diff --git a/src/options.c b/src/options.c index fcc160332..add57d481 100644 --- a/src/options.c +++ b/src/options.c @@ -273,6 +273,7 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); + _mi_fprintf(out, arg, "padding : %d\n", MI_PADDING ); } mi_decl_export void mi_options_print(void) mi_attr_noexcept { From d545167fb44d797e122c15ddecc3c9eede65c936 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 22:40:39 -0700 Subject: [PATCH 248/280] fix padding output --- include/mimalloc/types.h | 6 +++++- src/options.c | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 890623d74..21e787b43 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -113,8 +113,12 @@ terms of the MIT license. A copy of the license can be found in the file // Reserve extra padding at the end of each block to be more resilient against theap block overflows. // The padding can detect heap-block overflow on free, and provides byte-precise `mi_usable_size`. -#if !defined(MI_PADDING) && (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) +#if !defined(MI_PADDING) +#if (MI_SECURE>=3 || MI_DEBUG>=1 || (MI_TRACK_VALGRIND || MI_TRACK_ASAN || MI_TRACK_ETW)) #define MI_PADDING 1 +#else +#define MI_PADDING 0 +#endif #endif // Check for byte-precise buffer overflow? diff --git a/src/options.c b/src/options.c index add57d481..eaa2119de 100644 --- a/src/options.c +++ b/src/options.c @@ -273,7 +273,7 @@ mi_decl_export void mi_options_print_out(mi_output_fun* out, void* arg) mi_attr_ _mi_fprintf(out, arg, "mem tracking: %s\n", MI_TRACK_TOOL); _mi_fprintf(out, arg, "debug level : %d\n", MI_DEBUG ); _mi_fprintf(out, arg, "secure level: %d\n", MI_SECURE ); - _mi_fprintf(out, arg, "padding : %d\n", MI_PADDING ); + _mi_fprintf(out, arg, "padding : %s\n", (MI_PADDING==0 ? "none" : (MI_PADDING==1 ? "enabled" : "byte precise"))); } mi_decl_export void mi_options_print(void) mi_attr_noexcept { From d09cac69d9a15141b279e9080e49d3d3065572e6 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 22:49:00 -0700 Subject: [PATCH 249/280] fix alignment hint --- include/mimalloc/internal.h | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 93379eb7c..89530b87d 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1564,7 +1564,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); - #if 0 && MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) + #if MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) #if defined(_MSC_VER) && defined(__AVX2__) typedef __m128i __int128_t; @@ -1579,7 +1579,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); void* const adst = mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); __int128_t* const start = (__int128_t*)adst; - __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); + __int128_t* const end = mi_assume_aligned((__int128_t*)((uint8_t*)adst + bsize), MI_SIZE_SIZE); if mi_likely(bsize < 64) { const size_t ofs = (bsize>>5)&1; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); __int128_t* const end0 = end - ofs; From e4d2cc7a2ef8beaee2fa043aa86ad6e4240d1359 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 22:57:55 -0700 Subject: [PATCH 250/280] fix alignment hint --- include/mimalloc/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 89530b87d..b9c19b56d 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1579,7 +1579,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); void* const adst = mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); __int128_t* const start = (__int128_t*)adst; - __int128_t* const end = mi_assume_aligned((__int128_t*)((uint8_t*)adst + bsize), MI_SIZE_SIZE); + __int128_t* const end = (__int128_t*)((uint8_t*)dst + bsize); if mi_likely(bsize < 64) { const size_t ofs = (bsize>>5)&1; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); __int128_t* const end0 = end - ofs; From 209c435fe3426c4773628259160b186d2d0f9c95 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 23:05:44 -0700 Subject: [PATCH 251/280] fix alignment hint --- include/mimalloc/internal.h | 4 +++- src/alloc.c | 2 +- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index b9c19b56d..d255850e9 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1577,9 +1577,11 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize return dst; } mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); + mi_assert_internal(bsize%16 == 0); void* const adst = mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); __int128_t* const start = (__int128_t*)adst; - __int128_t* const end = (__int128_t*)((uint8_t*)dst + bsize); + __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); + mi_assert_internal(_mi_is_aligned(end,16)); if mi_likely(bsize < 64) { const size_t ofs = (bsize>>5)&1; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); __int128_t* const end0 = end - ofs; diff --git a/src/alloc.c b/src/alloc.c index 15184c35e..756d06c60 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -125,7 +125,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ #if !MI_PADDING return // use tail-call #endif - _mi_memzero_block(block,bsize); + _mi_memzero_block(block,bsize + MI_PADDING_SIZE); // ensure bsize is a MI_MAX_ALIGN_SIZE multiple } else { block->next = 0; From 8e507df7036cd4f4fb9a96283209e436ca5b2dd7 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 23:39:27 -0700 Subject: [PATCH 252/280] increase portability of the memzero128 code --- include/mimalloc/internal.h | 29 +++++++++++------------------ include/mimalloc/track.h | 8 +++----- src/alloc.c | 2 +- 3 files changed, 15 insertions(+), 24 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index d255850e9..1a4f73eca 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1566,33 +1566,26 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize #if MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) - #if defined(_MSC_VER) && defined(__AVX2__) - typedef __m128i __int128_t; - const __int128_t zero = _mm_setzero_si128(); - #else - const __int128_t zero = 0; - #endif + // assumes constant memset(p,0,N) gets optimized to fast simd stores. if mi_unlikely(bsize < 16) { *((uint64_t*)dst) = 0; return dst; } mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); - mi_assert_internal(bsize%16 == 0); - void* const adst = mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); - __int128_t* const start = (__int128_t*)adst; - __int128_t* const end = (__int128_t*)((uint8_t*)adst + bsize); + uint8_t* const start = mi_assume_aligned((uint8_t*)dst, MI_MAX_ALIGN_SIZE); + uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) mi_assert_internal(_mi_is_aligned(end,16)); if mi_likely(bsize < 64) { - const size_t ofs = (bsize>>5)&1; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==1); - __int128_t* const end0 = end - ofs; - start[0] = zero; start[ofs] = zero; - end0[-1] = zero; end[-1] = zero; - return adst; + const size_t ofs = (bsize>>1)&16; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==16); + uint8_t* const end0 = end - ofs; + memset(start,0,16); memset(start+ofs,0,16); + memset(end0-16,0,16); memset(end-16,0,16); + return dst; } if mi_likely(bsize <= 128) { - start[0] = zero; start[1] = zero; start[2] = zero; start[3] = zero; - end[-4] = zero; end[-3] = zero; end[-2] = zero; end[-1] = zero; - return adst; + memset(start,0,64); + memset(end-64,0,64); + return dst; } #endif diff --git a/include/mimalloc/track.h b/include/mimalloc/track.h index b391d8902..e6a57c8ec 100644 --- a/include/mimalloc/track.h +++ b/include/mimalloc/track.h @@ -136,16 +136,14 @@ defined, undefined, or not accessible at all: #if MI_PADDING_CHECK_BYTES && !MI_GUARDED #define mi_track_malloc(p,reqsize,zero) \ do { if ((p)!=NULL) { \ - const size_t __usize = mi_usable_size(p); MI_UNUSED(__usize); \ - mi_assert_internal(__usize==(reqsize)); \ + mi_assert_internal(mi_usable_size(p)==(reqsize)); \ mi_track_malloc_size(p,reqsize,reqsize,zero); \ } } while(0) #else #define mi_track_malloc(p,reqsize,zero) \ do { if ((p)!=NULL) { \ - const size_t __usize = mi_usable_size(p); MI_UNUSED(__usize); \ - mi_assert_internal(__usize>=(reqsize)); \ - mi_track_malloc_size(p,reqsize,__usize,zero); \ + mi_assert_internal(mi_usable_size(p)>=(reqsize)); \ + mi_track_malloc_size(p,reqsize,mi_usable_size(p),zero); \ } } while(0) #endif diff --git a/src/alloc.c b/src/alloc.c index 756d06c60..6815fa157 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -125,7 +125,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ #if !MI_PADDING return // use tail-call #endif - _mi_memzero_block(block,bsize + MI_PADDING_SIZE); // ensure bsize is a MI_MAX_ALIGN_SIZE multiple + _mi_memzero_block(block,bsize); } else { block->next = 0; From ee15bb8888444d14ebe51c103fbd8dcd09e6ab6c Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 23:42:10 -0700 Subject: [PATCH 253/280] remove wrong assertion --- include/mimalloc/internal.h | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 1a4f73eca..c4d13d2e4 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1573,8 +1573,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize } mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); uint8_t* const start = mi_assume_aligned((uint8_t*)dst, MI_MAX_ALIGN_SIZE); - uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) - mi_assert_internal(_mi_is_aligned(end,16)); + uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) if mi_likely(bsize < 64) { const size_t ofs = (bsize>>1)&16; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==16); uint8_t* const end0 = end - ofs; From 117d43919fbdb4c1ffc6e9148ee5ac6c4c814654 Mon Sep 17 00:00:00 2001 From: Daan Date: Thu, 10 Sep 2026 23:43:56 -0700 Subject: [PATCH 254/280] fix aligned assumption --- include/mimalloc/internal.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index c4d13d2e4..b0b345663 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1572,7 +1572,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize return dst; } mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); - uint8_t* const start = mi_assume_aligned((uint8_t*)dst, MI_MAX_ALIGN_SIZE); + uint8_t* const start = (uint8_t*)mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) if mi_likely(bsize < 64) { const size_t ofs = (bsize>>1)&16; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==16); From df377d0799a52e31a6bba15680ec51ac99c7fc2c Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 11 Sep 2026 09:52:09 +0000 Subject: [PATCH 255/280] init: release the dynamic thread locals after leaving the park, not before _mi_thread_done frees the thread's dynamic thread-local table (an mi_free into the meta heap) and must do that only after _mi_park_leave: a thread can reach its teardown while still parked, with the scavenger sweeping its pages. The call after the park leave was there, but the previous upstream merge (27d16368) had brought the original early call back in front of it, so the free still happened first and the later call was a no-op. --- src/init.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/init.c b/src/init.c index ef0f8bacc..f4ccf3784 100644 --- a/src/init.c +++ b/src/init.c @@ -530,8 +530,7 @@ void _mi_thread_done(mi_theap_t* _theap_main) // get the current tld mi_tld_t* const tld = _theap_main->tld; - // release dynamic thread_local's - _mi_thread_locals_thread_done(); + // (the dynamic thread_local's are released below, after the park leave) // adjust stats mi_subproc_stat_decrease(tld->subproc, threads, 1); // todo: or `_theap_main->heap`? From 9cc90aa80021cedba0485ca4808c3daf3d1a143e Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 11 Sep 2026 09:52:09 +0000 Subject: [PATCH 256/280] subproc: the meta theap of a new sub-process does not abandon pages and is not sampled Like the meta theap of the main sub-process (mi_heap_main_init_once). With allow_page_abandon left on, a meta allocation that filled a page abandoned it while holding theap_meta_lock, and abandoning allocates the per-bin abandoned bitmap (mi_arena_pages_abandoned_ensure) from the same meta theap: a self-deadlock on the non-recursive lock. Seen as a hang of test-stress-subprocs in a Debug MI_SECURE=FULL build (guarded allocations fill pages quickly there); also present before the upstream merge. --- src/subproc.c | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/src/subproc.c b/src/subproc.c index b14dcefcb..00ec3dc41 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -189,6 +189,13 @@ mi_subproc_id_t mi_subproc_new(void) { mi_assert_internal(parent->theap_meta->tld!=NULL); mi_assert_internal(parent->theap_meta->tld->thread_id == MI_THREADID_DETACHED); _mi_theap_init(theap_meta,heap_main,parent->theap_meta->tld /* detached tld */); + // as for the meta theap of the main sub-process (`init.c:mi_heap_main_init_once`): it is not shared with other threads. + // (abandoning one of its pages would also allocate the per-bin abandoned bitmap (`arena.c:mi_arena_pages_abandoned_ensure`) + // from the meta theap itself, under the `theap_meta_lock` that the allocation that filled the page still holds) + theap_meta->allow_page_abandon = false; + theap_meta->page_full_retain = 2; + theap_meta->sample_rate = 0; // no sampling for meta data + theap_meta->sample_countdown = 0; subproc->theap_meta = theap_meta; return _mi_subproc_to_id(subproc); From 8fa2762588aea2b6bb57009f449471205879b397 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 11 Sep 2026 10:15:56 +0000 Subject: [PATCH 257/280] windows: always compile as C++ with an MSVC compatible compiler Upstream now allows a plain C build with MSVC or clang-cl (MI_USE_CXX=OFF), through the Interlocked based wrapper in atomic.h. That wrapper only models word sized atomics; the park and scavenger state of this fork are 32-bit atomics, so a compare-exchange on them read and wrote 8 bytes (test-heap-teardown crashed in the parked test on the new C jobs of the CI, and the tests that include did not compile). cmake switches to C++ with a warning, atomic.h refuses the C wrapper unless MI_ALLOW_MSVC_C_ATOMICS is defined, and the CI matrix drops the Windows C variants of the basic jobs. --- .github/workflows/test.yaml | 7 +++++++ CMakeLists.txt | 8 ++++++++ include/mimalloc/atomic.h | 5 +++++ 3 files changed, 20 insertions(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 2c2ead2a1..8835a4c8c 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -32,6 +32,13 @@ jobs: tests: extra - c_or_cxx: cxx tests: extra + # this fork is always compiled as C++ on Windows (32-bit atomics that the MSVC C wrapper does not model) + - os: windows-latest + c_or_cxx: c + tests: basic + - os: windows-11-arm + c_or_cxx: c + tests: basic include: - os: ubuntu-latest tests: tsan diff --git a/CMakeLists.txt b/CMakeLists.txt index 3c0ec8dbf..311b102d7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -262,6 +262,14 @@ if(MI_USE_CXX STREQUAL "DEFAULT") endif() endif() +# The plain C wrapper for the MSVC atomics (`include/mimalloc/atomic.h`) only models word sized atomics, and this +# fork also uses 32-bit ones (the park and scavenger state). That wrapper is selected whenever `_MSC_VER` is defined +# in a C compilation: cl, clang-cl, and also plain clang when it targets the MSVC ABI. +if(NOT MI_USE_CXX AND WIN32 AND NOT (MINGW OR CYGWIN OR MSYS)) + message(WARNING "Cannot compile as C code with an MSVC compatible compiler (the C atomics wrapper is word-size only); compiling as C++ instead (MI_USE_CXX=ON)") + set(MI_USE_CXX "ON") +endif() + # enable C++ ? if(MI_USE_CXX) enable_language(CXX) diff --git a/include/mimalloc/atomic.h b/include/mimalloc/atomic.h index 15b012dc2..33d2d6230 100644 --- a/include/mimalloc/atomic.h +++ b/include/mimalloc/atomic.h @@ -161,6 +161,11 @@ static inline void mi_atomic_maxi64_relaxed(volatile int64_t* p, int64_t x) { // Deprecated: MSVC plain C compilation wrapper that uses Interlocked operations to model C11 atomics. // It is recommended to always compile as C++ when using MSVC. +// In this fork it is required: the wrapper only models word sized atomics, and the park and scavenger +// state (`types.h`: `park_state`, `scavenger_wake`, ...) are 32-bit atomics. Define MI_ALLOW_MSVC_C_ATOMICS to try anyway. +#if !defined(MI_ALLOW_MSVC_C_ATOMICS) +#error "compile mimalloc as C++ with MSVC, clang-cl, or clang targeting the MSVC ABI (cmake: -DMI_USE_CXX=ON)" +#endif #include #ifdef _WIN64 From 19c740063138b20d43eddf641abb9c0d569da2ff Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 11 Sep 2026 10:40:33 +0000 Subject: [PATCH 258/280] fork: the child does not inherit a running hole sweep; test-park-handoff says why the forked child failed tld->holes_sweeping stayed set in a child that was forked while the scavenger (or another thread) was sweeping, so that child never handed a discarded hole back to its free list, and its first own sweep asserted in a debug build. The fork-while-parked test now reports how the child failed (signal, or which check). --- src/subproc.c | 1 + test/test-park-handoff.c | 12 ++++++++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/src/subproc.c b/src/subproc.c index 00ec3dc41..55f3179cd 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -430,6 +430,7 @@ void _mi_process_fork_child(void) { mi_atomic_store_relaxed(&t->park_state, (uint32_t)MI_PARK_RUNNING); mi_atomic_store_relaxed(&t->park_reclaim, (uint32_t)0); mi_atomic_store_relaxed(&t->park_swept, (uint32_t)0); + t->holes_sweeping = false; // a sweep that was running at the fork (by the scavenger, or by a thread that is gone) does not continue here } for (mi_heap_t* h = sp->heaps; h != NULL; h = h->next) { mi_lock_init(&h->theaps_lock); diff --git a/test/test-park-handoff.c b/test/test-park-handoff.c index 7fd607fd7..28221a4e9 100644 --- a/test/test-park-handoff.c +++ b/test/test-park-handoff.c @@ -474,8 +474,16 @@ static void test_fork_while_parked(void) { else { int status = 0; // a corrupt free-list entry aborts the child (a signal), which is a failure too - if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { all_ok = false; } - if (first_corrupt_survivor(p) >= 0) { all_ok = false; } // and the parent stays intact + if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { + all_ok = false; + if (WIFSIGNALED(status)) { fprintf(stderr, "\n round %d: the child was killed by signal %d\n", r, WTERMSIG(status)); } + else if (WIFEXITED(status)) { fprintf(stderr, "\n round %d: the child failed with %d (1: out of memory, 2: a survivor changed, 3: two allocations alias)\n", r, WEXITSTATUS(status)); } + else { fprintf(stderr, "\n round %d: waitpid failed or the child stopped (status 0x%x)\n", r, (unsigned)status); } + } + if (first_corrupt_survivor(p) >= 0) { // and the parent stays intact + all_ok = false; + fprintf(stderr, "\n round %d: survivor %ld changed in the parent\n", r, first_corrupt_survivor(p)); + } } for (int i = 0; i < LIVE; i++) { if (p[i] != NULL) mi_free(p[i]); } free(p); From e98ec9619f6153ded17e9aaa0ce80e883d8317c3 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Fri, 11 Sep 2026 10:52:18 +0000 Subject: [PATCH 259/280] test-park-handoff: the forked child reports where it faulted --- test/test-park-handoff.c | 28 +++++++++++++++++++++++++++- 1 file changed, 27 insertions(+), 1 deletion(-) diff --git a/test/test-park-handoff.c b/test/test-park-handoff.c index 28221a4e9..bb2e8d002 100644 --- a/test/test-park-handoff.c +++ b/test/test-park-handoff.c @@ -22,6 +22,7 @@ int main(void) { printf("test-park-handoff: skipped on Windows (uses pthreads/fo #include "mimalloc.h" #include #include +#include #include #include #include @@ -439,6 +440,25 @@ static void test_park_inside_window_gets_swept(void) { // straight out of the churned pages' free lists without meeting a corrupt entry (which aborts) and // without two allocations aliasing. Refilling exactly the freed slots forces exactly that path. // --------------------------------------------------------------------------- +#if defined(__APPLE__) || defined(__GLIBC__) +#include +#define FORK_CHILD_HAS_BACKTRACE 1 +#endif + +// in the forked child: say where a fault happened (there is no core dump to look at on a CI machine) +static void fork_child_fault(int sig, siginfo_t* info, void* ctx) { + (void)ctx; + char buf[128]; + const int n = snprintf(buf, sizeof(buf), "\n forked child: signal %d at address %p (in a mimalloc heap: %d)\n", sig, info->si_addr, (int)mi_is_in_heap_region(info->si_addr)); + if (n > 0) { (void)!write(2, buf, (size_t)n); } + #if FORK_CHILD_HAS_BACKTRACE + void* frames[48]; + const int count = backtrace(frames, 48); + backtrace_symbols_fd(frames, count, 2); + #endif + _exit(64 + sig); +} + static void test_fork_while_parked(void) { enum { ROUNDS = 16 }; bool all_ok = true; @@ -451,6 +471,12 @@ static void test_fork_while_parked(void) { const pid_t pid = fork(); if (pid == 0) { // child: allocate out of the inherited free lists and check for aliasing + struct sigaction sa; + memset(&sa, 0, sizeof(sa)); + sa.sa_sigaction = &fork_child_fault; + sa.sa_flags = SA_SIGINFO; + sigaction(SIGSEGV, &sa, NULL); + sigaction(SIGBUS, &sa, NULL); const size_t ke = keep_every(); int bad = 0; for (int i = 0; i < LIVE; i++) { @@ -477,7 +503,7 @@ static void test_fork_while_parked(void) { if (waitpid(pid, &status, 0) < 0 || !WIFEXITED(status) || WEXITSTATUS(status) != 0) { all_ok = false; if (WIFSIGNALED(status)) { fprintf(stderr, "\n round %d: the child was killed by signal %d\n", r, WTERMSIG(status)); } - else if (WIFEXITED(status)) { fprintf(stderr, "\n round %d: the child failed with %d (1: out of memory, 2: a survivor changed, 3: two allocations alias)\n", r, WEXITSTATUS(status)); } + else if (WIFEXITED(status)) { fprintf(stderr, "\n round %d: the child failed with %d (1: out of memory, 2: a survivor changed, 3: two allocations alias, 64+n: signal n)\n", r, WEXITSTATUS(status)); } else { fprintf(stderr, "\n round %d: waitpid failed or the child stopped (status 0x%x)\n", r, (unsigned)status); } } if (first_corrupt_survivor(p) >= 0) { // and the parent stays intact From 49600a1122680195edc8ed95827015f080374662 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 08:16:58 -0700 Subject: [PATCH 260/280] enable generic memzero_block; remove rep stosb/movsb --- include/mimalloc/internal.h | 110 ++++++++++-------------------------- src/libc.c | 28 +-------- 2 files changed, 31 insertions(+), 107 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index b0b345663..3cece3093 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1465,60 +1465,8 @@ static inline size_t _mi_random_shuffle(size_t x) { // --------------------------------------------------------------------------------- // Provide our own `_mi_memcpy/set` for potential performance optimizations. -// -// For now, only on x64/x86 we optimize to `rep movsb/stosb`. -// Generally, we check for "fast short rep movsb/stosb" (FSRM/FSRS) or "fast enhanced rep movsb" (ERMS) support -// (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017). See also issue #201 and pr #253. -// Todo: we see improvements on win32 but less with glibc; we might want to only enable this on windows. // --------------------------------------------------------------------------------- -#if !MI_TRACK_ENABLED && (MI_ARCH_ARM64 || MI_ARCH_X64) && (defined(__SIZEOF_INT128__) || (defined(_MSC_VER) && defined(__AVX2__))) // any 64-bit platform with 128-bit stores can benefit. -#define MI_USE_MEMZERO128 1 -#endif - -#if !MI_TRACK_ENABLED && !MI_USE_MEMZERO128 && (MI_ARCH_X64 || MI_ARCH_X86) && (defined(_WIN32) || defined(__GNUC__)) - -extern mi_decl_hidden size_t _mi_cpu_movsb_max; // in init.c -extern mi_decl_hidden size_t _mi_cpu_stosb_max; - -static inline void mi_rep_movsb(void* dst, const void* src, size_t n) { - #if defined(__GNUC__) - __asm volatile("rep movsb" : "+D"(dst), "+c"(n), "+S"(src) : : "memory"); - #else - __movsb((unsigned char*)dst, (const unsigned char*)src, n); - #endif -} - -static inline void mi_rep_stosb(void* dst, uint8_t val, size_t n) { - #if defined(__GNUC__) - __asm volatile("rep stosb" : "+D"(dst), "+c"(n) : "a"(val) : "memory"); - #else - __stosb((unsigned char*)dst, val, n); - #endif -} - -static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { - if mi_likely(n <= _mi_cpu_movsb_max) { // has fsrm && n <= 127 (todo: and maybe has erms?) - mi_rep_movsb(dst, src, n); - return dst; - } - else { - return memcpy(dst, src, n); - } -} - -static inline void* _mi_memset(void* dst, int val, size_t n) { - if mi_likely(n <= _mi_cpu_stosb_max) { // has fsrs && n <= 127 - mi_rep_stosb(dst, (uint8_t)val, n); - return dst; - } - else { - return memset(dst, val, n); - } -} - -#else - static inline void* _mi_memcpy(void* dst, const void* src, size_t n) { return memcpy(dst, src, n); } @@ -1527,15 +1475,21 @@ static inline void* _mi_memset(void* dst, int val, size_t n) { return memset(dst, val, n); } -#endif +static inline void* _mi_memset_backward(void* dst, int val, size_t n) { + memset((uint8_t*)dst - n, val, n); + return dst; +} -// ------------------------------------------------------------------------------- -// The `_mi_memcpy_aligned` can be used if the pointers are machine-word (size_t) aligned -// This is used for example in `mi_realloc`. -// ------------------------------------------------------------------------------- +static inline void* _mi_memzero(void* dst, size_t n) { + return _mi_memset(dst, 0, n); +} + +static inline void* _mi_memzero_backward(void* dst, size_t n) { + return _mi_memset_backward(dst, 0, n); +} -// On GCC/CLang we provide a hint that the pointers are word aligned. static inline void* _mi_memcpy_aligned(void* dst, const void* src, size_t n) { + // on gcc/clang we can provide a hint that the pointers are word aligned. mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE) && _mi_is_aligned(src,MI_SIZE_SIZE)); void* adst = mi_assume_aligned(dst, MI_SIZE_SIZE); const void* asrc = mi_assume_aligned(src, MI_SIZE_SIZE); @@ -1548,15 +1502,10 @@ static inline void* _mi_memset_aligned(void* dst, int val, size_t n) { return _mi_memset(adst, val, n); } -static inline void* _mi_memzero(void* dst, size_t n) { - return _mi_memset(dst, 0, n); -} - static inline void* _mi_memzero_aligned(void* dst, size_t n) { return _mi_memset_aligned(dst, 0, n); } - // Zero a block: blocks are always aligned with a positive bsize in machine-word bytes. static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize) { mi_assert_internal(bsize%MI_SIZE_SIZE == 0); @@ -1564,33 +1513,34 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize mi_assert_internal(_mi_is_aligned(dst,MI_SIZE_SIZE)); mi_assert_internal(bsize < MI_MAX_ALIGN_SIZE || _mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); - #if MI_USE_MEMZERO128 // 64-bit with 128-bit stores (arm64 and x64) - // fast memzero based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) - // assumes constant memset(p,0,N) gets optimized to fast simd stores. - if mi_unlikely(bsize < 16) { - *((uint64_t*)dst) = 0; + // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) + // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler + // (compile with -DMI_USE_MEMZERO16X=0 to disable this) + #if !defined(MI_USE_MEMZERO16X) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE + if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8) + *((size_t*)dst) = 0; return dst; } mi_assert_internal(_mi_is_aligned(dst,MI_MAX_ALIGN_SIZE)); uint8_t* const start = (uint8_t*)mi_assume_aligned(dst, MI_MAX_ALIGN_SIZE); - uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) - if mi_likely(bsize < 64) { - const size_t ofs = (bsize>>1)&16; mi_assert_internal(bsize < 32 ? ofs==0 : ofs==16); - uint8_t* const end0 = end - ofs; - memset(start,0,16); memset(start+ofs,0,16); - memset(end0-16,0,16); memset(end-16,0,16); + uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) + if mi_likely(bsize < 8*MI_SIZE_SIZE) { // bsize < 64 (32) + const size_t ofs = (bsize>>1)&(2*MI_SIZE_SIZE); mi_assert_internal(bsize < 4*MI_SIZE_SIZE ? ofs==0 : ofs==2*MI_SIZE_SIZE); // ofs == 16 (8) + _mi_memzero(start, 2*MI_SIZE_SIZE); + _mi_memzero(start+ofs, 2*MI_SIZE_SIZE); + _mi_memzero_backward(end-ofs, 2*MI_SIZE_SIZE); + _mi_memzero_backward(end, 2*MI_SIZE_SIZE); return dst; } - if mi_likely(bsize <= 128) { - memset(start,0,64); - memset(end-64,0,64); + if mi_likely(bsize <= 16*MI_SIZE_SIZE) { // bsize < 128 (64) + _mi_memzero(start, 8*MI_SIZE_SIZE); + _mi_memzero_backward(end, 8*MI_SIZE_SIZE); return dst; } #endif - - // regular memset + // fallback to regular memset for larger sizes void* const wdst = mi_assume_aligned(dst,MI_SIZE_SIZE); - return _mi_memset_aligned(wdst, 0, bsize); + return _mi_memzero_aligned(wdst, bsize); } #endif // MI_INTERNAL_H diff --git a/src/libc.c b/src/libc.c index 17dd1d1dc..8f211230d 100644 --- a/src/libc.c +++ b/src/libc.c @@ -160,8 +160,6 @@ mi_decl_noinline bool _mi_pthread_key_create(pthread_key_t* pkey, void (*destruc // -------------------------------------------------------- // Detect CPU features // -------------------------------------------------------- -mi_decl_cache_align size_t _mi_cpu_movsb_max = 0; // for size <= max, rep movsb is fast -mi_decl_cache_align size_t _mi_cpu_stosb_max = 0; // for size <= max, rep stosb is fast mi_decl_cache_align bool _mi_cpu_has_popcnt = false; #if (MI_ARCH_X64 || MI_ARCH_X86) @@ -192,39 +190,15 @@ static bool mi_cpuid(uint32_t* regs4, uint32_t level, uint32_t sublevel) { #endif void _mi_detect_cpu_features(void) { - // FSRM for fast short rep movsb support (AMD Zen3+ (~2020) or Intel Ice Lake+ (~2017)) - // EMRS for fast enhanced rep movsb/stosb support (not used at the moment, memcpy always seems faster?) - // FSRS for fast short rep stosb - bool amd = false; - bool fsrm = false; - // bool erms = false; - bool fsrs = false; uint32_t cpu_info[4]; - if (mi_cpuid(cpu_info, 0, 0)) { - amd = (cpu_info[2]==0x444d4163); // (Auth enti cAMD) - } - if (mi_cpuid(cpu_info, 7, 0)) { - fsrm = ((cpu_info[3] & (1 << 4)) != 0); // bit 4 of EDX : see - // erms = ((cpu_info[1] & (1 << 9)) != 0); // bit 9 of EBX : see - } - if (mi_cpuid(cpu_info, 7, 1)) { - fsrs = ((cpu_info[1] & (1 << 11)) != 0); // bit 11 of EBX: see - } if (mi_cpuid(cpu_info, 1, 0)) { _mi_cpu_has_popcnt = ((cpu_info[2] & (1 << 23)) != 0); // bit 23 of ECX : see } - - if (fsrm) { - _mi_cpu_movsb_max = 127; - } - if (fsrs || (amd && fsrm)) { // fsrm on amd implies fsrs, see: https://marc.info/?l=git-commits-head&m=168186277717803 - _mi_cpu_stosb_max = 127; - } } #else void _mi_detect_cpu_features(void) { - #if MI_ARCH_ARM64 + #if MI_ARCH_ARM64 || defined(__riscv_zbb) || defined(__riscv_b) _mi_cpu_has_popcnt = true; #endif } From 19c93689af33d9a5b7f10066218ae72445d08e76 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 08:37:42 -0700 Subject: [PATCH 261/280] improve ptr_unalign check --- include/mimalloc/internal.h | 4 ++-- src/free.c | 16 +++++++++++----- src/sample-profile.c | 2 +- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 3cece3093..0c73413f9 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -349,7 +349,7 @@ mi_decl_restrict void* _mi_theap_malloc_sampled(mi_theap_t* theap, size_t req_si size_t _mi_theap_update_sample_rate(mi_theap_t* theap); mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; -void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p); +void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p); size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate); @@ -1516,7 +1516,7 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize // fast memzero for small sizes based on overlapping writes (and assuming non-zero size_t-multiple size, and size_t aligned) // assumes constant memset(p,0,N) gets optimized to fast simd stores by the compiler // (compile with -DMI_USE_MEMZERO16X=0 to disable this) - #if !defined(MI_USE_MEMZERO16X) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE + #if !defined(MI_USE_MEMZERO16X) || (MI_USE_MEMZERO16X != 0) // 16x MI_SIZE_SIZE (128 bytes on 64-bit) if mi_unlikely(bsize < 2*MI_SIZE_SIZE) { // bsize < 16 (8) *((size_t*)dst) = 0; return dst; diff --git a/src/free.c b/src/free.c index eff462b9f..40b545acf 100644 --- a/src/free.c +++ b/src/free.c @@ -97,7 +97,7 @@ static inline void mi_free_block_mt(mi_page_t* page, mi_block_t* block, bool was // note: this can be called from `mi_free_generic_mt` where a non-owning thread accesses the // `page_woffset` and `block_size` fields; however these are constant and the page won't be // deallocated (as the block we are freeing keeps it alive) and thus safe to read concurrently. -mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { +static inline mi_block_t* mi_page_ptr_unalign_ex(const mi_page_t* page, const void* p, size_t* poffset) { mi_assert_internal(page!=NULL && p!=NULL); const size_t diff = (uint8_t*)p - mi_page_start(page); @@ -106,9 +106,14 @@ mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { if mi_unlikely(!_mi_is_power_of_two(block_size)) { adjust = diff % block_size; } + if (poffset!=NULL) { *poffset = adjust; } return (mi_block_t*)((uintptr_t)p - adjust); } +mi_block_t* _mi_page_ptr_unalign(const mi_page_t* page, const void* p) { + return mi_page_ptr_unalign_ex(page,p,NULL); +} + static inline mi_block_t* mi_validate_block_from_ptr( const mi_page_t* page, const void* p ) { mi_assert(_mi_page_ptr_unalign(page,p) == (mi_block_t*)p); // should never be an interior pointer #if MI_SECURE > 0 @@ -126,13 +131,13 @@ static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool return mi_validate_block_from_ptr(page,p); } else { - mi_block_t* const block = _mi_page_ptr_unalign(page,p); + size_t offset; + mi_block_t* const block = mi_page_ptr_unalign_ex(page,p,&offset); #if MI_GUARDED || MI_PROFILE - const size_t offset = (uint8_t*)p - (uint8_t*)block; if (offset >= sizeof(mi_block_t)) { #if MI_PROFILE if (block->next == MI_BLOCK_TAG_PROFILED) { - _mi_page_profile_free(page,block,p); + _mi_page_profile_on_free(page,block,p); } else #endif @@ -141,8 +146,9 @@ static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool _mi_page_block_unguard(page, block, p); *was_guarded = true; } + else #else - { } + { /* nothing */} #endif } #endif diff --git a/src/sample-profile.c b/src/sample-profile.c index 084c8f5d1..f1b5db491 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -179,7 +179,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th return p; } -void _mi_page_profile_free(mi_page_t* page, mi_block_t* block, void* p) { +void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { mi_assert_internal(mi_block_ptr_is_sampled(block,p)); // get the heap and profiler From 6c1821d225a565d3b4156f59bbc020876943068a Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 09:04:18 -0700 Subject: [PATCH 262/280] fix 32-bit compilation warnings --- src/page.c | 11 ++++++----- src/stats.c | 4 ++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/src/page.c b/src/page.c index 0cf6648f2..aa669cdd0 100644 --- a/src/page.c +++ b/src/page.c @@ -171,7 +171,7 @@ static void mi_theap_adjust_sample_countdown(mi_theap_t* theap, mi_page_t* page, const size_t alloc_diff = alloc_count - last_alloc; const uint64_t requested = (uint64_t)alloc_diff * (uint64_t)(bsize - MI_PADDING_SIZE); if (requested <= SIZE_MAX && theap->sample_countdown >= (size_t)requested) { - theap->sample_countdown -= requested; + theap->sample_countdown -= (size_t)requested; } else { theap->sample_requested += (requested - theap->sample_countdown); @@ -247,6 +247,7 @@ static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) { const size_t alloc_count = mi_page_alloc_count(page); const size_t last_used = mi_page_last_used(page); mi_assert_internal(last_used + alloc_count >= used); + mi_assert_internal(used <= UINT16_MAX); const size_t free_count = last_used + alloc_count - used; #if MI_SAMPLE==1 // for ==2 it is already counted in every `alloc.c:mi_page_alloc_zero` @@ -264,7 +265,7 @@ static void mi_theap_page_update_stats(mi_theap_t* theap, mi_page_t* page) { page->xused.used_alloc = (used << 32) | used; #else page->xused.used_alloc = used; - page->xlast_used = used; + page->xlast_used = (uint16_t)used; page->xlast_alloc = 0; #endif mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); @@ -299,11 +300,11 @@ static void mi_page_update_sample_countdown(mi_page_t* page) if (theap==NULL) return; mi_theap_adjust_sample_countdown(theap,page,alloc_count); // update last_alloc to alloc_count + mi_assert_internal(alloc_count <= UINT16_MAX); #if MI_SIZE_SIZE >= 8 - mi_assert_internal(alloc_count <= UINT16_MAX); page->xused.used_alloc = (alloc_count << 48) | (page->xused.used_alloc & (~MI_ZU(0) >> 16)); #else - page->xlast_alloc = alloc_count; + page->xlast_alloc = (uint16_t)alloc_count; #endif mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); mi_assert_internal(mi_page_alloc_count(page) >= mi_page_last_alloc(page)); @@ -1367,4 +1368,4 @@ void* _mi_malloc_generic_no_sample(mi_theap_t* theap, size_t size, bool zero, mi theap->sample_rate = sample_rate; theap->sample_countdown = sample_countdown; return p; -} \ No newline at end of file +} diff --git a/src/stats.c b/src/stats.c index 854282b07..c763bfc3c 100644 --- a/src/stats.c +++ b/src/stats.c @@ -64,9 +64,9 @@ void __mi_stat_decrease(mi_stat_count_t* stat, uint64_t amount) { static void mi_stat_adjust_mt(mi_stat_count_t* stat, int64_t amount) { if (amount == 0) return; // adjust atomically - const size_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); + const int64_t peak = mi_atomic_loadi64_relaxed((_Atomic(int64_t)*)&stat->peak); mi_atomic_addi64_relaxed(&stat->current, amount); - const size_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); + const int64_t prev_total = mi_atomic_addi64_relaxed(&stat->total, amount); if (prev_total == peak) { mi_atomic_addi64_relaxed(&stat->peak, amount); } } From d68195c5045fe297abb6944132641b2c9979b00d Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 13:23:48 -0700 Subject: [PATCH 263/280] always unalign pointers passed to mi_usable_size --- src/free.c | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/src/free.c b/src/free.c index 40b545acf..cf352849c 100644 --- a/src/free.c +++ b/src/free.c @@ -147,9 +147,8 @@ static inline mi_block_t* mi_page_ptr_block_check(mi_page_t* page, void* p, bool *was_guarded = true; } else - #else - { /* nothing */} #endif + { /* nothing */ } } #endif return block; @@ -550,8 +549,8 @@ static void mi_decl_noinline mi_free_try_collect_mt(mi_page_t* page, mi_block_t* // ------------------------------------------------------ // Bytes available in a block -static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p) mi_attr_noexcept { - const mi_block_t* block = _mi_page_ptr_unalign(page, p); +static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* page, const void* p, const mi_block_t* block) mi_attr_noexcept { + // const mi_block_t* block = _mi_page_ptr_unalign(page, p); const bool is_guarded = mi_block_ptr_is_guarded(block,p); const size_t size = mi_page_usable_size_of(page, block, is_guarded); mi_assert_internal((void*)p >= (void*)block); @@ -564,13 +563,13 @@ static size_t mi_decl_noinline mi_page_usable_aligned_size_of(const mi_page_t* p size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexcept { if mi_unlikely(page==NULL) return 0; mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page); - if mi_likely(!mi_page_has_interior_pointers(page)) { - const mi_block_t* block = mi_validate_block_from_ptr(page,p); + const mi_block_t* block = _mi_page_ptr_unalign(page, p); // for safety, always unalign + if mi_likely((const void*)block==page) { return mi_page_usable_size_of(page, block, false /* is guarded */); } else { // split out to separate routine for improved code generation - return mi_page_usable_aligned_size_of(page, p); + return mi_page_usable_aligned_size_of(page, p, block); } } From 61d086fea053e43d050f3f43cc8d9ebf74127fb0 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 13:50:23 -0700 Subject: [PATCH 264/280] fix urealloc_invalid test --- src/alloc.c | 4 +--- src/sample-guarded.c | 2 +- test/test-api.c | 4 ++-- 3 files changed, 4 insertions(+), 6 deletions(-) diff --git a/src/alloc.c b/src/alloc.c index 6815fa157..ded0a00c5 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -30,10 +30,8 @@ terms of the MIT license. A copy of the license can be found in the file static mi_decl_noinline void mi_page_block_setup_padding(mi_page_t* page, mi_block_t* block, size_t size) mi_attr_noexcept { const size_t bsize = mi_page_usable_block_size(page); mi_padding_t* const padding = (mi_padding_t*)((uint8_t*)block + bsize); - ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); - #if (MI_DEBUG>=2) + const ptrdiff_t delta = ((uint8_t*)padding - (uint8_t*)block - (size - MI_PADDING_SIZE)); mi_assert_internal(delta >= 0 && bsize >= (size - MI_PADDING_SIZE + delta)); - #endif mi_track_mem_defined(padding,sizeof(mi_padding_t)); // note: re-enable since mi_page_usable_block_size may set noaccess padding->canary = mi_ptr_encode_canary(page,block,page->keys); padding->delta = (uint32_t)(delta); diff --git a/src/sample-guarded.c b/src/sample-guarded.c index 424dbff6d..f7c1fe81e 100644 --- a/src/sample-guarded.c +++ b/src/sample-guarded.c @@ -141,7 +141,7 @@ mi_decl_restrict void* _mi_theap_malloc_guarded(mi_theap_t* theap, size_t size, // Remove guard page when building with MI_GUARDED #if MI_GUARDED void _mi_page_block_unguard(mi_page_t* page, mi_block_t* block, void* p) { - if (!mi_block_ptr_is_guarded(block,p)) return; + // if (!mi_block_ptr_is_guarded(block,p)) return; mi_assert_internal(mi_block_ptr_is_guarded(block, p)); mi_assert_internal(mi_page_has_interior_pointers(page)); mi_assert_internal((uint8_t*)p - (uint8_t*)block >= (ptrdiff_t)sizeof(mi_block_t)); diff --git a/test/test-api.c b/test/test-api.c index f1a9ac1e4..70bba60b1 100644 --- a/test/test-api.c +++ b/test/test-api.c @@ -115,7 +115,7 @@ int main(void) { #endif CHECK_BODY("calloc-overflow") { // use (size_t)&mi_calloc to get some number without triggering compiler warnings - result = (mi_calloc((size_t)&mi_calloc,SIZE_MAX/1000) == NULL); + result = (mi_calloc((size_t)&mi_calloc,SIZE_MAX/1000) == NULL); }; CHECK_BODY("malloc-large") { // see PR #544. void* p = mi_malloc(67108872); @@ -132,7 +132,7 @@ int main(void) { CHECK_BODY("mi_urealloc_invalid") { void* p = mi_malloc(64); size_t pre, post; - void* q = mi_urealloc((char*)p + 3, 32, &pre, &post); + void* q = mi_urealloc((char*)p + 3, 40, &pre, &post); mi_free(p); result = (q==NULL || q==(uint8_t*)p+3); } From e35400bdf556ba51ce25b756abef4f324e1dcb53 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 14:02:58 -0700 Subject: [PATCH 265/280] gen asm for arm32 in CI --- .github/workflows/test.yaml | 12 ++++++++++-- include/mimalloc/internal.h | 6 +++--- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index e10ffb874..6801ac35a 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -77,7 +77,7 @@ jobs: ctest --test-dir out/release-${{ matrix.c_or_cxx }} --verbose --timeout 240 -C Release - name: Release, Asm - if: matrix.tests == 'basic' && runner.os == 'Linux' + if: matrix.tests == 'basic' && runner.os == 'Linux' && matrix.c_or_cxx == 'c' uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: ${{ matrix.os }}-${{ matrix.c_or_cxx }}-asm @@ -509,10 +509,18 @@ jobs: if: matrix.tests == 'alpine-arm32' shell: alpine-arm32.sh {0} run: | - cmake . -B out/release-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON + cmake . -B out/release-arm32 -DCMAKE_BUILD_TYPE=Release -DMI_OPT_ARCH=ON -DMI_SEE_ASM=ON cmake --build out/release-arm32 --parallel 4 --config Release ctest --test-dir out/release-arm32 --verbose --timeout 240 -C Release + - name: Alpine arm32, Asm + if: matrix.tests == 'alpine-arm32' + shell: alpine-arm32.sh {0} + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: alpine-arm32-c-asm + path: out/release-arm32/CMakeFiles/mimalloc-static.dir/src/*.s + - name: Alpine arm32, Secure if: matrix.tests == 'alpine-arm32' shell: alpine-arm32.sh {0} diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 0c73413f9..80d9106a0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -1526,14 +1526,14 @@ static mi_decl_forceinline void* _mi_memzero_block(mi_block_t* dst, size_t bsize uint8_t* const end = start + bsize; // note: if bsize is always a multiple of 16 then end is always aligned as well (but due to padding this does not hold) if mi_likely(bsize < 8*MI_SIZE_SIZE) { // bsize < 64 (32) const size_t ofs = (bsize>>1)&(2*MI_SIZE_SIZE); mi_assert_internal(bsize < 4*MI_SIZE_SIZE ? ofs==0 : ofs==2*MI_SIZE_SIZE); // ofs == 16 (8) - _mi_memzero(start, 2*MI_SIZE_SIZE); - _mi_memzero(start+ofs, 2*MI_SIZE_SIZE); + _mi_memzero(start, 2*MI_SIZE_SIZE); + _mi_memzero(start+ofs, 2*MI_SIZE_SIZE); _mi_memzero_backward(end-ofs, 2*MI_SIZE_SIZE); _mi_memzero_backward(end, 2*MI_SIZE_SIZE); return dst; } if mi_likely(bsize <= 16*MI_SIZE_SIZE) { // bsize < 128 (64) - _mi_memzero(start, 8*MI_SIZE_SIZE); + _mi_memzero(start, 8*MI_SIZE_SIZE); _mi_memzero_backward(end, 8*MI_SIZE_SIZE); return dst; } From e688ee5f598010396a95dc5fe4832c69d8e0e3cf Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 16:33:03 -0700 Subject: [PATCH 266/280] check for execinfo.h --- CMakeLists.txt | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2dafb80ed..3ed51dda3 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -722,6 +722,13 @@ if(MI_ARCH MATCHES "riscv") endif() endif() +if(NOT WIN32) + CHECK_INCLUDE_FILES("execinfo.h" MI_HAS_EXECINFOH) + if(MI_HAS_EXECINFOH) + list(APPEND mi_defines MI_HAS_EXECINFOH=1) + endif() +endif() + # On Haiku use `-DCMAKE_INSTALL_PREFIX` instead, issue #788 # if(CMAKE_SYSTEM_NAME MATCHES "Haiku") # SET(CMAKE_INSTALL_LIBDIR ~/config/non-packaged/lib) From 1fa2fd92ff9150ceeba69d4f2cbf3d90c05785be Mon Sep 17 00:00:00 2001 From: Daan Date: Tue, 1 Sep 2026 08:05:48 -0700 Subject: [PATCH 267/280] update release notes --- doc/release-notes.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/release-notes.md b/doc/release-notes.md index 1e59f4a53..867484268 100644 --- a/doc/release-notes.md +++ b/doc/release-notes.md @@ -1,7 +1,7 @@ Binary and source releases for mimalloc. -- __v3__: recommended: latest mimalloc design that tends to use less memory then v2 but should perform similarly. -- __v2__: stable: most widely used. +- __v3__: recommended: latest mimalloc design that tends to use less memory then v2. +- __v2__: stable legacy. - __v1__: legacy. Notes: @@ -10,6 +10,6 @@ Notes: - Generally it is recommended to download sources (or use `vcpkg` etc.) and build mimalloc as part of your project. - Source releases can also be downloaded directly from github by the tag. - For example . + For example . - Binary releases include a release-, debug-, and secure build. - Linux binaries are built on Ubuntu 22. From 439dc27bc8d835bd4bbdeedbf7e277dc56076033 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 16:56:16 -0700 Subject: [PATCH 268/280] associate process init theap with thread-done key, pr #1393 by @Brooooooklyn --- src/init.c | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/src/init.c b/src/init.c index a725fb704..30445a58c 100644 --- a/src/init.c +++ b/src/init.c @@ -446,6 +446,11 @@ static void mi_thread_theaps_done(mi_tld_t* tld) static void mi_process_setup_auto_thread_done(void) { mi_atomic_do_once { _mi_prim_thread_init_auto_done(); + mi_theap_t* theap = _mi_theap_default(); + mi_assert_internal(mi_theap_is_initialized(theap)); + if (mi_theap_is_initialized(theap)) { + _mi_theap_default_set(theap); + } } } @@ -454,7 +459,7 @@ void mi_thread_done(void) mi_attr_noexcept { } void _mi_thread_done(mi_theap_t* _theap_main) -{ +{ // NULL can be passed on some platforms if (_theap_main==NULL) { _theap_main = _mi_theap_default(); @@ -511,7 +516,6 @@ void _mi_auto_process_init(void) { os_preloading = false; mi_process_init(); - mi_process_setup_auto_thread_done(); _mi_options_post_init(); // now we can print to stderr if (_mi_is_redirected()) _mi_verbose_message("malloc is redirected.\n"); @@ -556,6 +560,7 @@ static void mi_process_init_once(void) { // the following can potentially allocate (on freeBSD for pthread keys) _mi_tls_slots_init(); // pthread key create _mi_thread_locals_init(); // pthread key create + mi_process_setup_auto_thread_done(); // after the above mi_thread_init so it can add the current theap _mi_process_is_initialized = true; #if defined(_WIN32) && defined(MI_WIN_INIT_USE_FLS) From 615a4365487b3f4e13d1400865391c0c4e3d0a42 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 17:01:55 -0700 Subject: [PATCH 269/280] upload arm32 asm --- .github/workflows/test.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index b83e5d0ec..8640938ec 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -515,7 +515,6 @@ jobs: - name: Alpine arm32, Asm if: matrix.tests == 'alpine-arm32' - shell: alpine-arm32.sh {0} uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 with: name: alpine-arm32-c-asm From cc82d81728ae5778bc0bf6a70b5e52218586d1e0 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 17:21:19 -0700 Subject: [PATCH 270/280] small fixes for multi-config cmake --- CMakeLists.txt | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 333e1998e..c12c77faa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -423,7 +423,7 @@ endif() if(MI_DEBUG STREQUAL "DEFAULT") if(CMAKE_CONFIGURATION_TYPES) # Multi-config generators: Choose debug flags using generator expression - list(APPEND mi_defines $,MI_DEBUG=2 MI_GUARDED=1,MI_DEBUG=0>) + list(APPEND mi_defines $,MI_DEBUG=2,MI_DEBUG=0 NDEBUG=1>) set(MI_DEBUG OFF) # don't go into further MI_DEBUG checks below... else() if(CMAKE_BUILD_TYPE MATCHES "Debug") @@ -447,7 +447,7 @@ elseif(MI_DEBUG STREQUAL "INTERNAL") elseif(MI_DEBUG) message(STATUS "Set debug level to assertion checking (MI_DEBUG=${MI_DEBUG})") list(APPEND mi_defines MI_DEBUG=1) # assertion checking (mi_assert) -else() +elseif(NOT CMAKE_CONFIGURATION_TYPES) message(STATUS "Disable assertion checks (MI_DEBUG=${MI_DEBUG})") list(APPEND mi_defines NDEBUG=1) endif() @@ -587,7 +587,7 @@ if(MI_DEBUG_TSAN) endif() if(MI_DEBUG_UBSAN) - if(CMAKE_BUILD_TYPE MATCHES "Debug") + if(CMAKE_BUILD_TYPE MATCHES "Debug" OR CMAKE_CONFIGURATION_TYPES) if(MI_USE_CXX) if(CMAKE_CXX_COMPILER_ID MATCHES "Clang") message(STATUS "Build with undefined-behavior sanitizer (MI_DEBUG_UBSAN=ON)") @@ -933,16 +933,15 @@ if(MI_TRACK STREQUAL "ASAN") endif() if (CMAKE_CONFIGURATION_TYPES) # multi-config builds - # generator expression lower-case build type - set(_mi_gx_build_type_lc $>) + set(mi_gx_build_type_lc $>) # generator expression for "is this a release build?" - set(_mi_gx_release $,$,$,$>) + set(mi_gx_release $,$,$,$>) - list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${_mi_gx_build_type_lc}") - list(APPEND mi_defines $<${_mi_gx_release}:MI_BUILD_RELEASE>) + list(APPEND mi_defines "MI_CMAKE_BUILD_TYPE=${mi_gx_build_type_lc}") + list(APPEND mi_defines $<${mi_gx_release}:MI_BUILD_RELEASE>) # append build type (e.g. -debug) if not a release version - set(mi_libname "${mi_libname}$<$:-${_mi_gx_build_type_lc}>") + set(mi_libname "${mi_libname}$<$:-${mi_gx_build_type_lc}>") else() # single-config builds string(TOLOWER "${CMAKE_BUILD_TYPE}" CMAKE_BUILD_TYPE_LC) From bd33550d182c4fc0bb2b9bcaa5c15c102178abe5 Mon Sep 17 00:00:00 2001 From: Daan Date: Fri, 11 Sep 2026 17:49:10 -0700 Subject: [PATCH 271/280] use $>) + set(mi_gx_build_type_lc $>) # generator expression for "is this a release build?" set(mi_gx_release $,$,$,$>) From 4db30553f77f2ad0bf9f8e72ff11583c5579b4fb Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 12 Sep 2026 12:28:44 -0700 Subject: [PATCH 272/280] bump version to v1.15.2 --- .github/workflows/release.yaml | 4 ++-- cmake/mimalloc-config-version.cmake | 2 +- include/mimalloc.h | 2 +- readme.md | 9 ++++++--- 4 files changed, 10 insertions(+), 7 deletions(-) diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index a65308a6c..b4b255602 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -9,7 +9,7 @@ permissions: contents: write env: - RELEASE: Release v3.5.1 + RELEASE: Release v3.5.2 FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true name: Release @@ -19,7 +19,7 @@ jobs: runs-on: ${{ matrix.os }} strategy: matrix: - branch: [v3.5.1, v2.5.1, v1.15.1] # [dev,dev2,dev3] + branch: [v3.5.2, v2.5.2, v1.15.2] # [dev,dev2,dev3] # we build on the oldest ubuntu version for better binary compatibility. os: [windows-latest, macOS-latest, macos-15-intel, ubuntu-22.04, ubuntu-22.04-arm] diff --git a/cmake/mimalloc-config-version.cmake b/cmake/mimalloc-config-version.cmake index 040238b9d..e23883136 100644 --- a/cmake/mimalloc-config-version.cmake +++ b/cmake/mimalloc-config-version.cmake @@ -1,6 +1,6 @@ set(mi_version_major 1) set(mi_version_minor 15) -set(mi_version_patch 1) +set(mi_version_patch 2) set(mi_version ${mi_version_major}.${mi_version_minor}) set(PACKAGE_VERSION ${mi_version}) diff --git a/include/mimalloc.h b/include/mimalloc.h index 19facfd05..dd8014680 100644 --- a/include/mimalloc.h +++ b/include/mimalloc.h @@ -8,7 +8,7 @@ terms of the MIT license. A copy of the license can be found in the file #ifndef MIMALLOC_H #define MIMALLOC_H -#define MI_MALLOC_VERSION 11501 // major + 2 digits minor + 2 digits patch +#define MI_MALLOC_VERSION 11502 // major + 2 digits minor + 2 digits patch // ------------------------------------------------------ // Compiler specific attributes diff --git a/readme.md b/readme.md index e77633269..9b8905601 100644 --- a/readme.md +++ b/readme.md @@ -15,9 +15,9 @@ is a general purpose allocator with excellent [performance](#performance) charac Initially developed by Daan Leijen for the runtime systems of the [Koka](https://koka-lang.github.io) and [Lean](https://github.com/leanprover/lean) languages. -Latest release : `v3.5.1` (2026-09-01) recommended. -Latest v2 release: `v2.5.1` (2026-09-01) stable, legacy. -Latest v1 release: `v1.15.1` (2026-09-01) legacy. +Latest release : `v3.5.2` (2026-09-12) recommended. +Latest v2 release: `v2.5.2` (2026-09-12) stable, legacy. +Latest v1 release: `v1.15.2` (2026-09-12) legacy. mimalloc is a drop-in replacement for `malloc` and can be used in other programs without code changes, for example, on dynamically linked ELF-based systems (Linux, BSD, etc.) you can use it as: @@ -89,6 +89,9 @@ New development is mostly on v3, while v1 and v2 are maintained with security an __Send PR's against this version if possible.__ ### Releases +* 2026-09-12, `v3.5.2`, `v2.5.2`, `v1.15.2`: (v3) Reduced cache contention, improved `mi_malloc_csize`, + improved zero'ing of small blocks, `mi_wmalloc` variants for runtime systems, always enable detailed + statistics. Initial experimental support for profiling hooks (`mimalloc-profile.h`). * 2026-09-01, `v3.5.1`, `v2.5.1`, `v1.15.1`: (v3) Yet better performance for `free` calls. (v3) Added `mi_free_small(_nonnull)` for runtimes, and `mi_free_csize(_nonnull)` for inlined constant size free-ing. (v3) Improved MI_OPT_ARCH options; on arm64, defaults to armv8.3 on Apple and armv8.1 otherwise. Improved cache From 636510a36ab743f76a582067142f29d15b024c90 Mon Sep 17 00:00:00 2001 From: Daan Date: Sat, 12 Sep 2026 13:33:54 -0700 Subject: [PATCH 273/280] fix wrong small free size alignment --- src/arena.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/arena.c b/src/arena.c index fd894c828..ffded1067 100644 --- a/src/arena.c +++ b/src/arena.c @@ -979,9 +979,9 @@ static mi_page_t* mi_arenas_page_alloc_fresh(mi_theap_t* theap, size_t slice_cou if (page_meta!=NULL) { mi_assert_internal(page_meta->block_size == 0); #if MI_PAGE_META_SMALL_IS_ALIGNED - // if `block_size <= MI_SMALL_SIZE_MAX` we put the page info in front of the slice, + // if `block_size <= MI_SMALL_MAX_OBJ_SIZE` we put the page info in front of the slice, // (note: it is important that `page_meta->block_size == 0` for `mi_arena_page_at_slice`) - if (block_size <= MI_SMALL_SIZE_MAX) { + if (!os_align && block_size <= MI_SMALL_MAX_OBJ_SIZE) { // put page info in front of the slice page = (mi_page_t*)slice_start; block_start = mi_page_block_start(block_size, os_align); From 3c1952bf9f54d856f432914f6d9453640137f487 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 13 Sep 2026 09:59:11 +0000 Subject: [PATCH 274/280] free: mi_usable_size takes its fast path again; an overflow is not reported as a double free Two slips in upstream code, both still in upstream dev3 at 636510a3: - `_mi_page_usable_size` (d68195c5) compares the unaligned block with `page` where it means `p`. A block is never at the address of its page, so every call went through the out-of-line `mi_page_usable_aligned_size_of`. The result was right (the offset is 0 there), only slower. - `mi_check_padding_on_free` read `is_double_free` without a value when the canary was intact and only the padding bytes behind the block were overwritten: `mi_page_decode_padding` sets it on a bad canary only. The message for a buffer overflow then depended on what was on the stack (gcc: -Wmaybe-uninitialized with MI_SECURE=FULL). --- src/free.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/free.c b/src/free.c index 39ebe63c8..281aa1199 100644 --- a/src/free.c +++ b/src/free.c @@ -595,7 +595,7 @@ size_t _mi_page_usable_size(const mi_page_t* page, const void* p) mi_attr_noexce if mi_unlikely(page==NULL) return 0; mi_assert_internal(mi_ptr_page_validate(p,"_mi_page_usable_size") == page); const mi_block_t* block = _mi_page_ptr_unalign(page, p); // for safety, always unalign - if mi_likely((const void*)block==page) { + if mi_likely((const void*)block==p) { return mi_page_usable_size_of(page, block, false /* is guarded */); } else { @@ -780,7 +780,7 @@ mi_decl_nodiscard static bool mi_check_padding_on_free(const mi_page_t* page, co } else { size_t wrong; - bool is_double_free; + bool is_double_free = false; // stays unset when the canary is fine and only the padding bytes are not if mi_unlikely(!mi_verify_padding(page,block,usable_size,&wrong,&is_double_free)) { if (is_double_free) { _mi_error_message(EAGAIN, "double free detected of heap block %p with size %zu\n", block, *usable_size); From 9e85e9fb63be472f6ecd0d0456ec05a1cc738b95 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 13 Sep 2026 09:59:11 +0000 Subject: [PATCH 275/280] test-heap-release-mt: free what stayed live in a released heap A third of the blocks of each heap stayed allocated for good after `mi_heap_delete` had moved them to the main heap. Each pins its page, so a release build grew by 3 GiB a second (38 GiB RSS at the end of the 6 s run on a 64 core machine), and with MI_SECURE=FULL, where a guard page follows every page, the process ran into `vm.max_map_count` (65530) within two seconds: `pthread_create` failed and the test crashed in `pthread_join`. The owner now frees those blocks after the delete, which also covers frees into the pages that the delete handed to the main heap. After a delete by another thread it waits for that delete to return first: frees by the thread that owns the theap are not safe against a concurrent `mi_heap_delete`, that is what the park is for. Release: 0.8 GiB RSS, and 3.4 times as many heaps released in the same time. --- test/test-heap-release-mt.c | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/test/test-heap-release-mt.c b/test/test-heap-release-mt.c index 67cc4e0be..a8ac65c24 100644 --- a/test/test-heap-release-mt.c +++ b/test/test-heap-release-mt.c @@ -7,8 +7,11 @@ terms of the MIT license. /* Release heaps on many threads at once while other threads free into them. - NCHURN threads loop: `mi_heap_new`, allocate mixed sizes, hand a third of the blocks to a shared - ring, free a third, leak a third, then `mi_heap_delete` (or `mi_heap_destroy` when nothing was - handed out). Some heaps are also published for a while so that short-lived threads allocate from + ring, free a third, keep a third live, then `mi_heap_delete` (or `mi_heap_destroy` when nothing was + handed out). What was kept live is freed after the delete, when it belongs to the main heap: left + in place it pins a page per block for the rest of the run (tens of GiB in a release build, and with + a guard page behind every page, MI_SECURE=FULL, more mappings than `vm.max_map_count` allows). + Some heaps are also published for a while so that short-lived threads allocate from them and exit with their blocks live (the abandon path, and the per-bin abandoned maps). - NFREE threads pop blocks from the ring and `mi_free` them: those frees land before, during and after the delete of the owning heap, so pages are freed and re-abandoned by a foreign thread while @@ -54,6 +57,7 @@ static _Atomic int stop; static _Atomic int park_mode; // second half: park around the release, see the header static _Atomic unsigned long heaps_done, frees_done, exits_done, parks_done, parks_handed_off, remote_deletes; static _Atomic(mi_heap_t*) to_delete[NCHURN]; // park mode: heap i's owner is parked and waits for a freer to delete it +static _Atomic int deleted[NCHURN]; // park mode: the freer that took `to_delete[i]` is done with the delete static const size_t sizes[] = {8,16,48,96,200,512,1024,2048,5000,8192,20000,40000,70000,200000}; #define NS (sizeof(sizes)/sizeof(sizes[0])) @@ -92,6 +96,7 @@ static void* churn(void* arg) { if (!atomic_compare_exchange_strong(&shared[slot], &expect, h)) slot = -1; } int handed = 0; + void* kept[450]; int nkept = 0; int n = 50 + rnd(&seed) % 400; for (int i = 0; i < n; i++) { size_t sz = sizes[rnd(&seed) % NS]; @@ -101,7 +106,7 @@ static void* churn(void* arg) { unsigned r = rnd(&seed) % 3; if (r == 0) { push(p); handed++; } else if (r == 1) mi_free(p); - // else leak into the heap; delete moves it to main / destroy frees it + else kept[nkept++] = p; // live in the heap when it is released: delete moves it to main / destroy frees it } if (slot >= 0) { // let exiting threads use it for a bit, then unpublish @@ -112,7 +117,7 @@ static void* churn(void* arg) { } if (atomic_load(&park_mode)) { const bool remote = ((rnd(&seed) & 1) != 0); - if (remote) { atomic_store(&to_delete[self], h); } // a freer deletes it while we are parked + if (remote) { atomic_store(&deleted[self], 0); atomic_store(&to_delete[self], h); } // a freer deletes it while we are parked const bool parked = mi_on_thread_idle_start(); if (parked) { atomic_fetch_add(&parks_handed_off, 1); } if (remote) { @@ -128,12 +133,17 @@ static void* churn(void* arg) { if (remote) { mi_heap_t* const left = atomic_exchange(&to_delete[self], NULL); // only non-NULL if we are stopping if (left != NULL) { mi_heap_delete(left); } + else { while (!atomic_load(&deleted[self])) usleep(10); } // our own frees are not thread-safe: not before the delete is over + for (int i = 0; i < nkept; i++) mi_free(kept[i]); atomic_fetch_add(&heaps_done, 1); continue; } } if (handed == 0 && (rnd(&seed) & 1)) mi_heap_destroy(h); - else mi_heap_delete(h); + else { + mi_heap_delete(h); + for (int i = 0; i < nkept; i++) mi_free(kept[i]); + } atomic_fetch_add(&heaps_done, 1); } return NULL; @@ -144,7 +154,7 @@ static void delete_parked_heaps(void) { for (int i = 0; i < NCHURN; i++) { if (atomic_load(&to_delete[i]) == NULL) continue; mi_heap_t* const h = atomic_exchange(&to_delete[i], NULL); - if (h != NULL) { mi_heap_delete(h); atomic_fetch_add(&remote_deletes, 1); } + if (h != NULL) { mi_heap_delete(h); atomic_store(&deleted[i], 1); atomic_fetch_add(&remote_deletes, 1); } } } From 7aae82a1fe1d0fa13d68498cefed10fb7c7ddf55 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 13 Sep 2026 10:10:06 +0000 Subject: [PATCH 276/280] test: the purge tests ask for a purge unit of one OS page, not 64 KiB 64 KiB is two slices on a 32-bit target (32 KiB slices), where a pass would again leave the odd slice at the edge of a run. The option value is rounded up to an OS page, and that is never more than a slice. No-Verification-Needed: test-only change (test/test-forced-purge.c, test/test-purge-behind-pass.c) --- test/test-forced-purge.c | 5 +++-- test/test-purge-behind-pass.c | 5 +++-- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/test/test-forced-purge.c b/test/test-forced-purge.c index dbaa8f134..63960d211 100644 --- a/test/test-forced-purge.c +++ b/test/test-forced-purge.c @@ -49,8 +49,9 @@ int main(void) { return 0; } // The rounds count bytes. With transparent huge pages left whole (`MI_ALLOW_THP=FULL`) a pass purges aligned 2 MiB - // units only and leaves the edges of each freed run for later, so purge by the slice here. - mi_option_set(mi_option_minimal_purge_size, 64 /* KiB */); + // units only and leaves the edges of each freed run for later, so purge by the slice here (the size is rounded up to an + // OS page, which is never more than a slice). + mi_option_set(mi_option_minimal_purge_size, 4 /* KiB */); // the scavenger starts when a second thread initializes or a thread first parks void* warm = mi_malloc(64); mi_free(warm); if (mi_on_thread_idle_start()) { mi_on_thread_idle_end(); } diff --git a/test/test-purge-behind-pass.c b/test/test-purge-behind-pass.c index af58e5bff..b4696fa01 100644 --- a/test/test-purge-behind-pass.c +++ b/test/test-purge-behind-pass.c @@ -156,8 +156,9 @@ int main(void) { return 0; } // The rounds count bytes. With transparent huge pages left whole (`MI_ALLOW_THP=FULL`) a pass purges aligned 2 MiB - // units only and leaves the edges of each freed run for later, so purge by the slice here. - mi_option_set(mi_option_minimal_purge_size, 64 /* KiB */); + // units only and leaves the edges of each freed run for later, so purge by the slice here (the size is rounded up to an + // OS page, which is never more than a slice). + mi_option_set(mi_option_minimal_purge_size, 4 /* KiB */); // the scavenger starts when a second thread initializes or a thread first parks void* warm = mi_malloc(64); mi_free(warm); if (mi_on_thread_idle_start()) { mi_on_thread_idle_end(); } From fe2a5e5c25dad98f222d0f9fd1bcb1f116cbed10 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 13 Sep 2026 10:12:29 +0000 Subject: [PATCH 277/280] test-heap-release-mt: the owner keeps working while another thread deletes its old heap The previous commit made the owner wait, after its park, for a delete by a freer thread to return before it freed the blocks that stayed live in that heap. That took away a case the test had: the owner allocating and freeing in its next heap while its old one is still being deleted elsewhere. It now sets those blocks aside, goes on, and frees them when `deleted[self]` says the delete is over (at the latest before it hands the next heap to a freer, and when it stops). The block count per heap and the size of the two arrays come from one pair of constants, and there is one place that frees what was kept. No-Verification-Needed: test-only change (test/test-heap-release-mt.c) --- test/test-heap-release-mt.c | 45 +++++++++++++++++++++++++++---------- 1 file changed, 33 insertions(+), 12 deletions(-) diff --git a/test/test-heap-release-mt.c b/test/test-heap-release-mt.c index a8ac65c24..053aa809f 100644 --- a/test/test-heap-release-mt.c +++ b/test/test-heap-release-mt.c @@ -11,6 +11,8 @@ terms of the MIT license. handed out). What was kept live is freed after the delete, when it belongs to the main heap: left in place it pins a page per block for the rest of the run (tens of GiB in a release build, and with a guard page behind every page, MI_SECURE=FULL, more mappings than `vm.max_map_count` allows). + After a delete by another thread the owner goes on with its next heap while that delete may still + run, and frees those blocks once it has returned. Some heaps are also published for a while so that short-lived threads allocate from them and exit with their blocks live (the abandon path, and the per-bin abandoned maps). - NFREE threads pop blocks from the ring and `mi_free` them: those frees land before, during and @@ -83,9 +85,22 @@ static unsigned rnd(unsigned* s){ *s = *s*1103515245u+12345u; return *s>>8; } static _Atomic(mi_heap_t*) shared[NSHARED]; static _Atomic int users[NSHARED]; +#define MIN_BLOCKS 50 +#define MAX_BLOCKS 450 // a heap gets MIN_BLOCKS up to (excluding) MAX_BLOCKS blocks + +static void free_all(void** blocks, int* count) { + for (int i = 0; i < *count; i++) mi_free(blocks[i]); + *count = 0; +} + static void* churn(void* arg) { const int self = (int)(uintptr_t)arg - 1; unsigned seed = (unsigned)(uintptr_t)arg * 7919u + 1; + void* kept[MAX_BLOCKS]; int nkept = 0; // live in the heap when it is released + // The kept blocks of the heap that we left to a freer. The delete may still be running when we are back from the park, + // and until it returns our own frees into that heap are not safe (they are not atomic, that is what the park is for). + // So go on with the next heap in the meantime and free these once `deleted[self]` says the delete is over. + void* behind[MAX_BLOCKS]; int nbehind = 0; while (!atomic_load(&stop)) { mi_heap_t* h = mi_heap_new(); int slot = -1; @@ -96,8 +111,7 @@ static void* churn(void* arg) { if (!atomic_compare_exchange_strong(&shared[slot], &expect, h)) slot = -1; } int handed = 0; - void* kept[450]; int nkept = 0; - int n = 50 + rnd(&seed) % 400; + int n = MIN_BLOCKS + rnd(&seed) % (MAX_BLOCKS - MIN_BLOCKS); for (int i = 0; i < n; i++) { size_t sz = sizes[rnd(&seed) % NS]; void* p = mi_heap_malloc(h, sz); @@ -106,7 +120,7 @@ static void* churn(void* arg) { unsigned r = rnd(&seed) % 3; if (r == 0) { push(p); handed++; } else if (r == 1) mi_free(p); - else kept[nkept++] = p; // live in the heap when it is released: delete moves it to main / destroy frees it + else kept[nkept++] = p; // delete moves it to main / destroy frees it } if (slot >= 0) { // let exiting threads use it for a bit, then unpublish @@ -115,9 +129,15 @@ static void* churn(void* arg) { while (atomic_load(&users[slot]) != 0) usleep(10); // in-flight allocators finish (contract: no alloc during delete) handed = 1; // exiting threads may hold blocks: must delete, not destroy } + bool released = false; if (atomic_load(&park_mode)) { const bool remote = ((rnd(&seed) & 1) != 0); - if (remote) { atomic_store(&deleted[self], 0); atomic_store(&to_delete[self], h); } // a freer deletes it while we are parked + if (remote) { + // one remote delete at a time: `deleted[self]` is about to be reused + if (nbehind > 0) { while (!atomic_load(&deleted[self])) usleep(10); free_all(behind, &nbehind); } + atomic_store(&deleted[self], 0); + atomic_store(&to_delete[self], h); // a freer deletes it while we are parked + } const bool parked = mi_on_thread_idle_start(); if (parked) { atomic_fetch_add(&parks_handed_off, 1); } if (remote) { @@ -133,19 +153,20 @@ static void* churn(void* arg) { if (remote) { mi_heap_t* const left = atomic_exchange(&to_delete[self], NULL); // only non-NULL if we are stopping if (left != NULL) { mi_heap_delete(left); } - else { while (!atomic_load(&deleted[self])) usleep(10); } // our own frees are not thread-safe: not before the delete is over - for (int i = 0; i < nkept; i++) mi_free(kept[i]); - atomic_fetch_add(&heaps_done, 1); - continue; + else { memcpy(behind, kept, (size_t)nkept * sizeof(void*)); nbehind = nkept; nkept = 0; } // a freer took it + released = true; } } - if (handed == 0 && (rnd(&seed) & 1)) mi_heap_destroy(h); - else { - mi_heap_delete(h); - for (int i = 0; i < nkept; i++) mi_free(kept[i]); + if (!released) { + if (handed == 0 && (rnd(&seed) & 1)) { mi_heap_destroy(h); nkept = 0; } + else { mi_heap_delete(h); } } + // What stayed live now belongs to the main heap. Left there, each block pins its page for the rest of the run. + free_all(kept, &nkept); + if (nbehind > 0 && atomic_load(&deleted[self])) { free_all(behind, &nbehind); } atomic_fetch_add(&heaps_done, 1); } + if (nbehind > 0) { while (!atomic_load(&deleted[self])) usleep(10); free_all(behind, &nbehind); } return NULL; } From b5bc53f99d1da573334d597d76cbddc85a80b7e4 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 13 Sep 2026 10:24:56 +0000 Subject: [PATCH 278/280] cmake: test-heap-release-mt runs with guarded sampling like the other tests It was run with MIMALLOC_GUARDED_SAMPLE_RATE=0 because it kept a third of its blocks for good and every guarded block is a mapping of its own. It frees them now: with MI_GUARDED a release build passes at sample rates 1024, 64 and 1, with and without the scavenger. So it takes the default rate of 1024, and the -no-scavenger variant whatever the job sets. No-Verification-Needed: test wiring only (CMakeLists.txt add_test environment) --- CMakeLists.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 66e17303f..9db485cd5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1173,8 +1173,6 @@ if (MI_BUILD_TESTS) set(mi_test_env "") if(MI_GUARDED AND TEST_NAME STREQUAL "purge-holes") list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0) # it knows the page layout, which guard pages change (set explicitly: the CI sets a rate for the whole job) - elseif(MI_GUARDED AND TEST_NAME STREQUAL "heap-release-mt") - list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0) # it leaks a third of its blocks on purpose, and every guarded block is a mapping of its own: that runs into vm.max_map_count elseif(MI_GUARDED AND NOT (TEST_NAME STREQUAL "profile")) list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=1024) elseif(TEST_NAME STREQUAL "stress-heaps") @@ -1201,7 +1199,7 @@ if (MI_BUILD_TESTS) # with no scavenger to hand off to, `mi_on_thread_idle_start` must do the work inline instead add_test(NAME test-park-handoff-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 $) - add_test(NAME test-heap-release-mt-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 MIMALLOC_GUARDED_SAMPLE_RATE=0 $) + add_test(NAME test-heap-release-mt-no-scavenger COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_SCAVENGER=0 $) # and sweep every park, so the reclaim path is hit as hard as possible add_test(NAME test-park-handoff-eager COMMAND ${CMAKE_COMMAND} -E env MIMALLOC_PURGE_HOLES_MIN_INTERVAL=0 $) From 93eb9c78bee2156bfaf45dd50674364c476185cf Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 13 Sep 2026 10:49:01 +0000 Subject: [PATCH 279/280] fork: no arena purge pass runs across a fork() The prepare handler takes the purge guard (last, under its locks: a pass takes none of them and waits for no one, so the one in progress ends), the parent and the child release it. test-park-handoff "fork while parked" kept failing now and then on macOS Debug builds: the forked child died with SIGBUS. The diagnostics the test got earlier now say where (CI run 34751251554, basic/macos-latest/cxx, Debug): forked child: signal 10 at address 0x20001210000 (in a mimalloc heap: 1) mi_block_set_next < mi_page_free_list_extend < mi_page_extend_free < _mi_page_init < mi_arenas_page_regular_alloc < _mi_arenas_page_alloc < mi_page_fresh_alloc < mi_page_fresh < mi_page_queue_find_free_ex < _mi_malloc_generic < mi_malloc Not in a free list that the interrupted hole sweep was rewriting, but on the first write into a page that the child had just allocated from the arena, at a slice aligned address: a range that the arena bitmaps call free and committed, and that is not writable. Whole slices become PROT_NONE in one place only, the decommit of an arena purge, and only with MI_DEBUG or MI_SECURE (`_mi_prim_decommit`), which fits "Debug only". A park ends in `_mi_arenas_purge_now`, so the scavenger is in an arena purge pass when the test forks a few hundred microseconds later. The prepare handler ended the park and took every lock, but a pass takes no lock, so it ran across the fork. A pass claims a range, decommits it, clears its commit bits and releases it; a consistent copy of that is harmless to the child (it sees the range claimed), a copy with the bitmaps from before and the protection from after is not. That the kernel produced such a copy is an inference, the missing quiescence is not. With the guard held by the fork no pass is cut off, so the child has nothing to repair: the release of a dead purger's guard and the "make a pass due" of 1b894afe are replaced by a plain release. `_mi_arenas_purge_guard_reset` becomes `_mi_arenas_purge_guard_acquire`/`_release`; the process exit on Windows uses the latter. test-purge-behind-pass "a fork in a pass" passes as it is: the fork waits for the pass, and the child purges the arena that was not yet due when its time comes. --- include/mimalloc/internal.h | 3 ++- src/arena.c | 24 ++++++++++++++++++------ src/init.c | 2 +- src/subproc.c | 6 +++--- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 0237db6e6..5e961dcb0 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -266,7 +266,8 @@ void _mi_arenas_free(mi_subproc_t* subproc, void* p, size_t size, mi_me void _mi_arenas_collect(bool force_purge, bool visit_all, mi_tld_t* tld); void _mi_arenas_purge_abandoned_holes(mi_heap_t* heap, mi_tld_t* tld); bool _mi_arenas_try_purge(bool force, bool visit_all, mi_subproc_t* subproc, size_t tseq); // false: dropped, another thread is purging -bool _mi_arenas_purge_guard_reset(void); // true: a purge pass was cut off (by a fork, or by the end of the process) +void _mi_arenas_purge_guard_acquire(void); // fork: wait for the purge pass in progress, and keep the next one out +void _mi_arenas_purge_guard_release(void); // (also on behalf of a holder that is gone: process exit on Windows) void _mi_arenas_unsafe_destroy_all(mi_subproc_t* subproc); mi_page_t* _mi_arenas_page_alloc(mi_theap_t* theap, size_t block_size, size_t page_alignment); diff --git a/src/arena.c b/src/arena.c index 681b010ee..2b844cd9d 100644 --- a/src/arena.c +++ b/src/arena.c @@ -2735,12 +2735,24 @@ void _mi_arenas_purge_now(mi_subproc_t* subproc) { // allow only one thread to purge at a time (todo: allow concurrent purging?) static mi_atomic_guard_t mi_arenas_purge_guard; -// Release the guard for a thread that is gone: the one that held it across fork() is not in the child, and at process exit -// on Windows every other thread is terminated before the detach callback runs. Nothing else would ever release it, and a -// forced purge waits for it. Returns true if it was held: that pass had reset the `purge_expire` of a sub-process, and -// it is not there to put back what was still pending (see `_mi_arenas_try_purge`). -bool _mi_arenas_purge_guard_reset(void) { - return (mi_atomic_exchange_acq_rel(&mi_arenas_purge_guard, (uintptr_t)0) != 0); +// No purge pass runs across a fork(): the prepare handler takes the guard, the parent and the child release it. A pass +// changes the protection of arena memory (a decommit is `mprotect(PROT_NONE)` in a debug or secure build) and the bits that +// describe that memory in separate steps, and a fork in between gives the child the one without the other. +// Taken last in the prepare handler, under its locks: a pass takes none of them and waits for no one, so it ends. +void _mi_arenas_purge_guard_acquire(void) { + size_t spin = 0; + uintptr_t expected = 0; + while (!mi_atomic_cas_weak_acq_rel(&mi_arenas_purge_guard, &expected, (uintptr_t)1)) { + expected = 0; + if (spin < 256) { mi_atomic_pause(); spin++; } + else { _mi_prim_thread_yield(); } + } +} + +// Also for a holder that is gone: at process exit on Windows every other thread is terminated before the detach callback +// runs. Nothing else would ever release it then, and a forced purge waits for it. +void _mi_arenas_purge_guard_release(void) { + mi_atomic_store_release(&mi_arenas_purge_guard, (uintptr_t)0); } // Make a purge pass over the arenas of `subproc` due at `expire`, unless one is due before that already. The scavenger diff --git a/src/init.c b/src/init.c index 3a1cbebcd..a6e8f8db4 100644 --- a/src/init.c +++ b/src/init.c @@ -691,7 +691,7 @@ static void mi_process_done_once(void) { #if defined(_WIN32) // This runs in the process detach callback, after the system terminated every other thread. One that was in a purge // pass then (the scavenger, as a rule) holds the purge guard for good, and the forced collect below would wait for it. - _mi_arenas_purge_guard_reset(); + _mi_arenas_purge_guard_release(); #endif _mi_heap_snapshot_on_exit(); // fork: heap snapshot at exit, before anything is torn down // only shutdown if we were initialized diff --git a/src/subproc.c b/src/subproc.c index e0e71760b..351e10f37 100644 --- a/src/subproc.c +++ b/src/subproc.c @@ -394,11 +394,13 @@ void _mi_process_fork_prepare(void) { mi_lock_acquire(&mi_subprocs_lock); _mi_thread_locals_fork_prepare(); for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { mi_subproc_fork_prepare(sp); } + _mi_arenas_purge_guard_acquire(); // last: a purge pass takes no lock, so the one in progress ends (see `arena.c`) } void _mi_process_fork_parent(void) { if (!_mi_process_is_initialized) return; if (mi_atomic_decrement_acq_rel(&mi_fork_depth) != 1) return; + _mi_arenas_purge_guard_release(); // release in reverse: last sub-process first (the registry is a stack: newest first, so walk it into a reversed order) size_t n = 0; for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { n++; } @@ -417,11 +419,9 @@ void _mi_process_fork_child(void) { if (mi_atomic_exchange_acq_rel(&mi_fork_depth, 0) == 0) return; _mi_process_is_forked_child = true; _mi_scavenger_forked_child(); // the scavenger thread did not survive the fork; clear the state that says it did - const bool purge_cut_off = _mi_arenas_purge_guard_reset(); // nor did a thread that was purging; release the guard it held + _mi_arenas_purge_guard_release(); // taken by the prepare handler: no purge pass was in progress, so each `purge_expire` is as a finished pass left it mi_lock_init(&mi_subprocs_lock); for (mi_subproc_t* sp = mi_subprocs; sp != NULL; sp = sp->next) { - // the purge pass that was cut off may have been over this one: make a pass due, for the arenas whose expire is still set - if (purge_cut_off) { mi_atomic_storei64_relaxed(&sp->purge_expire, (mi_msecs_t)1); } mi_lock_init(&sp->arena_reserve_lock); mi_lock_init(&sp->heaps_lock); mi_lock_init(&sp->tlds_lock); From 5af105d5f575751246a1d76255ab02811b5a49cf Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 13 Sep 2026 11:25:22 +0000 Subject: [PATCH 280/280] alloc, free: update the packed used count with one instruction on memory `page->used` became the low 16 bits of the packed word `mi_used_t xused` (used, alloc count, last used, last alloc) with the upstream merge. Both fast paths then load the word, change it, store it, and `mi_free` tests the low half, where they used to do `incl`/`decl` on a 32-bit field: three more instructions in `mi_free`, two more in `mi_malloc`. - `mi_free` decrements the 16-bit count in place, through the `used_count` member the union already had: `decw 0x18(%rdi); je`. Little-endian only; the code for the whole word stays for everything else. (On a used count of 0, a double free that nothing caught, the count wraps to 0xFFFF where it used to borrow from the alloc count.) - `mi_malloc` adds 0x10001 to the word in memory once it knows the free list is not empty: `addq $0x10001,0x18(%rcx)`. The load before the test and the compiler barrier that forced it are gone. Instructions for 25 M malloc + 25 M free of 16..1039 bytes (perf stat, user mode), clean bun-dev3-v2 / before / after: cmake Release, gcc: 2.4142 G / 2.5394 G / 2.4127 G Bun's flags (clang -O3 as C++): 2.2463 G / 2.3411 G / 2.2153 G (with MI_PROFILE=0: 2.3097 G / 2.1838 G) `mi_free` is 22 instructions (23 on bun-dev3-v2, 25 before), `mi_malloc` 18 (19, 20). Cycles do not differ between the three on this benchmark on a loaded machine (1.24 to 1.28 G each). --- include/mimalloc/types.h | 10 +++++++++- src/alloc.c | 7 +------ src/free.c | 6 ++---- 3 files changed, 12 insertions(+), 11 deletions(-) diff --git a/include/mimalloc/types.h b/include/mimalloc/types.h index 11fb93fe6..b0308d4bb 100644 --- a/include/mimalloc/types.h +++ b/include/mimalloc/types.h @@ -432,7 +432,7 @@ typedef uintptr_t mi_thread_free_t; // in a release build; this way we can update the stats in the slow path (`_mi_page_update_stats`). typedef union mi_used_s { size_t used_alloc; // used + alloc_count - // the following struct is unused but nice for debugging + // the same in parts, for a debugger and for `mi_xused_dec_used` struct { uint16_t used_count; uint16_t alloc_count; @@ -447,6 +447,14 @@ static inline size_t mi_xused_used_count(mi_used_t xused) { return (xused.use static inline size_t mi_xused_alloc_count(mi_used_t xused) { return ((xused.used_alloc>>16) & 0xFFFF); } static inline mi_used_t mi_xused_used_reset(mi_used_t xused) { xused.used_alloc = xused.used_alloc & ~0xFFFF; return xused; } +// Decrement the used count in place and return the new count (`mi_free`). Where its 16 bits can be addressed by themselves +// that is one `decw` on memory whose flags feed the branch; on the whole word it is a load, a decrement, a store and a test. +#if !MI_BIG_ENDIAN && ((defined(__BYTE_ORDER__) && defined(__ORDER_LITTLE_ENDIAN__) && (__BYTE_ORDER__ == __ORDER_LITTLE_ENDIAN__)) || defined(_WIN32)) +static inline size_t mi_xused_dec_used(mi_used_t* xused) { return --xused->debug_le.used_count; } +#else +static inline size_t mi_xused_dec_used(mi_used_t* xused) { xused->used_alloc--; return mi_xused_used_count(*xused); } +#endif + // A page contains blocks of one specific size (`block_size`). // Each page has three list of free blocks: // `free` for blocks that can be allocated, diff --git a/src/alloc.c b/src/alloc.c index 39e852ee1..f3b5717bf 100644 --- a/src/alloc.c +++ b/src/alloc.c @@ -59,11 +59,6 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ // check the free list mi_block_t* const block = page->free; - mi_used_t xused = page->xused; - xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count - #if defined(__GNUC__) - __asm("" : : : "memory" ); // always load the `used` field before the test - #endif if (block == NULL) { return _mi_malloc_generic(theap, size, (zero ? 1 : 0), ppage); } @@ -79,7 +74,7 @@ static mi_decl_forceinline void* mi_page_malloc_zero(mi_theap_t* theap, mi_page_ #endif page->free = next; - page->xused = xused; + page->xused.used_alloc += 0x10001; // increment both (16-bit) used count and alloc count: one add on memory mi_assert_internal(page->free == NULL || _mi_ptr_page(page->free) == page); mi_assert_internal(page->block_size < MI_MAX_ALIGN_SIZE || _mi_is_aligned(block, MI_MAX_ALIGN_SIZE)); diff --git a/src/free.c b/src/free.c index 281aa1199..56f5d3266 100644 --- a/src/free.c +++ b/src/free.c @@ -56,13 +56,11 @@ static inline void mi_free_block_local(mi_page_t* page, mi_block_t* block, bool #endif // actual free: push on the local free list - mi_used_t xused = page->xused; - xused.used_alloc--; // decrement used count mi_block_set_next(page, block, page->local_free); - page->xused = xused; page->local_free = block; + const size_t used = mi_xused_dec_used(&page->xused); mi_assert_internal(mi_page_alloc_count(page) + mi_page_last_used(page) >= mi_page_used(page)); - if mi_unlikely(mi_xused_used_count(xused) == 0) { // is used count zero ? + if mi_unlikely(used == 0) { if (page->retire_expire==0) { // no need to re-retire retired pages (happens when we alloc/free one block repeatedly in an empty page) _mi_page_retire(page); }