From 743469e488077f1617b0d1ea092d251f37606d64 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Sun, 13 Sep 2026 13:21:10 +0000 Subject: [PATCH 1/2] profile: sample the small allocation that refilled its page, new threads, and the blocks mi_heap_destroy frees - page.c: with coarse sampling the blocks the fast path took from a page are counted when its free list is refilled. If that uses up the countdown, the allocation in progress is the sample, not the next one through the generic path (which is an allocation above MI_SMALL_SIZE_MAX far more often than a small one). mi_page_extend_free counts the previous extension too. - sample-profile.c: a rate that on_alloc raised starts its period in full, so bytes_since_last_sample stays what was requested. - theap.c: a theap created while a profiler runs is sampled from its first allocation, and starts with a full period. - sample-profile.c, arena.c: mi_heap_destroy calls on_free for the sampled blocks still in use. A profiled block carries a check word (address xor a random key) so the walk does not take a regular block that starts with the tag's value for one; the word is cleared when the block is freed. No instruction is added to the mi_malloc/mi_free fast paths. --- include/mimalloc/internal.h | 1 + src/arena.c | 3 + src/page.c | 12 ++ src/sample-profile.c | 72 +++++++++++- src/theap.c | 15 +++ test/test-profile.c | 226 +++++++++++++++++++++++++++++++++++- 6 files changed, 321 insertions(+), 8 deletions(-) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 5e961dcb0..3a309d83e 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -408,6 +408,7 @@ size_t _mi_theap_update_sample_rate(mi_theap_t* theap); mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p); +void _mi_page_profile_free_all(const mi_heap_area_t* area, mi_page_t* page); size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate); diff --git a/src/arena.c b/src/arena.c index 2b844cd9d..60f6b72c6 100644 --- a/src/arena.c +++ b/src/arena.c @@ -3060,6 +3060,9 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are #if MI_GUARDED _mi_page_unguard_all(page); // remove potential interior guard pages #endif + #if MI_PROFILE + _mi_page_profile_free_all(area,page); // the sampled blocks that are still in use get their `on_free` + #endif // destroy the page // Drop what is (still) on the thread-free list first: `_mi_arenas_page_free` collects it, and with the used // count reset that reads as a corrupted list (more blocks freed than were in use). diff --git a/src/page.c b/src/page.c index bc0bed042..7becd5abf 100644 --- a/src/page.c +++ b/src/page.c @@ -1834,6 +1834,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { //uint8_t* page_start = mi_page_area(page, &page_size); mi_theap_stat_counter_increase(theap, pages_extended, 1); + mi_page_update_sample_countdown(page); // the blocks of the previous extension are handed out: count them // calculate the extend count const size_t bsize = mi_page_block_size(page); @@ -2368,6 +2369,17 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm page = mi_page_queue_find_free(theap,pq); // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); if (page!=NULL) { + #if MI_SAMPLE==1 + // The blocks that the fast path took from a page are counted against the sample countdown when its free + // list is refilled, which is what `mi_page_queue_find_free` just did. If that used up the countdown, the + // allocation in progress (of the size class of those blocks) is the sample. If we leave it to the next + // allocation that comes through here, that is one above `MI_SMALL_SIZE_MAX` (these always do) far more + // often than a small one: with one such allocation for every 40 blocks of 64 bytes, less than 2% of the + // sampled bytes went to the blocks of 64 bytes (`test-profile.c:test_profiler_small_attribution`). + if mi_unlikely(theap->sample_rate!=0 && mi_theap_should_sample(theap,req_size)) { + return _mi_theap_malloc_sampled(theap,req_size,zero,ppage); + } + #endif if (ppage!=NULL) { *ppage = page; } mi_assert_internal(mi_page_immediate_available(page)); // we should never recurse in _mi_page_malloc_zero return _mi_page_malloc_zero(theap,page,size,zero); diff --git a/src/sample-profile.c b/src/sample-profile.c index f1b5db491..c2c6a6656 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -115,6 +115,34 @@ static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { } } +//---------------------------------------------------------------------------- +// Layout of a profiled block: +// [MI_BLOCK_TAG_PROFILED] [check] [mi_profiler_sample_data_t ... user data ...] [ ... block as the program sees it ... ] +// `mi_free` recognizes it by the tag and by getting an interior pointer. `mi_heap_destroy` walks +// the blocks of a page and has no pointer to go by: the first word of a regular block in use is +// program data and can hold the value of the tag. The `check` word is the address of the block +// xor a random key, so a regular block is not mistaken for a profiled one. +//----------------------------------------------------------------------------- + +static _Atomic(uintptr_t) mi_profile_check_key; // = 0 + +static uintptr_t mi_profile_block_check(mi_theap_t* theap, const mi_block_t* block) { + uintptr_t key = mi_atomic_load_relaxed(&mi_profile_check_key); + if mi_unlikely(key==0) { + if (theap==NULL) return 0; // no block was profiled yet + uintptr_t expected = 0; + key = _mi_theap_random_next(theap) | 1; + if (!mi_atomic_cas_strong_acq_rel(&mi_profile_check_key, &expected, key)) { key = expected; } + } + return ((uintptr_t)block ^ key); +} + +#define MI_PROFILE_SAMPLE_DATA_OFFSET (sizeof(mi_block_t) + sizeof(uintptr_t)) + +static inline size_t mi_profile_user_offset(size_t sample_user_data_size) { + return _mi_align_up(MI_PROFILE_SAMPLE_DATA_OFFSET + sizeof(mi_profiler_sample_data_t) + sample_user_data_size, MI_MAX_ALIGN_SIZE); +} + //---------------------------------------------------------------------------- // Profile an allocation //----------------------------------------------------------------------------- @@ -142,11 +170,10 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th } else { // overallocate a larger block to store the profiler data - // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] - const size_t sample_data_offset = sizeof(mi_block_t); + // [MI_BLOCK_TAG_PROFILED] [check] [ ... profile data ... ] [... user data ...] + const size_t sample_data_offset = MI_PROFILE_SAMPLE_DATA_OFFSET; const size_t sample_user_data_size = _mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*)); - const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; // one void* too many just in case - const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); + const size_t user_offset = mi_profile_user_offset(sample_user_data_size); const size_t oversize = user_offset + size; mi_page_t* page = NULL; mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); @@ -163,6 +190,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free) mi_page_set_has_interior_pointers(page, true); block->next = MI_BLOCK_TAG_PROFILED; + *((uintptr_t*)(block + 1)) = mi_profile_block_check(theap,block); p = (uint8_t*)block + user_offset; mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); sample_data->user_data_size = sample_user_data_size; @@ -174,6 +202,11 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th } if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { _mi_theap_set_profile_sample_rate(theap,new_sample_rate); + // We are at a sample, where a period starts: start it with the new rate in full. (`_mi_theap_malloc_sampled` + // takes the period to be `sample_rate` long when it computes `bytes_since_last_sample`. A raised rate left the + // countdown at the previous rate, and each sample reported the difference on top of the bytes that were requested.) + theap->profile_sample_countdown = theap->profile_sample_rate; + theap->sample_countdown = theap->sample_rate; } mi_theap_stat_counter_increase(theap,profile_samples,1); return p; @@ -181,6 +214,9 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { mi_assert_internal(mi_block_ptr_is_sampled(block,p)); + // The free list only overwrites the tag. Clear the check word too, or the block could come back as a regular + // block that starts with the value of the tag and still has the check behind it (see `_mi_page_profile_free_all`). + *((uintptr_t*)(block + 1)) = 0; // get the heap and profiler mi_heap_t* const heap = mi_page_heap(page); @@ -189,10 +225,36 @@ void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; // call the on_free callback - mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t)); + mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + MI_PROFILE_SAMPLE_DATA_OFFSET); (*prof->on_free)(prof, sample_data, p, heap); } +// `mi_heap_destroy` frees the pages of a heap without a `mi_free` of each block: +// call `on_free` for the profiled blocks that are still in use in `page`. +static bool mi_page_profile_free_visitor(const mi_heap_t* heap, const mi_heap_area_t* area, void* vblock, size_t block_size, void* arg) { + MI_UNUSED(heap); MI_UNUSED(area); + mi_page_t* const page = (mi_page_t*)arg; + mi_block_t* const block = (mi_block_t*)vblock; + if (block==NULL || block_size < MI_PROFILE_SAMPLE_DATA_OFFSET + sizeof(mi_profiler_sample_data_t)) return true; + if (block->next != MI_BLOCK_TAG_PROFILED) return true; + const uintptr_t check = mi_profile_block_check(NULL,block); + if (check==0 || *((const uintptr_t*)(block + 1)) != check) return true; + const mi_profiler_sample_data_t* const sample_data = (const mi_profiler_sample_data_t*)((uint8_t*)block + MI_PROFILE_SAMPLE_DATA_OFFSET); + const size_t user_offset = mi_profile_user_offset(sample_data->user_data_size); + if (user_offset >= block_size) return true; + _mi_page_profile_on_free(page, block, (uint8_t*)block + user_offset); // clears the check word + return true; +} + +void _mi_page_profile_free_all(const mi_heap_area_t* area, mi_page_t* page) { + if mi_likely(!mi_page_has_interior_pointers(page)) return; + mi_heap_t* const heap = mi_page_heap(page); + if (heap==NULL) return; + mi_profiler_t* const prof = mi_heap_profiler(heap); + if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL || prof->sample_data_size==0) return; + _mi_theap_area_visit_blocks(area, page, &mi_page_profile_free_visitor, page); +} + //---------------------------------------------------------------------------- // Profiling API diff --git a/src/theap.c b/src/theap.c index a35a001d2..652c6fff1 100644 --- a/src/theap.c +++ b/src/theap.c @@ -552,6 +552,21 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) } // theap->cookie = _mi_theap_random_next(theap) | 1; _mi_theap_guarded_init(theap); // needs theap->random + #if MI_PROFILE + // A profiler that is running samples this theap from its first allocation on, and not only after + // `page.c:mi_malloc_generic_admin` has looked at it (every 1000 generic allocations): a thread that + // makes a few large allocations and ends would never be sampled. + if (!theap->is_detached) { + mi_profiler_t* const prof = mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); + if (prof!=NULL && mi_profiler_is_enabled(prof) && prof->on_alloc!=NULL) { + _mi_theap_set_profile_sample_rate(theap, (prof->initial_sample_rate==0 ? 1 : prof->initial_sample_rate)); + // and start with a full period: the countdowns are 0 here, and with that the first allocation would be + // a sample that reports a whole period of bytes that nobody requested. + theap->profile_sample_countdown = theap->profile_sample_rate; + theap->sample_countdown = theap->sample_rate; + } + } + #endif if (!theap->is_detached) { mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem } diff --git a/test/test-profile.c b/test/test-profile.c index 41c636622..3c7907b48 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -18,6 +18,9 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc/internal.h" #include "testhelper.h" +typedef void (thread_entry_fun_t)(intptr_t tid); +static void run_os_threads(size_t nthreads, thread_entry_fun_t* entry); + // --------------------------------------------------------------------------- // Shared callback state (not thread safe!) // --------------------------------------------------------------------------- @@ -29,8 +32,19 @@ typedef struct { size_t last_size; uint64_t last_upscaled; void* last_ptr; + // for the attribution, thread and heap-destroy tests + size_t threshold; // what `on_alloc` returns as the next sample rate + uint64_t jitter; // if not zero: return a random rate around `threshold` (and the state of the generator) + uint64_t upscaled_small; // sum of `bytes_since_last_sample` of the samples with a size <= SMALL_MAX + uint64_t upscaled_large; // .. and above + const mi_heap_t* watch_heap; // count the samples and frees of this heap + uint64_t watch_alloc_count; + uint64_t watch_free_count; + bool watch_bad_free; // `on_free` got something that `on_alloc` did not set up } my_profiler_t; +#define SMALL_MAX 1024 + static inline my_profiler_t* downcast( mi_profiler_t* prof ) { return (my_profiler_t*)prof; } @@ -51,13 +65,25 @@ static size_t mi_cdecl on_alloc(mi_profiler_t* profiler, mi_profiler_sample_data assert(data->user_data_size >= sizeof(void*)); assert(data->user_data_size >= prof->profiler.sample_data_size); data->user_data[0] = ptr; - return TEST_THRESHOLD; + if (requested_size <= SMALL_MAX) { prof->upscaled_small += bytes_since_last_sample; } + else { prof->upscaled_large += bytes_since_last_sample; } + if (heap == prof->watch_heap) { prof->watch_alloc_count++; } + if (prof->jitter != 0) { + // A fixed rate with a periodic allocation pattern always samples the same allocation of the period. + prof->jitter = prof->jitter * 6364136223846793005ULL + 1442695040888963407ULL; + return prof->threshold/2 + (size_t)((prof->jitter >> 33) % prof->threshold); + } + return prof->threshold; } static void mi_cdecl on_free(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, const mi_heap_t* heap) { - MI_UNUSED(heap); MI_UNUSED_RELEASE(data); MI_UNUSED_RELEASE(ptr); + MI_UNUSED_RELEASE(data); MI_UNUSED_RELEASE(ptr); my_profiler_t* prof = downcast(profiler); prof->free_count++; + if (heap == prof->watch_heap) { + prof->watch_free_count++; + if (data->user_data[0] != ptr) { prof->watch_bad_free = true; } + } // verify the user_data round-trip assert(data->user_data[0] == ptr); } @@ -71,7 +97,8 @@ static my_profiler_t my_profiler = { &on_free, NULL }, - 0, 0, 0, 0, NULL + 0, 0, 0, 0, NULL, + TEST_THRESHOLD, 0, 0, 0, NULL, 0, 0, false }; @@ -169,6 +196,144 @@ bool test_profiler_free_count_le_alloc_count(void) { } +// Small allocations take the fast path and are only counted against the sample countdown when +// their page comes through the generic path. The sample must then go to a small allocation, and +// not to the next allocation above `MI_SMALL_SIZE_MAX` (which always takes the generic path). +bool test_profiler_small_attribution(void) { + CHECK_BODY("profiler: bytes of small allocations are attributed to small allocations") { + const size_t small_size = 64; + const size_t large_size = SMALL_MAX + 76; // just over `MI_SMALL_SIZE_MAX`: generic path, a small page + const size_t rounds = 20000; + const size_t small_per_round = 40; + void** keep = (void**)mi_malloc(rounds * (small_per_round + 1) * sizeof(void*)); + size_t n = 0; + my_profiler.threshold = 64*1024; + my_profiler.jitter = 42; + // let the theap pick up the new rate + for (int i = 0; i < 2000; i++) { mi_free(mi_malloc(large_size)); } + my_profiler.upscaled_small = 0; + my_profiler.upscaled_large = 0; + for (size_t r = 0; r < rounds; r++) { + for (size_t i = 0; i < small_per_round; i++) { keep[n++] = mi_malloc(small_size); } + keep[n++] = mi_malloc(large_size); + } + const double small_bytes = (double)(rounds * small_per_round * small_size); + const double large_bytes = (double)(rounds * large_size); + const double expected = small_bytes / (small_bytes + large_bytes); // 0.70 + const double total = (double)(my_profiler.upscaled_small + my_profiler.upscaled_large); + const double share = (total > 0 ? (double)my_profiler.upscaled_small / total : 0.0); + for (size_t i = 0; i < n; i++) { mi_free(keep[i]); } + mi_free(keep); + my_profiler.threshold = TEST_THRESHOLD; + my_profiler.jitter = 0; + fprintf(stderr, " (small share %.3f, expected %.3f, sampled total %.1f MiB of %.1f MiB)\n", share, expected, total / (1024.0*1024.0), (small_bytes + large_bytes) / (1024.0*1024.0)); + // coarse sampling is not exact (the counts of a page lag behind), but without the re-check in the generic path the share is below 0.02 + result = (share > expected / 2 && share < 0.95); + // and the samples add up to what was requested (a small block counts for its size class, which rounds up) + result = result && (total > 0.95 * (small_bytes + large_bytes) && total < 1.15 * (small_bytes + large_bytes)); + } + return true; +} + +// A thread that starts while the profiler runs is sampled from its first allocations on. +static void new_thread_allocs(intptr_t tid) { + MI_UNUSED(tid); + for (int i = 0; i < 8; i++) { + void* p = mi_malloc(1024*1024); // far above the sample rate: each one is a sample + mi_free(p); + } +} + +bool test_profiler_new_thread(void) { + CHECK_BODY("profiler: a new thread is sampled from its first allocation") { + const uint64_t before = my_profiler.alloc_count; + run_os_threads(1, &new_thread_allocs); + fprintf(stderr, " (%llu samples on the new thread)\n", (unsigned long long)(my_profiler.alloc_count - before)); + result = (my_profiler.alloc_count - before >= 8); + } + return true; +} + +// ... and its first sample does not report a period of bytes that it never requested. +static void new_thread_small_alloc(intptr_t tid) { + MI_UNUSED(tid); + mi_free(mi_malloc(32)); +} + +bool test_profiler_new_thread_no_phantom_period(void) { + CHECK_BODY("profiler: a new thread does not start with a sample for bytes it did not request") { + my_profiler.threshold = 512*1024; + my_profiler.profiler.initial_sample_rate = 512*1024; // (no other thread runs now) + for (int i = 0; i < 2000; i++) { mi_free(mi_malloc(2000)); } // let this theap pick up the rate + const uint64_t before = my_profiler.upscaled_small + my_profiler.upscaled_large; + run_os_threads(4, &new_thread_small_alloc); + const uint64_t reported = (my_profiler.upscaled_small + my_profiler.upscaled_large) - before; + my_profiler.profiler.initial_sample_rate = 0; + my_profiler.threshold = TEST_THRESHOLD; + fprintf(stderr, " (%llu bytes reported for 4 x 32 bytes)\n", (unsigned long long)reported); + result = (reported < 64*1024); + } + return true; +} + +// `mi_heap_destroy` frees the blocks of a heap without a `mi_free`: the sampled ones still get `on_free`. +bool test_profiler_heap_destroy(void) { + CHECK_BODY("profiler: mi_heap_destroy calls on_free for the sampled blocks in use") { + mi_heap_t* heap = mi_heap_new(); + my_profiler.watch_heap = heap; + my_profiler.watch_alloc_count = 0; + my_profiler.watch_free_count = 0; + my_profiler.watch_bad_free = false; + const size_t sizes[3] = { 48, 2000, 40000 }; + for (int i = 0; i < 30000; i++) { + // a regular block that starts with the value of the profiled tag is not a profiled block + uintptr_t* p = (uintptr_t*)mi_heap_malloc(heap, sizes[i % 3]); + p[0] = 1; p[1] = (uintptr_t)p; + if (i % 7 == 0) { mi_free(p); } // freed the normal way: reported once, not again by the destroy + } + const uint64_t sampled = my_profiler.watch_alloc_count; + const uint64_t freed_before = my_profiler.watch_free_count; + mi_heap_destroy(heap); + my_profiler.watch_heap = NULL; + fprintf(stderr, " (%llu sampled, %llu freed before the destroy, %llu after)\n", (unsigned long long)sampled, (unsigned long long)freed_before, (unsigned long long)my_profiler.watch_free_count); + result = (sampled > 100 && freed_before < sampled && my_profiler.watch_free_count == sampled && !my_profiler.watch_bad_free); + } + return true; +} + +// A sampled block that was freed and is handed out again as a regular block is not reported by the destroy, +// also not if the program puts the value of the profiled tag in its first word and leaves the rest as it was. +bool test_profiler_heap_destroy_recycled(void) { + CHECK_BODY("profiler: mi_heap_destroy does not report a recycled block") { + mi_heap_t* heap = mi_heap_new(); + my_profiler.watch_heap = heap; + my_profiler.watch_alloc_count = 0; + my_profiler.watch_free_count = 0; + my_profiler.watch_bad_free = false; + void* blocks[400]; + for (int i = 0; i < 400; i++) { blocks[i] = mi_heap_malloc(heap, 40000); } // above the sample rate: all sampled + const uint64_t sampled = my_profiler.watch_alloc_count; + for (int i = 0; i < 400; i++) { mi_free(blocks[i]); } + const uint64_t freed = my_profiler.watch_free_count; + // the same size class as the (larger) blocks that carried the sample data, written to as little as possible + my_profiler.threshold = 64*1024*1024; + for (int i = 0; i < 3000; i++) { mi_free(mi_malloc(2000)); } // let this theap pick up the rate + const uint64_t resampled_before = my_profiler.watch_alloc_count; + for (int i = 0; i < 400; i++) { + uintptr_t* q = (uintptr_t*)mi_heap_malloc(heap, 40000 + 2*sizeof(void*) + sizeof(mi_profiler_sample_data_t)); + q[0] = 1; + } + const uint64_t resampled = my_profiler.watch_alloc_count - resampled_before; + mi_heap_destroy(heap); + my_profiler.watch_heap = NULL; + my_profiler.threshold = TEST_THRESHOLD; + fprintf(stderr, " (%llu sampled and freed, %llu sampled again, %llu reported by the destroy)\n", (unsigned long long)sampled, (unsigned long long)resampled, (unsigned long long)(my_profiler.watch_free_count - freed)); + result = (sampled >= 400 && freed == sampled && my_profiler.watch_free_count - freed == resampled && !my_profiler.watch_bad_free); + } + return true; +} + + // --------------------------------------------------------------------------- // main // --------------------------------------------------------------------------- @@ -182,8 +347,63 @@ int main(void) { test_profiler_record_fields(); test_profiler_on_free_called(); test_profiler_free_count_le_alloc_count(); + test_profiler_small_attribution(); + test_profiler_new_thread(); + test_profiler_new_thread_no_phantom_period(); + test_profiler_heap_destroy(); + test_profiler_heap_destroy_recycled(); mi_profiler_stop(&my_profiler.profiler); return print_test_summary(); } + + +// --------------------------------------------------------------------------- +// Portable threads (as in test-heap-mt.c) +// --------------------------------------------------------------------------- + +static thread_entry_fun_t* thread_entry_fun; + +#ifdef _WIN32 +#include + +static DWORD WINAPI thread_entry(LPVOID param) { + thread_entry_fun((intptr_t)param); + return 0; +} + +static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { + thread_entry_fun = fun; + HANDLE thandles[16]; + if (nthreads > 16) { nthreads = 16; } + for (size_t i = 0; i < nthreads; i++) { + DWORD tid; + thandles[i] = CreateThread(0, 64*1024L, &thread_entry, (void*)(i), 0, &tid); + } + for (size_t i = 0; i < nthreads; i++) { + WaitForSingleObject(thandles[i], INFINITE); + CloseHandle(thandles[i]); + } +} + +#else +#include + +static void* thread_entry(void* param) { + thread_entry_fun((intptr_t)param); + return NULL; +} + +static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { + thread_entry_fun = fun; + pthread_t threads[16]; + if (nthreads > 16) { nthreads = 16; } + for (size_t i = 0; i < nthreads; i++) { + pthread_create(&threads[i], NULL, &thread_entry, (void*)i); + } + for (size_t i = 0; i < nthreads; i++) { + pthread_join(threads[i], NULL); + } +} +#endif From 0cf802e6032f7b99d047ad39ebb79205f7d2bc76 Mon Sep 17 00:00:00 2001 From: Jarred Sumner Date: Mon, 14 Sep 2026 00:35:09 +0000 Subject: [PATCH 2/2] profile: on_free gets the pointer that on_alloc got, also for an aligned allocation mi_malloc_aligned aligns inside the block that _mi_theap_malloc_profiled returned, so mi_free of a sampled aligned block called on_free with the aligned pointer. test-profile's on_free asserts they are the same; it met such a block with guarded sampling on in the same process (the CI's guarded jobs set a rate for every test): samples then land on mimalloc's own mi_heap_zalloc_aligned of the per-heap arena page table, which mi_heap_destroy frees. _mi_page_profile_on_free computes the pointer from the block's layout, and checks the block's check word before it calls on_free. test-profile gets MIMALLOC_GUARDED_SAMPLE_RATE=0 explicitly in guarded builds (CMakeLists.txt meant it to have none: what the samples add up to is off with guarded samples in between, also without this branch). The new case turns guarded sampling on for itself: sampled, guarded and aligned blocks through mi_free, mi_free on another thread, mi_heap_realloc and mi_heap_destroy. --- CMakeLists.txt | 4 ++- src/sample-profile.c | 13 +++++++--- test/test-profile.c | 59 ++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 72 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9db485cd5..9417d7615 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1173,7 +1173,9 @@ if (MI_BUILD_TESTS) set(mi_test_env "") if(MI_GUARDED AND TEST_NAME STREQUAL "purge-holes") list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0) # it knows the page layout, which guard pages change (set explicitly: the CI sets a rate for the whole job) - elseif(MI_GUARDED AND NOT (TEST_NAME STREQUAL "profile")) + elseif(MI_GUARDED AND TEST_NAME STREQUAL "profile") + list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0) # it checks what the samples add up to, which guarded samples in between change (set explicitly: the CI sets a rate for the whole job); the case with guarded blocks sets its own rate + elseif(MI_GUARDED) list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=1024) elseif(TEST_NAME STREQUAL "stress-heaps") list(APPEND mi_test_env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=0) diff --git a/src/sample-profile.c b/src/sample-profile.c index c2c6a6656..9fdb3db13 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -213,10 +213,14 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th } void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED_RELEASE(p); mi_assert_internal(mi_block_ptr_is_sampled(block,p)); + uintptr_t* const pcheck = (uintptr_t*)(block + 1); + const uintptr_t check = mi_profile_block_check(NULL,block); + if mi_unlikely(check==0 || *pcheck != check) return; // not a block that `_mi_theap_malloc_profiled` set up // The free list only overwrites the tag. Clear the check word too, or the block could come back as a regular // block that starts with the value of the tag and still has the check behind it (see `_mi_page_profile_free_all`). - *((uintptr_t*)(block + 1)) = 0; + *pcheck = 0; // get the heap and profiler mi_heap_t* const heap = mi_page_heap(page); @@ -224,9 +228,12 @@ void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { mi_profiler_t* prof = mi_heap_profiler(heap); if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; - // call the on_free callback + // Call the on_free callback with the pointer that `on_alloc` got. `p` is another one if the block was allocated + // with an alignment: `alloc-aligned.c` aligns inside the block that `_mi_theap_malloc_profiled` returned. mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + MI_PROFILE_SAMPLE_DATA_OFFSET); - (*prof->on_free)(prof, sample_data, p, heap); + void* const sampled_p = (uint8_t*)block + mi_profile_user_offset(sample_data->user_data_size); + mi_assert_internal((uint8_t*)p >= (uint8_t*)sampled_p); + (*prof->on_free)(prof, sample_data, sampled_p, heap); } // `mi_heap_destroy` frees the pages of a heap without a `mi_free` of each block: diff --git a/test/test-profile.c b/test/test-profile.c index 3c7907b48..8ab6e20a4 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -15,6 +15,7 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-profile.h" +#include "mimalloc-stats.h" #include "mimalloc/internal.h" #include "testhelper.h" @@ -334,6 +335,63 @@ bool test_profiler_heap_destroy_recycled(void) { } +// Sampled blocks next to guarded ones (in a build with MI_GUARDED; both kinds make `mi_free` take its path for +// interior pointers, and a sample is profiled or guarded, never both), with and without an alignment, leave through +// `mi_free` on this thread, `mi_free` on another thread, a `mi_heap_realloc` that moves them, and `mi_heap_destroy`. +// Each sampled block is reported once, with the pointer `on_alloc` got (`on_free` asserts that). +#define MIXED_COUNT 6000 +static void* mixed_blocks[MIXED_COUNT]; + +static void mixed_free_on_other_thread(intptr_t tid) { + MI_UNUSED(tid); + for (int i = 1; i < MIXED_COUNT; i += 4) { mi_free(mixed_blocks[i]); mixed_blocks[i] = NULL; } +} + +bool test_profiler_guarded_mixed(void) { + CHECK_BODY("profiler: sampled, guarded and aligned blocks through free, realloc, another thread and mi_heap_destroy") { + const long guarded_rate = mi_option_get(mi_option_guarded_sample_rate); + mi_option_set(mi_option_guarded_sample_rate, 1024); // picked up by the theaps of the new heap and thread + my_profiler.threshold = 16*1024; + mi_heap_t* heap = mi_heap_new(); + my_profiler.watch_heap = heap; + my_profiler.watch_alloc_count = 0; + my_profiler.watch_free_count = 0; + my_profiler.watch_bad_free = false; + const size_t sizes[5] = { 40, 700, 3000, 20000, 70000 }; + const size_t alignments[3] = { 64, 1024, 16384 }; + size_t guarded = 0; + for (int i = 0; i < MIXED_COUNT; i++) { + const size_t size = sizes[i % 5]; + mixed_blocks[i] = (i % 3 == 0 ? mi_heap_malloc_aligned(heap, size, alignments[(i / 3) % 3]) : mi_heap_malloc(heap, size)); + memset(mixed_blocks[i], 1, (size < 64 ? size : 64)); + } + const uint64_t sampled = my_profiler.watch_alloc_count; + #if MI_GUARDED + mi_stats_t stats; mi_stats_init(&stats); + if (mi_heap_stats_get(heap, &stats)) { guarded = (size_t)stats.malloc_guarded_count.total; } + #endif + for (int i = 0; i < MIXED_COUNT; i += 4) { mi_free(mixed_blocks[i]); mixed_blocks[i] = NULL; } + run_os_threads(1, &mixed_free_on_other_thread); + for (int i = 2; i < MIXED_COUNT; i += 4) { + mixed_blocks[i] = mi_heap_realloc(heap, mixed_blocks[i], 2*sizes[i % 5] + 100000); // moves: a new block (that may be sampled) and a free + } + const uint64_t freed_before = my_profiler.watch_free_count; + const uint64_t sampled_all = my_profiler.watch_alloc_count; + mi_heap_destroy(heap); // the reallocated ones and every fourth of the first ones + my_profiler.watch_heap = NULL; + my_profiler.threshold = TEST_THRESHOLD; + mi_option_set(mi_option_guarded_sample_rate, guarded_rate); + fprintf(stderr, " (%llu of %d sampled, %zu guarded, %llu sampled with the moved ones; %llu freed before the destroy, %llu after)\n", (unsigned long long)sampled, MIXED_COUNT, guarded, (unsigned long long)sampled_all, (unsigned long long)freed_before, (unsigned long long)my_profiler.watch_free_count); + #if MI_GUARDED + if (guarded < 100) { result = false; } + else + #endif + result = (sampled > 500 && freed_before > sampled/2 && freed_before < sampled_all && my_profiler.watch_free_count == sampled_all && !my_profiler.watch_bad_free); + } + return true; +} + + // --------------------------------------------------------------------------- // main // --------------------------------------------------------------------------- @@ -352,6 +410,7 @@ int main(void) { test_profiler_new_thread_no_phantom_period(); test_profiler_heap_destroy(); test_profiler_heap_destroy_recycled(); + test_profiler_guarded_mixed(); mi_profiler_stop(&my_profiler.profiler);