diff --git a/CMakeLists.txt b/CMakeLists.txt index 9db485cd5..9417d7615 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1173,7 +1173,9 @@ if (MI_BUILD_TESTS) set(mi_test_env "") if(MI_GUARDED AND TEST_NAME STREQUAL "purge-holes") list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0) # it knows the page layout, which guard pages change (set explicitly: the CI sets a rate for the whole job) - elseif(MI_GUARDED AND NOT (TEST_NAME STREQUAL "profile")) + elseif(MI_GUARDED AND TEST_NAME STREQUAL "profile") + list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=0) # it checks what the samples add up to, which guarded samples in between change (set explicitly: the CI sets a rate for the whole job); the case with guarded blocks sets its own rate + elseif(MI_GUARDED) list(APPEND mi_test_env MIMALLOC_GUARDED_SAMPLE_RATE=1024) elseif(TEST_NAME STREQUAL "stress-heaps") list(APPEND mi_test_env MIMALLOC_ARENA_EAGER_COMMIT=0 MIMALLOC_PAGE_COMMIT_ON_DEMAND=0) diff --git a/include/mimalloc/internal.h b/include/mimalloc/internal.h index 5e961dcb0..3a309d83e 100644 --- a/include/mimalloc/internal.h +++ b/include/mimalloc/internal.h @@ -408,6 +408,7 @@ size_t _mi_theap_update_sample_rate(mi_theap_t* theap); mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* theap, size_t size, uint64_t requested_since_last_sample, bool zero, mi_page_t** ppage) mi_attr_noexcept; void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p); +void _mi_page_profile_free_all(const mi_heap_area_t* area, mi_page_t* page); size_t _mi_theap_set_profile_sample_rate(mi_theap_t* theap, size_t sample_rate); diff --git a/src/arena.c b/src/arena.c index 2b844cd9d..60f6b72c6 100644 --- a/src/arena.c +++ b/src/arena.c @@ -3060,6 +3060,9 @@ static bool mi_heap_delete_page(const mi_heap_t* heap, const mi_heap_area_t* are #if MI_GUARDED _mi_page_unguard_all(page); // remove potential interior guard pages #endif + #if MI_PROFILE + _mi_page_profile_free_all(area,page); // the sampled blocks that are still in use get their `on_free` + #endif // destroy the page // Drop what is (still) on the thread-free list first: `_mi_arenas_page_free` collects it, and with the used // count reset that reads as a corrupted list (more blocks freed than were in use). diff --git a/src/page.c b/src/page.c index bc0bed042..7becd5abf 100644 --- a/src/page.c +++ b/src/page.c @@ -1834,6 +1834,7 @@ static bool mi_page_extend_free(mi_theap_t* theap, mi_page_t* page) { //uint8_t* page_start = mi_page_area(page, &page_size); mi_theap_stat_counter_increase(theap, pages_extended, 1); + mi_page_update_sample_countdown(page); // the blocks of the previous extension are handed out: count them // calculate the extend count const size_t bsize = mi_page_block_size(page); @@ -2368,6 +2369,17 @@ void* _mi_malloc_generic(mi_theap_t* theap, size_t size, size_t zero_huge_alignm page = mi_page_queue_find_free(theap,pq); // mi_assert_internal(mi_page_block_size(page) <= MI_SMALL_MAX_OBJ_SIZE); if (page!=NULL) { + #if MI_SAMPLE==1 + // The blocks that the fast path took from a page are counted against the sample countdown when its free + // list is refilled, which is what `mi_page_queue_find_free` just did. If that used up the countdown, the + // allocation in progress (of the size class of those blocks) is the sample. If we leave it to the next + // allocation that comes through here, that is one above `MI_SMALL_SIZE_MAX` (these always do) far more + // often than a small one: with one such allocation for every 40 blocks of 64 bytes, less than 2% of the + // sampled bytes went to the blocks of 64 bytes (`test-profile.c:test_profiler_small_attribution`). + if mi_unlikely(theap->sample_rate!=0 && mi_theap_should_sample(theap,req_size)) { + return _mi_theap_malloc_sampled(theap,req_size,zero,ppage); + } + #endif if (ppage!=NULL) { *ppage = page; } mi_assert_internal(mi_page_immediate_available(page)); // we should never recurse in _mi_page_malloc_zero return _mi_page_malloc_zero(theap,page,size,zero); diff --git a/src/sample-profile.c b/src/sample-profile.c index f1b5db491..9fdb3db13 100644 --- a/src/sample-profile.c +++ b/src/sample-profile.c @@ -115,6 +115,34 @@ static mi_profiler_t* mi_theap_get_enabled_profiler(const mi_theap_t* theap) { } } +//---------------------------------------------------------------------------- +// Layout of a profiled block: +// [MI_BLOCK_TAG_PROFILED] [check] [mi_profiler_sample_data_t ... user data ...] [ ... block as the program sees it ... ] +// `mi_free` recognizes it by the tag and by getting an interior pointer. `mi_heap_destroy` walks +// the blocks of a page and has no pointer to go by: the first word of a regular block in use is +// program data and can hold the value of the tag. The `check` word is the address of the block +// xor a random key, so a regular block is not mistaken for a profiled one. +//----------------------------------------------------------------------------- + +static _Atomic(uintptr_t) mi_profile_check_key; // = 0 + +static uintptr_t mi_profile_block_check(mi_theap_t* theap, const mi_block_t* block) { + uintptr_t key = mi_atomic_load_relaxed(&mi_profile_check_key); + if mi_unlikely(key==0) { + if (theap==NULL) return 0; // no block was profiled yet + uintptr_t expected = 0; + key = _mi_theap_random_next(theap) | 1; + if (!mi_atomic_cas_strong_acq_rel(&mi_profile_check_key, &expected, key)) { key = expected; } + } + return ((uintptr_t)block ^ key); +} + +#define MI_PROFILE_SAMPLE_DATA_OFFSET (sizeof(mi_block_t) + sizeof(uintptr_t)) + +static inline size_t mi_profile_user_offset(size_t sample_user_data_size) { + return _mi_align_up(MI_PROFILE_SAMPLE_DATA_OFFSET + sizeof(mi_profiler_sample_data_t) + sample_user_data_size, MI_MAX_ALIGN_SIZE); +} + //---------------------------------------------------------------------------- // Profile an allocation //----------------------------------------------------------------------------- @@ -142,11 +170,10 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th } else { // overallocate a larger block to store the profiler data - // [MI_BLOCK_TAG_PROFILE] [usable size] [ ... profile data ... ] [... user data ...] - const size_t sample_data_offset = sizeof(mi_block_t); + // [MI_BLOCK_TAG_PROFILED] [check] [ ... profile data ... ] [... user data ...] + const size_t sample_data_offset = MI_PROFILE_SAMPLE_DATA_OFFSET; const size_t sample_user_data_size = _mi_align_up(prof->sample_data_size > MI_PROFILE_SAMPLE_DATA_MAX_SIZE ? MI_PROFILE_SAMPLE_DATA_MAX_SIZE : prof->sample_data_size, sizeof(void*)); - const size_t sample_data_size = sizeof(mi_profiler_sample_data_t) + sample_user_data_size; // one void* too many just in case - const size_t user_offset = _mi_align_up(sample_data_offset + sample_data_size, MI_MAX_ALIGN_SIZE); + const size_t user_offset = mi_profile_user_offset(sample_user_data_size); const size_t oversize = user_offset + size; mi_page_t* page = NULL; mi_block_t* const block = (mi_block_t*)_mi_malloc_generic_no_sample(theap,oversize,zero,&page); @@ -163,6 +190,7 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th // Set up the profiled block as an interior pointer so the interior "slow" path is taken on mi_free (where we catch it to call on_free) mi_page_set_has_interior_pointers(page, true); block->next = MI_BLOCK_TAG_PROFILED; + *((uintptr_t*)(block + 1)) = mi_profile_block_check(theap,block); p = (uint8_t*)block + user_offset; mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sample_data_offset); sample_data->user_data_size = sample_user_data_size; @@ -174,13 +202,25 @@ mi_decl_noinline mi_decl_restrict void* _mi_theap_malloc_profiled(mi_theap_t* th } if (new_sample_rate!=0 && new_sample_rate != (size_t)theap->profile_sample_rate) { _mi_theap_set_profile_sample_rate(theap,new_sample_rate); + // We are at a sample, where a period starts: start it with the new rate in full. (`_mi_theap_malloc_sampled` + // takes the period to be `sample_rate` long when it computes `bytes_since_last_sample`. A raised rate left the + // countdown at the previous rate, and each sample reported the difference on top of the bytes that were requested.) + theap->profile_sample_countdown = theap->profile_sample_rate; + theap->sample_countdown = theap->sample_rate; } mi_theap_stat_counter_increase(theap,profile_samples,1); return p; } void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { + MI_UNUSED_RELEASE(p); mi_assert_internal(mi_block_ptr_is_sampled(block,p)); + uintptr_t* const pcheck = (uintptr_t*)(block + 1); + const uintptr_t check = mi_profile_block_check(NULL,block); + if mi_unlikely(check==0 || *pcheck != check) return; // not a block that `_mi_theap_malloc_profiled` set up + // The free list only overwrites the tag. Clear the check word too, or the block could come back as a regular + // block that starts with the value of the tag and still has the check behind it (see `_mi_page_profile_free_all`). + *pcheck = 0; // get the heap and profiler mi_heap_t* const heap = mi_page_heap(page); @@ -188,9 +228,38 @@ void _mi_page_profile_on_free(mi_page_t* page, mi_block_t* block, void* p) { mi_profiler_t* prof = mi_heap_profiler(heap); if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL) return; - // call the on_free callback - mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + sizeof(mi_block_t)); - (*prof->on_free)(prof, sample_data, p, heap); + // Call the on_free callback with the pointer that `on_alloc` got. `p` is another one if the block was allocated + // with an alignment: `alloc-aligned.c` aligns inside the block that `_mi_theap_malloc_profiled` returned. + mi_profiler_sample_data_t* const sample_data = (mi_profiler_sample_data_t*)((uint8_t*)block + MI_PROFILE_SAMPLE_DATA_OFFSET); + void* const sampled_p = (uint8_t*)block + mi_profile_user_offset(sample_data->user_data_size); + mi_assert_internal((uint8_t*)p >= (uint8_t*)sampled_p); + (*prof->on_free)(prof, sample_data, sampled_p, heap); +} + +// `mi_heap_destroy` frees the pages of a heap without a `mi_free` of each block: +// call `on_free` for the profiled blocks that are still in use in `page`. +static bool mi_page_profile_free_visitor(const mi_heap_t* heap, const mi_heap_area_t* area, void* vblock, size_t block_size, void* arg) { + MI_UNUSED(heap); MI_UNUSED(area); + mi_page_t* const page = (mi_page_t*)arg; + mi_block_t* const block = (mi_block_t*)vblock; + if (block==NULL || block_size < MI_PROFILE_SAMPLE_DATA_OFFSET + sizeof(mi_profiler_sample_data_t)) return true; + if (block->next != MI_BLOCK_TAG_PROFILED) return true; + const uintptr_t check = mi_profile_block_check(NULL,block); + if (check==0 || *((const uintptr_t*)(block + 1)) != check) return true; + const mi_profiler_sample_data_t* const sample_data = (const mi_profiler_sample_data_t*)((uint8_t*)block + MI_PROFILE_SAMPLE_DATA_OFFSET); + const size_t user_offset = mi_profile_user_offset(sample_data->user_data_size); + if (user_offset >= block_size) return true; + _mi_page_profile_on_free(page, block, (uint8_t*)block + user_offset); // clears the check word + return true; +} + +void _mi_page_profile_free_all(const mi_heap_area_t* area, mi_page_t* page) { + if mi_likely(!mi_page_has_interior_pointers(page)) return; + mi_heap_t* const heap = mi_page_heap(page); + if (heap==NULL) return; + mi_profiler_t* const prof = mi_heap_profiler(heap); + if (prof==NULL || !mi_profiler_is_enabled(prof) || prof->on_free==NULL || prof->sample_data_size==0) return; + _mi_theap_area_visit_blocks(area, page, &mi_page_profile_free_visitor, page); } diff --git a/src/theap.c b/src/theap.c index a35a001d2..652c6fff1 100644 --- a/src/theap.c +++ b/src/theap.c @@ -552,6 +552,21 @@ void _mi_theap_init(mi_theap_t* theap, mi_heap_t* heap, mi_tld_t* tld) } // theap->cookie = _mi_theap_random_next(theap) | 1; _mi_theap_guarded_init(theap); // needs theap->random + #if MI_PROFILE + // A profiler that is running samples this theap from its first allocation on, and not only after + // `page.c:mi_malloc_generic_admin` has looked at it (every 1000 generic allocations): a thread that + // makes a few large allocations and ends would never be sampled. + if (!theap->is_detached) { + mi_profiler_t* const prof = mi_atomic_load_ptr_acquire(mi_profiler_t,&heap->profiler); + if (prof!=NULL && mi_profiler_is_enabled(prof) && prof->on_alloc!=NULL) { + _mi_theap_set_profile_sample_rate(theap, (prof->initial_sample_rate==0 ? 1 : prof->initial_sample_rate)); + // and start with a full period: the countdowns are 0 here, and with that the first allocation would be + // a sample that reports a whole period of bytes that nobody requested. + theap->profile_sample_countdown = theap->profile_sample_rate; + theap->sample_countdown = theap->sample_rate; + } + } + #endif if (!theap->is_detached) { mi_subproc_stat_increase(_mi_theap_subproc(theap),theaps,1); // on subproc to match theap_free_mem } diff --git a/test/test-profile.c b/test/test-profile.c index 41c636622..8ab6e20a4 100644 --- a/test/test-profile.c +++ b/test/test-profile.c @@ -15,9 +15,13 @@ terms of the MIT license. A copy of the license can be found in the file #include "mimalloc.h" #include "mimalloc-profile.h" +#include "mimalloc-stats.h" #include "mimalloc/internal.h" #include "testhelper.h" +typedef void (thread_entry_fun_t)(intptr_t tid); +static void run_os_threads(size_t nthreads, thread_entry_fun_t* entry); + // --------------------------------------------------------------------------- // Shared callback state (not thread safe!) // --------------------------------------------------------------------------- @@ -29,8 +33,19 @@ typedef struct { size_t last_size; uint64_t last_upscaled; void* last_ptr; + // for the attribution, thread and heap-destroy tests + size_t threshold; // what `on_alloc` returns as the next sample rate + uint64_t jitter; // if not zero: return a random rate around `threshold` (and the state of the generator) + uint64_t upscaled_small; // sum of `bytes_since_last_sample` of the samples with a size <= SMALL_MAX + uint64_t upscaled_large; // .. and above + const mi_heap_t* watch_heap; // count the samples and frees of this heap + uint64_t watch_alloc_count; + uint64_t watch_free_count; + bool watch_bad_free; // `on_free` got something that `on_alloc` did not set up } my_profiler_t; +#define SMALL_MAX 1024 + static inline my_profiler_t* downcast( mi_profiler_t* prof ) { return (my_profiler_t*)prof; } @@ -51,13 +66,25 @@ static size_t mi_cdecl on_alloc(mi_profiler_t* profiler, mi_profiler_sample_data assert(data->user_data_size >= sizeof(void*)); assert(data->user_data_size >= prof->profiler.sample_data_size); data->user_data[0] = ptr; - return TEST_THRESHOLD; + if (requested_size <= SMALL_MAX) { prof->upscaled_small += bytes_since_last_sample; } + else { prof->upscaled_large += bytes_since_last_sample; } + if (heap == prof->watch_heap) { prof->watch_alloc_count++; } + if (prof->jitter != 0) { + // A fixed rate with a periodic allocation pattern always samples the same allocation of the period. + prof->jitter = prof->jitter * 6364136223846793005ULL + 1442695040888963407ULL; + return prof->threshold/2 + (size_t)((prof->jitter >> 33) % prof->threshold); + } + return prof->threshold; } static void mi_cdecl on_free(mi_profiler_t* profiler, mi_profiler_sample_data_t* data, void* ptr, const mi_heap_t* heap) { - MI_UNUSED(heap); MI_UNUSED_RELEASE(data); MI_UNUSED_RELEASE(ptr); + MI_UNUSED_RELEASE(data); MI_UNUSED_RELEASE(ptr); my_profiler_t* prof = downcast(profiler); prof->free_count++; + if (heap == prof->watch_heap) { + prof->watch_free_count++; + if (data->user_data[0] != ptr) { prof->watch_bad_free = true; } + } // verify the user_data round-trip assert(data->user_data[0] == ptr); } @@ -71,7 +98,8 @@ static my_profiler_t my_profiler = { &on_free, NULL }, - 0, 0, 0, 0, NULL + 0, 0, 0, 0, NULL, + TEST_THRESHOLD, 0, 0, 0, NULL, 0, 0, false }; @@ -169,6 +197,201 @@ bool test_profiler_free_count_le_alloc_count(void) { } +// Small allocations take the fast path and are only counted against the sample countdown when +// their page comes through the generic path. The sample must then go to a small allocation, and +// not to the next allocation above `MI_SMALL_SIZE_MAX` (which always takes the generic path). +bool test_profiler_small_attribution(void) { + CHECK_BODY("profiler: bytes of small allocations are attributed to small allocations") { + const size_t small_size = 64; + const size_t large_size = SMALL_MAX + 76; // just over `MI_SMALL_SIZE_MAX`: generic path, a small page + const size_t rounds = 20000; + const size_t small_per_round = 40; + void** keep = (void**)mi_malloc(rounds * (small_per_round + 1) * sizeof(void*)); + size_t n = 0; + my_profiler.threshold = 64*1024; + my_profiler.jitter = 42; + // let the theap pick up the new rate + for (int i = 0; i < 2000; i++) { mi_free(mi_malloc(large_size)); } + my_profiler.upscaled_small = 0; + my_profiler.upscaled_large = 0; + for (size_t r = 0; r < rounds; r++) { + for (size_t i = 0; i < small_per_round; i++) { keep[n++] = mi_malloc(small_size); } + keep[n++] = mi_malloc(large_size); + } + const double small_bytes = (double)(rounds * small_per_round * small_size); + const double large_bytes = (double)(rounds * large_size); + const double expected = small_bytes / (small_bytes + large_bytes); // 0.70 + const double total = (double)(my_profiler.upscaled_small + my_profiler.upscaled_large); + const double share = (total > 0 ? (double)my_profiler.upscaled_small / total : 0.0); + for (size_t i = 0; i < n; i++) { mi_free(keep[i]); } + mi_free(keep); + my_profiler.threshold = TEST_THRESHOLD; + my_profiler.jitter = 0; + fprintf(stderr, " (small share %.3f, expected %.3f, sampled total %.1f MiB of %.1f MiB)\n", share, expected, total / (1024.0*1024.0), (small_bytes + large_bytes) / (1024.0*1024.0)); + // coarse sampling is not exact (the counts of a page lag behind), but without the re-check in the generic path the share is below 0.02 + result = (share > expected / 2 && share < 0.95); + // and the samples add up to what was requested (a small block counts for its size class, which rounds up) + result = result && (total > 0.95 * (small_bytes + large_bytes) && total < 1.15 * (small_bytes + large_bytes)); + } + return true; +} + +// A thread that starts while the profiler runs is sampled from its first allocations on. +static void new_thread_allocs(intptr_t tid) { + MI_UNUSED(tid); + for (int i = 0; i < 8; i++) { + void* p = mi_malloc(1024*1024); // far above the sample rate: each one is a sample + mi_free(p); + } +} + +bool test_profiler_new_thread(void) { + CHECK_BODY("profiler: a new thread is sampled from its first allocation") { + const uint64_t before = my_profiler.alloc_count; + run_os_threads(1, &new_thread_allocs); + fprintf(stderr, " (%llu samples on the new thread)\n", (unsigned long long)(my_profiler.alloc_count - before)); + result = (my_profiler.alloc_count - before >= 8); + } + return true; +} + +// ... and its first sample does not report a period of bytes that it never requested. +static void new_thread_small_alloc(intptr_t tid) { + MI_UNUSED(tid); + mi_free(mi_malloc(32)); +} + +bool test_profiler_new_thread_no_phantom_period(void) { + CHECK_BODY("profiler: a new thread does not start with a sample for bytes it did not request") { + my_profiler.threshold = 512*1024; + my_profiler.profiler.initial_sample_rate = 512*1024; // (no other thread runs now) + for (int i = 0; i < 2000; i++) { mi_free(mi_malloc(2000)); } // let this theap pick up the rate + const uint64_t before = my_profiler.upscaled_small + my_profiler.upscaled_large; + run_os_threads(4, &new_thread_small_alloc); + const uint64_t reported = (my_profiler.upscaled_small + my_profiler.upscaled_large) - before; + my_profiler.profiler.initial_sample_rate = 0; + my_profiler.threshold = TEST_THRESHOLD; + fprintf(stderr, " (%llu bytes reported for 4 x 32 bytes)\n", (unsigned long long)reported); + result = (reported < 64*1024); + } + return true; +} + +// `mi_heap_destroy` frees the blocks of a heap without a `mi_free`: the sampled ones still get `on_free`. +bool test_profiler_heap_destroy(void) { + CHECK_BODY("profiler: mi_heap_destroy calls on_free for the sampled blocks in use") { + mi_heap_t* heap = mi_heap_new(); + my_profiler.watch_heap = heap; + my_profiler.watch_alloc_count = 0; + my_profiler.watch_free_count = 0; + my_profiler.watch_bad_free = false; + const size_t sizes[3] = { 48, 2000, 40000 }; + for (int i = 0; i < 30000; i++) { + // a regular block that starts with the value of the profiled tag is not a profiled block + uintptr_t* p = (uintptr_t*)mi_heap_malloc(heap, sizes[i % 3]); + p[0] = 1; p[1] = (uintptr_t)p; + if (i % 7 == 0) { mi_free(p); } // freed the normal way: reported once, not again by the destroy + } + const uint64_t sampled = my_profiler.watch_alloc_count; + const uint64_t freed_before = my_profiler.watch_free_count; + mi_heap_destroy(heap); + my_profiler.watch_heap = NULL; + fprintf(stderr, " (%llu sampled, %llu freed before the destroy, %llu after)\n", (unsigned long long)sampled, (unsigned long long)freed_before, (unsigned long long)my_profiler.watch_free_count); + result = (sampled > 100 && freed_before < sampled && my_profiler.watch_free_count == sampled && !my_profiler.watch_bad_free); + } + return true; +} + +// A sampled block that was freed and is handed out again as a regular block is not reported by the destroy, +// also not if the program puts the value of the profiled tag in its first word and leaves the rest as it was. +bool test_profiler_heap_destroy_recycled(void) { + CHECK_BODY("profiler: mi_heap_destroy does not report a recycled block") { + mi_heap_t* heap = mi_heap_new(); + my_profiler.watch_heap = heap; + my_profiler.watch_alloc_count = 0; + my_profiler.watch_free_count = 0; + my_profiler.watch_bad_free = false; + void* blocks[400]; + for (int i = 0; i < 400; i++) { blocks[i] = mi_heap_malloc(heap, 40000); } // above the sample rate: all sampled + const uint64_t sampled = my_profiler.watch_alloc_count; + for (int i = 0; i < 400; i++) { mi_free(blocks[i]); } + const uint64_t freed = my_profiler.watch_free_count; + // the same size class as the (larger) blocks that carried the sample data, written to as little as possible + my_profiler.threshold = 64*1024*1024; + for (int i = 0; i < 3000; i++) { mi_free(mi_malloc(2000)); } // let this theap pick up the rate + const uint64_t resampled_before = my_profiler.watch_alloc_count; + for (int i = 0; i < 400; i++) { + uintptr_t* q = (uintptr_t*)mi_heap_malloc(heap, 40000 + 2*sizeof(void*) + sizeof(mi_profiler_sample_data_t)); + q[0] = 1; + } + const uint64_t resampled = my_profiler.watch_alloc_count - resampled_before; + mi_heap_destroy(heap); + my_profiler.watch_heap = NULL; + my_profiler.threshold = TEST_THRESHOLD; + fprintf(stderr, " (%llu sampled and freed, %llu sampled again, %llu reported by the destroy)\n", (unsigned long long)sampled, (unsigned long long)resampled, (unsigned long long)(my_profiler.watch_free_count - freed)); + result = (sampled >= 400 && freed == sampled && my_profiler.watch_free_count - freed == resampled && !my_profiler.watch_bad_free); + } + return true; +} + + +// Sampled blocks next to guarded ones (in a build with MI_GUARDED; both kinds make `mi_free` take its path for +// interior pointers, and a sample is profiled or guarded, never both), with and without an alignment, leave through +// `mi_free` on this thread, `mi_free` on another thread, a `mi_heap_realloc` that moves them, and `mi_heap_destroy`. +// Each sampled block is reported once, with the pointer `on_alloc` got (`on_free` asserts that). +#define MIXED_COUNT 6000 +static void* mixed_blocks[MIXED_COUNT]; + +static void mixed_free_on_other_thread(intptr_t tid) { + MI_UNUSED(tid); + for (int i = 1; i < MIXED_COUNT; i += 4) { mi_free(mixed_blocks[i]); mixed_blocks[i] = NULL; } +} + +bool test_profiler_guarded_mixed(void) { + CHECK_BODY("profiler: sampled, guarded and aligned blocks through free, realloc, another thread and mi_heap_destroy") { + const long guarded_rate = mi_option_get(mi_option_guarded_sample_rate); + mi_option_set(mi_option_guarded_sample_rate, 1024); // picked up by the theaps of the new heap and thread + my_profiler.threshold = 16*1024; + mi_heap_t* heap = mi_heap_new(); + my_profiler.watch_heap = heap; + my_profiler.watch_alloc_count = 0; + my_profiler.watch_free_count = 0; + my_profiler.watch_bad_free = false; + const size_t sizes[5] = { 40, 700, 3000, 20000, 70000 }; + const size_t alignments[3] = { 64, 1024, 16384 }; + size_t guarded = 0; + for (int i = 0; i < MIXED_COUNT; i++) { + const size_t size = sizes[i % 5]; + mixed_blocks[i] = (i % 3 == 0 ? mi_heap_malloc_aligned(heap, size, alignments[(i / 3) % 3]) : mi_heap_malloc(heap, size)); + memset(mixed_blocks[i], 1, (size < 64 ? size : 64)); + } + const uint64_t sampled = my_profiler.watch_alloc_count; + #if MI_GUARDED + mi_stats_t stats; mi_stats_init(&stats); + if (mi_heap_stats_get(heap, &stats)) { guarded = (size_t)stats.malloc_guarded_count.total; } + #endif + for (int i = 0; i < MIXED_COUNT; i += 4) { mi_free(mixed_blocks[i]); mixed_blocks[i] = NULL; } + run_os_threads(1, &mixed_free_on_other_thread); + for (int i = 2; i < MIXED_COUNT; i += 4) { + mixed_blocks[i] = mi_heap_realloc(heap, mixed_blocks[i], 2*sizes[i % 5] + 100000); // moves: a new block (that may be sampled) and a free + } + const uint64_t freed_before = my_profiler.watch_free_count; + const uint64_t sampled_all = my_profiler.watch_alloc_count; + mi_heap_destroy(heap); // the reallocated ones and every fourth of the first ones + my_profiler.watch_heap = NULL; + my_profiler.threshold = TEST_THRESHOLD; + mi_option_set(mi_option_guarded_sample_rate, guarded_rate); + fprintf(stderr, " (%llu of %d sampled, %zu guarded, %llu sampled with the moved ones; %llu freed before the destroy, %llu after)\n", (unsigned long long)sampled, MIXED_COUNT, guarded, (unsigned long long)sampled_all, (unsigned long long)freed_before, (unsigned long long)my_profiler.watch_free_count); + #if MI_GUARDED + if (guarded < 100) { result = false; } + else + #endif + result = (sampled > 500 && freed_before > sampled/2 && freed_before < sampled_all && my_profiler.watch_free_count == sampled_all && !my_profiler.watch_bad_free); + } + return true; +} + + // --------------------------------------------------------------------------- // main // --------------------------------------------------------------------------- @@ -182,8 +405,64 @@ int main(void) { test_profiler_record_fields(); test_profiler_on_free_called(); test_profiler_free_count_le_alloc_count(); + test_profiler_small_attribution(); + test_profiler_new_thread(); + test_profiler_new_thread_no_phantom_period(); + test_profiler_heap_destroy(); + test_profiler_heap_destroy_recycled(); + test_profiler_guarded_mixed(); mi_profiler_stop(&my_profiler.profiler); return print_test_summary(); } + + +// --------------------------------------------------------------------------- +// Portable threads (as in test-heap-mt.c) +// --------------------------------------------------------------------------- + +static thread_entry_fun_t* thread_entry_fun; + +#ifdef _WIN32 +#include + +static DWORD WINAPI thread_entry(LPVOID param) { + thread_entry_fun((intptr_t)param); + return 0; +} + +static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { + thread_entry_fun = fun; + HANDLE thandles[16]; + if (nthreads > 16) { nthreads = 16; } + for (size_t i = 0; i < nthreads; i++) { + DWORD tid; + thandles[i] = CreateThread(0, 64*1024L, &thread_entry, (void*)(i), 0, &tid); + } + for (size_t i = 0; i < nthreads; i++) { + WaitForSingleObject(thandles[i], INFINITE); + CloseHandle(thandles[i]); + } +} + +#else +#include + +static void* thread_entry(void* param) { + thread_entry_fun((intptr_t)param); + return NULL; +} + +static void run_os_threads(size_t nthreads, thread_entry_fun_t* fun) { + thread_entry_fun = fun; + pthread_t threads[16]; + if (nthreads > 16) { nthreads = 16; } + for (size_t i = 0; i < nthreads; i++) { + pthread_create(&threads[i], NULL, &thread_entry, (void*)i); + } + for (size_t i = 0; i < nthreads; i++) { + pthread_join(threads[i], NULL); + } +} +#endif