parallelize sweeping of object pools (#51282)

Sweeping of object pools will either construct a free list through dead objects (if there is at least one live object in a given page) or return the page to the OS (if there are no live objects whatsoever). With this PR, we're basically constructing the free-lists for each GC page in parallel.

Author: d-netto

src/gc-debug.c

@@ -115,7 +115,7 @@ static void gc_clear_mark_outer(int bits)
 {
     for (int i = 0; i < gc_n_threads; i++) {
         jl_ptls_t ptls2 = gc_all_tls_states[i];
-        jl_gc_pagemeta_t *pg = ptls2->page_metadata_allocd;
+        jl_gc_pagemeta_t *pg = jl_atomic_load_relaxed(&ptls2->page_metadata_allocd.bottom);
         while (pg != NULL) {
             gc_clear_mark_page(pg, bits);
             pg = pg->next;
@@ -1153,7 +1153,7 @@ static void gc_count_pool_pagetable(void)
 {
     for (int i = 0; i < gc_n_threads; i++) {
         jl_ptls_t ptls2 = gc_all_tls_states[i];
-        jl_gc_pagemeta_t *pg = ptls2->page_metadata_allocd;
+        jl_gc_pagemeta_t *pg = jl_atomic_load_relaxed(&ptls2->page_metadata_allocd.bottom);
         while (pg != NULL) {
             if (gc_alloc_map_is_set(pg->data)) {
                 gc_count_pool_page(pg);

src/gc-pages.c

@@ -100,22 +100,22 @@ NOINLINE jl_gc_pagemeta_t *jl_gc_alloc_page(void) JL_NOTSAFEPOINT
     jl_gc_pagemeta_t *meta = NULL;
 
     // try to get page from `pool_lazily_freed`
-    meta = pop_lf_page_metadata_back(&global_page_pool_lazily_freed);
+    meta = pop_lf_back(&global_page_pool_lazily_freed);
     if (meta != NULL) {
         gc_alloc_map_set(meta->data, GC_PAGE_ALLOCATED);
         // page is already mapped
         return meta;
     }
 
     // try to get page from `pool_clean`
-    meta = pop_lf_page_metadata_back(&global_page_pool_clean);
+    meta = pop_lf_back(&global_page_pool_clean);
     if (meta != NULL) {
         gc_alloc_map_set(meta->data, GC_PAGE_ALLOCATED);
         goto exit;
     }
 
     // try to get page from `pool_freed`
-    meta = pop_lf_page_metadata_back(&global_page_pool_freed);
+    meta = pop_lf_back(&global_page_pool_freed);
     if (meta != NULL) {
         jl_atomic_fetch_add_relaxed(&gc_heap_stats.bytes_resident, GC_PAGE_SZ);
         gc_alloc_map_set(meta->data, GC_PAGE_ALLOCATED);
@@ -124,7 +124,7 @@ NOINLINE jl_gc_pagemeta_t *jl_gc_alloc_page(void) JL_NOTSAFEPOINT
 
     uv_mutex_lock(&gc_perm_lock);
     // another thread may have allocated a large block while we were waiting...
-    meta = pop_lf_page_metadata_back(&global_page_pool_clean);
+    meta = pop_lf_back(&global_page_pool_clean);
     if (meta != NULL) {
         uv_mutex_unlock(&gc_perm_lock);
         gc_alloc_map_set(meta->data, GC_PAGE_ALLOCATED);
@@ -138,10 +138,10 @@ NOINLINE jl_gc_pagemeta_t *jl_gc_alloc_page(void) JL_NOTSAFEPOINT
         pg->data = data + GC_PAGE_SZ * i;
         gc_alloc_map_maybe_create(pg->data);
         if (i == 0) {
-            gc_alloc_map_set(pg->data, 1);
+            gc_alloc_map_set(pg->data, GC_PAGE_ALLOCATED);
         }
         else {
-            push_lf_page_metadata_back(&global_page_pool_clean, pg);
+            push_lf_back(&global_page_pool_clean, pg);
         }
     }
     uv_mutex_unlock(&gc_perm_lock);