simple mechanism to adjust number of threads sweeping

Author: d-netto

src/gc.c

@@ -895,7 +895,13 @@ STATIC_INLINE void gc_setmark_pool_(jl_ptls_t ptls, jl_taggedvalue_t *o,
     }
     objprofile_count(jl_typeof(jl_valueof(o)),
                      mark_mode == GC_OLD_MARKED, page->osize);
-    page->has_marked = 1;
+    // Test-exchange on `has_marked` to avoid too many strong atomic ops
+    _Atomic(uint8_t) *phas_marked = (_Atomic(uint8_t) *)&page->has_marked;
+    if (jl_atomic_load_relaxed(phas_marked) == 0) {
+        if (jl_atomic_exchange(phas_marked, 1) == 0) {
+            jl_atomic_fetch_add(&gc_heap_stats.marked_pages, 1);
+        }
+    }
 #endif
 }
 
@@ -1520,8 +1526,9 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
         }
     }
     gc_time_count_page(freedall, pg_skpd);
+    jl_atomic_fetch_add(&gc_heap_stats.marked_pages, pg->has_marked);
+    jl_atomic_fetch_add((_Atomic(int64_t) *)&pool_live_bytes, GC_PAGE_SZ - GC_PAGE_OFFSET - nfree * osize);
     jl_atomic_fetch_add((_Atomic(int64_t) *)&gc_num.freed, (nfree - old_nfree) * osize);
-    pool_live_bytes += GC_PAGE_SZ - GC_PAGE_OFFSET - nfree * osize;
 }
 
 // the actual sweeping over all allocated pages in a memory pool
@@ -1559,10 +1566,13 @@ static void gc_pool_sync_nfree(jl_gc_pagemeta_t *pg, jl_taggedvalue_t *last) JL_
     pg->nfree = nfree;
 }
 
-void gc_sweep_wake_all(void)
+void gc_sweep_wake_all(size_t n_workers)
 {
+    if (n_workers > jl_n_markthreads) {
+        n_workers = jl_n_markthreads;
+    }
     uv_mutex_lock(&gc_threads_lock);
-    for (int i = gc_first_tid; i < gc_first_tid + jl_n_gcthreads; i++) {
+    for (int i = gc_first_tid; i < gc_first_tid + jl_n_markthreads; i++) {
         jl_ptls_t ptls2 = gc_all_tls_states[i];
         jl_atomic_fetch_add(&ptls2->gc_sweeps_requested, 1);
     }
@@ -1618,6 +1628,8 @@ void gc_free_pages(void)
     }
 }
 
+#define GC_MIN_PAGES_PER_WORKER (1 << 6)
+
 // setup the data-structures for a sweep over all memory pools
 static void gc_sweep_pool(void)
 {
@@ -1674,7 +1686,9 @@ static void gc_sweep_pool(void)
     jl_gc_page_stack_t *tmp = (jl_gc_page_stack_t *)alloca(n_threads * sizeof(jl_gc_page_stack_t));
     memset(tmp, 0, n_threads * sizeof(jl_gc_page_stack_t));
     jl_atomic_store(&gc_allocd_scratch, tmp);
-    gc_sweep_wake_all();
+    size_t n_workers = jl_atomic_load_relaxed(&gc_heap_stats.marked_pages) / GC_MIN_PAGES_PER_WORKER;
+    jl_atomic_store_relaxed(&gc_heap_stats.marked_pages, 0);
+    gc_sweep_wake_all(n_workers);
     gc_sweep_pool_parallel();
     gc_sweep_wait_for_all();
 
@@ -1730,7 +1744,6 @@ static void gc_sweep_pool(void)
 #else
     gc_free_pages();
 #endif
-
     gc_time_pool_end(current_sweep_full);
 }
 

src/gc.h

@@ -268,6 +268,7 @@ typedef struct {
     _Atomic(size_t) bytes_resident;
     _Atomic(size_t) heap_size;
     _Atomic(size_t) heap_target;
+    _Atomic(size_t) marked_pages;
 } gc_heapstatus_t;
 
 #define GC_PAGE_UNMAPPED        0