return pages to the OS as the last step

Author: d-netto

src/gc.c

@@ -1301,7 +1301,7 @@ static NOINLINE jl_taggedvalue_t *gc_add_page(jl_gc_pool_t *p) JL_NOTSAFEPOINT
     // Do not pass in `ptls` as argument. This slows down the fast path
     // in pool_alloc significantly
     jl_ptls_t ptls = jl_current_task->ptls;
-    jl_gc_pagemeta_t *pg = pop_lf_back(&ptls->page_metadata_lazily_freed);
+    jl_gc_pagemeta_t *pg = pop_lf_back(&ptls->page_metadata_buffered);
     if (pg != NULL) {
         gc_alloc_map_set(pg->data, GC_PAGE_ALLOCATED);
     }
@@ -1410,10 +1410,10 @@ int jl_gc_classify_pools(size_t sz, int *osize)
 
 // sweep phase
 
-int64_t lazy_freed_pages = 0;
+int64_t buffered_pages = 0;
 
 // Returns pointer to terminal pointer of list rooted at *pfl.
-static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *lazily_freed,
+static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_page_stack_t *buffered,
                           jl_gc_pagemeta_t *pg, int osize) JL_NOTSAFEPOINT
 {
     char *data = pg->data;
@@ -1427,7 +1427,7 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
     size_t nfree;
 
     int re_use_page = 1;
-    int freed_lazily = 0;
+    int keep_as_local_buffer = 0;
     int freedall = 1;
     int pg_skpd = 1;
     if (!pg->has_marked) {
@@ -1438,9 +1438,9 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
         // the eager one uses less memory.
         // FIXME - need to do accounting on a per-thread basis
         // on quick sweeps, keep a few pages empty but allocated for performance
-        if (!current_sweep_full && lazy_freed_pages <= default_collect_interval / GC_PAGE_SZ) {
-            lazy_freed_pages++;
-            freed_lazily = 1;
+        if (!current_sweep_full && buffered_pages <= default_collect_interval / GC_PAGE_SZ) {
+            buffered_pages++;
+            keep_as_local_buffer = 1;
         }
     #endif
         nfree = (GC_PAGE_SZ - GC_PAGE_OFFSET) / osize;
@@ -1509,26 +1509,15 @@ static void gc_sweep_page(jl_gc_pool_t *p, jl_gc_page_stack_t *allocd, jl_gc_pag
     if (re_use_page) {
         push_lf_back(allocd, pg);
     }
-    else if (freed_lazily) {
-        gc_alloc_map_set(pg->data, GC_PAGE_LAZILY_FREED);
-        push_lf_back(lazily_freed, pg);
-        jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -GC_PAGE_SZ);
-    }
     else {
+        gc_alloc_map_set(pg->data, GC_PAGE_LAZILY_FREED);
         jl_atomic_fetch_add_relaxed(&gc_heap_stats.heap_size, -GC_PAGE_SZ);
-    #ifdef _P64 // only enable concurrent sweeping on 64bit
-        if (jl_n_sweepthreads == 0) {
-            jl_gc_free_page(pg);
-            push_lf_back(&global_page_pool_freed, pg);
+        if (keep_as_local_buffer) {
+            push_lf_back(buffered, pg);
         }
         else {
-            gc_alloc_map_set(pg->data, GC_PAGE_LAZILY_FREED);
             push_lf_back(&global_page_pool_lazily_freed, pg);
         }
-    #else
-        jl_gc_free_page(pg);
-        push_lf_back(&global_page_pool_freed, pg);
-    #endif
     }
     gc_time_count_page(freedall, pg_skpd);
     jl_atomic_fetch_add((_Atomic(int64_t) *)&gc_num.freed, (nfree - old_nfree) * osize);
@@ -1598,7 +1587,7 @@ void gc_sweep_pool_parallel(void)
                 if (pg == NULL) {
                     continue;
                 }
-                gc_sweep_pool_page(allocd, &ptls2->page_metadata_lazily_freed, pg);
+                gc_sweep_pool_page(allocd, &ptls2->page_metadata_buffered, pg);
                 found_pg = 1;
             }
             if (!found_pg) {
@@ -1617,11 +1606,23 @@ void gc_sweep_wait_for_all(void)
     }
 }
 
+void gc_free_pages(void)
+{
+    while (1) {
+        jl_gc_pagemeta_t *pg = pop_lf_back(&global_page_pool_lazily_freed);
+        if (pg == NULL) {
+            break;
+        }
+        jl_gc_free_page(pg);
+        push_lf_back(&global_page_pool_freed, pg);
+    }
+}
+
 // setup the data-structures for a sweep over all memory pools
 static void gc_sweep_pool(void)
 {
     gc_time_pool_start();
-    lazy_freed_pages = 0;
+    buffered_pages = 0;
 
     // For the benefit of the analyzer, which doesn't know that gc_n_threads
     // doesn't change over the course of this function
@@ -1661,10 +1662,10 @@ static void gc_sweep_pool(void)
                 pg->has_young = 1;
             }
         }
-        jl_gc_pagemeta_t *pg = jl_atomic_load_relaxed(&ptls2->page_metadata_lazily_freed.bottom);
+        jl_gc_pagemeta_t *pg = jl_atomic_load_relaxed(&ptls2->page_metadata_buffered.bottom);
         while (pg != NULL) {
             jl_gc_pagemeta_t *pg2 = pg->next;
-            lazy_freed_pages++;
+            buffered_pages++;
             pg = pg2;
         }
     }
@@ -1723,6 +1724,11 @@ static void gc_sweep_pool(void)
     if (jl_n_sweepthreads > 0) {
         uv_sem_post(&gc_sweep_assists_needed);
     }
+    else {
+        gc_free_pages();
+    }
+#else
+    gc_free_pages();
 #endif
 
     gc_time_pool_end(current_sweep_full);

src/gc.h

@@ -383,7 +383,7 @@ extern jl_gc_num_t gc_num;
 extern bigval_t *big_objects_marked;
 extern arraylist_t finalizer_list_marked;
 extern arraylist_t to_finalize;
-extern int64_t lazy_freed_pages;
+extern int64_t buffered_pages;
 extern int gc_first_tid;
 extern int gc_n_threads;
 extern jl_ptls_t* gc_all_tls_states;
@@ -460,6 +460,7 @@ void gc_mark_loop_serial_(jl_ptls_t ptls, jl_gc_markqueue_t *mq);
 void gc_mark_loop_serial(jl_ptls_t ptls);
 void gc_mark_loop_parallel(jl_ptls_t ptls, int master);
 void gc_sweep_pool_parallel(void);
+void gc_free_pages(void);
 void sweep_stack_pools(void);
 void jl_gc_debug_init(void);
 

src/julia_threads.h

@@ -266,7 +266,7 @@ typedef struct _jl_tls_states_t {
     jl_thread_t system_id;
     arraylist_t finalizers;
     jl_gc_page_stack_t page_metadata_allocd;
-    jl_gc_page_stack_t page_metadata_lazily_freed;
+    jl_gc_page_stack_t page_metadata_buffered;
     jl_gc_markqueue_t mark_queue;
     jl_gc_mark_cache_t gc_cache;
     arraylist_t sweep_objs;

src/partr.c

@@ -165,14 +165,7 @@ void jl_concurrent_gc_threadfun(void *arg)
 
     while (1) {
         uv_sem_wait(&gc_sweep_assists_needed);
-        while (1) {
-            jl_gc_pagemeta_t *pg = pop_lf_back(&global_page_pool_lazily_freed);
-            if (pg == NULL) {
-                break;
-            }
-            jl_gc_free_page(pg);
-            push_lf_back(&global_page_pool_freed, pg);
-        }
+        gc_free_pages();
     }
 }