WIP: Incremental GC

base/util.jl

@@ -12,6 +12,9 @@ gc_time_ns() = ccall(:jl_gc_total_hrtime, Uint64, ())
 # total number of bytes allocated so far
 gc_bytes() = ccall(:jl_gc_total_bytes, Int64, ())
 
+gc_num_pause() = ccall(:jl_gc_num_pause, Int64, ())
+gc_num_full_sweep() = ccall(:jl_gc_num_full_sweep, Int64, ())
+
 # reset the malloc log. Used to avoid counting memory allocated during compilation.
 clear_malloc_data() = ccall(:jl_clear_malloc_data, Void, ())
 
@@ -39,12 +42,17 @@ function toc()
 end
 
 # print elapsed time, return expression value
-
-function time_print(t, b, g)
+const _units = ["bytes", "kB", "MB"]
+function time_print(t, b, g, np, nfs)
+    i = 1
+    while b > 1024 && i < length(_units)
+        b = div(b, 1024)
+        i += 1
+    end
     if 0 < g
-        @printf("elapsed time: %s seconds (%d bytes allocated, %.2f%% gc time)\n", t/1e9, b, 100*g/t)
+        @printf("elapsed time: %s seconds (%d %s allocated, %.2f%% gc time in %d pauses with %d full sweep)\n", t/1e9, b, _units[i], 100*g/t, np, nfs)
     else
-        @printf("elapsed time: %s seconds (%d bytes allocated)\n", t/1e9, b)
+        @printf("elapsed time: %s seconds (%d %s allocated)\n", t/1e9, b, _units[i])
     end
 end
 
@@ -53,11 +61,15 @@ macro time(ex)
         local b0 = gc_bytes()
         local t0 = time_ns()
         local g0 = gc_time_ns()
+        local n0 = gc_num_pause()
+        local nfs0 = gc_num_full_sweep()
         local val = $(esc(ex))
+        local nfs1 = gc_num_full_sweep()
+        local n1 = gc_num_pause()
         local g1 = gc_time_ns()
         local t1 = time_ns()
         local b1 = gc_bytes()
-        time_print(t1-t0, b1-b0, g1-g0)
+        time_print(t1-t0, b1-b0, g1-g0, n1-n0, nfs1-nfs0)
         val
     end
 end

src/alloc.c

@@ -249,6 +249,7 @@ void jl_set_nth_field(jl_value_t *v, size_t i, jl_value_t *rhs)
     size_t offs = jl_field_offset(st,i) + sizeof(void*);
     if (st->fields[i].isptr) {
         *(jl_value_t**)((char*)v + offs) = rhs;
+        if(rhs != NULL) gc_wb(v, rhs);
     }
     else {
         jl_assign_bits((char*)v + offs, rhs);
@@ -488,17 +489,18 @@ static jl_sym_t *mk_symbol(const char *str)
 static void unmark_symbols_(jl_sym_t *root)
 {
     while (root != NULL) {
-        root->type = (jl_value_t*)(((uptrint_t)root->type)&~1UL);
+        root->type = (jl_value_t*)(((uptrint_t)root->type)&~3UL);
         unmark_symbols_(root->left);
         root = root->right;
     }
 }
 
 void jl_unmark_symbols(void) { unmark_symbols_(symtab); }
 
-static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
+static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str, jl_sym_t **parent)
 {
     int x;
+    if (parent != NULL) *parent = NULL;
     uptrint_t h = hash_symbol(str, strlen(str));
 
     // Tree nodes sorted by major key of (int(hash)) and minor key o (str).
@@ -509,6 +511,7 @@ static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
             if (x == 0)
                 return ptree;
         }
+        if (parent != NULL) *parent = *ptree;
         if (x < 0)
             ptree = &(*ptree)->left;
         else
@@ -520,16 +523,19 @@ static jl_sym_t **symtab_lookup(jl_sym_t **ptree, const char *str)
 jl_sym_t *jl_symbol(const char *str)
 {
     jl_sym_t **pnode;
-
-    pnode = symtab_lookup(&symtab, str);
-    if (*pnode == NULL)
+    jl_sym_t *parent;
+    pnode = symtab_lookup(&symtab, str, &parent);
+    if (*pnode == NULL) {
         *pnode = mk_symbol(str);
+        if (parent != NULL)
+            gc_wb(parent, *pnode);
+    }
     return *pnode;
 }
 
 jl_sym_t *jl_symbol_lookup(const char *str)
 {
-    return *symtab_lookup(&symtab, str);
+    return *symtab_lookup(&symtab, str, NULL);
 }
 
 DLLEXPORT jl_sym_t *jl_symbol_n(const char *str, int32_t len)
@@ -627,6 +633,7 @@ void jl_add_constructors(jl_datatype_t *t)
             cfactory = jl_instantiate_method((jl_function_t*)t->name->ctor_factory, env);
             cfactory->linfo->ast = jl_prepare_ast(cfactory->linfo,
                                                   cfactory->linfo->sparams);
+            gc_wb(cfactory->linfo, cfactory->linfo->ast);
         }
         else {
             cfactory = ((jl_datatype_t*)t)->name->static_ctor_factory;
@@ -715,25 +722,31 @@ jl_datatype_t *jl_new_datatype(jl_sym_t *name, jl_datatype_t *super,
             t = jl_bool_type;
     }
     if (t == NULL) {
-        t = jl_new_uninitialized_datatype(jl_tuple_len(fnames));
         if (jl_is_typename(name))
             tn = (jl_typename_t*)name;
         else
             tn = jl_new_typename((jl_sym_t*)name);
+        t = jl_new_uninitialized_datatype(jl_tuple_len(fnames));
         t->name = tn;
+       gc_wb(t, t->name);
     }
 
     if (t->name->primary == NULL)
         t->name->primary = (jl_value_t*)t;
     t->super = super;
+    if(super != NULL) gc_wb(t, t->super);
     t->parameters = parameters;
+    gc_wb(t, t->parameters);
     t->names = fnames;
+    gc_wb(t, t->names);
     t->types = ftypes;
+    if(ftypes != NULL) gc_wb(t, t->types);
     t->abstract = abstract;
     t->mutabl = mutabl;
     t->pointerfree = 0;
     t->fptr = jl_f_no_function;
     t->env = (jl_value_t*)t;
+    gc_wb(t, t->env);
     t->linfo = NULL;
     t->instance = NULL;
     t->struct_decl = NULL;

src/array.c

@@ -99,6 +99,7 @@ static jl_array_t *_new_array_(jl_value_t *atype, uint32_t ndims, size_t *dims,
             memset(data, 0, tot);
         JL_GC_POP();
     }
+    a->pooled = tsz <= 2048;
 
     a->data = data;
     if (elsz == 1) ((char*)data)[tot-1] = '\0';
@@ -147,8 +148,10 @@ jl_array_t *jl_reshape_array(jl_value_t *atype, jl_array_t *data, jl_tuple_t *di
     size_t ndims = jl_tuple_len(dims);
 
     int ndimwords = jl_array_ndimwords(ndims);
-    a = (jl_array_t*)allocobj((sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16);
+    int tsz = (sizeof(jl_array_t) + sizeof(void*) + ndimwords*sizeof(size_t) + 15)&-16;
+    a = (jl_array_t*)allocobj(tsz);
     a->type = atype;
+    a->pooled = tsz <= 2048;
     a->ndims = ndims;
     a->offset = 0;
     a->data = NULL;
@@ -211,8 +214,9 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel,
         elsz = jl_datatype_size(el_type);
     else
         elsz = sizeof(void*);
-
-    a = (jl_array_t*)allocobj((sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16);
+    int tsz = (sizeof(jl_array_t)+jl_array_ndimwords(1)*sizeof(size_t)+15)&-16;
+    a = (jl_array_t*)allocobj(tsz);
+    a->pooled = tsz <= 2048;
     a->type = atype;
     a->data = data;
 #ifdef STORE_ARRAY_LEN
@@ -226,6 +230,7 @@ jl_array_t *jl_ptr_to_array_1d(jl_value_t *atype, void *data, size_t nel,
     if (own_buffer) {
         a->how = 2;
         jl_gc_track_malloced_array(a);
+        jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
     }
     else {
         a->how = 0;
@@ -260,7 +265,9 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims,
         elsz = sizeof(void*);
 
     int ndimwords = jl_array_ndimwords(ndims);
-    a = (jl_array_t*)allocobj((sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16);
+    int tsz = (sizeof(jl_array_t) + ndimwords*sizeof(size_t)+15)&-16;
+    a = (jl_array_t*)allocobj(tsz);
+    a->pooled = tsz <= 2048;
     a->type = atype;
     a->data = data;
 #ifdef STORE_ARRAY_LEN
@@ -275,6 +282,7 @@ jl_array_t *jl_ptr_to_array(jl_value_t *atype, void *data, jl_tuple_t *dims,
     if (own_buffer) {
         a->how = 2;
         jl_gc_track_malloced_array(a);
+        jl_gc_count_allocd(nel*elsz + (elsz == 1 ? 1 : 0));
     }
     else {
         a->how = 0;
@@ -500,6 +508,7 @@ void jl_arrayset(jl_array_t *a, jl_value_t *rhs, size_t i)
     }
     else {
         ((jl_value_t**)a->data)[i] = rhs;
+        gc_wb(a, rhs);
     }
 }
 
@@ -569,6 +578,8 @@ static void array_resize_buffer(jl_array_t *a, size_t newlen, size_t oldlen, siz
     a->isshared = 0;
     if (a->ptrarray || es==1)
         memset(newdata+offsnb+oldnbytes, 0, nbytes-oldnbytes-offsnb);
+    if (a->how == 1)
+        gc_wb_buf(a, newdata);
     a->maxsize = newlen;
 }
 

src/ast.c

@@ -720,6 +720,7 @@ static jl_value_t *copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
         // of a top-level thunk that gets type inferred.
         li->def = li;
         li->ast = jl_prepare_ast(li, li->sparams);
+        gc_wb(li, li->ast);
         JL_GC_POP();
         return (jl_value_t*)li;
     }
@@ -738,17 +739,18 @@ static jl_value_t *copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
         jl_expr_t *ne = jl_exprn(e->head, jl_array_len(e->args));
         JL_GC_PUSH1(&ne);
         if (e->head == lambda_sym) {
-            jl_exprarg(ne, 0) = copy_ast(jl_exprarg(e,0), sp, 0);
-            jl_exprarg(ne, 1) = copy_ast(jl_exprarg(e,1), sp, 0);
-            jl_exprarg(ne, 2) = copy_ast(jl_exprarg(e,2), sp, 1);
+            jl_exprargset(ne, 0, copy_ast(jl_exprarg(e,0), sp, 0));
+            jl_exprargset(ne, 1, copy_ast(jl_exprarg(e,1), sp, 0));
+            jl_exprargset(ne, 2, copy_ast(jl_exprarg(e,2), sp, 1));
         }
         else if (e->head == assign_sym) {
-            jl_exprarg(ne, 0) = copy_ast(jl_exprarg(e,0), sp, 0);
-            jl_exprarg(ne, 1) = copy_ast(jl_exprarg(e,1), sp, 1);
+            jl_exprargset(ne, 0, copy_ast(jl_exprarg(e,0), sp, 0));
+            jl_exprargset(ne, 1, copy_ast(jl_exprarg(e,1), sp, 1));
         }
         else {
-            for(size_t i=0; i < jl_array_len(e->args); i++)
-                jl_exprarg(ne, i) = copy_ast(jl_exprarg(e,i), sp, 1);
+            for(size_t i=0; i < jl_array_len(e->args); i++) {
+                jl_exprargset(ne, i, copy_ast(jl_exprarg(e,i), sp, 1));
+            }
         }
         JL_GC_POP();
         return (jl_value_t*)ne;
@@ -766,10 +768,12 @@ DLLEXPORT jl_value_t *jl_copy_ast(jl_value_t *expr)
         ne = jl_exprn(e->head, l);
         if (l == 0) {
             ne->args = jl_alloc_cell_1d(0);
+            gc_wb(ne, ne->args);
         }
         else {
-            for(i=0; i < l; i++)
-                jl_exprarg(ne, i) = jl_copy_ast(jl_exprarg(e,i));
+            for(i=0; i < l; i++) {
+                jl_exprargset(ne, i, jl_copy_ast(jl_exprarg(e,i)));
+            }
         }
         JL_GC_POP();
         return (jl_value_t*)ne;
@@ -806,17 +810,18 @@ static jl_value_t *dont_copy_ast(jl_value_t *expr, jl_tuple_t *sp, int do_sp)
     else if (jl_is_expr(expr)) {
         jl_expr_t *e = (jl_expr_t*)expr;
         if (e->head == lambda_sym) {
-            jl_exprarg(e, 0) = dont_copy_ast(jl_exprarg(e,0), sp, 0);
-            jl_exprarg(e, 1) = dont_copy_ast(jl_exprarg(e,1), sp, 0);
-            jl_exprarg(e, 2) = dont_copy_ast(jl_exprarg(e,2), sp, 1);
+            jl_exprargset(e, 0, dont_copy_ast(jl_exprarg(e,0), sp, 0));
+            jl_exprargset(e, 1, dont_copy_ast(jl_exprarg(e,1), sp, 0));
+            jl_exprargset(e, 2, dont_copy_ast(jl_exprarg(e,2), sp, 1));
         }
         else if (e->head == assign_sym) {
-            jl_exprarg(e, 0) = dont_copy_ast(jl_exprarg(e,0), sp, 0);
-            jl_exprarg(e, 1) = dont_copy_ast(jl_exprarg(e,1), sp, 1);
+            jl_exprargset(e, 0, dont_copy_ast(jl_exprarg(e,0), sp, 0));
+            jl_exprargset(e, 1, dont_copy_ast(jl_exprarg(e,1), sp, 1));
         }
         else {
-            for(size_t i=0; i < jl_array_len(e->args); i++)
-                jl_exprarg(e, i) = dont_copy_ast(jl_exprarg(e,i), sp, 1);
+            for(size_t i=0; i < jl_array_len(e->args); i++) {
+                jl_exprargset(e, i, dont_copy_ast(jl_exprarg(e,i), sp, 1));
+            }
         }
         return (jl_value_t*)e;
     }
@@ -834,6 +839,7 @@ static void eval_decl_types(jl_array_t *vi, jl_value_t *ast, jl_tuple_t *spenv)
                                         (jl_value_t*)spenv, (jl_expr_t*)ast, 1, 1);
         if (ty != NULL && (jl_is_type(ty) || jl_is_typevar(ty))) {
             jl_cellref(v, 1) = ty;
+            gc_wb(v, ty);
         }
         else {
             jl_cellref(v, 1) = (jl_value_t*)jl_any_type;