Skip to content

Commit

Permalink
Only add big objarray to remset once (#49315)
Browse files Browse the repository at this point in the history 
Fixes #49205
  • Loading branch information
@gbaraldi
gbaraldi authored Apr 20, 2023
1 parent 4f035fa commit 1f94d2e
Showing 1 changed file with 112 additions and 29 deletions.
141 changes: 112 additions & 29 deletions src/gc.c
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2087,24 +2087,47 @@ STATIC_INLINE void gc_mark_objarray(jl_ptls_t ptls, jl_value_t *obj_parent, jl_v
jl_value_t *new_obj;
// Decide whether need to chunk objary
(void)jl_assume(step > 0);
size_t nobjs = (obj_end - obj_begin) / step;
if (nobjs > MAX_REFS_AT_ONCE) {
jl_gc_chunk_t c = {GC_objary_chunk, obj_parent, obj_begin + step * MAX_REFS_AT_ONCE,
obj_end, NULL, NULL,
step, nptr};
gc_chunkqueue_push(mq, &c);
obj_end = obj_begin + step * MAX_REFS_AT_ONCE;
if ((nptr & 0x2) == 0x2) {
// pre-scan this object: most of this object should be old, so look for
// the first young object before starting this chunk
// (this also would be valid for young objects, but probably less beneficial)
for (; obj_begin < obj_end; obj_begin += step) {
new_obj = *obj_begin;
if (new_obj != NULL) {
verify_parent2("obj array", obj_parent, obj_begin, "elem(%d)",
gc_slot_to_arrayidx(obj_parent, obj_begin));
jl_taggedvalue_t *o = jl_astaggedvalue(new_obj);
if (!gc_old(o->header))
nptr |= 1;
if (!gc_marked(o->header))
break;
gc_heap_snapshot_record_array_edge(obj_parent, &new_obj);
}
}
}
size_t too_big = (obj_end - obj_begin) / MAX_REFS_AT_ONCE > step; // use this order of operations to avoid idiv
jl_value_t **scan_end = obj_end;
if (too_big) {
scan_end = obj_begin + step * MAX_REFS_AT_ONCE;
}
for (; obj_begin < obj_end; obj_begin += step) {
for (; obj_begin < scan_end; obj_begin += step) {
new_obj = *obj_begin;
if (new_obj != NULL) {
verify_parent2("obj array", obj_parent, obj_begin, "elem(%d)",
gc_slot_to_arrayidx(obj_parent, obj_begin));
gc_slot_to_arrayidx(obj_parent, obj_begin));
gc_try_claim_and_push(mq, new_obj, &nptr);
gc_heap_snapshot_record_array_edge(obj_parent, &new_obj);
}
}
gc_mark_push_remset(ptls, obj_parent, nptr);
if (too_big) {
jl_gc_chunk_t c = {GC_objary_chunk, obj_parent, scan_end,
obj_end, NULL, NULL,
step, nptr};
gc_chunkqueue_push(mq, &c);
}
else {
gc_mark_push_remset(ptls, obj_parent, nptr);
}
}

// Mark array with 8bit field descriptors
Expand All @@ -2116,14 +2139,36 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va
jl_value_t *new_obj;
size_t elsize = ((jl_array_t *)ary8_parent)->elsize / sizeof(jl_value_t *);
assert(elsize > 0);
// Decide whether need to chunk ary8
size_t nrefs = (ary8_end - ary8_begin) / elsize;
if (nrefs > MAX_REFS_AT_ONCE) {
jl_gc_chunk_t c = {GC_ary8_chunk, ary8_parent, ary8_begin + elsize * MAX_REFS_AT_ONCE,
ary8_end, elem_begin, elem_end,
0, nptr};
gc_chunkqueue_push(mq, &c);
ary8_end = ary8_begin + elsize * MAX_REFS_AT_ONCE;
// Decide whether need to chunk objary
if ((nptr & 0x2) == 0x2) {
// pre-scan this object: most of this object should be old, so look for
// the first young object before starting this chunk
// (this also would be valid for young objects, but probably less beneficial)
for (; ary8_begin < ary8_end; ary8_begin += elsize) {
int early_end = 0;
for (uint8_t *pindex = elem_begin; pindex < elem_end; pindex++) {
new_obj = ary8_begin[*pindex];
if (new_obj != NULL) {
verify_parent2("array", ary8_parent, &new_obj, "elem(%d)",
gc_slot_to_arrayidx(ary8_parent, ary8_begin));
jl_taggedvalue_t *o = jl_astaggedvalue(new_obj);
if (!gc_old(o->header))
nptr |= 1;
if (!gc_marked(o->header)){
early_end = 1;
break;
}
gc_heap_snapshot_record_array_edge(ary8_parent, &new_obj);
}
}
if (early_end)
break;
}
}
size_t too_big = (ary8_end - ary8_begin) / MAX_REFS_AT_ONCE > elsize; // use this order of operations to avoid idiv
jl_value_t **scan_end = ary8_end;
if (too_big) {
scan_end = ary8_begin + elsize * MAX_REFS_AT_ONCE;
}
for (; ary8_begin < ary8_end; ary8_begin += elsize) {
for (uint8_t *pindex = elem_begin; pindex < elem_end; pindex++) {
Expand All @@ -2136,7 +2181,15 @@ STATIC_INLINE void gc_mark_array8(jl_ptls_t ptls, jl_value_t *ary8_parent, jl_va
}
}
}
gc_mark_push_remset(ptls, ary8_parent, nptr);
if (too_big) {
jl_gc_chunk_t c = {GC_objary_chunk, ary8_parent, scan_end,
ary8_end, elem_begin, elem_end,
0, nptr};
gc_chunkqueue_push(mq, &c);
}
else {
gc_mark_push_remset(ptls, ary8_parent, nptr);
}
}

// Mark array with 16bit field descriptors
Expand All @@ -2148,16 +2201,38 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_
jl_value_t *new_obj;
size_t elsize = ((jl_array_t *)ary16_parent)->elsize / sizeof(jl_value_t *);
assert(elsize > 0);
// Decide whether need to chunk ary16
size_t nrefs = (ary16_end - ary16_begin) / elsize;
if (nrefs > MAX_REFS_AT_ONCE) {
jl_gc_chunk_t c = {GC_ary16_chunk, ary16_parent, ary16_begin + elsize * MAX_REFS_AT_ONCE,
ary16_end, elem_begin, elem_end,
0, nptr};
gc_chunkqueue_push(mq, &c);
ary16_end = ary16_begin + elsize * MAX_REFS_AT_ONCE;
// Decide whether need to chunk objary
if ((nptr & 0x2) == 0x2) {
// pre-scan this object: most of this object should be old, so look for
// the first young object before starting this chunk
// (this also would be valid for young objects, but probably less beneficial)
for (; ary16_begin < ary16_end; ary16_begin += elsize) {
int early_end = 0;
for (uint16_t *pindex = elem_begin; pindex < elem_end; pindex++) {
new_obj = ary16_begin[*pindex];
if (new_obj != NULL) {
verify_parent2("array", ary16_parent, &new_obj, "elem(%d)",
gc_slot_to_arrayidx(ary16_parent, ary16_begin));
jl_taggedvalue_t *o = jl_astaggedvalue(new_obj);
if (!gc_old(o->header))
nptr |= 1;
if (!gc_marked(o->header)){
early_end = 1;
break;
}
gc_heap_snapshot_record_array_edge(ary16_parent, &new_obj);
}
}
if (early_end)
break;
}
}
size_t too_big = (ary16_end - ary16_begin) / MAX_REFS_AT_ONCE > elsize; // use this order of operations to avoid idiv
jl_value_t **scan_end = ary16_end;
if (too_big) {
scan_end = ary16_begin + elsize * MAX_REFS_AT_ONCE;
}
for (; ary16_begin < ary16_end; ary16_begin += elsize) {
for (; ary16_begin < scan_end; ary16_begin += elsize) {
for (uint16_t *pindex = elem_begin; pindex < elem_end; pindex++) {
new_obj = ary16_begin[*pindex];
if (new_obj != NULL) {
Expand All @@ -2168,7 +2243,15 @@ STATIC_INLINE void gc_mark_array16(jl_ptls_t ptls, jl_value_t *ary16_parent, jl_
}
}
}
gc_mark_push_remset(ptls, ary16_parent, nptr);
if (too_big) {
jl_gc_chunk_t c = {GC_objary_chunk, ary16_parent, scan_end,
ary16_end, elem_begin, elem_end,
elsize, nptr};
gc_chunkqueue_push(mq, &c);
}
else {
gc_mark_push_remset(ptls, ary16_parent, nptr);
}
}

// Mark chunk of large array
Expand Down

0 comments on commit 1f94d2e

Please sign in to comment.