Merge pull request #150 from shwestrick/cc-work-list-opt

Better space performance for CC work list

Author: shwestrick

runtime/gc/cc-work-list.c

@@ -10,7 +10,7 @@ void CC_workList_init(
   HM_chunkList c = &(w->storage);
   HM_initChunkList(c);
   // arbitrary, just need an initial chunk
-  w->currentChunk = HM_allocateChunk(c, sizeof(objptr));
+  w->currentChunk = HM_allocateChunk(c, sizeof(struct CC_workList_elem));
 }
 
 
@@ -34,41 +34,240 @@ bool CC_workList_isEmpty(
 }
 
 
+pointer findStackNonEmptyFrameTop(GC_state s, pointer bottom, pointer top) {
+  assert(bottom <= top);
+
+  while (top > bottom) {
+    /* Invariant: top points just past a "return address". */
+    GC_returnAddress returnAddress =
+      *((GC_returnAddress*)(top - GC_RETURNADDRESS_SIZE));
+    GC_frameInfo frameInfo = getFrameInfoFromReturnAddress(s, returnAddress);
+    // index zero of this array is size
+    GC_frameOffsets frameOffsets = frameInfo->offsets;
+
+    if (frameOffsets[0] > 0) {
+      return top;
+    }
+
+    top -= frameInfo->size;
+  }
+
+  return NULL;
+}
+
+
+// returns FALSE if object has no objptrs and doesn't need to be traced
+bool makeInitialElem(GC_state s, objptr op, CC_workList_elem result) {
+  GC_header header;
+  uint16_t numObjptrs;
+  GC_objectTypeTag tag;
+  header = getHeader(objptrToPointer(op, NULL));
+  splitHeader(s, header, &tag, NULL, NULL, &numObjptrs);
+
+  if (NORMAL_TAG == tag) {
+    if (0 == numObjptrs) return FALSE;
+
+    result->op = op;
+    result->data.normal.objptrIdx = 0;
+    return TRUE;
+  }
+
+  if (SEQUENCE_TAG == tag) {
+    if (0 == numObjptrs) return FALSE;
+    if (0 == getSequenceLength(objptrToPointer(op, NULL))) return FALSE;
+
+    result->op = op;
+    result->data.sequence.cellIdx = 0;
+    result->data.sequence.objptrIdx = 0;
+    return TRUE;
+  }
+
+  if (STACK_TAG == tag) {
+    // printf("makeInitialElem stack\n");
+
+    GC_stack stack = (GC_stack)objptrToPointer(op, NULL);
+    assert (stack->used <= stack->reserved);
+    pointer bottom = getStackBottom(s, stack);
+    pointer top = getStackTop(s, stack);
+    pointer firstTop = findStackNonEmptyFrameTop(s, bottom, top);
+
+    if (NULL == firstTop) {
+      // printf("makeInitialElem: skipping stack "FMTOBJPTR"\n", op);
+      return FALSE;
+    }
+
+    assert(firstTop > bottom);
+
+    result->op = op;
+    result->data.stack.topCursor = firstTop;
+    result->data.stack.frameOffsetsIdx = 0;
+
+    // printf("makeInitialElem: push stack "FMTOBJPTR"\n", op);
+    return TRUE;
+  }
+
+  DIE("makeInitialElem: cannot handle tag %u", tag);
+  return FALSE;
+}
+
+
 void CC_workList_push(
-  __attribute__((unused)) GC_state s,
+  GC_state s,
   CC_workList w,
   objptr op)
 {
+  struct CC_workList_elem elem;
+  if (!makeInitialElem(s, op, &elem))
+    return;
+
   HM_chunkList list = &(w->storage);
   HM_chunk chunk = w->currentChunk;
-  size_t opsz = sizeof(objptr);
+  size_t elemSize = sizeof(struct CC_workList_elem);
 
-  if (HM_getChunkSizePastFrontier(chunk) < opsz) {
+  if (HM_getChunkSizePastFrontier(chunk) < elemSize) {
     if (chunk->nextChunk != NULL) {
       chunk = chunk->nextChunk; // this will be an empty chunk
     } else {
-      chunk = HM_allocateChunk(list, opsz);
+      chunk = HM_allocateChunk(list, elemSize);
     }
     w->currentChunk = chunk;
   }
 
   assert(NULL != chunk);
-  assert(HM_getChunkSizePastFrontier(chunk) >= opsz);
+  assert(HM_getChunkSizePastFrontier(chunk) >= elemSize);
   assert(chunk == w->currentChunk);
 
   pointer frontier = HM_getChunkFrontier(chunk);
   HM_updateChunkFrontierInList(
     list,
     chunk,
-    frontier + opsz);
+    frontier + elemSize);
+
+  *(CC_workList_elem)frontier = elem;
+  return;
+}
 
-  *((objptr*)frontier) = op;
 
+struct advanceOneFieldResult {
+  objptr* field;
+  bool objectDone;
+};
+
+void advanceOneField(
+  GC_state s,
+  CC_workList_elem elem,
+  struct advanceOneFieldResult * result)
+{
+  pointer p = objptrToPointer(elem->op, NULL);
+
+  // inspect the object
+  GC_header header;
+  uint16_t bytesNonObjptrs;
+  uint16_t numObjptrs;
+  GC_objectTypeTag tag;
+  header = getHeader(p);
+  splitHeader(s, header, &tag, NULL, &bytesNonObjptrs, &numObjptrs);
+
+  // ======================== NORMAL OBJECTS ========================
+
+  if (NORMAL_TAG == tag) {
+    uint16_t objptrIdx = elem->data.normal.objptrIdx;
+    assert(objptrIdx < numObjptrs);
+    result->field = (objptr*)(p + bytesNonObjptrs + (objptrIdx * OBJPTR_SIZE));
+    result->objectDone = (objptrIdx+1 == numObjptrs);
+    elem->data.normal.objptrIdx++;
+    return;
+  }
+
+  // ======================== SEQUENCE OBJECTS ========================
+
+  if (SEQUENCE_TAG == tag) {
+    GC_sequenceLength numCells = getSequenceLength(p);
+    size_t bytesPerCell = bytesNonObjptrs + (numObjptrs * OBJPTR_SIZE);
+
+    size_t cellIdx = elem->data.sequence.cellIdx;
+    uint16_t objptrIdx = elem->data.sequence.objptrIdx;
+    assert(cellIdx < numCells);
+    assert(objptrIdx < numObjptrs);
+
+    result->field =
+      (objptr*)(
+        p                            // object start
+        + (cellIdx * bytesPerCell)   // cell offset
+        + bytesNonObjptrs            // objptrs offset
+        + (objptrIdx * OBJPTR_SIZE)  // current objptr offset
+      );
+
+    result->objectDone = FALSE;
+
+    elem->data.sequence.objptrIdx++;
+    if (objptrIdx+1 == numObjptrs) {
+      elem->data.sequence.cellIdx++;
+      elem->data.sequence.objptrIdx = 0;
+      if (cellIdx+1 == numCells)
+        result->objectDone = TRUE;
+    }
+    return;
+  }
+
+  // ======================== STACK OBJECTS ========================
+
+  if (STACK_TAG == tag) {
+    GC_stack stack = (GC_stack)p;
+    pointer bottom = getStackBottom(s, stack);
+    pointer top = elem->data.stack.topCursor;
+    unsigned int i = elem->data.stack.frameOffsetsIdx;
+
+    // printf(
+    //   "advanceOneField: stack "FMTOBJPTR" top="FMTPTR" bottom="FMTPTR" i=%u\n",
+    //   (uintptr_t)elem->op,
+    //   (uintptr_t)top,
+    //   (uintptr_t)bottom,
+    //   i
+    // );
+
+    /* Invariant: top points just past a "return address". */
+    GC_returnAddress returnAddress =
+      *((GC_returnAddress*)(top - GC_RETURNADDRESS_SIZE));
+    GC_frameInfo frameInfo = getFrameInfoFromReturnAddress(s, returnAddress);
+    // index zero of this array is size
+    GC_frameOffsets frameOffsets = frameInfo->offsets;
+    pointer frameStart = top - frameInfo->size;
+
+    assert(frameOffsets[0] > 0);
+    assert(i < frameOffsets[0]);
+    assert(frameStart >= bottom);
+
+    result->field = (objptr*)(frameStart + frameOffsets[i+1]);
+    elem->data.stack.frameOffsetsIdx++;
+
+    result->objectDone = FALSE;
+
+    if (i+1 == frameOffsets[0]) {
+      /** walk backwards until we find a frame that has at least one objptr
+        * or, until we find the end of the stack.
+        */
+      pointer newTop = findStackNonEmptyFrameTop(s, bottom, frameStart);
+
+      if (NULL == newTop) {
+        result->objectDone = TRUE;
+      } else {
+        elem->data.stack.topCursor = newTop;
+        elem->data.stack.frameOffsetsIdx = 0;
+      }
+    }
+
+    return;
+  }
+
+  // ======================== OTHERWISE... ========================
+
+  DIE("advanceOneField: cannot handle tag %u", tag);
   return;
 }
 
 
-objptr CC_workList_pop(
+objptr* CC_workList_pop(
   GC_state s,
   CC_workList w)
 {
@@ -81,7 +280,7 @@ objptr CC_workList_pop(
     HM_chunk prevChunk = chunk->prevChunk;
     if (prevChunk == NULL) {
       // whole worklist is empty
-      return BOGUS_OBJPTR;
+      return NULL;
     }
 
     /** Otherwise, there is a chunk before us. It's now safe (for cost
@@ -101,19 +300,25 @@ objptr CC_workList_pop(
   }
 
   assert(w->currentChunk == chunk);
-  assert(HM_getChunkFrontier(chunk) >= HM_getChunkStart(chunk) + sizeof(objptr));
+  assert(HM_getChunkFrontier(chunk) >= HM_getChunkStart(chunk) + sizeof(struct CC_workList_elem));
 
   pointer frontier = HM_getChunkFrontier(chunk);
-  pointer newFrontier = frontier - sizeof(objptr);
+  pointer elemPtr = frontier - sizeof(struct CC_workList_elem);
+  CC_workList_elem elem = (CC_workList_elem)elemPtr;
 
-  HM_updateChunkFrontierInList(
-    list,
-    chunk,
-    newFrontier);
+  struct advanceOneFieldResult r;
+  advanceOneField(s, elem, &r);
 
-  objptr result = *((objptr*)newFrontier);
+  if (r.objectDone) {
+    pointer newFrontier = elemPtr;
+    HM_updateChunkFrontierInList(
+      list,
+      chunk,
+      newFrontier);
+  }
 
-  return result;
+  assert(NULL != r.field);
+  return r.field;
 }
 
 #endif /* MLTON_GC_INTERNAL_FUNCS */

runtime/gc/cc-work-list.h

@@ -8,11 +8,34 @@ typedef struct CC_workList {
   HM_chunk currentChunk;
 } * CC_workList;
 
+typedef struct CC_workList_elem {
+  objptr op;    // object front
+  union data {
+    struct normal {
+      uint16_t objptrIdx;
+    } normal;
+    struct sequence {
+      size_t cellIdx;
+      uint16_t objptrIdx;
+    } sequence;
+    struct stack {
+      pointer topCursor;
+      unsigned int frameOffsetsIdx;
+    } stack;
+  } data;
+} * CC_workList_elem;
+
 #else
 
 struct CC_workList;
 typedef struct CC_workList * CC_workList;
 
+struct CC_workList_elem;
+typedef struct CC_workList_elem * CC_workList_elem;
+
+struct CC_workList_range;
+typedef struct CC_workList_range * CC_workList_range;
+
 #endif /* MLTON_GC_INTERNAL_TYPES */
 
 #if (defined (MLTON_GC_INTERNAL_FUNCS))
@@ -21,8 +44,11 @@ bool CC_workList_isEmpty(GC_state s, CC_workList w);
 void CC_workList_init(GC_state s, CC_workList w);
 void CC_workList_push(GC_state s, CC_workList w, objptr op);
 
-// returns BOGUS_OBJPTR if empty
-objptr CC_workList_pop(GC_state s, CC_workList w);
+/** Returns a single field of an object that still needs to be traced.
+  * So, note that a single push can result in many pops.
+  *
+  * Returns NULL if work list is empty */
+objptr* CC_workList_pop(GC_state s, CC_workList w);
 
 #endif /* MLTON_GC_INTERNAL_FUNCS */