Adding support for red/black roots

Cargo.toml

@@ -85,6 +85,9 @@ global_alloc_bit = []
 # conservative garbage collection support
 is_mmtk_object = ["global_alloc_bit"]
 
+# Enable object pinning, in particular, enable pinning/unpinning, and its metadata
+object_pinning = []
+
 # The following two features are useful for using Immix for VMs that do not support moving GC.
 
 # Disable defragmentation for ImmixSpace.  This makes Immix a non-moving plan.

src/memory_manager.rs

@@ -761,6 +761,50 @@ pub fn add_finalizer<VM: VMBinding>(
     mmtk.finalizable_processor.lock().unwrap().add(object);
 }
 
+/// Pin an object. MMTk will make sure that the object does not move
+/// during GC. Note that action cannot happen in some plans, eg, semispace.
+/// It returns true if the pinning operation has been performed, i.e.,
+/// the object status changed from non-pinned to pinned
+///
+/// Arguments:
+/// * `object`: The object to be pinned
+#[cfg(feature = "object_pinning")]
+pub fn pin_object<VM: VMBinding>(object: ObjectReference) -> bool {
+    use crate::mmtk::SFT_MAP;
+    use crate::policy::sft_map::SFTMap;
+    SFT_MAP
+        .get_checked(object.to_address::<VM>())
+        .pin_object(object)
+}
+
+/// Unpin an object.
+/// Returns true if the unpinning operation has been performed, i.e.,
+/// the object status changed from pinned to non-pinned
+///
+/// Arguments:
+/// * `object`: The object to be pinned
+#[cfg(feature = "object_pinning")]
+pub fn unpin_object<VM: VMBinding>(object: ObjectReference) -> bool {
+    use crate::mmtk::SFT_MAP;
+    use crate::policy::sft_map::SFTMap;
+    SFT_MAP
+        .get_checked(object.to_address::<VM>())
+        .unpin_object(object)
+}
+
+/// Check whether an object is currently pinned
+///
+/// Arguments:
+/// * `object`: The object to be checked
+#[cfg(feature = "object_pinning")]
+pub fn is_pinned<VM: VMBinding>(object: ObjectReference) -> bool {
+    use crate::mmtk::SFT_MAP;
+    use crate::policy::sft_map::SFTMap;
+    SFT_MAP
+        .get_checked(object.to_address::<VM>())
+        .is_object_pinned(object)
+}
+
 /// Get an object that is ready for finalization. After each GC, if any registered object is not
 /// alive, this call will return one of the objects. MMTk will retain the liveness of those objects
 /// until they are popped through this call. Once an object is popped, it is the responsibility of

src/plan/generational/gc_work.rs

@@ -22,8 +22,13 @@ impl<VM: VMBinding> ProcessEdgesWork for GenNurseryProcessEdges<VM> {
     type VM = VM;
     type ScanObjectsWorkType = ScanObjects<Self>;
 
-    fn new(edges: Vec<EdgeOf<Self>>, roots: bool, mmtk: &'static MMTK<VM>) -> Self {
-        let base = ProcessEdgesBase::new(edges, roots, mmtk);
+    fn new(
+        edges: Vec<EdgeOf<Self>>,
+        roots: bool,
+        mmtk: &'static MMTK<VM>,
+        is_immovable: bool,
+    ) -> Self {
+        let base = ProcessEdgesBase::new(edges, roots, mmtk, is_immovable);
         let gen = base.plan().generational();
         Self { gen, base }
     }
@@ -46,8 +51,13 @@ impl<VM: VMBinding> ProcessEdgesWork for GenNurseryProcessEdges<VM> {
     }
 
     #[inline(always)]
-    fn create_scan_work(&self, nodes: Vec<ObjectReference>, roots: bool) -> ScanObjects<Self> {
-        ScanObjects::<Self>::new(nodes, false, roots)
+    fn create_scan_work(
+        &self,
+        nodes: Vec<ObjectReference>,
+        roots: bool,
+        is_immovable: bool,
+    ) -> ScanObjects<Self> {
+        ScanObjects::<Self>::new(nodes, false, roots, is_immovable)
     }
 }
 
@@ -99,7 +109,7 @@ impl<E: ProcessEdgesWork> GCWork<E::VM> for ProcessModBuf<E> {
             // Scan objects in the modbuf and forward pointers
             let modbuf = std::mem::take(&mut self.modbuf);
             GCWork::do_work(
-                &mut ScanObjects::<E>::new(modbuf, false, false),
+                &mut ScanObjects::<E>::new(modbuf, false, false, false),
                 worker,
                 mmtk,
             )
@@ -138,7 +148,7 @@ impl<E: ProcessEdgesWork> GCWork<E::VM> for ProcessRegionModBuf<E> {
                 }
             }
             // Forward entries
-            GCWork::do_work(&mut E::new(edges, false, mmtk), worker, mmtk)
+            GCWork::do_work(&mut E::new(edges, false, mmtk, false), worker, mmtk)
         }
     }
 }

src/plan/tracing.rs

@@ -78,23 +78,32 @@ impl ObjectQueue for VectorQueue<ObjectReference> {
 pub struct ObjectsClosure<'a, E: ProcessEdgesWork> {
     buffer: VectorQueue<EdgeOf<E>>,
     worker: &'a mut GCWorker<E::VM>,
+    is_immovable: bool,
 }
 
 impl<'a, E: ProcessEdgesWork> ObjectsClosure<'a, E> {
-    pub fn new(worker: &'a mut GCWorker<E::VM>) -> Self {
+    pub fn new(worker: &'a mut GCWorker<E::VM>, is_immovable: bool) -> Self {
         Self {
             buffer: VectorQueue::new(),
             worker,
+            is_immovable,
         }
     }
 
     fn flush(&mut self) {
         let buf = self.buffer.take();
         if !buf.is_empty() {
-            self.worker.add_work(
-                WorkBucketStage::Closure,
-                E::new(buf, false, self.worker.mmtk),
-            );
+            if self.is_immovable {
+                self.worker.add_work(
+                    WorkBucketStage::ClosureImmovable,
+                    E::new(buf, false, self.worker.mmtk, true),
+                );
+            } else {
+                self.worker.add_work(
+                    WorkBucketStage::Closure,
+                    E::new(buf, false, self.worker.mmtk, false),
+                );
+            }
         }
     }
 }

src/policy/copyspace.rs

@@ -36,6 +36,21 @@ impl<VM: VMBinding> SFT for CopySpace<VM> {
         !self.is_from_space() || object_forwarding::is_forwarded::<VM>(object)
     }
 
+    #[cfg(feature = "object_pinning")]
+    fn pin_object(&self, _object: ObjectReference) -> bool {
+        panic!("Cannot pin/unpin objects of CopySpace.")
+    }
+
+    #[cfg(feature = "object_pinning")]
+    fn unpin_object(&self, _object: ObjectReference) -> bool {
+        panic!("Cannot pin/unpin objects of CopySpace.")
+    }
+
+    #[cfg(feature = "object_pinning")]
+    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
+        false
+    }
+
     fn is_movable(&self) -> bool {
         true
     }

src/policy/immix/immixspace.rs

@@ -63,9 +63,22 @@ impl<VM: VMBinding> SFT for ImmixSpace<VM> {
             self.is_marked(object, self.mark_state) || ForwardingWord::is_forwarded::<VM>(object)
         }
     }
+    #[cfg(feature = "object_pinning")]
+    fn pin_object(&self, object: ObjectReference) -> bool {
+        VM::VMObjectModel::LOCAL_PINNING_BIT_SPEC.pin_object::<VM>(object)
+    }
+    #[cfg(feature = "object_pinning")]
+    fn unpin_object(&self, object: ObjectReference) -> bool {
+        VM::VMObjectModel::LOCAL_PINNING_BIT_SPEC.unpin_object::<VM>(object)
+    }
+    #[cfg(feature = "object_pinning")]
+    fn is_object_pinned(&self, object: ObjectReference) -> bool {
+        VM::VMObjectModel::LOCAL_PINNING_BIT_SPEC.is_object_pinned::<VM>(object)
+    }
     fn is_movable(&self) -> bool {
         super::DEFRAG
     }
+
     #[cfg(feature = "sanity")]
     fn is_sane(&self) -> bool {
         true
@@ -164,6 +177,8 @@ impl<VM: VMBinding> ImmixSpace<VM> {
                 MetadataSpec::OnSide(Block::MARK_TABLE),
                 MetadataSpec::OnSide(ChunkMap::ALLOC_TABLE),
                 *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
+                #[cfg(feature = "object_pinning")]
+                *VM::VMObjectModel::LOCAL_PINNING_BIT_SPEC,
             ]
         } else {
             vec![
@@ -172,6 +187,8 @@ impl<VM: VMBinding> ImmixSpace<VM> {
                 MetadataSpec::OnSide(Block::MARK_TABLE),
                 MetadataSpec::OnSide(ChunkMap::ALLOC_TABLE),
                 *VM::VMObjectModel::LOCAL_MARK_BIT_SPEC,
+                #[cfg(feature = "object_pinning")]
+                *VM::VMObjectModel::LOCAL_PINNING_BIT_SPEC,
             ]
         })
     }
@@ -483,7 +500,7 @@ impl<VM: VMBinding> ImmixSpace<VM> {
             {
                 if new_object == object {
                     debug_assert!(
-                        self.is_marked(object, self.mark_state) || self.defrag.space_exhausted() || Self::is_pinned(object),
+                        self.is_marked(object, self.mark_state) || self.defrag.space_exhausted() || self.is_pinned(object),
                         "Forwarded object is the same as original object {} even though it should have been copied",
                         object,
                     );
@@ -501,17 +518,12 @@ impl<VM: VMBinding> ImmixSpace<VM> {
         } else if self.is_marked(object, self.mark_state) {
             // We won the forwarding race but the object is already marked so we clear the
             // forwarding status and return the unmoved object
-            debug_assert!(
-                self.defrag.space_exhausted() || Self::is_pinned(object),
-                "Forwarded object is the same as original object {} even though it should have been copied",
-                object,
-            );
             ForwardingWord::clear_forwarding_bits::<VM>(object);
             object
         } else {
             // We won the forwarding race; actually forward and copy the object if it is not pinned
             // and we have sufficient space in our copy allocator
-            let new_object = if Self::is_pinned(object) || self.defrag.space_exhausted() {
+            let new_object = if self.is_pinned(object) || self.defrag.space_exhausted() {
                 self.attempt_mark(object, self.mark_state);
                 ForwardingWord::clear_forwarding_bits::<VM>(object);
                 Block::containing::<VM>(object).set_state(BlockState::Marked);
@@ -582,8 +594,11 @@ impl<VM: VMBinding> ImmixSpace<VM> {
 
     /// Check if an object is pinned.
     #[inline(always)]
-    fn is_pinned(_object: ObjectReference) -> bool {
-        // TODO(wenyuzhao): Object pinning not supported yet.
+    fn is_pinned(&self, _object: ObjectReference) -> bool {
+        #[cfg(feature = "object_pinning")]
+        return self.is_object_pinned(_object);
+
+        #[cfg(not(feature = "object_pinning"))]
         false
     }
 

src/policy/immortalspace.rs

@@ -45,6 +45,18 @@ impl<VM: VMBinding> SFT for ImmortalSpace<VM> {
         );
         old_value == self.mark_state
     }
+    #[cfg(feature = "object_pinning")]
+    fn pin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+    #[cfg(feature = "object_pinning")]
+    fn unpin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+    #[cfg(feature = "object_pinning")]
+    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
+        true
+    }
     fn is_movable(&self) -> bool {
         false
     }

src/policy/largeobjectspace.rs

@@ -43,6 +43,18 @@ impl<VM: VMBinding> SFT for LargeObjectSpace<VM> {
     fn is_live(&self, object: ObjectReference) -> bool {
         self.test_mark_bit(object, self.mark_state)
     }
+    #[cfg(feature = "object_pinning")]
+    fn pin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+    #[cfg(feature = "object_pinning")]
+    fn unpin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+    #[cfg(feature = "object_pinning")]
+    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
+        true
+    }
     fn is_movable(&self) -> bool {
         false
     }

src/policy/lockfreeimmortalspace.rs

@@ -50,6 +50,18 @@ impl<VM: VMBinding> SFT for LockFreeImmortalSpace<VM> {
     fn is_live(&self, _object: ObjectReference) -> bool {
         unimplemented!()
     }
+    #[cfg(feature = "object_pinning")]
+    fn pin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+    #[cfg(feature = "object_pinning")]
+    fn unpin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+    #[cfg(feature = "object_pinning")]
+    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
+        true
+    }
     fn is_movable(&self) -> bool {
         unimplemented!()
     }

src/policy/markcompactspace.rs

@@ -52,6 +52,21 @@ impl<VM: VMBinding> SFT for MarkCompactSpace<VM> {
         Self::is_marked(object)
     }
 
+    #[cfg(feature = "object_pinning")]
+    fn pin_object(&self, _object: ObjectReference) -> bool {
+        panic!("Cannot pin/unpin objects of MarkCompactSpace.")
+    }
+
+    #[cfg(feature = "object_pinning")]
+    fn unpin_object(&self, _object: ObjectReference) -> bool {
+        panic!("Cannot pin/unpin objects of MarkCompactSpace.")
+    }
+
+    #[cfg(feature = "object_pinning")]
+    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
+        false
+    }
+
     fn is_movable(&self) -> bool {
         true
     }

src/policy/marksweepspace/malloc_ms/global.rs

@@ -71,6 +71,21 @@ impl<VM: VMBinding> SFT for MallocSpace<VM> {
         is_marked::<VM>(object, Ordering::SeqCst)
     }
 
+    #[cfg(feature = "object_pinning")]
+    fn pin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+
+    #[cfg(feature = "object_pinning")]
+    fn unpin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+
+    #[cfg(feature = "object_pinning")]
+    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
+        false
+    }
+
     fn is_movable(&self) -> bool {
         false
     }

src/policy/marksweepspace/native_ms/global.rs

@@ -89,6 +89,21 @@ impl<VM: VMBinding> SFT for MarkSweepSpace<VM> {
         VM::VMObjectModel::LOCAL_MARK_BIT_SPEC.is_marked::<VM>(object, Ordering::SeqCst)
     }
 
+    #[cfg(feature = "object_pinning")]
+    fn pin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+
+    #[cfg(feature = "object_pinning")]
+    fn unpin_object(&self, _object: ObjectReference) -> bool {
+        false
+    }
+
+    #[cfg(feature = "object_pinning")]
+    fn is_object_pinned(&self, _object: ObjectReference) -> bool {
+        false
+    }
+
     fn is_movable(&self) -> bool {
         false
     }