This PR enables transitively pinning (TP) objects from particular roots for Immix/StickyImmix

Cargo.toml

@@ -107,6 +107,10 @@ object_pinning = []
 # Disable any object copying in Immix. This makes Immix a non-moving policy.
 immix_non_moving = []
 
+# Disable any object copying in nursery GC for Sticky Immix.
+sticky_immix_non_moving_nursery = []
+
+
 # Reduce block size for ImmixSpace.  This mitigates fragmentation when defrag is disabled.
 immix_smaller_block = []
 # Zero the unmarked lines after a GC cycle in immix. This helps debug untraced objects.

docs/tutorial/code/mygc_semispace/gc_work.rs

@@ -1,6 +1,6 @@
 // ANCHOR: imports
 use super::global::MyGC;
-use crate::scheduler::gc_work::*;
+use crate::scheduler::{gc_work::*, WorkBucketStage};
 use crate::vm::VMBinding;
 use std::ops::{Deref, DerefMut};
 // ANCHOR_END: imports
@@ -11,24 +11,26 @@ impl<VM: VMBinding> crate::scheduler::GCWorkContext for MyGCWorkContext<VM> {
     type VM = VM;
     type PlanType = MyGC<VM>;
     type ProcessEdgesWorkType = SFTProcessEdges<Self::VM>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<Self::VM>;
 }
 // ANCHOR_END: workcontext_sft
 
 // ANCHOR: workcontext_plan
-use crate::scheduler::gc_work::PlanProcessEdges;
 use crate::policy::gc_work::DEFAULT_TRACE;
+use crate::scheduler::gc_work::PlanProcessEdges;
 pub struct MyGCWorkContext2<VM: VMBinding>(std::marker::PhantomData<VM>);
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for MyGCWorkContext2<VM> {
     type VM = VM;
     type PlanType = MyGC<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<Self::VM, MyGC<VM>, DEFAULT_TRACE>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<Self::VM>;
 }
 // ANCHOR: workcontext_plan
 
-use crate::util::ObjectReference;
+use crate::policy::space::Space;
 use crate::util::copy::CopySemantics;
+use crate::util::ObjectReference;
 use crate::MMTK;
-use crate::policy::space::Space;
 
 // ANCHOR: mygc_process_edges
 pub struct MyGCProcessEdges<VM: VMBinding> {
@@ -38,12 +40,17 @@ pub struct MyGCProcessEdges<VM: VMBinding> {
 // ANCHOR_END: mygc_process_edges
 
 // ANCHOR: mygc_process_edges_impl
-impl<VM:VMBinding> ProcessEdgesWork for MyGCProcessEdges<VM> {
+impl<VM: VMBinding> ProcessEdgesWork for MyGCProcessEdges<VM> {
     type VM = VM;
     type ScanObjectsWorkType = ScanObjects<Self>;
 
-    fn new(edges: Vec<EdgeOf<Self>>, roots: bool, mmtk: &'static MMTK<VM>) -> Self {
-        let base = ProcessEdgesBase::new(edges, roots, mmtk);
+    fn new(
+        edges: Vec<EdgeOf<Self>>,
+        roots: bool,
+        mmtk: &'static MMTK<VM>,
+        bucket: WorkBucketStage,
+    ) -> Self {
+        let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket);
         let plan = base.plan().downcast_ref::<MyGC<VM>>().unwrap();
         Self { base, plan }
     }
@@ -74,7 +81,7 @@ impl<VM:VMBinding> ProcessEdgesWork for MyGCProcessEdges<VM> {
     }
 
     fn create_scan_work(&self, nodes: Vec<ObjectReference>, roots: bool) -> ScanObjects<Self> {
-        ScanObjects::<Self>::new(nodes, false, roots)
+        ScanObjects::<Self>::new(nodes, false, roots, self.bucket)
     }
 }
 // ANCHOR_END: mygc_process_edges_impl
@@ -100,5 +107,6 @@ impl<VM: VMBinding> crate::scheduler::GCWorkContext for MyGCWorkContext3<VM> {
     type VM = VM;
     type PlanType = MyGC<VM>;
     type ProcessEdgesWorkType = MyGCProcessEdges<Self::VM>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<Self::VM>;
 }
 // ANCHOR: workcontext_mygc

src/plan/generational/copying/gc_work.rs

@@ -3,18 +3,20 @@ use crate::plan::generational::gc_work::GenNurseryProcessEdges;
 use crate::vm::*;
 
 use crate::policy::gc_work::DEFAULT_TRACE;
-use crate::scheduler::gc_work::PlanProcessEdges;
+use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges};
 
 pub struct GenCopyNurseryGCWorkContext<VM: VMBinding>(std::marker::PhantomData<VM>);
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for GenCopyNurseryGCWorkContext<VM> {
     type VM = VM;
     type PlanType = GenCopy<VM>;
     type ProcessEdgesWorkType = GenNurseryProcessEdges<Self::VM, Self::PlanType>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<VM>;
 }
 
 pub struct GenCopyGCWorkContext<VM: VMBinding>(std::marker::PhantomData<VM>);
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for GenCopyGCWorkContext<VM> {
     type VM = VM;
     type PlanType = GenCopy<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<Self::VM, GenCopy<VM>, DEFAULT_TRACE>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<VM>;
 }

src/plan/generational/gc_work.rs

@@ -1,7 +1,7 @@
 use atomic::Ordering;
 
 use crate::plan::PlanTraceObject;
-use crate::scheduler::{gc_work::*, GCWork, GCWorker};
+use crate::scheduler::{gc_work::*, GCWork, GCWorker, WorkBucketStage};
 use crate::util::ObjectReference;
 use crate::vm::edge_shape::{Edge, MemorySlice};
 use crate::vm::*;
@@ -25,8 +25,13 @@ impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>> ProcessEdg
     type VM = VM;
     type ScanObjectsWorkType = PlanScanObjects<Self, P>;
 
-    fn new(edges: Vec<EdgeOf<Self>>, roots: bool, mmtk: &'static MMTK<VM>) -> Self {
-        let base = ProcessEdgesBase::new(edges, roots, mmtk);
+    fn new(
+        edges: Vec<EdgeOf<Self>>,
+        roots: bool,
+        mmtk: &'static MMTK<VM>,
+        bucket: WorkBucketStage,
+    ) -> Self {
+        let base = ProcessEdgesBase::new(edges, roots, mmtk, bucket);
         let plan = base.plan().downcast_ref().unwrap();
         Self { plan, base }
     }
@@ -51,7 +56,7 @@ impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>> ProcessEdg
         nodes: Vec<ObjectReference>,
         roots: bool,
     ) -> Self::ScanObjectsWorkType {
-        PlanScanObjects::new(self.plan, nodes, false, roots)
+        PlanScanObjects::new(self.plan, nodes, false, roots, self.bucket)
     }
 }
 
@@ -106,7 +111,7 @@ impl<E: ProcessEdgesWork> GCWork<E::VM> for ProcessModBuf<E> {
             // Scan objects in the modbuf and forward pointers
             let modbuf = std::mem::take(&mut self.modbuf);
             GCWork::do_work(
-                &mut ScanObjects::<E>::new(modbuf, false, false),
+                &mut ScanObjects::<E>::new(modbuf, false, false, WorkBucketStage::Closure),
                 worker,
                 mmtk,
             )
@@ -144,7 +149,11 @@ impl<E: ProcessEdgesWork> GCWork<E::VM> for ProcessRegionModBuf<E> {
                 }
             }
             // Forward entries
-            GCWork::do_work(&mut E::new(edges, false, mmtk), worker, mmtk)
+            GCWork::do_work(
+                &mut E::new(edges, false, mmtk, WorkBucketStage::Closure),
+                worker,
+                mmtk,
+            )
         }
     }
 }

src/plan/generational/immix/gc_work.rs

@@ -1,6 +1,7 @@
 use super::global::GenImmix;
 use crate::plan::generational::gc_work::GenNurseryProcessEdges;
 use crate::policy::gc_work::TraceKind;
+use crate::policy::gc_work::TRACE_KIND_IMMOVABLE;
 use crate::scheduler::gc_work::PlanProcessEdges;
 use crate::vm::VMBinding;
 
@@ -9,6 +10,7 @@ impl<VM: VMBinding> crate::scheduler::GCWorkContext for GenImmixNurseryGCWorkCon
     type VM = VM;
     type PlanType = GenImmix<VM>;
     type ProcessEdgesWorkType = GenNurseryProcessEdges<VM, Self::PlanType>;
+    type ImmovableProcessEdges = GenNurseryProcessEdges<VM, Self::PlanType>;
 }
 
 pub(super) struct GenImmixMatureGCWorkContext<VM: VMBinding, const KIND: TraceKind>(
@@ -20,4 +22,5 @@ impl<VM: VMBinding, const KIND: TraceKind> crate::scheduler::GCWorkContext
     type VM = VM;
     type PlanType = GenImmix<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<VM, GenImmix<VM>, KIND>;
+    type ImmovableProcessEdges = PlanProcessEdges<VM, GenImmix<VM>, TRACE_KIND_IMMOVABLE>;
 }

src/plan/generational/mod.rs

@@ -23,7 +23,7 @@ use super::mutator_context::ReservedAllocators;
 pub mod barrier;
 /// Generational copying (GenCopy)
 pub mod copying;
-/// Generational immix (GenImmix)
+// /// Generational immix (GenImmix)
 pub mod immix;
 
 // Common generational code

src/plan/immix/gc_work.rs

@@ -1,5 +1,6 @@
 use super::global::Immix;
 use crate::policy::gc_work::TraceKind;
+use crate::policy::gc_work::TRACE_KIND_IMMOVABLE;
 use crate::scheduler::gc_work::PlanProcessEdges;
 use crate::vm::VMBinding;
 
@@ -12,4 +13,5 @@ impl<VM: VMBinding, const KIND: TraceKind> crate::scheduler::GCWorkContext
     type VM = VM;
     type PlanType = Immix<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<VM, Immix<VM>, KIND>;
+    type ImmovableProcessEdges = PlanProcessEdges<VM, Immix<VM>, TRACE_KIND_IMMOVABLE>;
 }

src/plan/immix/global.rs

@@ -173,8 +173,8 @@ impl<VM: VMBinding> Immix<VM> {
     /// to schedule a full heap collection. A plan must call set_collection_kind and set_gc_status before this method.
     pub(crate) fn schedule_immix_full_heap_collection<
         PlanType: Plan<VM = VM>,
-        FastContext: 'static + GCWorkContext<VM = VM, PlanType = PlanType>,
-        DefragContext: 'static + GCWorkContext<VM = VM, PlanType = PlanType>,
+        FastContext: Send + 'static + GCWorkContext<VM = VM, PlanType = PlanType>,
+        DefragContext: Send + 'static + GCWorkContext<VM = VM, PlanType = PlanType>,
     >(
         plan: &'static DefragContext::PlanType,
         immix_space: &ImmixSpace<VM>,

src/plan/markcompact/gc_work.rs

@@ -50,12 +50,13 @@ impl<VM: VMBinding> GCWork<VM> for UpdateReferences<VM> {
             .get_and_clear_worker_live_bytes();
 
         for mutator in VM::VMActivePlan::mutators() {
-            mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots]
-                .add(ScanMutatorRoots::<ForwardingProcessEdges<VM>>(mutator));
+            mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots].add(ScanMutatorRoots::<
+                MarkCompactForwardingGCWorkContext<VM>,
+            >(mutator));
         }
 
         mmtk.scheduler.work_buckets[WorkBucketStage::SecondRoots]
-            .add(ScanVMSpecificRoots::<ForwardingProcessEdges<VM>>::new());
+            .add(ScanVMSpecificRoots::<MarkCompactForwardingGCWorkContext<VM>>::new());
     }
 }
 
@@ -92,4 +93,13 @@ impl<VM: VMBinding> crate::scheduler::GCWorkContext for MarkCompactGCWorkContext
     type VM = VM;
     type PlanType = MarkCompact<VM>;
     type ProcessEdgesWorkType = MarkingProcessEdges<VM>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<VM>;
+}
+
+pub struct MarkCompactForwardingGCWorkContext<VM: VMBinding>(std::marker::PhantomData<VM>);
+impl<VM: VMBinding> crate::scheduler::GCWorkContext for MarkCompactForwardingGCWorkContext<VM> {
+    type VM = VM;
+    type PlanType = MarkCompact<VM>;
+    type ProcessEdgesWorkType = ForwardingProcessEdges<VM>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<VM>;
 }

src/plan/markcompact/global.rs

@@ -1,8 +1,8 @@
-use super::gc_work::MarkCompactGCWorkContext;
 use super::gc_work::{
     CalculateForwardingAddress, Compact, ForwardingProcessEdges, MarkingProcessEdges,
     UpdateReferences,
 };
+use super::gc_work::{MarkCompactForwardingGCWorkContext, MarkCompactGCWorkContext};
 use crate::plan::global::CommonPlan;
 use crate::plan::global::GcStatus;
 use crate::plan::global::{BasePlan, CreateGeneralPlanArgs, CreateSpecificPlanArgs};
@@ -97,7 +97,7 @@ impl<VM: VMBinding> Plan for MarkCompact<VM> {
 
         // Stop & scan mutators (mutator scanning can happen before STW)
         scheduler.work_buckets[WorkBucketStage::Unconstrained]
-            .add(StopMutators::<MarkingProcessEdges<VM>>::new());
+            .add(StopMutators::<MarkCompactForwardingGCWorkContext<VM>>::new());
 
         // Prepare global/collectors/mutators
         scheduler.work_buckets[WorkBucketStage::Prepare]

src/plan/marksweep/gc_work.rs

@@ -8,4 +8,5 @@ impl<VM: VMBinding> crate::scheduler::GCWorkContext for MSGCWorkContext<VM> {
     type VM = VM;
     type PlanType = MarkSweep<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<Self::VM, MarkSweep<VM>, DEFAULT_TRACE>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<VM>;
 }

src/plan/mod.rs

@@ -40,9 +40,9 @@ pub use plan_constraints::DEFAULT_PLAN_CONSTRAINTS;
 mod tracing;
 pub use tracing::{ObjectQueue, ObjectsClosure, VectorObjectQueue, VectorQueue};
 
-/// Generational plans (with a copying nursery)
+// /// Generational plans (with a copying nursery)
 mod generational;
-/// Sticky plans (using sticky marks for generational behaviors without a copying nursery)
+// /// Sticky plans (using sticky marks for generational behaviors without a copying nursery)
 mod sticky;
 
 mod immix;

src/plan/pageprotect/gc_work.rs

@@ -1,11 +1,12 @@
 use super::global::PageProtect;
 use crate::policy::gc_work::DEFAULT_TRACE;
-use crate::scheduler::gc_work::PlanProcessEdges;
+use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges};
 use crate::vm::VMBinding;
 
 pub struct PPGCWorkContext<VM: VMBinding>(std::marker::PhantomData<VM>);
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for PPGCWorkContext<VM> {
     type VM = VM;
     type PlanType = PageProtect<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<Self::VM, PageProtect<VM>, DEFAULT_TRACE>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<VM>;
 }

src/plan/semispace/gc_work.rs

@@ -1,11 +1,12 @@
 use super::global::SemiSpace;
 use crate::policy::gc_work::DEFAULT_TRACE;
-use crate::scheduler::gc_work::PlanProcessEdges;
+use crate::scheduler::gc_work::{PlanProcessEdges, UnsupportedProcessEdges};
 use crate::vm::VMBinding;
 
 pub struct SSGCWorkContext<VM: VMBinding>(std::marker::PhantomData<VM>);
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for SSGCWorkContext<VM> {
     type VM = VM;
     type PlanType = SemiSpace<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<Self::VM, SemiSpace<VM>, DEFAULT_TRACE>;
+    type ImmovableProcessEdges = UnsupportedProcessEdges<VM>;
 }

src/plan/sticky/immix/gc_work.rs

@@ -1,4 +1,5 @@
 use crate::policy::gc_work::TraceKind;
+use crate::policy::gc_work::TRACE_KIND_IMMOVABLE;
 use crate::scheduler::gc_work::PlanProcessEdges;
 use crate::{plan::generational::gc_work::GenNurseryProcessEdges, vm::VMBinding};
 
@@ -9,6 +10,7 @@ impl<VM: VMBinding> crate::scheduler::GCWorkContext for StickyImmixNurseryGCWork
     type VM = VM;
     type PlanType = StickyImmix<VM>;
     type ProcessEdgesWorkType = GenNurseryProcessEdges<VM, Self::PlanType>;
+    type ImmovableProcessEdges = GenNurseryProcessEdges<VM, Self::PlanType>;
 }
 
 pub struct StickyImmixMatureGCWorkContext<VM: VMBinding, const KIND: TraceKind>(
@@ -20,4 +22,5 @@ impl<VM: VMBinding, const KIND: TraceKind> crate::scheduler::GCWorkContext
     type VM = VM;
     type PlanType = StickyImmix<VM>;
     type ProcessEdgesWorkType = PlanProcessEdges<VM, Self::PlanType, KIND>;
+    type ImmovableProcessEdges = PlanProcessEdges<VM, Self::PlanType, TRACE_KIND_IMMOVABLE>;
 }

src/plan/tracing.rs

@@ -75,22 +75,24 @@ impl ObjectQueue for VectorQueue<ObjectReference> {
 pub struct ObjectsClosure<'a, E: ProcessEdgesWork> {
     buffer: VectorQueue<EdgeOf<E>>,
     pub(crate) worker: &'a mut GCWorker<E::VM>,
+    bucket: WorkBucketStage,
 }
 
 impl<'a, E: ProcessEdgesWork> ObjectsClosure<'a, E> {
-    pub fn new(worker: &'a mut GCWorker<E::VM>) -> Self {
+    pub fn new(worker: &'a mut GCWorker<E::VM>, bucket: WorkBucketStage) -> Self {
         Self {
             buffer: VectorQueue::new(),
             worker,
+            bucket,
         }
     }
 
     fn flush(&mut self) {
         let buf = self.buffer.take();
         if !buf.is_empty() {
             self.worker.add_work(
-                WorkBucketStage::Closure,
-                E::new(buf, false, self.worker.mmtk),
+                self.bucket,
+                E::new(buf, false, self.worker.mmtk, self.bucket),
             );
         }
     }

src/policy/copyspace.rs

@@ -1,5 +1,6 @@
 use crate::plan::{ObjectQueue, VectorObjectQueue};
 use crate::policy::copy_context::PolicyCopyContext;
+use crate::policy::gc_work::TRACE_KIND_IMMOVABLE;
 use crate::policy::sft::GCWorkerMutRef;
 use crate::policy::sft::SFT;
 use crate::policy::space::{CommonSpace, Space};
@@ -126,6 +127,10 @@ impl<VM: VMBinding> crate::policy::gc_work::PolicyTraceObject<VM> for CopySpace<
         copy: Option<CopySemantics>,
         worker: &mut GCWorker<VM>,
     ) -> ObjectReference {
+        assert!(
+            KIND != TRACE_KIND_IMMOVABLE,
+            "Copyspace does not support immovable trace."
+        );
         self.trace_object(queue, object, copy, worker)
     }
 

src/policy/gc_work.rs

@@ -3,6 +3,7 @@
 pub(crate) type TraceKind = u8;
 
 pub const DEFAULT_TRACE: u8 = u8::MAX;
+pub const TRACE_KIND_IMMOVABLE: u8 = DEFAULT_TRACE - 1;
 
 use crate::plan::ObjectQueue;
 use crate::scheduler::GCWorker;