Allow roots to be pinned for StickyImmix nursery collections

src/plan/generational/barrier.rs

@@ -3,6 +3,7 @@
 use crate::plan::barriers::BarrierSemantics;
 use crate::plan::PlanTraceObject;
 use crate::plan::VectorQueue;
+use crate::policy::gc_work::DEFAULT_TRACE;
 use crate::scheduler::WorkBucketStage;
 use crate::util::constants::BYTES_IN_INT;
 use crate::util::*;
@@ -45,7 +46,7 @@ impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>>
         let buf = self.modbuf.take();
         if !buf.is_empty() {
             self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure]
-                .add(ProcessModBuf::<GenNurseryProcessEdges<VM, P>>::new(buf));
+                .add(ProcessModBuf::<GenNurseryProcessEdges<VM, P, DEFAULT_TRACE>>::new(buf));
         }
     }
 
@@ -54,7 +55,7 @@ impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>>
         if !buf.is_empty() {
             debug_assert!(!buf.is_empty());
             self.mmtk.scheduler.work_buckets[WorkBucketStage::Closure].add(ProcessRegionModBuf::<
-                GenNurseryProcessEdges<VM, P>,
+                GenNurseryProcessEdges<VM, P, DEFAULT_TRACE>,
             >::new(buf));
         }
     }

src/plan/generational/copying/gc_work.rs

@@ -9,7 +9,7 @@ pub struct GenCopyNurseryGCWorkContext<VM: VMBinding>(std::marker::PhantomData<V
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for GenCopyNurseryGCWorkContext<VM> {
     type VM = VM;
     type PlanType = GenCopy<VM>;
-    type DefaultProcessEdges = GenNurseryProcessEdges<Self::VM, Self::PlanType>;
+    type DefaultProcessEdges = GenNurseryProcessEdges<Self::VM, Self::PlanType, DEFAULT_TRACE>;
     type PinningProcessEdges = UnsupportedProcessEdges<VM>;
 }
 

src/plan/generational/copying/global.rs

@@ -12,6 +12,7 @@ use crate::plan::AllocationSemantics;
 use crate::plan::Plan;
 use crate::plan::PlanConstraints;
 use crate::policy::copyspace::CopySpace;
+use crate::policy::gc_work::TraceKind;
 use crate::policy::space::Space;
 use crate::scheduler::*;
 use crate::util::alloc::allocators::AllocatorSelector;
@@ -183,13 +184,14 @@ impl<VM: VMBinding> GenerationalPlan for GenCopy<VM> {
 }
 
 impl<VM: VMBinding> GenerationalPlanExt<VM> for GenCopy<VM> {
-    fn trace_object_nursery<Q: ObjectQueue>(
+    fn trace_object_nursery<Q: ObjectQueue, const KIND: TraceKind>(
         &self,
         queue: &mut Q,
         object: ObjectReference,
         worker: &mut GCWorker<VM>,
     ) -> ObjectReference {
-        self.gen.trace_object_nursery(queue, object, worker)
+        self.gen
+            .trace_object_nursery::<Q, KIND>(queue, object, worker)
     }
 }
 

src/plan/generational/gc_work.rs

@@ -1,6 +1,8 @@
 use atomic::Ordering;
 
 use crate::plan::PlanTraceObject;
+use crate::plan::VectorObjectQueue;
+use crate::policy::gc_work::TraceKind;
 use crate::scheduler::{gc_work::*, GCWork, GCWorker, WorkBucketStage};
 use crate::util::ObjectReference;
 use crate::vm::edge_shape::{Edge, MemorySlice};
@@ -14,13 +16,17 @@ use super::global::GenerationalPlanExt;
 /// Process edges for a nursery GC. This type is provided if a generational plan does not use
 /// [`crate::scheduler::gc_work::SFTProcessEdges`]. If a plan uses `SFTProcessEdges`,
 /// it does not need to use this type.
-pub struct GenNurseryProcessEdges<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>> {
+pub struct GenNurseryProcessEdges<
+    VM: VMBinding,
+    P: GenerationalPlanExt<VM> + PlanTraceObject<VM>,
+    const KIND: TraceKind,
+> {
     plan: &'static P,
     base: ProcessEdgesBase<VM>,
 }
 
-impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>> ProcessEdgesWork
-    for GenNurseryProcessEdges<VM, P>
+impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>, const KIND: TraceKind>
+    ProcessEdgesWork for GenNurseryProcessEdges<VM, P, KIND>
 {
     type VM = VM;
     type ScanObjectsWorkType = PlanScanObjects<Self, P>;
@@ -35,14 +41,19 @@ impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>> ProcessEdg
         let plan = base.plan().downcast_ref().unwrap();
         Self { plan, base }
     }
+
     fn trace_object(&mut self, object: ObjectReference) -> ObjectReference {
         debug_assert!(!object.is_null());
 
         // We cannot borrow `self` twice in a call, so we extract `worker` as a local variable.
         let worker = self.worker();
-        self.plan
-            .trace_object_nursery(&mut self.base.nodes, object, worker)
+        self.plan.trace_object_nursery::<VectorObjectQueue, KIND>(
+            &mut self.base.nodes,
+            object,
+            worker,
+        )
     }
+
     fn process_edge(&mut self, slot: EdgeOf<Self>) {
         let object = slot.load();
         if object.is_null() {
@@ -62,17 +73,17 @@ impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>> ProcessEdg
     }
 }
 
-impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>> Deref
-    for GenNurseryProcessEdges<VM, P>
+impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>, const KIND: TraceKind> Deref
+    for GenNurseryProcessEdges<VM, P, KIND>
 {
     type Target = ProcessEdgesBase<VM>;
     fn deref(&self) -> &Self::Target {
         &self.base
     }
 }
 
-impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>> DerefMut
-    for GenNurseryProcessEdges<VM, P>
+impl<VM: VMBinding, P: GenerationalPlanExt<VM> + PlanTraceObject<VM>, const KIND: TraceKind>
+    DerefMut for GenNurseryProcessEdges<VM, P, KIND>
 {
     fn deref_mut(&mut self) -> &mut Self::Target {
         &mut self.base

src/plan/generational/global.rs

@@ -3,6 +3,7 @@ use crate::plan::global::CreateSpecificPlanArgs;
 use crate::plan::ObjectQueue;
 use crate::plan::Plan;
 use crate::policy::copyspace::CopySpace;
+use crate::policy::gc_work::{TraceKind, TRACE_KIND_TRANSITIVE_PIN};
 use crate::policy::space::Space;
 use crate::scheduler::*;
 use crate::util::copy::CopySemantics;
@@ -226,12 +227,17 @@ impl<VM: VMBinding> CommonGenPlan<VM> {
     }
 
     /// Trace objects for spaces in generational and common plans for a nursery GC.
-    pub fn trace_object_nursery<Q: ObjectQueue>(
+    pub fn trace_object_nursery<Q: ObjectQueue, const KIND: TraceKind>(
         &self,
         queue: &mut Q,
         object: ObjectReference,
         worker: &mut GCWorker<VM>,
     ) -> ObjectReference {
+        assert!(
+            KIND != TRACE_KIND_TRANSITIVE_PIN,
+            "A copying nursery cannot pin objects"
+        );
+
         // Evacuate nursery objects
         if self.nursery.in_space(object) {
             return self.nursery.trace_object::<Q>(
@@ -327,7 +333,7 @@ pub trait GenerationalPlan: Plan {
 pub trait GenerationalPlanExt<VM: VMBinding>: GenerationalPlan<VM = VM> {
     /// Trace an object in nursery collection. If the object is in nursery, we should call `trace_object`
     /// on the space. Otherwise, we can just return the object.
-    fn trace_object_nursery<Q: ObjectQueue>(
+    fn trace_object_nursery<Q: ObjectQueue, const KIND: TraceKind>(
         &self,
         queue: &mut Q,
         object: ObjectReference,

src/plan/generational/immix/gc_work.rs

@@ -1,6 +1,7 @@
 use super::global::GenImmix;
 use crate::plan::generational::gc_work::GenNurseryProcessEdges;
 use crate::policy::gc_work::TraceKind;
+use crate::policy::gc_work::DEFAULT_TRACE;
 use crate::scheduler::gc_work::PlanProcessEdges;
 use crate::scheduler::gc_work::UnsupportedProcessEdges;
 use crate::vm::VMBinding;
@@ -9,7 +10,7 @@ pub struct GenImmixNurseryGCWorkContext<VM: VMBinding>(std::marker::PhantomData<
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for GenImmixNurseryGCWorkContext<VM> {
     type VM = VM;
     type PlanType = GenImmix<VM>;
-    type DefaultProcessEdges = GenNurseryProcessEdges<VM, Self::PlanType>;
+    type DefaultProcessEdges = GenNurseryProcessEdges<VM, Self::PlanType, DEFAULT_TRACE>;
     type PinningProcessEdges = UnsupportedProcessEdges<VM>;
 }
 

src/plan/generational/immix/global.rs

@@ -9,6 +9,7 @@ use crate::plan::global::CreateSpecificPlanArgs;
 use crate::plan::AllocationSemantics;
 use crate::plan::Plan;
 use crate::plan::PlanConstraints;
+use crate::policy::gc_work::TraceKind;
 use crate::policy::immix::ImmixSpace;
 use crate::policy::immix::ImmixSpaceArgs;
 use crate::policy::immix::{TRACE_KIND_DEFRAG, TRACE_KIND_FAST};
@@ -215,13 +216,14 @@ impl<VM: VMBinding> GenerationalPlan for GenImmix<VM> {
 }
 
 impl<VM: VMBinding> crate::plan::generational::global::GenerationalPlanExt<VM> for GenImmix<VM> {
-    fn trace_object_nursery<Q: ObjectQueue>(
+    fn trace_object_nursery<Q: ObjectQueue, const KIND: TraceKind>(
         &self,
         queue: &mut Q,
         object: ObjectReference,
         worker: &mut GCWorker<VM>,
     ) -> ObjectReference {
-        self.gen.trace_object_nursery(queue, object, worker)
+        self.gen
+            .trace_object_nursery::<Q, KIND>(queue, object, worker)
     }
 }
 

src/plan/sticky/immix/gc_work.rs

@@ -1,16 +1,19 @@
 use crate::policy::gc_work::TraceKind;
+use crate::policy::gc_work::DEFAULT_TRACE;
 use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN;
 use crate::scheduler::gc_work::PlanProcessEdges;
 use crate::{plan::generational::gc_work::GenNurseryProcessEdges, vm::VMBinding};
 
 use super::global::StickyImmix;
 
 pub struct StickyImmixNurseryGCWorkContext<VM: VMBinding>(std::marker::PhantomData<VM>);
+
 impl<VM: VMBinding> crate::scheduler::GCWorkContext for StickyImmixNurseryGCWorkContext<VM> {
     type VM = VM;
     type PlanType = StickyImmix<VM>;
-    type DefaultProcessEdges = GenNurseryProcessEdges<VM, Self::PlanType>;
-    type PinningProcessEdges = GenNurseryProcessEdges<VM, Self::PlanType>;
+    type DefaultProcessEdges = GenNurseryProcessEdges<VM, Self::PlanType, DEFAULT_TRACE>;
+    type PinningProcessEdges =
+        GenNurseryProcessEdges<VM, Self::PlanType, TRACE_KIND_TRANSITIVE_PIN>;
 }
 
 pub struct StickyImmixMatureGCWorkContext<VM: VMBinding, const KIND: TraceKind>(

src/plan/sticky/immix/global.rs

@@ -4,7 +4,10 @@ use crate::plan::global::CreateGeneralPlanArgs;
 use crate::plan::global::CreateSpecificPlanArgs;
 use crate::plan::immix;
 use crate::plan::PlanConstraints;
+use crate::policy::gc_work::TraceKind;
+use crate::policy::gc_work::TRACE_KIND_TRANSITIVE_PIN;
 use crate::policy::immix::ImmixSpace;
+use crate::policy::immix::TRACE_KIND_FAST;
 use crate::policy::sft::SFT;
 use crate::policy::space::Space;
 use crate::util::copy::CopyConfig;
@@ -91,7 +94,7 @@ impl<VM: VMBinding> Plan for StickyImmix<VM> {
         } else {
             info!("Full heap GC");
             use crate::plan::immix::Immix;
-            use crate::policy::immix::{TRACE_KIND_DEFRAG, TRACE_KIND_FAST};
+            use crate::policy::immix::TRACE_KIND_DEFRAG;
             Immix::schedule_immix_full_heap_collection::<
                 StickyImmix<VM>,
                 StickyImmixMatureGCWorkContext<VM, TRACE_KIND_FAST>,
@@ -227,7 +230,7 @@ impl<VM: VMBinding> GenerationalPlan for StickyImmix<VM> {
 }
 
 impl<VM: VMBinding> crate::plan::generational::global::GenerationalPlanExt<VM> for StickyImmix<VM> {
-    fn trace_object_nursery<Q: crate::ObjectQueue>(
+    fn trace_object_nursery<Q: crate::ObjectQueue, const KIND: TraceKind>(
         &self,
         queue: &mut Q,
         object: crate::util::ObjectReference,
@@ -239,7 +242,15 @@ impl<VM: VMBinding> crate::plan::generational::global::GenerationalPlanExt<VM> f
                 trace!("Immix mature object {}, skip", object);
                 return object;
             } else {
-                let object = if crate::policy::immix::PREFER_COPY_ON_NURSERY_GC {
+                let object = if KIND == TRACE_KIND_TRANSITIVE_PIN || KIND == TRACE_KIND_FAST {
+                    trace!(
+                        "Immix nursery object {} is being traced without moving",
+                        object
+                    );
+                    self.immix
+                        .immix_space
+                        .trace_object_without_moving(queue, object)
+                } else if crate::policy::immix::PREFER_COPY_ON_NURSERY_GC {
                     let ret = self.immix.immix_space.trace_object_with_opportunistic_copy(
                         queue,
                         object,
@@ -255,7 +266,7 @@ impl<VM: VMBinding> crate::plan::generational::global::GenerationalPlanExt<VM> f
                         if ret == object {
                             "".to_string()
                         } else {
-                            format!(" -> new object {}", ret)
+                            format!("-> new object {}", ret)
                         }
                     );
                     ret