Speed up marking of local-immune table

rm155 · Oct 26, 2024 · 356359b · 356359b
1 parent d0739e6
commit 356359b
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 7 deletions.
diff --git a/gc.c b/gc.c
@@ -617,6 +617,7 @@ typedef struct gc_function_map {
     void (*mark_and_pin)(void *objspace_ptr, VALUE obj);
     void (*mark_maybe)(void *objspace_ptr, VALUE obj);
     void (*stack_location_mark_maybe)(void *objspace_ptr, VALUE obj);
+    void (*mark_in_range)(void *objspace_ptr, VALUE obj);
     void (*mark_weak)(void *objspace_ptr, VALUE *ptr);
     void (*remove_weak)(void *objspace_ptr, VALUE parent_obj, VALUE *ptr);
     bool (*object_marked_p)(void *objspace_ptr, VALUE obj);
@@ -773,6 +774,7 @@ ruby_external_gc_init(void)
     load_external_gc_func(mark_and_pin);
     load_external_gc_func(mark_maybe);
     load_external_gc_func(stack_location_mark_maybe);
+    load_external_gc_func(mark_in_range);
     load_external_gc_func(mark_weak);
     load_external_gc_func(remove_weak);
     load_external_gc_func(object_marked_p);
@@ -865,6 +867,7 @@ ruby_external_gc_init(void)
 # define rb_gc_impl_mark_and_pin rb_gc_functions.mark_and_pin
 # define rb_gc_impl_mark_maybe rb_gc_functions.mark_maybe
 # define rb_gc_impl_stack_location_mark_maybe rb_gc_functions.stack_location_mark_maybe
+# define rb_gc_impl_mark_in_range rb_gc_functions.mark_in_range
 # define rb_gc_impl_mark_weak rb_gc_functions.mark_weak
 # define rb_gc_impl_remove_weak rb_gc_functions.remove_weak
 # define rb_gc_impl_object_marked_p rb_gc_functions.object_marked_p
@@ -2352,6 +2355,27 @@ rb_mark_set(st_table *tbl)
     st_foreach(tbl, mark_key, (st_data_t)rb_gc_get_objspace());
 }
 
+static int
+mark_local_key_no_traversal(st_data_t key, st_data_t value, st_data_t data)
+{
+    rb_gc_impl_mark_in_range(data, key);
+
+    return ST_CONTINUE;
+}
+
+void
+rb_mark_local_set(st_table *tbl)
+{
+    if (!tbl) return;
+
+        if (LIKELY(!MARK_FUNC_IN_USE(GET_RACTOR()))) {
+	    st_foreach(tbl, mark_local_key_no_traversal, (st_data_t)rb_gc_get_objspace());
+	}
+	else {
+	    st_foreach(tbl, mark_key, (st_data_t)rb_gc_get_objspace());
+	}
+}
+
 static int
 mark_keyvalue(st_data_t key, st_data_t value, st_data_t data)
 {

diff --git a/gc/default.c b/gc/default.c
@@ -4946,7 +4946,7 @@ gc_grey(rb_objspace_t *objspace, VALUE obj)
 }
 
 static bool
-in_marking_range(rb_objspace_t *objspace, VALUE obj)
+in_local_marking_range(rb_objspace_t *objspace, VALUE obj)
 {
     return !FL_TEST_RAW(obj, FL_SHAREABLE) || GET_OBJSPACE_OF_VALUE(obj) == objspace;
 }
@@ -4983,7 +4983,7 @@ confirm_global_connections(rb_objspace_t *objspace, VALUE obj)
 	    }
 	}
 	else {
-	    if (!in_marking_range(objspace, obj)) {
+	    if (!in_local_marking_range(objspace, obj)) {
 		if (is_full_marking(objspace)) {
 		    check_not_tnone(obj);
 		    mark_in_external_reference_tbl(objspace->local_gate, obj);
@@ -4996,12 +4996,8 @@ confirm_global_connections(rb_objspace_t *objspace, VALUE obj)
 }
 
 static void
-gc_mark(rb_objspace_t *objspace, VALUE obj)
+gc_mark_in_range(rb_objspace_t *objspace, VALUE obj)
 {
-    VM_ASSERT(GET_OBJSPACE_OF_VALUE(obj) == objspace || FL_TEST(obj, FL_SHAREABLE) || !using_local_limits(objspace));
-
-    GC_ASSERT(during_gc);
-    if (!confirm_global_connections(objspace, obj)) return;
     rgengc_check_relation(objspace, obj);
     if (!gc_mark_set(objspace, obj)) return; /* already marked */
 
@@ -5021,6 +5017,23 @@ gc_mark(rb_objspace_t *objspace, VALUE obj)
     gc_grey(objspace, obj);
 }
 
+static void
+gc_mark(rb_objspace_t *objspace, VALUE obj)
+{
+    VM_ASSERT(GET_OBJSPACE_OF_VALUE(obj) == objspace || FL_TEST(obj, FL_SHAREABLE) || !using_local_limits(objspace));
+
+    GC_ASSERT(during_gc);
+    if (!confirm_global_connections(objspace, obj)) return;
+    gc_mark_in_range(objspace, obj);
+}
+
+void
+rb_gc_impl_mark_in_range(void *objspace_ptr, VALUE obj)
+{
+    rb_objspace_t *objspace = objspace_ptr;
+    gc_mark_in_range(objspace, obj);
+}
+
 static inline void
 gc_pin(rb_objspace_t *objspace, VALUE obj)
 {

diff --git a/gc/gc_impl.h b/gc/gc_impl.h
@@ -69,6 +69,7 @@ GC_IMPL_FN void rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr);
 GC_IMPL_FN void rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj);
 GC_IMPL_FN void rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj);
 GC_IMPL_FN void rb_gc_impl_stack_location_mark_maybe(void *objspace_ptr, VALUE obj);
+GC_IMPL_FN void rb_gc_impl_mark_in_range(void *objspace_ptr, VALUE obj);
 GC_IMPL_FN void rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr);
 GC_IMPL_FN void rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr);
 GC_IMPL_FN bool rb_gc_impl_object_marked_p(void *objspace_ptr, VALUE obj);