From 356359b0755637ba3b3fb73c8ee1ef64c51bcdcc Mon Sep 17 00:00:00 2001 From: rm155 Date: Sat, 26 Oct 2024 17:32:46 -0400 Subject: [PATCH] Speed up marking of local-immune table --- gc.c | 24 ++++++++++++++++++++++++ gc/default.c | 27 ++++++++++++++++++++------- gc/gc_impl.h | 1 + 3 files changed, 45 insertions(+), 7 deletions(-) diff --git a/gc.c b/gc.c index b332ea7f65ebb8..ddd5805e224e4c 100644 --- a/gc.c +++ b/gc.c @@ -617,6 +617,7 @@ typedef struct gc_function_map { void (*mark_and_pin)(void *objspace_ptr, VALUE obj); void (*mark_maybe)(void *objspace_ptr, VALUE obj); void (*stack_location_mark_maybe)(void *objspace_ptr, VALUE obj); + void (*mark_in_range)(void *objspace_ptr, VALUE obj); void (*mark_weak)(void *objspace_ptr, VALUE *ptr); void (*remove_weak)(void *objspace_ptr, VALUE parent_obj, VALUE *ptr); bool (*object_marked_p)(void *objspace_ptr, VALUE obj); @@ -773,6 +774,7 @@ ruby_external_gc_init(void) load_external_gc_func(mark_and_pin); load_external_gc_func(mark_maybe); load_external_gc_func(stack_location_mark_maybe); + load_external_gc_func(mark_in_range); load_external_gc_func(mark_weak); load_external_gc_func(remove_weak); load_external_gc_func(object_marked_p); @@ -865,6 +867,7 @@ ruby_external_gc_init(void) # define rb_gc_impl_mark_and_pin rb_gc_functions.mark_and_pin # define rb_gc_impl_mark_maybe rb_gc_functions.mark_maybe # define rb_gc_impl_stack_location_mark_maybe rb_gc_functions.stack_location_mark_maybe +# define rb_gc_impl_mark_in_range rb_gc_functions.mark_in_range # define rb_gc_impl_mark_weak rb_gc_functions.mark_weak # define rb_gc_impl_remove_weak rb_gc_functions.remove_weak # define rb_gc_impl_object_marked_p rb_gc_functions.object_marked_p @@ -2352,6 +2355,27 @@ rb_mark_set(st_table *tbl) st_foreach(tbl, mark_key, (st_data_t)rb_gc_get_objspace()); } +static int +mark_local_key_no_traversal(st_data_t key, st_data_t value, st_data_t data) +{ + rb_gc_impl_mark_in_range(data, key); + + return ST_CONTINUE; +} + +void +rb_mark_local_set(st_table *tbl) +{ + if (!tbl) return; + + if (LIKELY(!MARK_FUNC_IN_USE(GET_RACTOR()))) { + st_foreach(tbl, mark_local_key_no_traversal, (st_data_t)rb_gc_get_objspace()); + } + else { + st_foreach(tbl, mark_key, (st_data_t)rb_gc_get_objspace()); + } +} + static int mark_keyvalue(st_data_t key, st_data_t value, st_data_t data) { diff --git a/gc/default.c b/gc/default.c index 949315f1b9c918..3e1cae6078a70d 100644 --- a/gc/default.c +++ b/gc/default.c @@ -4946,7 +4946,7 @@ gc_grey(rb_objspace_t *objspace, VALUE obj) } static bool -in_marking_range(rb_objspace_t *objspace, VALUE obj) +in_local_marking_range(rb_objspace_t *objspace, VALUE obj) { return !FL_TEST_RAW(obj, FL_SHAREABLE) || GET_OBJSPACE_OF_VALUE(obj) == objspace; } @@ -4983,7 +4983,7 @@ confirm_global_connections(rb_objspace_t *objspace, VALUE obj) } } else { - if (!in_marking_range(objspace, obj)) { + if (!in_local_marking_range(objspace, obj)) { if (is_full_marking(objspace)) { check_not_tnone(obj); mark_in_external_reference_tbl(objspace->local_gate, obj); @@ -4996,12 +4996,8 @@ confirm_global_connections(rb_objspace_t *objspace, VALUE obj) } static void -gc_mark(rb_objspace_t *objspace, VALUE obj) +gc_mark_in_range(rb_objspace_t *objspace, VALUE obj) { - VM_ASSERT(GET_OBJSPACE_OF_VALUE(obj) == objspace || FL_TEST(obj, FL_SHAREABLE) || !using_local_limits(objspace)); - - GC_ASSERT(during_gc); - if (!confirm_global_connections(objspace, obj)) return; rgengc_check_relation(objspace, obj); if (!gc_mark_set(objspace, obj)) return; /* already marked */ @@ -5021,6 +5017,23 @@ gc_mark(rb_objspace_t *objspace, VALUE obj) gc_grey(objspace, obj); } +static void +gc_mark(rb_objspace_t *objspace, VALUE obj) +{ + VM_ASSERT(GET_OBJSPACE_OF_VALUE(obj) == objspace || FL_TEST(obj, FL_SHAREABLE) || !using_local_limits(objspace)); + + GC_ASSERT(during_gc); + if (!confirm_global_connections(objspace, obj)) return; + gc_mark_in_range(objspace, obj); +} + +void +rb_gc_impl_mark_in_range(void *objspace_ptr, VALUE obj) +{ + rb_objspace_t *objspace = objspace_ptr; + gc_mark_in_range(objspace, obj); +} + static inline void gc_pin(rb_objspace_t *objspace, VALUE obj) { diff --git a/gc/gc_impl.h b/gc/gc_impl.h index 4f9cd31c103ae5..910e406a5f0d25 100644 --- a/gc/gc_impl.h +++ b/gc/gc_impl.h @@ -69,6 +69,7 @@ GC_IMPL_FN void rb_gc_impl_mark_and_move(void *objspace_ptr, VALUE *ptr); GC_IMPL_FN void rb_gc_impl_mark_and_pin(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_mark_maybe(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_stack_location_mark_maybe(void *objspace_ptr, VALUE obj); +GC_IMPL_FN void rb_gc_impl_mark_in_range(void *objspace_ptr, VALUE obj); GC_IMPL_FN void rb_gc_impl_mark_weak(void *objspace_ptr, VALUE *ptr); GC_IMPL_FN void rb_gc_impl_remove_weak(void *objspace_ptr, VALUE parent_obj, VALUE *ptr); GC_IMPL_FN bool rb_gc_impl_object_marked_p(void *objspace_ptr, VALUE obj);