From 48dccdb5263c274ee44523063acd0aa5c0673a86 Mon Sep 17 00:00:00 2001 From: Koichi Sasada Date: Mon, 31 Jul 2023 16:17:55 +0900 Subject: [PATCH] use inline cache for refinements From Ruby 3.0, refined method invocations are slow because resolved methods are not cached by inline cache because of conservertive strategy. However, `using` clears all caches so that it seems safe to cache resolved method entries. This patch caches resolved method entries in inline cache and clear all of inline method caches when `using` is called. fix [Bug #18572] ```ruby # without refinements class C def foo = :C end N = 1_000_000 obj = C.new require 'benchmark' Benchmark.bm{|x| x.report{N.times{ obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; }} } _END__ user system total real master 0.362859 0.002544 0.365403 ( 0.365424) modified 0.357251 0.000000 0.357251 ( 0.357258) ``` ```ruby # with refinment but without using class C def foo = :C end module R refine C do def foo = :R end end N = 1_000_000 obj = C.new require 'benchmark' Benchmark.bm{|x| x.report{N.times{ obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; }} } __END__ user system total real master 0.957182 0.000000 0.957182 ( 0.957212) modified 0.359228 0.000000 0.359228 ( 0.359238) ``` ```ruby # with using class C def foo = :C end module R refine C do def foo = :R end end N = 1_000_000 using R obj = C.new require 'benchmark' Benchmark.bm{|x| x.report{N.times{ obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; obj.foo; }} } --- eval.c | 2 +- gc.c | 2 ++ method.h | 2 +- test/ruby/test_refinement.rb | 18 ++++++++++++++++++ vm_callinfo.h | 17 ++++++++++++++--- vm_insnhelper.c | 19 ++++++++++++++----- vm_method.c | 17 ++++++++--------- 7 files changed, 58 insertions(+), 19 deletions(-) diff --git a/eval.c b/eval.c index 6fc396969021f1..efac69735bdce7 100644 --- a/eval.c +++ b/eval.c @@ -1341,7 +1341,7 @@ rb_using_module(const rb_cref_t *cref, VALUE module) { Check_Type(module, T_MODULE); using_module_recursive(cref, module); - rb_clear_method_cache_all(); + rb_clear_all_refinement_method_cache(); } /* diff --git a/gc.c b/gc.c index 916e08eb039707..b686bbbaa43a3a 100644 --- a/gc.c +++ b/gc.c @@ -3187,6 +3187,8 @@ vm_ccs_free(struct rb_class_cc_entries *ccs, int alive, rb_objspace_t *objspace, asan_poison_object((VALUE)cc); } } + + VM_ASSERT(!vm_cc_super_p(cc) && !vm_cc_refinement_p(cc)); vm_cc_invalidate(cc); } ruby_xfree(ccs->entries); diff --git a/method.h b/method.h index d33ab5053cbcaa..6b60a49a3aac4d 100644 --- a/method.h +++ b/method.h @@ -249,6 +249,6 @@ void rb_scope_visibility_set(rb_method_visibility_t); VALUE rb_unnamed_parameters(int arity); void rb_clear_method_cache(VALUE klass_or_module, ID mid); -void rb_clear_method_cache_all(void); +void rb_clear_all_refinement_method_cache(void); #endif /* RUBY_METHOD_H */ diff --git a/test/ruby/test_refinement.rb b/test/ruby/test_refinement.rb index 56f33ae00a6d0d..7cddfce6bd9d80 100644 --- a/test/ruby/test_refinement.rb +++ b/test/ruby/test_refinement.rb @@ -2626,6 +2626,24 @@ def test_inherit_singleton_methods_of_module assert_equal([], Refinement.used_modules) end + def test_inlinecache + assert_separately([], <<-"end;") + module R + refine String do + def to_s = :R + end + end + + 2.times{|i| + s = ''.to_s + assert_equal '', s if i == 0 + assert_equal :R, s if i == 1 + using R if i == 0 + assert_equal :R, ''.to_s + } + end; + end + private def eval_using(mod, s) diff --git a/vm_callinfo.h b/vm_callinfo.h index 914f1eafdfd9ae..91c92854d7849b 100644 --- a/vm_callinfo.h +++ b/vm_callinfo.h @@ -296,13 +296,15 @@ struct rb_callcache { #define VM_CALLCACHE_UNMARKABLE FL_FREEZE #define VM_CALLCACHE_ON_STACK FL_EXIVAR -#define VM_CALLCACHE_IVAR IMEMO_FL_USER0 -#define VM_CALLCACHE_BF IMEMO_FL_USER1 -#define VM_CALLCACHE_SUPER IMEMO_FL_USER2 +#define VM_CALLCACHE_IVAR IMEMO_FL_USER0 +#define VM_CALLCACHE_BF IMEMO_FL_USER1 +#define VM_CALLCACHE_SUPER IMEMO_FL_USER2 +#define VM_CALLCACHE_REFINEMENT IMEMO_FL_USER3 enum vm_cc_type { cc_type_normal, // chained from ccs cc_type_super, + cc_type_refinement, }; extern const struct rb_callcache *rb_vm_empty_cc(void); @@ -332,6 +334,9 @@ vm_cc_new(VALUE klass, case cc_type_super: *(VALUE *)&cc->flags |= VM_CALLCACHE_SUPER; break; + case cc_type_refinement: + *(VALUE *)&cc->flags |= VM_CALLCACHE_REFINEMENT; + break; } vm_cc_attr_index_initialize(cc, INVALID_SHAPE_ID); @@ -345,6 +350,12 @@ vm_cc_super_p(const struct rb_callcache *cc) return (cc->flags & VM_CALLCACHE_SUPER) != 0; } +static inline bool +vm_cc_refinement_p(const struct rb_callcache *cc) +{ + return (cc->flags & VM_CALLCACHE_REFINEMENT) != 0; +} + #define VM_CC_ON_STACK(clazz, call, aux, cme) \ (struct rb_callcache) { \ .flags = T_IMEMO | \ diff --git a/vm_insnhelper.c b/vm_insnhelper.c index 5e52207d81b36d..6291e13fec3e0c 100644 --- a/vm_insnhelper.c +++ b/vm_insnhelper.c @@ -2003,6 +2003,8 @@ vm_ccs_verify(struct rb_class_cc_entries *ccs, ID mid, VALUE klass) VM_ASSERT(IMEMO_TYPE_P(cc, imemo_callcache)); VM_ASSERT(vm_cc_class_check(cc, klass)); VM_ASSERT(vm_cc_check_cme(cc, ccs->cme)); + VM_ASSERT(!vm_cc_super_p(cc)); + VM_ASSERT(!vm_cc_refinement_p(cc)); } return TRUE; } @@ -4193,12 +4195,19 @@ search_refined_method(rb_execution_context_t *ec, rb_control_frame_t *cfp, struc static VALUE vm_call_refined(rb_execution_context_t *ec, rb_control_frame_t *cfp, struct rb_calling_info *calling) { - struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, - search_refined_method(ec, cfp, calling)); + const rb_callable_method_entry_t *ref_cme = search_refined_method(ec, cfp, calling); - if (vm_cc_cme(ref_cc)) { - calling->cc= ref_cc; - return vm_call_method(ec, cfp, calling); + if (ref_cme) { + if (calling->cd->cc) { + const struct rb_callcache *cc = calling->cc = vm_cc_new(vm_cc_cme(calling->cc)->defined_class, ref_cme, vm_call_general, cc_type_refinement); + RB_OBJ_WRITE(cfp->iseq, &calling->cd->cc, cc); + return vm_call_method(ec, cfp, calling); + } + else { + struct rb_callcache *ref_cc = &VM_CC_ON_STACK(Qundef, vm_call_general, {{ 0 }}, ref_cme); + calling->cc= ref_cc; + return vm_call_method(ec, cfp, calling); + } } else { return vm_call_method_nome(ec, cfp, calling); diff --git a/vm_method.c b/vm_method.c index 221f184267ed36..a65c676cfcee79 100644 --- a/vm_method.c +++ b/vm_method.c @@ -307,19 +307,19 @@ rb_clear_method_cache(VALUE klass_or_module, ID mid) void rb_cc_table_free(VALUE klass); static int -invalidate_all_cc(void *vstart, void *vend, size_t stride, void *data) +invalidate_all_refinement_cc(void *vstart, void *vend, size_t stride, void *data) { VALUE v = (VALUE)vstart; for (; v != (VALUE)vend; v += stride) { void *ptr = asan_poisoned_object_p(v); asan_unpoison_object(v, false); + if (RBASIC(v)->flags) { // liveness check - if (RB_TYPE_P(v, T_CLASS) || - RB_TYPE_P(v, T_ICLASS)) { - if (RCLASS_CC_TBL(v)) { - rb_cc_table_free(v); + if (imemo_type_p(v, imemo_callcache)) { + const struct rb_callcache *cc = (const struct rb_callcache *)v; + if (vm_cc_refinement_p(cc) && cc->klass) { + vm_cc_invalidate(cc); } - RCLASS_CC_TBL(v) = NULL; } } @@ -331,10 +331,9 @@ invalidate_all_cc(void *vstart, void *vend, size_t stride, void *data) } void -rb_clear_method_cache_all(void) +rb_clear_all_refinement_method_cache(void) { - rb_objspace_each_objects(invalidate_all_cc, NULL); - + rb_objspace_each_objects(invalidate_all_refinement_cc, NULL); rb_yjit_invalidate_all_method_lookup_assumptions(); }