Introduce disposable call-cache.

This patch contains several ideas:

(1) Disposable inline method cache (IMC) for race-free inline method cache
    * Making call-cache (CC) as a RVALUE (GC target object) and allocate new
      CC on cache miss.
    * This technique allows race-free access from parallel processing
      elements like RCU.
(2) Introduce per-Class method cache (pCMC)
    * Instead of fixed-size global method cache (GMC), pCMC allows flexible
      cache size.
    * Caching CCs reduces CC allocation and allow sharing CC's fast-path
      between same call-info (CI) call-sites.
(3) Invalidate an inline method cache by invalidating corresponding method
    entries (MEs)
    * Instead of using class serials, we set "invalidated" flag for method
      entry itself to represent cache invalidation.
    * Compare with using class serials, the impact of method modification
      (add/overwrite/delete) is small.
    * Updating class serials invalidate all method caches of the class and
      sub-classes.
    * Proposed approach only invalidate the method cache of only one ME.

See [Feature #16614] for more details.

1 files changed, 54 insertions, 38 deletions

diff --git a/debug_counter.h b/debug_counter.h
index 19909fbb29..e38b5c6b79 100644
--- a/debug_counter.h
+++ b/debug_counter.h
@@ -14,46 +14,45 @@
 
 #ifdef RB_DEBUG_COUNTER
 
-/*
- * method cache (mc) counts.
- *
- * * mc_inline_hit/miss: inline mc hit/miss counts (VM send insn)
- * * mc_global_hit/miss: global method cache hit/miss counts
- *                       two types: (1) inline cache miss (VM send insn)
- *                                  (2) called from C (rb_funcall).
- * * mc_global_state_miss: inline mc miss by global_state miss.
- * * mc_class_serial_miss:            ... by mc_class_serial_miss
- * * mc_cme_complement: callable_method_entry complement counts.
- * * mc_cme_complement_hit: callable_method_entry cache hit counts.
- * * mc_search_super: search_method() call counts.
- * * mc_miss_by_nome: inline mc miss by no ment.
- * * mc_miss_by_distinct:        ... by distinct ment.
- * * mc_miss_by_refine:          ... by ment being refined.
- * * mc_miss_by_visi:            ... by visibility change.
- * * mc_miss_spurious: spurious inline mc misshit.
- * * mc_miss_reuse_call: count of reuse of cc->call.
- */
-RB_DEBUG_COUNTER(mc_inline_hit)
-RB_DEBUG_COUNTER(mc_inline_miss)
-RB_DEBUG_COUNTER(mc_global_hit)
-RB_DEBUG_COUNTER(mc_global_miss)
-RB_DEBUG_COUNTER(mc_global_state_miss)
-RB_DEBUG_COUNTER(mc_class_serial_miss)
-RB_DEBUG_COUNTER(mc_cme_complement)
-RB_DEBUG_COUNTER(mc_cme_complement_hit)
-RB_DEBUG_COUNTER(mc_search_super)
-RB_DEBUG_COUNTER(mc_miss_by_nome)
-RB_DEBUG_COUNTER(mc_miss_by_distinct)
-RB_DEBUG_COUNTER(mc_miss_by_refine)
-RB_DEBUG_COUNTER(mc_miss_by_visi)
-RB_DEBUG_COUNTER(mc_miss_spurious)
-RB_DEBUG_COUNTER(mc_miss_reuse_call)
+// method cache (IMC: inline method cache)
+RB_DEBUG_COUNTER(mc_inline_hit)              // IMC hit
+RB_DEBUG_COUNTER(mc_inline_miss_klass)       // IMC miss by different class
+RB_DEBUG_COUNTER(mc_inline_miss_invalidated) // IMC miss by invalidated ME
+RB_DEBUG_COUNTER(mc_cme_complement)          // number of acquiring complement CME
+RB_DEBUG_COUNTER(mc_cme_complement_hit)      // number of cahche hit for complemented CME
+
+RB_DEBUG_COUNTER(mc_search)                  // count for method lookup in class tree
+RB_DEBUG_COUNTER(mc_search_notfound)         //           method lookup, but not found
+RB_DEBUG_COUNTER(mc_search_super)            // total traversed classes
 
 // callinfo
-RB_DEBUG_COUNTER(ci_packed)
-RB_DEBUG_COUNTER(ci_kw)
-RB_DEBUG_COUNTER(ci_nokw)
-RB_DEBUG_COUNTER(ci_runtime)
+RB_DEBUG_COUNTER(ci_packed)  // number of packed CI
+RB_DEBUG_COUNTER(ci_kw)      //           non-packed CI w/ keywords
+RB_DEBUG_COUNTER(ci_nokw)    //           non-packed CI w/o keywords
+RB_DEBUG_COUNTER(ci_runtime) //           creating temporary CI
+
+// callcache
+RB_DEBUG_COUNTER(cc_new)        // number of CC
+RB_DEBUG_COUNTER(cc_temp)       //           dummy CC (stack-allocated)
+RB_DEBUG_COUNTER(cc_found_ccs)  // count for CC lookup sucess in CCS
+
+RB_DEBUG_COUNTER(cc_ent_invalidate) // count for invalidating cc (cc->klass = 0)
+RB_DEBUG_COUNTER(cc_cme_invalidate) // coutn for invalidating CME
+
+RB_DEBUG_COUNTER(cc_invalidate_leaf)          // count for invalidating klass if klass has no-sublcasses
+RB_DEBUG_COUNTER(cc_invalidate_leaf_ccs)      //                        corresponding CCS
+RB_DEBUG_COUNTER(cc_invalidate_leaf_callable) //                        complimented cache (no-subclasses)
+RB_DEBUG_COUNTER(cc_invalidate_tree)          // count for invalidating klass if klass has sublcasses
+RB_DEBUG_COUNTER(cc_invalidate_tree_cme)      //                        cme if cme is found in this class or superclasses
+RB_DEBUG_COUNTER(cc_invalidate_tree_callable) //                        complimented cache (subclasses)
+
+RB_DEBUG_COUNTER(ccs_free)   // count for free'ing ccs
+RB_DEBUG_COUNTER(ccs_maxlen) // maximum length of ccs
+RB_DEBUG_COUNTER(ccs_found)  // count for finding corresponding ccs on method lookup
+
+// iseq
+RB_DEBUG_COUNTER(iseq_num)    // number of total created iseq
+RB_DEBUG_COUNTER(iseq_cd_num) // number of total created cd (call_data)
 
 /*
  * call cache fastpath usage
@@ -289,6 +288,7 @@ RB_DEBUG_COUNTER(obj_imemo_ifunc)
 RB_DEBUG_COUNTER(obj_imemo_memo)
 RB_DEBUG_COUNTER(obj_imemo_parser_strterm)
 RB_DEBUG_COUNTER(obj_imemo_callinfo)
+RB_DEBUG_COUNTER(obj_imemo_callcache)
 
 /* ar_table */
 RB_DEBUG_COUNTER(artable_hint_hit)
@@ -375,17 +375,33 @@ rb_debug_counter_add(enum rb_debug_counter_type type, int add, int cond)
     return cond;
 }
 
+inline static int
+rb_debug_counter_max(enum rb_debug_counter_type type, unsigned int num)
+{
+    if (rb_debug_counter[(int)type] < num) {
+        rb_debug_counter[(int)type] = num;
+        return 1;
+    }
+    else {
+        return 0;
+    }
+}
+
 VALUE rb_debug_counter_reset(VALUE klass);
 VALUE rb_debug_counter_show(VALUE klass);
 
 #define RB_DEBUG_COUNTER_INC(type)                rb_debug_counter_add(RB_DEBUG_COUNTER_##type, 1, 1)
 #define RB_DEBUG_COUNTER_INC_UNLESS(type, cond) (!rb_debug_counter_add(RB_DEBUG_COUNTER_##type, 1, !(cond)))
 #define RB_DEBUG_COUNTER_INC_IF(type, cond)       rb_debug_counter_add(RB_DEBUG_COUNTER_##type, 1, (cond))
+#define RB_DEBUG_COUNTER_ADD(type, num)           rb_debug_counter_add(RB_DEBUG_COUNTER_##type, (num), 1)
+#define RB_DEBUG_COUNTER_SETMAX(type, num)        rb_debug_counter_max(RB_DEBUG_COUNTER_##type, (unsigned int)(num))
 
 #else
 #define RB_DEBUG_COUNTER_INC(type)              ((void)0)
 #define RB_DEBUG_COUNTER_INC_UNLESS(type, cond) (cond)
 #define RB_DEBUG_COUNTER_INC_IF(type, cond)     (cond)
+#define RB_DEBUG_COUNTER_ADD(type, num)         ((void)0)
+#define RB_DEBUG_COUNTER_SETMAX(type, num)      0
 #endif
 
 void rb_debug_counter_show_results(const char *msg);