WT-1592 Add per-dhandle current cache usage statistics (#3062)

Exposed via a new 'cache_walk' statistics configuration option.

6 files changed, 65 insertions, 15 deletions

diff --git a/dist/api_data.py b/dist/api_data.py
index d02d7e4b985..df3577f56b8 100644
--- a/dist/api_data.py
+++ b/dist/api_data.py
@@ -502,7 +502,8 @@ connection_runtime_config = [
         is used to gather statistics, as well as each time statistics
         are logged using the \c statistics_log configuration.  See
         @ref statistics for more information''',
-        type='list', choices=['all', 'fast', 'none', 'clear']),
+        type='list',
+        choices=['all', 'cache_walk', 'fast', 'none', 'clear', 'tree_walk']),
     Config('verbose', '', r'''
         enable messages for various events. Only available if WiredTiger
         is configured with --enable-verbose. Options are given as a
@@ -976,7 +977,8 @@ methods = {
         gathering them, where appropriate (for example, a cache size statistic
         is not cleared, while the count of cursor insert operations will be
         cleared).  See @ref statistics for more information''',
-        type='list', choices=['all', 'fast', 'clear', 'size']),
+        type='list',
+        choices=['all', 'cache_walk', 'fast', 'clear', 'size', 'tree_walk']),
     Config('target', '', r'''
         if non-empty, backup the list of objects; valid only for a
         backup data source''',
diff --git a/dist/filelist b/dist/filelist
index 32e4231c5f2..fe9a17b7799 100644
--- a/dist/filelist
+++ b/dist/filelist
@@ -90,6 +90,7 @@ src/cursor/cur_table.c
 src/evict/evict_file.c
 src/evict/evict_lru.c
 src/evict/evict_page.c
+src/evict/evict_stat.c
 src/log/log.c
 src/log/log_auto.c
 src/log/log_slot.c
diff --git a/dist/flags.py b/dist/flags.py
index 93b6e0cbbf4..e200f95fba6 100644
--- a/dist/flags.py
+++ b/dist/flags.py
@@ -133,6 +133,16 @@ flags = {
         'SESSION_QUIET_CORRUPT_FILE',
         'SESSION_SERVER_ASYNC',
     ],
+    'stat' : [
+        'STAT_CLEAR',
+        'STAT_JSON',
+        'STAT_ON_CLOSE',
+        'STAT_TYPE_ALL',
+        'STAT_TYPE_CACHE_WALK',
+        'STAT_TYPE_FAST',
+        'STAT_TYPE_SIZE',
+        'STAT_TYPE_TREE_WALK',
+    ],
 }
 
 flag_cnt = {}    # Dictionary [flag] : [reference count]
diff --git a/dist/s_string.ok b/dist/s_string.ok
index 6e629c8cd89..e8723de9810 100644
--- a/dist/s_string.ok
+++ b/dist/s_string.ok
@@ -993,6 +993,7 @@ qdown
 qrrSS
 qsort
 quartile
+queueable
 qup
 rN
 rS
diff --git a/dist/stat.py b/dist/stat.py
index c3c85bbe9b4..e42585c1b8c 100644
--- a/dist/stat.py
+++ b/dist/stat.py
@@ -42,8 +42,11 @@ compare_srcfile(tmp_file, '../src/include/stat.h')
 def print_defines_one(capname, base, stats):
     for v, l in enumerate(stats, base):
         desc = l.desc
-        if 'all_only' in l.flags:
-            desc += ', only reported if statistics=all is set'
+        if 'cache_walk' in l.flags:
+            desc += \
+                ', only reported if cache_walk or all statistics are enabled'
+        if 'tree_walk' in l.flags:
+            desc += ', only reported if tree_walk or all statistics are enabled'
         if len(textwrap.wrap(desc, 70)) > 1:
             f.write('/*!\n')
             f.write(' * %s\n' % '\n * '.join(textwrap.wrap(desc, 70)))
diff --git a/dist/stat_data.py b/dist/stat_data.py
index 5b4d3fa7709..ef70ca89eb0 100644
--- a/dist/stat_data.py
+++ b/dist/stat_data.py
@@ -9,7 +9,8 @@
 #
 # Data-source statistics are normally aggregated across the set of underlying
 # objects. Additional optional configuration flags are available:
-#       all_only        Only gets reported when statistics=all set
+#       cache_walk      Only reported when statistics=cache_walk is set
+#       tree_walk       Only reported when statistics=tree_walk is set
 #       max_aggregate   Take the maximum value when aggregating statistics
 #       no_clear        Value not cleared when statistics cleared
 #       no_scale        Don't scale value per second in the logging tool script
@@ -46,6 +47,11 @@ class CacheStat(Stat):
     prefix = 'cache'
     def __init__(self, name, desc, flags=''):
         Stat.__init__(self, name, CacheStat.prefix, desc, flags)
+class CacheWalkStat(Stat):
+    prefix = 'cache_walk'
+    def __init__(self, name, desc, flags=''):
+        flags += ',cache_walk'
+        Stat.__init__(self, name, CacheWalkStat.prefix, desc, flags)
 class CompressStat(Stat):
     prefix = 'compression'
     def __init__(self, name, desc, flags=''):
@@ -109,11 +115,16 @@ groups['cursor'] = [CursorStat.prefix, SessionStat.prefix]
 groups['evict'] = [
     BlockStat.prefix,
     CacheStat.prefix,
+    CacheWalkStat.prefix,
     ConnStat.prefix,
     ThreadStat.prefix
 ]
 groups['lsm'] = [LSMStat.prefix, TxnStat.prefix]
-groups['memory'] = [CacheStat.prefix, ConnStat.prefix, RecStat.prefix]
+groups['memory'] = [
+    CacheStat.prefix,
+    CacheWalkStat.prefix,
+    ConnStat.prefix,
+    RecStat.prefix]
 groups['system'] = [
     ConnStat.prefix,
     DhandleStat.prefix,
@@ -438,13 +449,13 @@ dsrc_stats = [
     # Btree statistics
     ##########################################
     BtreeStat('btree_checkpoint_generation', 'btree checkpoint generation', 'no_clear,no_scale'),
-    BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,all_only'),
-    BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,all_only'),
-    BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,all_only'),
-    BtreeStat('btree_column_rle', 'column-store variable-size RLE encoded values', 'no_scale,all_only'),
-    BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,all_only'),
+    BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,tree_walk'),
+    BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,tree_walk'),
+    BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,tree_walk'),
+    BtreeStat('btree_column_rle', 'column-store variable-size RLE encoded values', 'no_scale,tree_walk'),
+    BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,tree_walk'),
     BtreeStat('btree_compact_rewrite', 'pages rewritten by compaction'),
-    BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,all_only'),
+    BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,tree_walk'),
     BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale,size'),
     BtreeStat('btree_maximum_depth', 'maximum tree depth', 'max_aggregate,no_scale'),
     BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale,size'),
@@ -452,9 +463,9 @@ dsrc_stats = [
     BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale,size'),
     BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale,size'),
     BtreeStat('btree_maxleafvalue', 'maximum leaf page value size', 'max_aggregate,no_scale,size'),
-    BtreeStat('btree_overflow', 'overflow pages', 'no_scale,all_only'),
-    BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale,all_only'),
-    BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale,all_only'),
+    BtreeStat('btree_overflow', 'overflow pages', 'no_scale,tree_walk'),
+    BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale,tree_walk'),
+    BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale,tree_walk'),
 
     ##########################################
     # Cache and eviction statistics
@@ -483,6 +494,28 @@ dsrc_stats = [
     CacheStat('cache_write_restore', 'pages written requiring in-memory restoration'),
 
     ##########################################
+    # Cache content statistics
+    ##########################################
+    CacheWalkStat('cache_state_avg_written_size', 'Average on-disk page image size seen', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_gen_avg_gap', 'Average difference between current eviction generation when the page was last considered', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_gen_current', 'Current eviction generation', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_gen_max_gap', 'Maximum difference between current eviction generation when the page was last considered', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_max_pagesize', 'Maximum page size seen', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_memory', 'Pages created in memory and never written', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_min_written_size', 'Minimum on-disk page image size seen', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_not_queueable', 'Pages that could not be queued for eviction', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_pages', 'Total number of pages currently in cache', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_pages_clean', 'Clean pages currently in cache', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_pages_dirty', 'Dirty pages currently in cache', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_pages_internal', 'Internal pages currently in cache', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_pages_leaf', 'Leaf pages currently in cache', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_queued', 'Pages currently queued for eviction', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_refs_skipped', 'Refs skipped during cache traversal', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_root_entries', 'Entries in the root page', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_root_size', 'Size of the root page', 'no_clear,no_scale'),
+    CacheWalkStat('cache_state_smaller_alloc_size', 'On-disk page image sizes smaller than a single allocation unit', 'no_clear,no_scale'),
+
+    ##########################################
     # Compression statistics
     ##########################################
     CompressStat('compress_raw_fail', 'raw compression call failed, no additional data available'),