diff options
-rw-r--r-- | dist/api_data.py | 6 | ||||
-rw-r--r-- | dist/filelist | 1 | ||||
-rw-r--r-- | dist/flags.py | 10 | ||||
-rw-r--r-- | dist/s_string.ok | 1 | ||||
-rw-r--r-- | dist/stat.py | 7 | ||||
-rw-r--r-- | dist/stat_data.py | 55 | ||||
-rw-r--r-- | src/btree/bt_stat.c | 29 | ||||
-rw-r--r-- | src/cache/cache_las.c | 2 | ||||
-rw-r--r-- | src/config/config_def.c | 18 | ||||
-rw-r--r-- | src/conn/conn_api.c | 48 | ||||
-rw-r--r-- | src/conn/conn_stat.c | 10 | ||||
-rw-r--r-- | src/cursor/cur_stat.c | 77 | ||||
-rw-r--r-- | src/evict/evict_lru.c | 2 | ||||
-rw-r--r-- | src/evict/evict_stat.c | 150 | ||||
-rw-r--r-- | src/include/btmem.h | 2 | ||||
-rw-r--r-- | src/include/cache.h | 1 | ||||
-rw-r--r-- | src/include/connection.h | 8 | ||||
-rw-r--r-- | src/include/cursor.h | 2 | ||||
-rw-r--r-- | src/include/extern.h | 1 | ||||
-rw-r--r-- | src/include/flags.h | 8 | ||||
-rw-r--r-- | src/include/stat.h | 18 | ||||
-rw-r--r-- | src/include/wiredtiger.in | 217 | ||||
-rw-r--r-- | src/lsm/lsm_stat.c | 22 | ||||
-rw-r--r-- | src/schema/schema_stat.c | 2 | ||||
-rw-r--r-- | src/support/stat.c | 90 | ||||
-rw-r--r-- | test/suite/test_stat02.py | 70 | ||||
-rw-r--r-- | tools/wtstats/stat_data.py | 39 | ||||
-rwxr-xr-x | tools/wtstats/wtstats.py | 3 |
28 files changed, 752 insertions, 147 deletions
diff --git a/dist/api_data.py b/dist/api_data.py index d02d7e4b985..df3577f56b8 100644 --- a/dist/api_data.py +++ b/dist/api_data.py @@ -502,7 +502,8 @@ connection_runtime_config = [ is used to gather statistics, as well as each time statistics are logged using the \c statistics_log configuration. See @ref statistics for more information''', - type='list', choices=['all', 'fast', 'none', 'clear']), + type='list', + choices=['all', 'cache_walk', 'fast', 'none', 'clear', 'tree_walk']), Config('verbose', '', r''' enable messages for various events. Only available if WiredTiger is configured with --enable-verbose. Options are given as a @@ -976,7 +977,8 @@ methods = { gathering them, where appropriate (for example, a cache size statistic is not cleared, while the count of cursor insert operations will be cleared). See @ref statistics for more information''', - type='list', choices=['all', 'fast', 'clear', 'size']), + type='list', + choices=['all', 'cache_walk', 'fast', 'clear', 'size', 'tree_walk']), Config('target', '', r''' if non-empty, backup the list of objects; valid only for a backup data source''', diff --git a/dist/filelist b/dist/filelist index 32e4231c5f2..fe9a17b7799 100644 --- a/dist/filelist +++ b/dist/filelist @@ -90,6 +90,7 @@ src/cursor/cur_table.c src/evict/evict_file.c src/evict/evict_lru.c src/evict/evict_page.c +src/evict/evict_stat.c src/log/log.c src/log/log_auto.c src/log/log_slot.c diff --git a/dist/flags.py b/dist/flags.py index 93b6e0cbbf4..e200f95fba6 100644 --- a/dist/flags.py +++ b/dist/flags.py @@ -133,6 +133,16 @@ flags = { 'SESSION_QUIET_CORRUPT_FILE', 'SESSION_SERVER_ASYNC', ], + 'stat' : [ + 'STAT_CLEAR', + 'STAT_JSON', + 'STAT_ON_CLOSE', + 'STAT_TYPE_ALL', + 'STAT_TYPE_CACHE_WALK', + 'STAT_TYPE_FAST', + 'STAT_TYPE_SIZE', + 'STAT_TYPE_TREE_WALK', + ], } flag_cnt = {} # Dictionary [flag] : [reference count] diff --git a/dist/s_string.ok b/dist/s_string.ok index 6e629c8cd89..e8723de9810 100644 --- a/dist/s_string.ok +++ b/dist/s_string.ok @@ -993,6 +993,7 @@ qdown qrrSS qsort quartile +queueable qup rN rS diff --git a/dist/stat.py b/dist/stat.py index c3c85bbe9b4..e42585c1b8c 100644 --- a/dist/stat.py +++ b/dist/stat.py @@ -42,8 +42,11 @@ compare_srcfile(tmp_file, '../src/include/stat.h') def print_defines_one(capname, base, stats): for v, l in enumerate(stats, base): desc = l.desc - if 'all_only' in l.flags: - desc += ', only reported if statistics=all is set' + if 'cache_walk' in l.flags: + desc += \ + ', only reported if cache_walk or all statistics are enabled' + if 'tree_walk' in l.flags: + desc += ', only reported if tree_walk or all statistics are enabled' if len(textwrap.wrap(desc, 70)) > 1: f.write('/*!\n') f.write(' * %s\n' % '\n * '.join(textwrap.wrap(desc, 70))) diff --git a/dist/stat_data.py b/dist/stat_data.py index 5b4d3fa7709..ef70ca89eb0 100644 --- a/dist/stat_data.py +++ b/dist/stat_data.py @@ -9,7 +9,8 @@ # # Data-source statistics are normally aggregated across the set of underlying # objects. Additional optional configuration flags are available: -# all_only Only gets reported when statistics=all set +# cache_walk Only reported when statistics=cache_walk is set +# tree_walk Only reported when statistics=tree_walk is set # max_aggregate Take the maximum value when aggregating statistics # no_clear Value not cleared when statistics cleared # no_scale Don't scale value per second in the logging tool script @@ -46,6 +47,11 @@ class CacheStat(Stat): prefix = 'cache' def __init__(self, name, desc, flags=''): Stat.__init__(self, name, CacheStat.prefix, desc, flags) +class CacheWalkStat(Stat): + prefix = 'cache_walk' + def __init__(self, name, desc, flags=''): + flags += ',cache_walk' + Stat.__init__(self, name, CacheWalkStat.prefix, desc, flags) class CompressStat(Stat): prefix = 'compression' def __init__(self, name, desc, flags=''): @@ -109,11 +115,16 @@ groups['cursor'] = [CursorStat.prefix, SessionStat.prefix] groups['evict'] = [ BlockStat.prefix, CacheStat.prefix, + CacheWalkStat.prefix, ConnStat.prefix, ThreadStat.prefix ] groups['lsm'] = [LSMStat.prefix, TxnStat.prefix] -groups['memory'] = [CacheStat.prefix, ConnStat.prefix, RecStat.prefix] +groups['memory'] = [ + CacheStat.prefix, + CacheWalkStat.prefix, + ConnStat.prefix, + RecStat.prefix] groups['system'] = [ ConnStat.prefix, DhandleStat.prefix, @@ -438,13 +449,13 @@ dsrc_stats = [ # Btree statistics ########################################## BtreeStat('btree_checkpoint_generation', 'btree checkpoint generation', 'no_clear,no_scale'), - BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,all_only'), - BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,all_only'), - BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,all_only'), - BtreeStat('btree_column_rle', 'column-store variable-size RLE encoded values', 'no_scale,all_only'), - BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,all_only'), + BtreeStat('btree_column_deleted', 'column-store variable-size deleted values', 'no_scale,tree_walk'), + BtreeStat('btree_column_fix', 'column-store fixed-size leaf pages', 'no_scale,tree_walk'), + BtreeStat('btree_column_internal', 'column-store internal pages', 'no_scale,tree_walk'), + BtreeStat('btree_column_rle', 'column-store variable-size RLE encoded values', 'no_scale,tree_walk'), + BtreeStat('btree_column_variable', 'column-store variable-size leaf pages', 'no_scale,tree_walk'), BtreeStat('btree_compact_rewrite', 'pages rewritten by compaction'), - BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,all_only'), + BtreeStat('btree_entries', 'number of key/value pairs', 'no_scale,tree_walk'), BtreeStat('btree_fixed_len', 'fixed-record size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maximum_depth', 'maximum tree depth', 'max_aggregate,no_scale'), BtreeStat('btree_maxintlkey', 'maximum internal page key size', 'max_aggregate,no_scale,size'), @@ -452,9 +463,9 @@ dsrc_stats = [ BtreeStat('btree_maxleafkey', 'maximum leaf page key size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maxleafpage', 'maximum leaf page size', 'max_aggregate,no_scale,size'), BtreeStat('btree_maxleafvalue', 'maximum leaf page value size', 'max_aggregate,no_scale,size'), - BtreeStat('btree_overflow', 'overflow pages', 'no_scale,all_only'), - BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale,all_only'), - BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale,all_only'), + BtreeStat('btree_overflow', 'overflow pages', 'no_scale,tree_walk'), + BtreeStat('btree_row_internal', 'row-store internal pages', 'no_scale,tree_walk'), + BtreeStat('btree_row_leaf', 'row-store leaf pages', 'no_scale,tree_walk'), ########################################## # Cache and eviction statistics @@ -483,6 +494,28 @@ dsrc_stats = [ CacheStat('cache_write_restore', 'pages written requiring in-memory restoration'), ########################################## + # Cache content statistics + ########################################## + CacheWalkStat('cache_state_avg_written_size', 'Average on-disk page image size seen', 'no_clear,no_scale'), + CacheWalkStat('cache_state_gen_avg_gap', 'Average difference between current eviction generation when the page was last considered', 'no_clear,no_scale'), + CacheWalkStat('cache_state_gen_current', 'Current eviction generation', 'no_clear,no_scale'), + CacheWalkStat('cache_state_gen_max_gap', 'Maximum difference between current eviction generation when the page was last considered', 'no_clear,no_scale'), + CacheWalkStat('cache_state_max_pagesize', 'Maximum page size seen', 'no_clear,no_scale'), + CacheWalkStat('cache_state_memory', 'Pages created in memory and never written', 'no_clear,no_scale'), + CacheWalkStat('cache_state_min_written_size', 'Minimum on-disk page image size seen', 'no_clear,no_scale'), + CacheWalkStat('cache_state_not_queueable', 'Pages that could not be queued for eviction', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages', 'Total number of pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages_clean', 'Clean pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages_dirty', 'Dirty pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages_internal', 'Internal pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_pages_leaf', 'Leaf pages currently in cache', 'no_clear,no_scale'), + CacheWalkStat('cache_state_queued', 'Pages currently queued for eviction', 'no_clear,no_scale'), + CacheWalkStat('cache_state_refs_skipped', 'Refs skipped during cache traversal', 'no_clear,no_scale'), + CacheWalkStat('cache_state_root_entries', 'Entries in the root page', 'no_clear,no_scale'), + CacheWalkStat('cache_state_root_size', 'Size of the root page', 'no_clear,no_scale'), + CacheWalkStat('cache_state_smaller_alloc_size', 'On-disk page image sizes smaller than a single allocation unit', 'no_clear,no_scale'), + + ########################################## # Compression statistics ########################################## CompressStat('compress_raw_fail', 'raw compression call failed, no additional data available'), diff --git a/src/btree/bt_stat.c b/src/btree/bt_stat.c index d3ddf33446e..ad3d94a856f 100644 --- a/src/btree/bt_stat.c +++ b/src/btree/bt_stat.c @@ -8,6 +8,7 @@ #include "wt_internal.h" +static int __stat_tree_walk(WT_SESSION_IMPL *); static int __stat_page(WT_SESSION_IMPL *, WT_PAGE *, WT_DSRC_STATS **); static void __stat_page_col_var(WT_SESSION_IMPL *, WT_PAGE *, WT_DSRC_STATS **); static void __stat_page_row_int(WT_SESSION_IMPL *, WT_PAGE *, WT_DSRC_STATS **); @@ -23,9 +24,7 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) { WT_BM *bm; WT_BTREE *btree; - WT_DECL_RET; WT_DSRC_STATS **stats; - WT_REF *next_walk; btree = S2BT(session); bm = btree->bm; @@ -44,9 +43,29 @@ __wt_btree_stat_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) WT_STAT_SET(session, stats, cache_bytes_inuse, __wt_btree_bytes_inuse(session)); - /* Everything else is really, really expensive. */ - if (!F_ISSET(cst, WT_CONN_STAT_ALL)) - return (0); + if (F_ISSET(cst, WT_STAT_TYPE_CACHE_WALK)) + WT_RET(__wt_curstat_cache_walk(session)); + + if (F_ISSET(cst, WT_STAT_TYPE_TREE_WALK)) + WT_RET(__stat_tree_walk(session)); + + return (0); +} + +/* + * __stat_tree_walk -- + * Gather btree statistics that require traversing the tree. + */ +static int +__stat_tree_walk(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_DECL_RET; + WT_DSRC_STATS **stats; + WT_REF *next_walk; + + btree = S2BT(session); + stats = btree->dhandle->stats; /* * Clear the statistics we're about to count. diff --git a/src/cache/cache_las.c b/src/cache/cache_las.c index 4c338bc6ad9..41f50957809 100644 --- a/src/cache/cache_las.c +++ b/src/cache/cache_las.c @@ -49,7 +49,7 @@ __wt_las_stats_update(WT_SESSION_IMPL *session) * don't have to worry about users seeing inconsistent data source * information. */ - if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_CLEAR)) { + if (FLD_ISSET(conn->stat_flags, WT_STAT_CLEAR)) { WT_STAT_SET(session, dstats, cursor_insert, 0); WT_STAT_SET(session, dstats, cursor_remove, 0); } diff --git a/src/config/config_def.c b/src/config/config_def.c index 7bce4bc9cef..018cc7a8ac4 100644 --- a/src/config/config_def.c +++ b/src/config/config_def.c @@ -138,7 +138,8 @@ static const WT_CONFIG_CHECK confchk_WT_CONNECTION_reconfigure[] = { NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, { "statistics", "list", - NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\"," + "\"clear\",\"tree_walk\"]", NULL, 0 }, { "statistics_log", "category", NULL, NULL, @@ -334,7 +335,8 @@ static const WT_CONFIG_CHECK confchk_WT_SESSION_open_cursor[] = { { "readonly", "boolean", NULL, NULL, NULL, 0 }, { "skip_sort_check", "boolean", NULL, NULL, NULL, 0 }, { "statistics", "list", - NULL, "choices=[\"all\",\"fast\",\"clear\",\"size\"]", + NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"clear\"," + "\"size\",\"tree_walk\"]", NULL, 0 }, { "target", "list", NULL, NULL, NULL, 0 }, { NULL, NULL, NULL, NULL, NULL, 0 } @@ -709,7 +711,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open[] = { NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, { "statistics", "list", - NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\"," + "\"clear\",\"tree_walk\"]", NULL, 0 }, { "statistics_log", "category", NULL, NULL, @@ -793,7 +796,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_all[] = { NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, { "statistics", "list", - NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\"," + "\"clear\",\"tree_walk\"]", NULL, 0 }, { "statistics_log", "category", NULL, NULL, @@ -874,7 +878,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_basecfg[] = { NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, { "statistics", "list", - NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\"," + "\"clear\",\"tree_walk\"]", NULL, 0 }, { "statistics_log", "category", NULL, NULL, @@ -953,7 +958,8 @@ static const WT_CONFIG_CHECK confchk_wiredtiger_open_usercfg[] = { NULL, NULL, confchk_wiredtiger_open_shared_cache_subconfigs, 5 }, { "statistics", "list", - NULL, "choices=[\"all\",\"fast\",\"none\",\"clear\"]", + NULL, "choices=[\"all\",\"cache_walk\",\"fast\",\"none\"," + "\"clear\",\"tree_walk\"]", NULL, 0 }, { "statistics_log", "category", NULL, NULL, diff --git a/src/conn/conn_api.c b/src/conn/conn_api.c index 74c675d794d..8a87aa9c7da 100644 --- a/src/conn/conn_api.c +++ b/src/conn/conn_api.c @@ -1668,32 +1668,60 @@ __conn_statistics_config(WT_SESSION_IMPL *session, const char *cfg[]) if ((ret = __wt_config_subgets( session, &cval, "fast", &sval)) == 0 && sval.val != 0) { - LF_SET(WT_CONN_STAT_FAST); + LF_SET(WT_STAT_TYPE_FAST); ++set; } WT_RET_NOTFOUND_OK(ret); if ((ret = __wt_config_subgets( session, &cval, "all", &sval)) == 0 && sval.val != 0) { - LF_SET(WT_CONN_STAT_ALL | WT_CONN_STAT_FAST); + LF_SET( + WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK | + WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK); ++set; } WT_RET_NOTFOUND_OK(ret); + if (set > 1) + WT_RET_MSG(session, EINVAL, + "Only one of all, fast, none configuration values should " + "be specified"); + + /* + * Now that we've parsed general statistics categories, process + * sub-categories. + */ + if ((ret = __wt_config_subgets( + session, &cval, "cache_walk", &sval)) == 0 && sval.val != 0) + /* + * Configuring cache walk statistics implies fast statistics. + * Keep that knowledge internal for now - it may change in the + * future. + */ + LF_SET(WT_STAT_TYPE_FAST | WT_STAT_TYPE_CACHE_WALK); + WT_RET_NOTFOUND_OK(ret); + + if ((ret = __wt_config_subgets( + session, &cval, "tree_walk", &sval)) == 0 && sval.val != 0) + /* + * Configuring tree walk statistics implies fast statistics. + * Keep that knowledge internal for now - it may change in the + * future. + */ + LF_SET(WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK); + WT_RET_NOTFOUND_OK(ret); + if ((ret = __wt_config_subgets( session, &cval, "clear", &sval)) == 0 && sval.val != 0) { - if (!LF_ISSET(WT_CONN_STAT_FAST | WT_CONN_STAT_ALL)) + if (!LF_ISSET(WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK | + WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK)) WT_RET_MSG(session, EINVAL, - "the value \"clear\" can be specified only if " - "either \"all\" or \"fast\" is specified"); - LF_SET(WT_CONN_STAT_CLEAR); + "the value \"clear\" can only be specified if " + "statistics are enabled"); + LF_SET(WT_STAT_CLEAR); } WT_RET_NOTFOUND_OK(ret); - if (set > 1) - WT_RET_MSG(session, EINVAL, - "only one statistics configuration value may be specified"); - /* Configuring statistics clears any existing values. */ conn->stat_flags = flags; diff --git a/src/conn/conn_stat.c b/src/conn/conn_stat.c index d5a31c671c0..0715a035807 100644 --- a/src/conn/conn_stat.c +++ b/src/conn/conn_stat.c @@ -130,12 +130,12 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) WT_RET(__wt_config_gets(session, cfg, "statistics_log.json", &cval)); if (cval.val != 0) - FLD_SET(conn->stat_flags, WT_CONN_STAT_JSON); + FLD_SET(conn->stat_flags, WT_STAT_JSON); WT_RET(__wt_config_gets( session, cfg, "statistics_log.on_close", &cval)); if (cval.val != 0) - FLD_SET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE); + FLD_SET(conn->stat_flags, WT_STAT_ON_CLOSE); /* * We don't allow the log path to be reconfigured for security reasons. @@ -206,7 +206,7 @@ __statlog_config(WT_SESSION_IMPL *session, const char **cfg, bool *runp) #define WT_TIMESTAMP_JSON_DEFAULT "%Y-%m-%dT%H:%M:%S.000Z" WT_ERR(__wt_config_gets( session, cfg, "statistics_log.timestamp", &cval)); - if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON) && + if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON) && WT_STRING_MATCH(WT_TIMESTAMP_DEFAULT, cval.str, cval.len)) WT_ERR(__wt_strdup( session, WT_TIMESTAMP_JSON_DEFAULT, &conn->stat_format)); @@ -264,7 +264,7 @@ __statlog_dump(WT_SESSION_IMPL *session, const char *name, bool conn_stats) goto err; } - if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_JSON)) { + if (FLD_ISSET(conn->stat_flags, WT_STAT_JSON)) { WT_ERR(__wt_fprintf(session, conn->stat_fs, "{\"version\":\"%s\",\"localTime\":\"%s\"", WIREDTIGER_VERSION_STRING, conn->stat_stamp)); @@ -482,7 +482,7 @@ __wt_statlog_log_one(WT_SESSION_IMPL *session) conn = S2C(session); - if (!FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ON_CLOSE)) + if (!FLD_ISSET(conn->stat_flags, WT_STAT_ON_CLOSE)) return (0); if (F_ISSET(conn, WT_CONN_SERVER_RUN) && diff --git a/src/cursor/cur_stat.c b/src/cursor/cur_stat.c index 700cc366ff0..b36416debe1 100644 --- a/src/cursor/cur_stat.c +++ b/src/cursor/cur_stat.c @@ -354,7 +354,7 @@ __curstat_conn_init(WT_SESSION_IMPL *session, WT_CURSOR_STAT *cst) */ __wt_conn_stat_init(session); __wt_stat_connection_aggregate(conn->stats, &cst->u.conn_stats); - if (F_ISSET(cst, WT_CONN_STAT_CLEAR)) + if (F_ISSET(cst, WT_STAT_CLEAR)) __wt_stat_connection_clear_all(conn->stats); cst->stats = (int64_t *)&cst->u.conn_stats; @@ -380,7 +380,7 @@ __curstat_file_init(WT_SESSION_IMPL *session, * If we are only getting the size of the file, we don't need to open * the tree. */ - if (F_ISSET(cst, WT_CONN_STAT_SIZE)) { + if (F_ISSET(cst, WT_STAT_TYPE_SIZE)) { filename = uri; if (!WT_PREFIX_SKIP(filename, "file:")) return (EINVAL); @@ -401,7 +401,7 @@ __curstat_file_init(WT_SESSION_IMPL *session, if ((ret = __wt_btree_stat_init(session, cst)) == 0) { __wt_stat_dsrc_init_single(&cst->u.dsrc_stats); __wt_stat_dsrc_aggregate(dhandle->stats, &cst->u.dsrc_stats); - if (F_ISSET(cst, WT_CONN_STAT_CLEAR)) + if (F_ISSET(cst, WT_STAT_CLEAR)) __wt_stat_dsrc_clear_all(dhandle->stats); __wt_curstat_dsrc_final(cst); } @@ -604,50 +604,79 @@ __wt_curstat_open(WT_SESSION_IMPL *session, if ((ret = __wt_config_gets(session, cfg, "statistics", &cval)) == 0) { if ((ret = __wt_config_subgets( session, &cval, "all", &sval)) == 0 && sval.val != 0) { - if (!FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ALL)) + if (!FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_ALL)) goto config_err; - F_SET(cst, WT_CONN_STAT_ALL | WT_CONN_STAT_FAST); + F_SET(cst, WT_STAT_TYPE_ALL | WT_STAT_TYPE_CACHE_WALK | + WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK); } WT_ERR_NOTFOUND_OK(ret); if ((ret = __wt_config_subgets( session, &cval, "fast", &sval)) == 0 && sval.val != 0) { - if (F_ISSET(cst, WT_CONN_STAT_ALL)) + if (F_ISSET(cst, WT_STAT_TYPE_ALL)) WT_ERR_MSG(session, EINVAL, - "only one statistics configuration value " - "may be specified"); - F_SET(cst, WT_CONN_STAT_FAST); + "Only one of all, fast, none " + "configuration values should be specified"); + F_SET(cst, WT_STAT_TYPE_FAST); } WT_ERR_NOTFOUND_OK(ret); + + if ((ret = __wt_config_subgets(session, + &cval, "cache_walk", &sval)) == 0 && sval.val != 0) { + /* + * Configuring cache walk statistics implies fast + * statistics. Keep that knowledge internal for now - + * it may change in the future. + */ + F_SET(cst, WT_STAT_TYPE_CACHE_WALK | WT_STAT_TYPE_FAST); + } + WT_ERR_NOTFOUND_OK(ret); + + if ((ret = __wt_config_subgets(session, + &cval, "tree_walk", &sval)) == 0 && sval.val != 0) { + /* + * Configuring tree walk statistics implies fast + * statistics. Keep that knowledge internal for now - + * it may change in the future. + */ + F_SET(cst, WT_STAT_TYPE_FAST | WT_STAT_TYPE_TREE_WALK); + } + WT_ERR_NOTFOUND_OK(ret); + if ((ret = __wt_config_subgets( session, &cval, "size", &sval)) == 0 && sval.val != 0) { - if (F_ISSET(cst, WT_CONN_STAT_FAST | WT_CONN_STAT_ALL)) + if (F_ISSET(cst, WT_STAT_TYPE_FAST | WT_STAT_TYPE_ALL)) WT_ERR_MSG(session, EINVAL, - "only one statistics configuration value " - "may be specified"); - F_SET(cst, WT_CONN_STAT_SIZE); + "Only one of all, fast, none " + "configuration values should be specified"); + F_SET(cst, WT_STAT_TYPE_SIZE); } WT_ERR_NOTFOUND_OK(ret); if ((ret = __wt_config_subgets( session, &cval, "clear", &sval)) == 0 && sval.val != 0) { - if (F_ISSET(cst, WT_CONN_STAT_SIZE)) + if (F_ISSET(cst, WT_STAT_TYPE_SIZE)) WT_ERR_MSG(session, EINVAL, "clear is incompatible with size " "statistics"); - F_SET(cst, WT_CONN_STAT_CLEAR); + F_SET(cst, WT_STAT_CLEAR); } WT_ERR_NOTFOUND_OK(ret); /* If no configuration, use the connection's configuration. */ if (cst->flags == 0) { - if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_ALL)) - F_SET(cst, WT_CONN_STAT_ALL); - if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_FAST)) - F_SET(cst, WT_CONN_STAT_FAST); + if (FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_ALL)) + F_SET(cst, WT_STAT_TYPE_ALL); + if (FLD_ISSET( + conn->stat_flags, WT_STAT_TYPE_CACHE_WALK)) + F_SET(cst, WT_STAT_TYPE_CACHE_WALK); + if (FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_FAST)) + F_SET(cst, WT_STAT_TYPE_FAST); + if (FLD_ISSET(conn->stat_flags, WT_STAT_TYPE_TREE_WALK)) + F_SET(cst, WT_STAT_TYPE_TREE_WALK); } /* If the connection configures clear, so do we. */ - if (FLD_ISSET(conn->stat_flags, WT_CONN_STAT_CLEAR)) - F_SET(cst, WT_CONN_STAT_CLEAR); + if (FLD_ISSET(conn->stat_flags, WT_STAT_CLEAR)) + F_SET(cst, WT_STAT_CLEAR); } /* @@ -670,9 +699,9 @@ __wt_curstat_open(WT_SESSION_IMPL *session, /* * Do the initial statistics snapshot: there won't be cursor operations - * to trigger initialization when aggregating statistics for upper-level - * objects like tables, we need to a valid set of statistics when before - * the open returns. + * to trigger initialization with aggregating statistics for upper-level + * objects like tables so we need a valid set of statistics before the + * open returns. */ WT_ERR(__wt_curstat_init(session, uri, other, cst->cfg, cst)); cst->notinitialized = false; diff --git a/src/evict/evict_lru.c b/src/evict/evict_lru.c index db1d19d60d7..8ca73160cd5 100644 --- a/src/evict/evict_lru.c +++ b/src/evict/evict_lru.c @@ -548,6 +548,7 @@ __evict_pass(WT_SESSION_IMPL *session) * does need to do some work. */ __wt_cache_read_gen_incr(session); + ++cache->evict_pass_gen; /* * Update the oldest ID: we use it to decide whether pages are @@ -1411,6 +1412,7 @@ __evict_walk_file(WT_SESSION_IMPL *session, page = ref->page; modified = __wt_page_is_modified(page); + page->evict_pass_gen = cache->evict_pass_gen; /* * Use the EVICT_LRU flag to avoid putting pages onto the list diff --git a/src/evict/evict_stat.c b/src/evict/evict_stat.c new file mode 100644 index 00000000000..cbb21d0bce6 --- /dev/null +++ b/src/evict/evict_stat.c @@ -0,0 +1,150 @@ +/*- + * Copyright (c) 2014-2016 MongoDB, Inc. + * Copyright (c) 2008-2014 WiredTiger, Inc. + * All rights reserved. + * + * See the file LICENSE for redistribution information. + */ + +#include "wt_internal.h" + +/* + * __evict_stat_walk -- + * Walk all the pages in cache for a dhandle gathering stats information + */ +static int +__evict_stat_walk(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_PAGE *page; + WT_REF *next_walk; + uint64_t dsk_size, gen_gap, size; + uint64_t bytes_inmem; + uint64_t written_size_cnt, written_size_sum; + uint64_t gen_gap_cnt, gen_gap_max, gen_gap_sum; + uint64_t max_pagesize, min_written_size; + uint64_t num_memory, num_queued, num_not_queueable, num_smaller_allocsz; + uint64_t pages_clean, pages_dirty, pages_internal, pages_leaf; + uint64_t seen_count, walk_count; + + btree = S2BT(session); + next_walk = NULL; + bytes_inmem = written_size_cnt = written_size_sum = 0; + gen_gap_cnt = gen_gap_max = gen_gap_sum = 0; + max_pagesize = 0; + num_memory = num_queued = num_not_queueable = num_smaller_allocsz = 0; + pages_clean = pages_dirty = pages_internal = pages_leaf = 0; + seen_count = walk_count = 0; + min_written_size = UINT64_MAX; + + while (__wt_tree_walk_count(session, &next_walk, &walk_count, + WT_READ_CACHE | WT_READ_NO_EVICT | WT_READ_NO_WAIT) == 0 && + next_walk != NULL) { + ++seen_count; + page = next_walk->page; + size = page->memory_footprint; + + bytes_inmem += size; + + if (__wt_page_is_modified(page)) + ++pages_dirty; + else + ++pages_clean; + + if (!__wt_ref_is_root(next_walk) && + !__wt_page_can_evict(session, next_walk, NULL)) + ++num_not_queueable; + + if (F_ISSET_ATOMIC(page, WT_PAGE_EVICT_LRU)) + ++num_queued; + + if (size > max_pagesize) + max_pagesize = size; + + dsk_size = page->dsk != NULL ? page->dsk->mem_size : 0; + if (dsk_size != 0) { + if (dsk_size < btree->allocsize) + ++num_smaller_allocsz; + if (dsk_size < min_written_size) + min_written_size = dsk_size; + ++written_size_cnt; + written_size_sum += dsk_size; + } else + ++num_memory; + + if (WT_PAGE_IS_INTERNAL(page)) { + ++pages_internal; + } else { + ++pages_leaf; + } + + /* Skip root pages since they are never considered */ + if (__wt_ref_is_root(next_walk)) + continue; + + gen_gap = + S2C(session)->cache->evict_pass_gen - page->evict_pass_gen; + if (gen_gap > gen_gap_max) + gen_gap_max = gen_gap; + gen_gap_sum += gen_gap; + ++gen_gap_cnt; + } + + WT_STAT_DATA_SET(session, + cache_state_avg_written_size, written_size_cnt == 0 ? + 0 : written_size_sum / written_size_cnt); + WT_STAT_DATA_SET(session, + cache_state_gen_avg_gap, gen_gap_cnt == 0 ? + 0 : gen_gap_sum / gen_gap_cnt); + + WT_STAT_DATA_SET(session, cache_state_gen_max_gap, gen_gap_max); + WT_STAT_DATA_SET(session, cache_state_max_pagesize, max_pagesize); + WT_STAT_DATA_SET(session, + cache_state_min_written_size, min_written_size); + WT_STAT_DATA_SET(session, cache_state_memory, num_memory); + WT_STAT_DATA_SET(session, cache_state_queued, num_queued); + WT_STAT_DATA_SET(session, + cache_state_not_queueable, num_not_queueable); + WT_STAT_DATA_SET(session, + cache_state_smaller_alloc_size, num_smaller_allocsz); + WT_STAT_DATA_SET(session, cache_state_pages, walk_count); + WT_STAT_DATA_SET(session, cache_state_pages_clean, pages_clean); + WT_STAT_DATA_SET(session, cache_state_pages_dirty, pages_dirty); + WT_STAT_DATA_SET(session, cache_state_pages_internal, pages_internal); + WT_STAT_DATA_SET(session, cache_state_pages_leaf, pages_leaf); + WT_STAT_DATA_SET(session, + cache_state_refs_skipped, walk_count - seen_count); + + return (0); +} + +/* + * __wt_curstat_cache_walk -- + * Initialize the statistics for a cache cache_walk pass. + */ +int +__wt_curstat_cache_walk(WT_SESSION_IMPL *session) +{ + WT_BTREE *btree; + WT_CONNECTION_IMPL *conn; + WT_DECL_RET; + WT_PAGE_INDEX *root_idx; + + btree = S2BT(session); + conn = S2C(session); + + /* Set statistics that don't require walking the cache. */ + WT_STAT_DATA_SET(session, + cache_state_gen_current, conn->cache->evict_pass_gen); + + /* Root page statistics */ + root_idx = WT_INTL_INDEX_GET_SAFE(btree->root.page); + WT_STAT_DATA_SET(session, + cache_state_root_entries, root_idx->entries); + WT_STAT_DATA_SET(session, + cache_state_root_size, btree->root.page->memory_footprint); + + WT_WITH_HANDLE_LIST_LOCK(session, ret = __evict_stat_walk(session)); + + return (ret); +} diff --git a/src/include/btmem.h b/src/include/btmem.h index b4ca937e7ed..278d0e50ee8 100644 --- a/src/include/btmem.h +++ b/src/include/btmem.h @@ -619,6 +619,8 @@ struct __wt_page { #define WT_READGEN_START_VALUE 100 #define WT_READGEN_STEP 100 uint64_t read_gen; + /* The evict pass generation for the page */ + uint64_t evict_pass_gen; size_t memory_footprint; /* Memory attached to the page */ diff --git a/src/include/cache.h b/src/include/cache.h index b24b625aec4..9a2b83b5b57 100644 --- a/src/include/cache.h +++ b/src/include/cache.h @@ -91,6 +91,7 @@ struct __wt_cache { uint64_t read_gen; /* Current page read generation */ uint64_t read_gen_oldest; /* Oldest read generation the eviction * server saw in its last queue load */ + uint64_t evict_pass_gen; /* Number of eviction passes */ /* * Eviction thread information. diff --git a/src/include/connection.h b/src/include/connection.h index ce81dcf5976..d7c3bf69686 100644 --- a/src/include/connection.h +++ b/src/include/connection.h @@ -285,13 +285,7 @@ struct __wt_connection_impl { uint64_t ckpt_time_recent; /* Checkpoint time recent/total */ uint64_t ckpt_time_total; -#define WT_CONN_STAT_ALL 0x01 /* "all" statistics configured */ -#define WT_CONN_STAT_CLEAR 0x02 /* clear after gathering */ -#define WT_CONN_STAT_FAST 0x04 /* "fast" statistics configured */ -#define WT_CONN_STAT_JSON 0x08 /* output JSON format */ -#define WT_CONN_STAT_ON_CLOSE 0x10 /* output statistics on close */ -#define WT_CONN_STAT_SIZE 0x20 /* "size" statistics configured */ - uint32_t stat_flags; + uint32_t stat_flags; /* Options declared in flags.py */ /* Connection statistics */ WT_CONNECTION_STATS *stats[WT_COUNTER_SLOTS]; diff --git a/src/include/cursor.h b/src/include/cursor.h index f1fa4d193ac..e322a53a65d 100644 --- a/src/include/cursor.h +++ b/src/include/cursor.h @@ -467,7 +467,7 @@ struct __wt_cursor_stat { uint64_t v; /* Current stats value */ WT_ITEM pv; /* Current stats value (string) */ - /* Uses the same values as WT_CONNECTION::stat_flags field */ + /* Options declared in flags.py, shared by WT_CONNECTION::stat_flags */ uint32_t flags; }; diff --git a/src/include/extern.h b/src/include/extern.h index 9660e641e50..94b1feede47 100644 --- a/src/include/extern.h +++ b/src/include/extern.h @@ -353,6 +353,7 @@ extern void __wt_evict_priority_clear(WT_SESSION_IMPL *session); extern int __wt_cache_dump(WT_SESSION_IMPL *session, const char *ofile) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_page_release_evict(WT_SESSION_IMPL *session, WT_REF *ref) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern int __wt_evict(WT_SESSION_IMPL *session, WT_REF *ref, bool closing) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); +extern int __wt_curstat_cache_walk(WT_SESSION_IMPL *session) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_ckpt(WT_SESSION_IMPL *session, WT_LSN *ckp_lsn); extern int __wt_log_flush_lsn(WT_SESSION_IMPL *session, WT_LSN *lsn, bool start) WT_GCC_FUNC_DECL_ATTRIBUTE((warn_unused_result)); extern void __wt_log_background(WT_SESSION_IMPL *session, WT_LSN *lsn); diff --git a/src/include/flags.h b/src/include/flags.h index 5d718da473d..b0d167525b2 100644 --- a/src/include/flags.h +++ b/src/include/flags.h @@ -70,6 +70,14 @@ #define WT_SESSION_NO_SCHEMA_LOCK 0x00020000 #define WT_SESSION_QUIET_CORRUPT_FILE 0x00040000 #define WT_SESSION_SERVER_ASYNC 0x00080000 +#define WT_STAT_CLEAR 0x00000001 +#define WT_STAT_JSON 0x00000002 +#define WT_STAT_ON_CLOSE 0x00000004 +#define WT_STAT_TYPE_ALL 0x00000008 +#define WT_STAT_TYPE_CACHE_WALK 0x00000010 +#define WT_STAT_TYPE_FAST 0x00000020 +#define WT_STAT_TYPE_SIZE 0x00000040 +#define WT_STAT_TYPE_TREE_WALK 0x00000080 #define WT_TXN_LOG_CKPT_CLEANUP 0x00000001 #define WT_TXN_LOG_CKPT_PREPARE 0x00000002 #define WT_TXN_LOG_CKPT_START 0x00000004 diff --git a/src/include/stat.h b/src/include/stat.h index 45c21d89cb3..15edce9ec85 100644 --- a/src/include/stat.h +++ b/src/include/stat.h @@ -555,6 +555,24 @@ struct __wt_dsrc_stats { int64_t cache_write; int64_t cache_write_restore; int64_t cache_eviction_clean; + int64_t cache_state_gen_avg_gap; + int64_t cache_state_avg_written_size; + int64_t cache_state_pages_clean; + int64_t cache_state_gen_current; + int64_t cache_state_pages_dirty; + int64_t cache_state_root_entries; + int64_t cache_state_pages_internal; + int64_t cache_state_pages_leaf; + int64_t cache_state_gen_max_gap; + int64_t cache_state_max_pagesize; + int64_t cache_state_min_written_size; + int64_t cache_state_smaller_alloc_size; + int64_t cache_state_memory; + int64_t cache_state_queued; + int64_t cache_state_not_queueable; + int64_t cache_state_refs_skipped; + int64_t cache_state_root_size; + int64_t cache_state_pages; int64_t compress_read; int64_t compress_write; int64_t compress_write_fail; diff --git a/src/include/wiredtiger.in b/src/include/wiredtiger.in index 5f61e86b561..f3156609fed 100644 --- a/src/include/wiredtiger.in +++ b/src/include/wiredtiger.in @@ -962,8 +962,9 @@ struct __wt_session { * where appropriate (for example\, a cache size statistic is not * cleared\, while the count of cursor insert operations will be * cleared). See @ref statistics for more information., a list\, with - * values chosen from the following options: \c "all"\, \c "fast"\, \c - * "clear"\, \c "size"; default empty.} + * values chosen from the following options: \c "all"\, \c + * "cache_walk"\, \c "fast"\, \c "clear"\, \c "size"\, \c "tree_walk"; + * default empty.} * @config{target, if non-empty\, backup the list of objects; valid only * for a backup data source., a list of strings; default empty.} * @configend @@ -1903,8 +1904,9 @@ struct __wt_connection { * reset each time a statistics cursor is used to gather statistics\, as * well as each time statistics are logged using the \c statistics_log * configuration. See @ref statistics for more information., a list\, - * with values chosen from the following options: \c "all"\, \c "fast"\, - * \c "none"\, \c "clear"; default \c none.} + * with values chosen from the following options: \c "all"\, \c + * "cache_walk"\, \c "fast"\, \c "none"\, \c "clear"\, \c "tree_walk"; + * default \c none.} * @config{statistics_log = (, log any statistics the database is * configured to maintain\, to a file. See @ref statistics for more * information. Enabling the statistics log server uses a session from @@ -2406,8 +2408,9 @@ struct __wt_connection { * statistics are reset each time a statistics cursor is used to gather * statistics\, as well as each time statistics are logged using the \c * statistics_log configuration. See @ref statistics for more information., a - * list\, with values chosen from the following options: \c "all"\, \c "fast"\, - * \c "none"\, \c "clear"; default \c none.} + * list\, with values chosen from the following options: \c "all"\, \c + * "cache_walk"\, \c "fast"\, \c "none"\, \c "clear"\, \c "tree_walk"; default + * \c none.} * @config{statistics_log = (, log any statistics the database is configured to * maintain\, to a file. See @ref statistics for more information. Enabling * the statistics log server uses a session from the configured session_max., a @@ -4757,28 +4760,28 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); /*! btree: btree checkpoint generation */ #define WT_STAT_DSRC_BTREE_CHECKPOINT_GENERATION 2022 /*! - * btree: column-store fixed-size leaf pages, only reported if - * statistics=all is set + * btree: column-store fixed-size leaf pages, only reported if tree_walk + * or all statistics are enabled */ #define WT_STAT_DSRC_BTREE_COLUMN_FIX 2023 /*! - * btree: column-store internal pages, only reported if statistics=all is - * set + * btree: column-store internal pages, only reported if tree_walk or all + * statistics are enabled */ #define WT_STAT_DSRC_BTREE_COLUMN_INTERNAL 2024 /*! * btree: column-store variable-size RLE encoded values, only reported if - * statistics=all is set + * tree_walk or all statistics are enabled */ #define WT_STAT_DSRC_BTREE_COLUMN_RLE 2025 /*! * btree: column-store variable-size deleted values, only reported if - * statistics=all is set + * tree_walk or all statistics are enabled */ #define WT_STAT_DSRC_BTREE_COLUMN_DELETED 2026 /*! * btree: column-store variable-size leaf pages, only reported if - * statistics=all is set + * tree_walk or all statistics are enabled */ #define WT_STAT_DSRC_BTREE_COLUMN_VARIABLE 2027 /*! btree: fixed-record size */ @@ -4796,20 +4799,26 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); /*! btree: maximum tree depth */ #define WT_STAT_DSRC_BTREE_MAXIMUM_DEPTH 2034 /*! - * btree: number of key/value pairs, only reported if statistics=all is - * set + * btree: number of key/value pairs, only reported if tree_walk or all + * statistics are enabled */ #define WT_STAT_DSRC_BTREE_ENTRIES 2035 -/*! btree: overflow pages, only reported if statistics=all is set */ +/*! + * btree: overflow pages, only reported if tree_walk or all statistics + * are enabled + */ #define WT_STAT_DSRC_BTREE_OVERFLOW 2036 /*! btree: pages rewritten by compaction */ #define WT_STAT_DSRC_BTREE_COMPACT_REWRITE 2037 /*! - * btree: row-store internal pages, only reported if statistics=all is - * set + * btree: row-store internal pages, only reported if tree_walk or all + * statistics are enabled */ #define WT_STAT_DSRC_BTREE_ROW_INTERNAL 2038 -/*! btree: row-store leaf pages, only reported if statistics=all is set */ +/*! + * btree: row-store leaf pages, only reported if tree_walk or all + * statistics are enabled + */ #define WT_STAT_DSRC_BTREE_ROW_LEAF 2039 /*! cache: bytes currently in the cache */ #define WT_STAT_DSRC_CACHE_BYTES_INUSE 2040 @@ -4855,87 +4864,179 @@ extern int wiredtiger_extension_terminate(WT_CONNECTION *connection); #define WT_STAT_DSRC_CACHE_WRITE_RESTORE 2060 /*! cache: unmodified pages evicted */ #define WT_STAT_DSRC_CACHE_EVICTION_CLEAN 2061 +/*! + * cache_walk: Average difference between current eviction generation + * when the page was last considered, only reported if cache_walk or all + * statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_GEN_AVG_GAP 2062 +/*! + * cache_walk: Average on-disk page image size seen, only reported if + * cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_AVG_WRITTEN_SIZE 2063 +/*! + * cache_walk: Clean pages currently in cache, only reported if + * cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_PAGES_CLEAN 2064 +/*! + * cache_walk: Current eviction generation, only reported if cache_walk + * or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_GEN_CURRENT 2065 +/*! + * cache_walk: Dirty pages currently in cache, only reported if + * cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_PAGES_DIRTY 2066 +/*! + * cache_walk: Entries in the root page, only reported if cache_walk or + * all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_ROOT_ENTRIES 2067 +/*! + * cache_walk: Internal pages currently in cache, only reported if + * cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_PAGES_INTERNAL 2068 +/*! + * cache_walk: Leaf pages currently in cache, only reported if cache_walk + * or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_PAGES_LEAF 2069 +/*! + * cache_walk: Maximum difference between current eviction generation + * when the page was last considered, only reported if cache_walk or all + * statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_GEN_MAX_GAP 2070 +/*! + * cache_walk: Maximum page size seen, only reported if cache_walk or all + * statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_MAX_PAGESIZE 2071 +/*! + * cache_walk: Minimum on-disk page image size seen, only reported if + * cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_MIN_WRITTEN_SIZE 2072 +/*! + * cache_walk: On-disk page image sizes smaller than a single allocation + * unit, only reported if cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_SMALLER_ALLOC_SIZE 2073 +/*! + * cache_walk: Pages created in memory and never written, only reported + * if cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_MEMORY 2074 +/*! + * cache_walk: Pages currently queued for eviction, only reported if + * cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_QUEUED 2075 +/*! + * cache_walk: Pages that could not be queued for eviction, only reported + * if cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_NOT_QUEUEABLE 2076 +/*! + * cache_walk: Refs skipped during cache traversal, only reported if + * cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_REFS_SKIPPED 2077 +/*! + * cache_walk: Size of the root page, only reported if cache_walk or all + * statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_ROOT_SIZE 2078 +/*! + * cache_walk: Total number of pages currently in cache, only reported if + * cache_walk or all statistics are enabled + */ +#define WT_STAT_DSRC_CACHE_STATE_PAGES 2079 /*! compression: compressed pages read */ -#define WT_STAT_DSRC_COMPRESS_READ 2062 +#define WT_STAT_DSRC_COMPRESS_READ 2080 /*! compression: compressed pages written */ -#define WT_STAT_DSRC_COMPRESS_WRITE 2063 +#define WT_STAT_DSRC_COMPRESS_WRITE 2081 /*! compression: page written failed to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2064 +#define WT_STAT_DSRC_COMPRESS_WRITE_FAIL 2082 /*! compression: page written was too small to compress */ -#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2065 +#define WT_STAT_DSRC_COMPRESS_WRITE_TOO_SMALL 2083 /*! compression: raw compression call failed, additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2066 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL_TEMPORARY 2084 /*! compression: raw compression call failed, no additional data available */ -#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2067 +#define WT_STAT_DSRC_COMPRESS_RAW_FAIL 2085 /*! compression: raw compression call succeeded */ -#define WT_STAT_DSRC_COMPRESS_RAW_OK 2068 +#define WT_STAT_DSRC_COMPRESS_RAW_OK 2086 /*! cursor: bulk-loaded cursor-insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2069 +#define WT_STAT_DSRC_CURSOR_INSERT_BULK 2087 /*! cursor: create calls */ -#define WT_STAT_DSRC_CURSOR_CREATE 2070 +#define WT_STAT_DSRC_CURSOR_CREATE 2088 /*! cursor: cursor-insert key and value bytes inserted */ -#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2071 +#define WT_STAT_DSRC_CURSOR_INSERT_BYTES 2089 /*! cursor: cursor-remove key bytes removed */ -#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2072 +#define WT_STAT_DSRC_CURSOR_REMOVE_BYTES 2090 /*! cursor: cursor-update value bytes updated */ -#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2073 +#define WT_STAT_DSRC_CURSOR_UPDATE_BYTES 2091 /*! cursor: insert calls */ -#define WT_STAT_DSRC_CURSOR_INSERT 2074 +#define WT_STAT_DSRC_CURSOR_INSERT 2092 /*! cursor: next calls */ -#define WT_STAT_DSRC_CURSOR_NEXT 2075 +#define WT_STAT_DSRC_CURSOR_NEXT 2093 /*! cursor: prev calls */ -#define WT_STAT_DSRC_CURSOR_PREV 2076 +#define WT_STAT_DSRC_CURSOR_PREV 2094 /*! cursor: remove calls */ -#define WT_STAT_DSRC_CURSOR_REMOVE 2077 +#define WT_STAT_DSRC_CURSOR_REMOVE 2095 /*! cursor: reset calls */ -#define WT_STAT_DSRC_CURSOR_RESET 2078 +#define WT_STAT_DSRC_CURSOR_RESET 2096 /*! cursor: restarted searches */ -#define WT_STAT_DSRC_CURSOR_RESTART 2079 +#define WT_STAT_DSRC_CURSOR_RESTART 2097 /*! cursor: search calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH 2080 +#define WT_STAT_DSRC_CURSOR_SEARCH 2098 /*! cursor: search near calls */ -#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2081 +#define WT_STAT_DSRC_CURSOR_SEARCH_NEAR 2099 /*! cursor: truncate calls */ -#define WT_STAT_DSRC_CURSOR_TRUNCATE 2082 +#define WT_STAT_DSRC_CURSOR_TRUNCATE 2100 /*! cursor: update calls */ -#define WT_STAT_DSRC_CURSOR_UPDATE 2083 +#define WT_STAT_DSRC_CURSOR_UPDATE 2101 /*! reconciliation: dictionary matches */ -#define WT_STAT_DSRC_REC_DICTIONARY 2084 +#define WT_STAT_DSRC_REC_DICTIONARY 2102 /*! reconciliation: fast-path pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2085 +#define WT_STAT_DSRC_REC_PAGE_DELETE_FAST 2103 /*! * reconciliation: internal page key bytes discarded using suffix * compression */ -#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2086 +#define WT_STAT_DSRC_REC_SUFFIX_COMPRESSION 2104 /*! reconciliation: internal page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2087 +#define WT_STAT_DSRC_REC_MULTIBLOCK_INTERNAL 2105 /*! reconciliation: internal-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2088 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_INTERNAL 2106 /*! reconciliation: leaf page key bytes discarded using prefix compression */ -#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2089 +#define WT_STAT_DSRC_REC_PREFIX_COMPRESSION 2107 /*! reconciliation: leaf page multi-block writes */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2090 +#define WT_STAT_DSRC_REC_MULTIBLOCK_LEAF 2108 /*! reconciliation: leaf-page overflow keys */ -#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2091 +#define WT_STAT_DSRC_REC_OVERFLOW_KEY_LEAF 2109 /*! reconciliation: maximum blocks required for a page */ -#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2092 +#define WT_STAT_DSRC_REC_MULTIBLOCK_MAX 2110 /*! reconciliation: overflow values written */ -#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2093 +#define WT_STAT_DSRC_REC_OVERFLOW_VALUE 2111 /*! reconciliation: page checksum matches */ -#define WT_STAT_DSRC_REC_PAGE_MATCH 2094 +#define WT_STAT_DSRC_REC_PAGE_MATCH 2112 /*! reconciliation: page reconciliation calls */ -#define WT_STAT_DSRC_REC_PAGES 2095 +#define WT_STAT_DSRC_REC_PAGES 2113 /*! reconciliation: page reconciliation calls for eviction */ -#define WT_STAT_DSRC_REC_PAGES_EVICTION 2096 +#define WT_STAT_DSRC_REC_PAGES_EVICTION 2114 /*! reconciliation: pages deleted */ -#define WT_STAT_DSRC_REC_PAGE_DELETE 2097 +#define WT_STAT_DSRC_REC_PAGE_DELETE 2115 /*! session: object compaction */ -#define WT_STAT_DSRC_SESSION_COMPACT 2098 +#define WT_STAT_DSRC_SESSION_COMPACT 2116 /*! session: open cursor count */ -#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2099 +#define WT_STAT_DSRC_SESSION_CURSOR_OPEN 2117 /*! transaction: update conflicts */ -#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2100 +#define WT_STAT_DSRC_TXN_UPDATE_CONFLICT 2118 /*! * @} diff --git a/src/lsm/lsm_stat.c b/src/lsm/lsm_stat.c index f4f5a0acce8..3fe3ca1ba81 100644 --- a/src/lsm/lsm_stat.c +++ b/src/lsm/lsm_stat.c @@ -42,11 +42,11 @@ __curstat_lsm_init( if (cst->flags != 0) { (void)snprintf(config, sizeof(config), "statistics=(%s%s%s%s)", - F_ISSET(cst, WT_CONN_STAT_ALL) ? "all," : "", - F_ISSET(cst, WT_CONN_STAT_CLEAR) ? "clear," : "", - !F_ISSET(cst, WT_CONN_STAT_ALL) && - F_ISSET(cst, WT_CONN_STAT_FAST) ? "fast," : "", - F_ISSET(cst, WT_CONN_STAT_SIZE) ? "size," : ""); + F_ISSET(cst, WT_STAT_TYPE_ALL) ? "all," : "", + F_ISSET(cst, WT_STAT_CLEAR) ? "clear," : "", + !F_ISSET(cst, WT_STAT_TYPE_ALL) && + F_ISSET(cst, WT_STAT_TYPE_FAST) ? "fast," : "", + F_ISSET(cst, WT_STAT_TYPE_SIZE) ? "size," : ""); cfg[1] = disk_cfg[1] = config; } @@ -132,26 +132,26 @@ __curstat_lsm_init( /* Include, and optionally clear, LSM-level specific information. */ WT_STAT_WRITE(session, stats, bloom_miss, lsm_tree->bloom_miss); - if (F_ISSET(cst, WT_CONN_STAT_CLEAR)) + if (F_ISSET(cst, WT_STAT_CLEAR)) lsm_tree->bloom_miss = 0; WT_STAT_WRITE(session, stats, bloom_hit, lsm_tree->bloom_hit); - if (F_ISSET(cst, WT_CONN_STAT_CLEAR)) + if (F_ISSET(cst, WT_STAT_CLEAR)) lsm_tree->bloom_hit = 0; WT_STAT_WRITE(session, stats, bloom_false_positive, lsm_tree->bloom_false_positive); - if (F_ISSET(cst, WT_CONN_STAT_CLEAR)) + if (F_ISSET(cst, WT_STAT_CLEAR)) lsm_tree->bloom_false_positive = 0; WT_STAT_WRITE(session, stats, lsm_lookup_no_bloom, lsm_tree->lsm_lookup_no_bloom); - if (F_ISSET(cst, WT_CONN_STAT_CLEAR)) + if (F_ISSET(cst, WT_STAT_CLEAR)) lsm_tree->lsm_lookup_no_bloom = 0; WT_STAT_WRITE(session, stats, lsm_checkpoint_throttle, lsm_tree->lsm_checkpoint_throttle); - if (F_ISSET(cst, WT_CONN_STAT_CLEAR)) + if (F_ISSET(cst, WT_STAT_CLEAR)) lsm_tree->lsm_checkpoint_throttle = 0; WT_STAT_WRITE(session, stats, lsm_merge_throttle, lsm_tree->lsm_merge_throttle); - if (F_ISSET(cst, WT_CONN_STAT_CLEAR)) + if (F_ISSET(cst, WT_STAT_CLEAR)) lsm_tree->lsm_merge_throttle = 0; __wt_curstat_dsrc_final(cst); diff --git a/src/schema/schema_stat.c b/src/schema/schema_stat.c index 1cd39d97364..345f9164e9b 100644 --- a/src/schema/schema_stat.c +++ b/src/schema/schema_stat.c @@ -137,7 +137,7 @@ __wt_curstat_table_init(WT_SESSION_IMPL *session, * If only gathering table size statistics, try a fast path that * avoids the schema and table list locks. */ - if (F_ISSET(cst, WT_CONN_STAT_SIZE)) { + if (F_ISSET(cst, WT_STAT_TYPE_SIZE)) { WT_RET(__curstat_size_only(session, uri, &was_fast, cst)); if (was_fast) return (0); diff --git a/src/support/stat.c b/src/support/stat.c index 8fafd2982e5..d6457ed64bb 100644 --- a/src/support/stat.c +++ b/src/support/stat.c @@ -65,6 +65,24 @@ static const char * const __stats_dsrc_desc[] = { "cache: pages written from cache", "cache: pages written requiring in-memory restoration", "cache: unmodified pages evicted", + "cache_walk: Average difference between current eviction generation when the page was last considered", + "cache_walk: Average on-disk page image size seen", + "cache_walk: Clean pages currently in cache", + "cache_walk: Current eviction generation", + "cache_walk: Dirty pages currently in cache", + "cache_walk: Entries in the root page", + "cache_walk: Internal pages currently in cache", + "cache_walk: Leaf pages currently in cache", + "cache_walk: Maximum difference between current eviction generation when the page was last considered", + "cache_walk: Maximum page size seen", + "cache_walk: Minimum on-disk page image size seen", + "cache_walk: On-disk page image sizes smaller than a single allocation unit", + "cache_walk: Pages created in memory and never written", + "cache_walk: Pages currently queued for eviction", + "cache_walk: Pages that could not be queued for eviction", + "cache_walk: Refs skipped during cache traversal", + "cache_walk: Size of the root page", + "cache_walk: Total number of pages currently in cache", "compression: compressed pages read", "compression: compressed pages written", "compression: page written failed to compress", @@ -196,6 +214,24 @@ __wt_stat_dsrc_clear_single(WT_DSRC_STATS *stats) stats->cache_write = 0; stats->cache_write_restore = 0; stats->cache_eviction_clean = 0; + /* not clearing cache_state_gen_avg_gap */ + /* not clearing cache_state_avg_written_size */ + /* not clearing cache_state_pages_clean */ + /* not clearing cache_state_gen_current */ + /* not clearing cache_state_pages_dirty */ + /* not clearing cache_state_root_entries */ + /* not clearing cache_state_pages_internal */ + /* not clearing cache_state_pages_leaf */ + /* not clearing cache_state_gen_max_gap */ + /* not clearing cache_state_max_pagesize */ + /* not clearing cache_state_min_written_size */ + /* not clearing cache_state_smaller_alloc_size */ + /* not clearing cache_state_memory */ + /* not clearing cache_state_queued */ + /* not clearing cache_state_not_queueable */ + /* not clearing cache_state_refs_skipped */ + /* not clearing cache_state_root_size */ + /* not clearing cache_state_pages */ stats->compress_read = 0; stats->compress_write = 0; stats->compress_write_fail = 0; @@ -325,6 +361,27 @@ __wt_stat_dsrc_aggregate_single( to->cache_write += from->cache_write; to->cache_write_restore += from->cache_write_restore; to->cache_eviction_clean += from->cache_eviction_clean; + to->cache_state_gen_avg_gap += from->cache_state_gen_avg_gap; + to->cache_state_avg_written_size += + from->cache_state_avg_written_size; + to->cache_state_pages_clean += from->cache_state_pages_clean; + to->cache_state_gen_current += from->cache_state_gen_current; + to->cache_state_pages_dirty += from->cache_state_pages_dirty; + to->cache_state_root_entries += from->cache_state_root_entries; + to->cache_state_pages_internal += from->cache_state_pages_internal; + to->cache_state_pages_leaf += from->cache_state_pages_leaf; + to->cache_state_gen_max_gap += from->cache_state_gen_max_gap; + to->cache_state_max_pagesize += from->cache_state_max_pagesize; + to->cache_state_min_written_size += + from->cache_state_min_written_size; + to->cache_state_smaller_alloc_size += + from->cache_state_smaller_alloc_size; + to->cache_state_memory += from->cache_state_memory; + to->cache_state_queued += from->cache_state_queued; + to->cache_state_not_queueable += from->cache_state_not_queueable; + to->cache_state_refs_skipped += from->cache_state_refs_skipped; + to->cache_state_root_size += from->cache_state_root_size; + to->cache_state_pages += from->cache_state_pages; to->compress_read += from->compress_read; to->compress_write += from->compress_write; to->compress_write_fail += from->compress_write_fail; @@ -467,6 +524,39 @@ __wt_stat_dsrc_aggregate( to->cache_write += WT_STAT_READ(from, cache_write); to->cache_write_restore += WT_STAT_READ(from, cache_write_restore); to->cache_eviction_clean += WT_STAT_READ(from, cache_eviction_clean); + to->cache_state_gen_avg_gap += + WT_STAT_READ(from, cache_state_gen_avg_gap); + to->cache_state_avg_written_size += + WT_STAT_READ(from, cache_state_avg_written_size); + to->cache_state_pages_clean += + WT_STAT_READ(from, cache_state_pages_clean); + to->cache_state_gen_current += + WT_STAT_READ(from, cache_state_gen_current); + to->cache_state_pages_dirty += + WT_STAT_READ(from, cache_state_pages_dirty); + to->cache_state_root_entries += + WT_STAT_READ(from, cache_state_root_entries); + to->cache_state_pages_internal += + WT_STAT_READ(from, cache_state_pages_internal); + to->cache_state_pages_leaf += + WT_STAT_READ(from, cache_state_pages_leaf); + to->cache_state_gen_max_gap += + WT_STAT_READ(from, cache_state_gen_max_gap); + to->cache_state_max_pagesize += + WT_STAT_READ(from, cache_state_max_pagesize); + to->cache_state_min_written_size += + WT_STAT_READ(from, cache_state_min_written_size); + to->cache_state_smaller_alloc_size += + WT_STAT_READ(from, cache_state_smaller_alloc_size); + to->cache_state_memory += WT_STAT_READ(from, cache_state_memory); + to->cache_state_queued += WT_STAT_READ(from, cache_state_queued); + to->cache_state_not_queueable += + WT_STAT_READ(from, cache_state_not_queueable); + to->cache_state_refs_skipped += + WT_STAT_READ(from, cache_state_refs_skipped); + to->cache_state_root_size += + WT_STAT_READ(from, cache_state_root_size); + to->cache_state_pages += WT_STAT_READ(from, cache_state_pages); to->compress_read += WT_STAT_READ(from, compress_read); to->compress_write += WT_STAT_READ(from, compress_write); to->compress_write_fail += WT_STAT_READ(from, compress_write_fail); diff --git a/test/suite/test_stat02.py b/test/suite/test_stat02.py index 3d2a83d1c3c..047d2c74499 100644 --- a/test/suite/test_stat02.py +++ b/test/suite/test_stat02.py @@ -165,7 +165,7 @@ class test_stat_cursor_conn_error(wttest.WiredTigerTestCase): args = ['none', 'all', 'fast'] for i in list(itertools.permutations(args, 2)): config = 'create,statistics=(' + i[0] + ',' + i[1] + ')' - msg = '/only one statistics configuration value/' + msg = '/Only one of/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.wiredtiger_open('.', config), msg) @@ -188,10 +188,76 @@ class test_stat_cursor_dsrc_error(wttest.WiredTigerTestCase): args = ['all', 'fast'] for i in list(itertools.permutations(args, 2)): config = 'statistics=(' + i[0] + ',' + i[1] + ')' - msg = '/only one statistics configuration value/' + msg = '/Only one of/' self.assertRaisesWithMessage(wiredtiger.WiredTigerError, lambda: self.session.open_cursor( 'statistics:' + self.uri, None, config), msg) +# Test data-source cache walk statistics +class test_stat_cursor_dsrc_cache_walk(wttest.WiredTigerTestCase): + uri = 'file:test_stat_cursor_dsrc_cache_walk' + + conn_config = 'statistics=(none)' + + def test_stat_cursor_dsrc_cache_walk(self): + simple_populate(self, self.uri, 'key_format=S', 100) + # Ensure that it's an error to get cache_walk stats if none is set + msg = '/doesn\'t match the database statistics/' + self.assertRaisesWithMessage(wiredtiger.WiredTigerError, + lambda: self.session.open_cursor( + 'statistics:' + self.uri, None, None), msg) + + # Test configurations that are valid but should not collect + # cache walk information. Do these first since the cache walk + # statistics are mostly marked as not cleared - so once they are + # populated the values will always be returned + self.conn.reconfigure('statistics=(cache_walk,fast,clear)') + c = self.session.open_cursor( + 'statistics:' + self.uri, None, 'statistics=(fast)') + self.assertEqual(c[stat.dsrc.cache_state_root_size][2], 0) + c.close() + + self.conn.reconfigure('statistics=(all,clear)') + c = self.session.open_cursor( + 'statistics:' + self.uri, None, 'statistics=(fast)') + self.assertEqual(c[stat.dsrc.cache_state_root_size][2], 0) + c.close() + + self.conn.reconfigure('statistics=(cache_walk,fast,clear)') + c = self.session.open_cursor('statistics:' + self.uri, None, None) + self.assertGreater(c[stat.dsrc.cache_state_root_size][2], 0) + # Verify that cache_walk didn't imply tree_walk + self.assertEqual(c[stat.dsrc.btree_entries][2], 0) + c.close() + + self.conn.reconfigure('statistics=(cache_walk,tree_walk,fast,clear)') + c = self.session.open_cursor('statistics:' + self.uri, None, None) + self.assertGreater(c[stat.dsrc.cache_state_root_size][2], 0) + # Verify that cache_walk didn't exclude tree_walk + self.assertGreater(c[stat.dsrc.btree_entries][2], 0) + c.close() + + self.conn.reconfigure('statistics=(all,clear)') + c = self.session.open_cursor( + 'statistics:' + self.uri, None, 'statistics=(all)') + self.assertGreater(c[stat.dsrc.cache_state_root_size][2], 0) + self.assertGreater(c[stat.dsrc.btree_entries][2], 0) + c.close() + + # Verify that cache and tree walk can operate independantly + self.conn.reconfigure('statistics=(all,clear)') + c = self.session.open_cursor( + 'statistics:' + self.uri, None, 'statistics=(cache_walk,fast)') + self.assertGreater(c[stat.dsrc.cache_state_root_size][2], 0) + self.assertEqual(c[stat.dsrc.btree_entries][2], 0) + c.close() + + self.conn.reconfigure('statistics=(all,clear)') + c = self.session.open_cursor( + 'statistics:' + self.uri, None, 'statistics=(tree_walk,fast)') + # Don't check the cache walk stats for empty - they won't be cleared + self.assertGreater(c[stat.dsrc.btree_entries][2], 0) + c.close() + if __name__ == '__main__': wttest.run() diff --git a/tools/wtstats/stat_data.py b/tools/wtstats/stat_data.py index a7202788a95..635e710c469 100644 --- a/tools/wtstats/stat_data.py +++ b/tools/wtstats/stat_data.py @@ -91,6 +91,24 @@ no_scale_per_second_list = [ 'btree: row-store leaf pages', 'cache: bytes currently in the cache', 'cache: overflow values cached in memory', + 'cache_walk: Average difference between current eviction generation when the page was last considered', + 'cache_walk: Average on-disk page image size seen', + 'cache_walk: Clean pages currently in cache', + 'cache_walk: Current eviction generation', + 'cache_walk: Dirty pages currently in cache', + 'cache_walk: Entries in the root page', + 'cache_walk: Internal pages currently in cache', + 'cache_walk: Leaf pages currently in cache', + 'cache_walk: Maximum difference between current eviction generation when the page was last considered', + 'cache_walk: Maximum page size seen', + 'cache_walk: Minimum on-disk page image size seen', + 'cache_walk: On-disk page image sizes smaller than a single allocation unit', + 'cache_walk: Pages created in memory and never written', + 'cache_walk: Pages currently queued for eviction', + 'cache_walk: Pages that could not be queued for eviction', + 'cache_walk: Refs skipped during cache traversal', + 'cache_walk: Size of the root page', + 'cache_walk: Total number of pages currently in cache', 'LSM: bloom filters in the LSM tree', 'LSM: chunks in the LSM tree', 'LSM: highest merge generation in the LSM tree', @@ -162,6 +180,24 @@ no_clear_list = [ 'transaction: transaction range of IDs currently pinned by named snapshots', 'btree: btree checkpoint generation', 'cache: bytes currently in the cache', + 'cache_walk: Average difference between current eviction generation when the page was last considered', + 'cache_walk: Average on-disk page image size seen', + 'cache_walk: Clean pages currently in cache', + 'cache_walk: Current eviction generation', + 'cache_walk: Dirty pages currently in cache', + 'cache_walk: Entries in the root page', + 'cache_walk: Internal pages currently in cache', + 'cache_walk: Leaf pages currently in cache', + 'cache_walk: Maximum difference between current eviction generation when the page was last considered', + 'cache_walk: Maximum page size seen', + 'cache_walk: Minimum on-disk page image size seen', + 'cache_walk: On-disk page image sizes smaller than a single allocation unit', + 'cache_walk: Pages created in memory and never written', + 'cache_walk: Pages currently queued for eviction', + 'cache_walk: Pages that could not be queued for eviction', + 'cache_walk: Refs skipped during cache traversal', + 'cache_walk: Size of the root page', + 'cache_walk: Total number of pages currently in cache', 'session: open cursor count', ] prefix_list = [ @@ -177,9 +213,10 @@ prefix_list = [ 'session', 'block-manager', 'thread-yield', + 'cache_walk', 'async', 'btree', 'thread-state', 'compression', ] -groups = {'cursor': ['cursor', 'session'], 'lsm': ['LSM', 'transaction'], 'system': ['connection', 'data-handle', 'session', 'thread-state'], 'evict': ['block-manager', 'cache', 'connection', 'thread-state'], 'memory': ['cache', 'connection', 'reconciliation']}
\ No newline at end of file +groups = {'cursor': ['cursor', 'session'], 'lsm': ['LSM', 'transaction'], 'system': ['connection', 'data-handle', 'session', 'thread-state'], 'evict': ['block-manager', 'cache', 'cache_walk', 'connection', 'thread-state'], 'memory': ['cache', 'cache_walk', 'connection', 'reconciliation']}
\ No newline at end of file diff --git a/tools/wtstats/wtstats.py b/tools/wtstats/wtstats.py index 3549031c30f..bf5557d12f4 100755 --- a/tools/wtstats/wtstats.py +++ b/tools/wtstats/wtstats.py @@ -115,6 +115,9 @@ def parse_wtstats_file(file, result): # Parse file for line in open(file, 'rU'): month, day, time, v, title = line.strip('\n').split(" ", 4) + # The colon in the URI confuses parsing, strip it out. + if "cache_walk" in title: + title = title.replace("file:", "", 1) result[title].append((month + " " + day + " " + time, v)) |